timetable-extension #1
@ -23,7 +23,7 @@ func CheckCif(cfg *helpers.Configuration) {
|
||||
// Load and read metadata from database
|
||||
metadata, err := dbAccess.GetCifMetadata()
|
||||
if err != nil {
|
||||
log.Msg.Error("Unable to read last update time")
|
||||
log.Msg.Error("Unable to read last update time", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
|
39
cif/metadata.go
Normal file
39
cif/metadata.go
Normal file
@ -0,0 +1,39 @@
|
||||
package cif
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||
)
|
||||
|
||||
// Evaluates last and new metadata and determines whether an update is required.
|
||||
// // If last update == "full" && lastupdate after yesterday at 0600, run update.
|
||||
// // Beyond that condition, comparisons are made between old and new metadata.
|
||||
func checkMetadata(oldMeta *dbAccess.CifMetadata, newMeta *upstreamApi.JsonTimetableV1) (reason string, updateRequired bool) {
|
||||
// Handle nil pointer - although this should be resolved in the calling function in ideal situations
|
||||
if oldMeta == nil || newMeta == nil {
|
||||
return "nil-pointer", false
|
||||
}
|
||||
|
||||
// Handle non-matching sequence numbers between full and daily download types
|
||||
if isAfterYesterdayAt0600(oldMeta.LastUpdate) {
|
||||
return "last-update-full", true
|
||||
}
|
||||
|
||||
// Check that the new metadata sequence is as expected.
|
||||
if newMeta.Metadata.Sequence == oldMeta.LastSequence+1 {
|
||||
return "correct-sequence", true
|
||||
} else {
|
||||
s := fmt.Sprintf("incorrect sequence, Old: %d, New: %d, Expected New: %d", oldMeta.LastSequence, newMeta.Metadata.Sequence, oldMeta.LastSequence+1)
|
||||
return s, false
|
||||
}
|
||||
}
|
||||
|
||||
// Evaluates whether the given time is after yesterday at 0600
|
||||
func isAfterYesterdayAt0600(t time.Time) bool {
|
||||
yesterday0600 := time.Now().In(time.UTC).AddDate(0, 0, -1)
|
||||
yesterday0600 = time.Date(yesterday0600.Year(), yesterday0600.Month(), yesterday0600.Day(), 6, 0, 0, 0, time.UTC)
|
||||
return t.After(yesterday0600)
|
||||
}
|
@ -13,6 +13,7 @@ import (
|
||||
// Processes parsed CIF data and applies the data to the database
|
||||
func processParsedCif(data *parsedData) error {
|
||||
log.Msg.Debug("Starting CIF Processing")
|
||||
log.Msg.Info("Processing CIF Data", zap.Int("schedule-count", len(data.sched)))
|
||||
|
||||
// Batch size for processing
|
||||
batchSize := 250 // Needs optimising for better RAM use. 1000 ~ 5.7GB, 500 ~ 5.5GB, 750 ~ 5.2GB
|
||||
|
@ -1,7 +1,6 @@
|
||||
package cif
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
@ -73,36 +72,24 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
||||
return err
|
||||
}
|
||||
|
||||
// Check CIF Metadata
|
||||
log.Msg.Debug("Starting metadata checks")
|
||||
|
||||
// If last update type is == fullUpdateType and LastUpdate less than twodays? ago then run the update regardless of sequence
|
||||
// this is because the sequence number of full and daily updates to not appear to match up although I will need to check this!
|
||||
//if metadata.LastUpdateType == fullUpdateType && metadata.LastUpdate //after_yesterday_at_0600 {
|
||||
// Somehow skip the below checks! Maybe I need a checkMetadata function!
|
||||
//}
|
||||
|
||||
// Check CIF Sequence
|
||||
// Skip if LastSequence is >= to this sequence
|
||||
if metadata.LastSequence >= parsed.header.Metadata.Sequence {
|
||||
log.Msg.Info("Skipping CIF file, already processed", zap.Int64("LastSequence", metadata.LastSequence), zap.Int64("New sequence", parsed.header.Metadata.Sequence))
|
||||
reason, update := checkMetadata(metadata, &parsed.header)
|
||||
if !update {
|
||||
log.Msg.Warn("Update file not processed", zap.String("reason", reason))
|
||||
continue
|
||||
}
|
||||
|
||||
// Fail if sequence number is not as expected
|
||||
if parsed.header.Metadata.Sequence != metadata.LastSequence+1 {
|
||||
err := errors.New("out of sequence CIF data")
|
||||
log.Msg.Error("CIF sequence not as expected", zap.Error(err), zap.Int64("LastSequence", metadata.LastSequence), zap.Int64("New Sequence", parsed.header.Metadata.Sequence))
|
||||
return err
|
||||
}
|
||||
log.Msg.Debug("Metadata checks complete")
|
||||
log.Msg.Info("CIF Data is suitable for processing", zap.String("reason", reason))
|
||||
|
||||
// Do further sequence checks - parsed.header.Metadata.Sequence MUST = metadata.LastSequence + 1
|
||||
|
||||
log.Msg.Debug("CIF Data has passed checks and should now be processed <<<<<<")
|
||||
// Process CIF file
|
||||
err = processParsedCif(parsed)
|
||||
if err != nil {
|
||||
log.Msg.Error("Error processing CIF data", zap.Error(err))
|
||||
}
|
||||
|
||||
// Temporarily disable METADATA GENERATION AND WRITING
|
||||
// metadata = generateMetadata(&parsed.header)
|
||||
metadata = generateMetadata(&parsed.header)
|
||||
}
|
||||
|
||||
ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType)
|
||||
|
Loading…
Reference in New Issue
Block a user