From 6ec8cd889572cb40b38971407b006b7d3d84b2eb Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Fri, 12 Apr 2024 20:43:03 +0100 Subject: [PATCH] Complete CIF Metadata parsing logic --- cif/check.go | 2 +- cif/metadata.go | 39 +++++++++++++++++++++++++++++++++++++++ cif/process.go | 1 + cif/update.go | 33 ++++++++++----------------------- 4 files changed, 51 insertions(+), 24 deletions(-) create mode 100644 cif/metadata.go diff --git a/cif/check.go b/cif/check.go index ca4d7d3..1bc7edd 100644 --- a/cif/check.go +++ b/cif/check.go @@ -23,7 +23,7 @@ func CheckCif(cfg *helpers.Configuration) { // Load and read metadata from database metadata, err := dbAccess.GetCifMetadata() if err != nil { - log.Msg.Error("Unable to read last update time") + log.Msg.Error("Unable to read last update time", zap.Error(err)) return } diff --git a/cif/metadata.go b/cif/metadata.go new file mode 100644 index 0000000..8f6e380 --- /dev/null +++ b/cif/metadata.go @@ -0,0 +1,39 @@ +package cif + +import ( + "fmt" + "time" + + "git.fjla.uk/owlboard/go-types/pkg/upstreamApi" + "git.fjla.uk/owlboard/timetable-mgr/dbAccess" +) + +// Evaluates last and new metadata and determines whether an update is required. +// // If last update == "full" && lastupdate after yesterday at 0600, run update. +// // Beyond that condition, comparisons are made between old and new metadata. +func checkMetadata(oldMeta *dbAccess.CifMetadata, newMeta *upstreamApi.JsonTimetableV1) (reason string, updateRequired bool) { + // Handle nil pointer - although this should be resolved in the calling function in ideal situations + if oldMeta == nil || newMeta == nil { + return "nil-pointer", false + } + + // Handle non-matching sequence numbers between full and daily download types + if isAfterYesterdayAt0600(oldMeta.LastUpdate) { + return "last-update-full", true + } + + // Check that the new metadata sequence is as expected. + if newMeta.Metadata.Sequence == oldMeta.LastSequence+1 { + return "correct-sequence", true + } else { + s := fmt.Sprintf("incorrect sequence, Old: %d, New: %d, Expected New: %d", oldMeta.LastSequence, newMeta.Metadata.Sequence, oldMeta.LastSequence+1) + return s, false + } +} + +// Evaluates whether the given time is after yesterday at 0600 +func isAfterYesterdayAt0600(t time.Time) bool { + yesterday0600 := time.Now().In(time.UTC).AddDate(0, 0, -1) + yesterday0600 = time.Date(yesterday0600.Year(), yesterday0600.Month(), yesterday0600.Day(), 6, 0, 0, 0, time.UTC) + return t.After(yesterday0600) +} diff --git a/cif/process.go b/cif/process.go index cf83a82..d5bd2ec 100644 --- a/cif/process.go +++ b/cif/process.go @@ -13,6 +13,7 @@ import ( // Processes parsed CIF data and applies the data to the database func processParsedCif(data *parsedData) error { log.Msg.Debug("Starting CIF Processing") + log.Msg.Info("Processing CIF Data", zap.Int("schedule-count", len(data.sched))) // Batch size for processing batchSize := 250 // Needs optimising for better RAM use. 1000 ~ 5.7GB, 500 ~ 5.5GB, 750 ~ 5.2GB diff --git a/cif/update.go b/cif/update.go index 8093b52..3da3d62 100644 --- a/cif/update.go +++ b/cif/update.go @@ -1,7 +1,6 @@ package cif import ( - "errors" "io" "time" @@ -73,36 +72,24 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta return err } + // Check CIF Metadata log.Msg.Debug("Starting metadata checks") - // If last update type is == fullUpdateType and LastUpdate less than twodays? ago then run the update regardless of sequence - // this is because the sequence number of full and daily updates to not appear to match up although I will need to check this! - //if metadata.LastUpdateType == fullUpdateType && metadata.LastUpdate //after_yesterday_at_0600 { - // Somehow skip the below checks! Maybe I need a checkMetadata function! - //} - - // Check CIF Sequence - // Skip if LastSequence is >= to this sequence - if metadata.LastSequence >= parsed.header.Metadata.Sequence { - log.Msg.Info("Skipping CIF file, already processed", zap.Int64("LastSequence", metadata.LastSequence), zap.Int64("New sequence", parsed.header.Metadata.Sequence)) + reason, update := checkMetadata(metadata, &parsed.header) + if !update { + log.Msg.Warn("Update file not processed", zap.String("reason", reason)) continue } - // Fail if sequence number is not as expected - if parsed.header.Metadata.Sequence != metadata.LastSequence+1 { - err := errors.New("out of sequence CIF data") - log.Msg.Error("CIF sequence not as expected", zap.Error(err), zap.Int64("LastSequence", metadata.LastSequence), zap.Int64("New Sequence", parsed.header.Metadata.Sequence)) - return err - } - log.Msg.Debug("Metadata checks complete") + log.Msg.Info("CIF Data is suitable for processing", zap.String("reason", reason)) - // Do further sequence checks - parsed.header.Metadata.Sequence MUST = metadata.LastSequence + 1 - - log.Msg.Debug("CIF Data has passed checks and should now be processed <<<<<<") // Process CIF file + err = processParsedCif(parsed) + if err != nil { + log.Msg.Error("Error processing CIF data", zap.Error(err)) + } - // Temporarily disable METADATA GENERATION AND WRITING - // metadata = generateMetadata(&parsed.header) + metadata = generateMetadata(&parsed.header) } ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType)