From 76ae25fab6413d58df3401864bbede5d37c6966a Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Thu, 11 Apr 2024 20:59:02 +0100 Subject: [PATCH] Add LastUpdateType to CIF Metadata --- cif/constants.go | 4 ++++ cif/update.go | 20 +++++++++----------- dbAccess/cif.go | 20 +++++++++++--------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/cif/constants.go b/cif/constants.go index 782c33d..49c6861 100644 --- a/cif/constants.go +++ b/cif/constants.go @@ -11,5 +11,9 @@ const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAu // The time at which CIF Data is expected to be available for download (full hour) const dataAvailable = 6 +// Define update type strings to pass into metadata +const fullUpdateType = "full" +const dailyUpdateType = "daily" + // An object representing the Europe/London timezone var londonTimezone, _ = time.LoadLocation("Europe/London") diff --git a/cif/update.go b/cif/update.go index 77b4846..8093b52 100644 --- a/cif/update.go +++ b/cif/update.go @@ -43,7 +43,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error { } newMeta := generateMetadata(&parsed.header) - ok := dbAccess.PutCifMetadata(newMeta) + ok := dbAccess.PutCifMetadata(newMeta, fullUpdateType) if !ok { log.Msg.Warn("CIF Data updated, but metadata write failed") } @@ -66,12 +66,6 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta return err } - // DOES NOT WORK WITH NEW NROD STREAMER - // If debug mode is on, call debugWriteDownload - // if helpers.Runtime == "debug" { - // debugWriteDownload(data) - // } - // Parse CIF file parsed, err := parseCifDataStream(data) if err != nil { @@ -79,10 +73,14 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta return err } - // Make `data` a nil pointer as it is no longer required - data = nil - log.Msg.Debug("Starting metadata checks") + + // If last update type is == fullUpdateType and LastUpdate less than twodays? ago then run the update regardless of sequence + // this is because the sequence number of full and daily updates to not appear to match up although I will need to check this! + //if metadata.LastUpdateType == fullUpdateType && metadata.LastUpdate //after_yesterday_at_0600 { + // Somehow skip the below checks! Maybe I need a checkMetadata function! + //} + // Check CIF Sequence // Skip if LastSequence is >= to this sequence if metadata.LastSequence >= parsed.header.Metadata.Sequence { @@ -107,7 +105,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta // metadata = generateMetadata(&parsed.header) } - ok := dbAccess.PutCifMetadata(metadata) + ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType) if !ok { log.Msg.Warn("CIF Data updated, but metadata write failed.") } diff --git a/dbAccess/cif.go b/dbAccess/cif.go index a44274e..d10028b 100644 --- a/dbAccess/cif.go +++ b/dbAccess/cif.go @@ -18,10 +18,11 @@ const Doctype = "CifMetadata" // The type describing the CifMetadata 'type' in the database. // This type will be moved to owlboard/go-types type CifMetadata struct { - Doctype string `bson:"type"` - LastUpdate time.Time `bson:"lastUpdate"` - LastTimestamp int64 `bson:"lastTimestamp"` - LastSequence int64 `bson:"lastSequence"` + Doctype string `bson:"type"` + LastUpdate time.Time `bson:"lastUpdate"` + LastTimestamp int64 `bson:"lastTimestamp"` + LastSequence int64 `bson:"lastSequence"` + LastUpdateType string `bson:"lastUpdateType"` } // Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example. @@ -45,17 +46,18 @@ func GetCifMetadata() (*CifMetadata, error) { } // Uses upsert to Insert/Update the CifMetadata in the database -func PutCifMetadata(metadata *CifMetadata) bool { +func PutCifMetadata(metadata *CifMetadata, lastUpdateType string) bool { database := MongoClient.Database(databaseName) collection := database.Collection(metaCollection) options := options.Update().SetUpsert(true) filter := bson.M{"type": Doctype} update := bson.M{ "$set": bson.M{ - "type": Doctype, - "lastUpdate": metadata.LastUpdate, - "lastTimestamp": metadata.LastTimestamp, - "lastSequence": metadata.LastSequence, + "type": Doctype, + "lastUpdate": metadata.LastUpdate, + "lastTimestamp": metadata.LastTimestamp, + "lastSequence": metadata.LastSequence, + "lastUpdateType": lastUpdateType, }, }