From d16634f07fa073fdfd9c0e17236e4b6eb0833cf0 Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Thu, 28 Mar 2024 22:47:08 +0000 Subject: [PATCH] Add CIF checking logic --- src/cif/check.go | 70 ++++++++++++++++++++++-------------- src/cif/constants.go | 5 +++ src/cif/helpers.go | 6 +--- src/cif/types.go | 6 ++++ src/cif/update.go | 11 +++--- src/corpus/check.go | 2 +- src/dbAccess/cif.go | 60 +++++++++++++++++++++++++++++++ src/dbAccess/client.go | 1 + src/helpers/config_loader.go | 2 ++ 9 files changed, 127 insertions(+), 36 deletions(-) create mode 100644 src/cif/constants.go create mode 100644 src/cif/types.go create mode 100644 src/dbAccess/cif.go diff --git a/src/cif/check.go b/src/cif/check.go index 8e12768..fbbb6fb 100644 --- a/src/cif/check.go +++ b/src/cif/check.go @@ -9,37 +9,55 @@ import ( "go.uber.org/zap" ) -func CifCheck(cfg *helpers.Configuration) { - log.Msg.Info("Checking age of CIF Data") - utime, err := dbAccess.CheckUpdateTime(dbAccess.TimetableCollection) +// Break this down in to smaller, simpler functions +func CifCheck(cfg *helpers.Configuration) error { + log.Msg.Debug("Checking age of CIF Data") + + metadata, err := dbAccess.GetCifMetadata() if err != nil { - log.Msg.Error("Error checking last timetable update", zap.Error(err)) + log.Msg.Error("Unable to fetch CifMetadata", zap.Error(err)) + return err + } + if metadata == nil { + log.Msg.Info("No metadata found for last CIF Update, recreating timetable") + newMeta, err := runUpdate("full", nil) + if err != nil { + log.Msg.Error("CIF Update failed", zap.Error(err)) + return err + } + ok := dbAccess.PutCifMetadata(*newMeta) + if !ok { + log.Msg.Warn("CIF Update Successful but metadata update failed") + return nil + } } - lastUpdate := time.Unix(utime, 0) - currentTime := time.Now().In(time.UTC) london, _ := time.LoadLocation("Europe/London") - londonTime := currentTime.In(london) - dataAge := currentTime.Sub(lastUpdate) - maxAge := 22 * time.Hour + londonTimeNow := time.Now().In(london) + day := 12 * time.Hour + updateThreshold := londonTimeNow.Add(-day) + availableHour := 6 - log.Msg.Debug("CIF Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", maxAge)) - - if dataAge >= maxAge { - log.Msg.Warn("Timetable data is more than 24 hours old") - if londonTime.Hour() >= updateHour { - runUpdate("daily") - } else { - log.Msg.Debug("Timetable update is required but data is not yet available") + if londonTimeNow.Hour() >= availableHour { + if metadata.LastUpdate.Before(updateThreshold) || metadata.LastUpdate.Equal(updateThreshold) { + newMeta, err := runUpdate("full", metadata) + if err != nil { + log.Msg.Error("CIF Update failed", zap.Error(err)) + return err + } + if newMeta == nil { + log.Msg.Info("CIF Update requirements not met, will retry") + return nil + } + ok := dbAccess.PutCifMetadata(*newMeta) + if !ok { + log.Msg.Warn("CIF Update Successful but metadata update failed") + return nil + } + return nil } - } else if dataAge > 48 { - log.Msg.Warn("Timetable data is more than 48 hours old") - if londonTime.Hour() >= updateHour { - runUpdate("full") - } else { - log.Msg.Debug("Waiting until todays data is available before updating") - } - } else { - log.Msg.Info("Timetable data is less than 24 hours old") } + log.Msg.Info("CIF Data does not require updating at this time", zap.Time("Last Update", metadata.LastUpdate)) + + return nil } diff --git a/src/cif/constants.go b/src/cif/constants.go new file mode 100644 index 0000000..4d934e0 --- /dev/null +++ b/src/cif/constants.go @@ -0,0 +1,5 @@ +package cif + +const updateHour = 7 +const dailyUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-" +const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full" diff --git a/src/cif/helpers.go b/src/cif/helpers.go index 66faf5f..7109568 100644 --- a/src/cif/helpers.go +++ b/src/cif/helpers.go @@ -8,10 +8,6 @@ import ( "go.uber.org/zap" ) -const updateHour = 7 -const dailyUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-" -const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full" - func getDayString() string { london, err := time.LoadLocation("Europe/London") if err != nil { @@ -32,6 +28,6 @@ func getUpdateUrl(updateType string) (string, error) { } else if updateType == "full" { return fullUpdateUrl, nil } - err := errors.New("Invalid update type provided, must be one of 'daily' or 'full'") + err := errors.New("invalid update type provided, must be one of 'daily' or 'full'") return "", err } diff --git a/src/cif/types.go b/src/cif/types.go new file mode 100644 index 0000000..9c8be6f --- /dev/null +++ b/src/cif/types.go @@ -0,0 +1,6 @@ +package cif + +type CIFUpdate struct { + Timestamp int64 + Sequence int64 +} diff --git a/src/cif/update.go b/src/cif/update.go index 7a4d178..ece9950 100644 --- a/src/cif/update.go +++ b/src/cif/update.go @@ -3,20 +3,23 @@ package cif import ( "errors" + "git.fjla.uk/owlboard/timetable-mgr/dbAccess" "git.fjla.uk/owlboard/timetable-mgr/log" "go.uber.org/zap" ) -func runUpdate(updateType string) error { +func runUpdate(updateType string, metadata *dbAccess.CifMetadata) (*dbAccess.CifMetadata, error) { url, err := getUpdateUrl(updateType) if err != nil { log.Msg.Error("Unable to get the update URL", zap.Error(err)) - return err + return nil, err } - return errors.New("This function is not yet defined") + log.Msg.Debug("", zap.String("URL", url)) + return nil, errors.New("function is not yet defined") // Fetch Data - // Check that the data is not too old. Maybe aim for less than two days? + // Use the values in metadata to determine which day to attempt to update. + // Before running any actions on the data, check the sequence number and timestamp againse previous updates // Write a parsing function that can handle VSTP as well as SCHEDULE data // Handle database management } diff --git a/src/corpus/check.go b/src/corpus/check.go index 992882d..418d6f3 100644 --- a/src/corpus/check.go +++ b/src/corpus/check.go @@ -10,7 +10,7 @@ import ( ) func CheckCorpus(cfg *helpers.Configuration) { - log.Msg.Info("Checking age of CORPUS Data") + log.Msg.Debug("Checking age of CORPUS Data") utime, err := dbAccess.CheckUpdateTime(dbAccess.CorpusCollection) if err != nil { log.Msg.Error("Error checking last CORPUS update", zap.Error(err)) diff --git a/src/dbAccess/cif.go b/src/dbAccess/cif.go new file mode 100644 index 0000000..f829217 --- /dev/null +++ b/src/dbAccess/cif.go @@ -0,0 +1,60 @@ +package dbAccess + +import ( + "context" + "errors" + "time" + + "git.fjla.uk/owlboard/timetable-mgr/log" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.uber.org/zap" +) + +const Doctype = "CifMetadata" + +type CifMetadata struct { + Doctype string `json:"type"` + LastUpdate time.Time `json:"lastUpdate"` + LastTimestamp int64 `json:"lastTimestamp"` + LastSequence int64 `json:"lastSequence"` +} + +func GetCifMetadata() (*CifMetadata, error) { + database := MongoClient.Database(databaseName) + collection := database.Collection(metaCollection) + filter := bson.M{"type": Doctype} + var result CifMetadata + err := collection.FindOne(context.Background(), filter).Decode(&result) + if err != nil { + if errors.Is(err, mongo.ErrNoDocuments) { + return nil, nil + } + log.Msg.Error("Error fetching CIF Metadata") + return nil, err + } + + return &result, nil +} + +func PutCifMetadata(metadata CifMetadata) bool { + database := MongoClient.Database(databaseName) + collection := database.Collection(metaCollection) + options := options.Update().SetUpsert(true) + filter := bson.M{"type": Doctype} + update := bson.M{ + "type": Doctype, + "LastUpdate": metadata.LastUpdate, + "LastTimestamp": metadata.LastTimestamp, + "LastSequence": metadata.LastSequence, + } + + _, err := collection.UpdateOne(context.Background(), filter, update, options) + + if err != nil { + log.Msg.Error("Error updating CIF Metadata", zap.Error(err)) + return false + } + return true +} diff --git a/src/dbAccess/client.go b/src/dbAccess/client.go index fbf6d89..3bf38a1 100644 --- a/src/dbAccess/client.go +++ b/src/dbAccess/client.go @@ -16,6 +16,7 @@ const databaseName string = "owlboard" const CorpusCollection string = "corpus" const StationsCollection string = "stations" const metaCollection string = "meta" +const cifMetaCollection string = "cifMeta" const TimetableCollection string = "timetable" // Provide the DB Connection to other functions diff --git a/src/helpers/config_loader.go b/src/helpers/config_loader.go index e83000f..a3012da 100644 --- a/src/helpers/config_loader.go +++ b/src/helpers/config_loader.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "strings" ) type ConfigParameter struct { @@ -92,6 +93,7 @@ func LoadConfig() (*Configuration, error) { } func (c *Configuration) setConfigValue(key, value string) { + value = strings.TrimSpace(value) switch key { case "nrod_user": c.NrodUser = value