Add CIF checking logic

src/cif/check.go

@@ -9,37 +9,55 @@ import (
 	"go.uber.org/zap"
 )
 
-func CifCheck(cfg *helpers.Configuration) {
+// Break this down in to smaller, simpler functions
-	log.Msg.Info("Checking age of CIF Data")
+func CifCheck(cfg *helpers.Configuration) error {
-	utime, err := dbAccess.CheckUpdateTime(dbAccess.TimetableCollection)
+	log.Msg.Debug("Checking age of CIF Data")
+
+	metadata, err := dbAccess.GetCifMetadata()
 	if err != nil {
-		log.Msg.Error("Error checking last timetable update", zap.Error(err))
+		log.Msg.Error("Unable to fetch CifMetadata", zap.Error(err))
+		return err
+	}
+	if metadata == nil {
+		log.Msg.Info("No metadata found for last CIF Update, recreating timetable")
+		newMeta, err := runUpdate("full", nil)
+		if err != nil {
+			log.Msg.Error("CIF Update failed", zap.Error(err))
+			return err
+		}
+		ok := dbAccess.PutCifMetadata(*newMeta)
+		if !ok {
+			log.Msg.Warn("CIF Update Successful but metadata update failed")
+			return nil
+		}
 	}
 
-	lastUpdate := time.Unix(utime, 0)
-	currentTime := time.Now().In(time.UTC)
 	london, _ := time.LoadLocation("Europe/London")
-	londonTime := currentTime.In(london)
+	londonTimeNow := time.Now().In(london)
-	dataAge := currentTime.Sub(lastUpdate)
+	day := 12 * time.Hour
-	maxAge := 22 * time.Hour
+	updateThreshold := londonTimeNow.Add(-day)
+	availableHour := 6
 
-	log.Msg.Debug("CIF Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", maxAge))
+	if londonTimeNow.Hour() >= availableHour {
+		if metadata.LastUpdate.Before(updateThreshold) || metadata.LastUpdate.Equal(updateThreshold) {
+			newMeta, err := runUpdate("full", metadata)
+			if err != nil {
+				log.Msg.Error("CIF Update failed", zap.Error(err))
+				return err
+			}
+			if newMeta == nil {
+				log.Msg.Info("CIF Update requirements not met, will retry")
+				return nil
+			}
+			ok := dbAccess.PutCifMetadata(*newMeta)
+			if !ok {
+				log.Msg.Warn("CIF Update Successful but metadata update failed")
+				return nil
+			}
+			return nil
+		}
+	}
+	log.Msg.Info("CIF Data does not require updating at this time", zap.Time("Last Update", metadata.LastUpdate))
 
-	if dataAge >= maxAge {
+	return nil
-		log.Msg.Warn("Timetable data is more than 24 hours old")
-		if londonTime.Hour() >= updateHour {
-			runUpdate("daily")
-		} else {
-			log.Msg.Debug("Timetable update is required but data is not yet available")
-		}
-	} else if dataAge > 48 {
-		log.Msg.Warn("Timetable data is more than 48 hours old")
-		if londonTime.Hour() >= updateHour {
-			runUpdate("full")
-		} else {
-			log.Msg.Debug("Waiting until todays data is available before updating")
-		}
-	} else {
-		log.Msg.Info("Timetable data is less than 24 hours old")
-	}
 }

src/cif/constants.go

@@ -0,0 +1,5 @@
+package cif
+
+const updateHour = 7
+const dailyUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-"
+const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full"

src/cif/helpers.go

@@ -8,10 +8,6 @@ import (
 	"go.uber.org/zap"
 )
 
-const updateHour = 7
-const dailyUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-"
-const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full"
-
 func getDayString() string {
 	london, err := time.LoadLocation("Europe/London")
 	if err != nil {
@@ -32,6 +28,6 @@ func getUpdateUrl(updateType string) (string, error) {
 	} else if updateType == "full" {
 		return fullUpdateUrl, nil
 	}
-	err := errors.New("Invalid update type provided, must be one of 'daily' or 'full'")
+	err := errors.New("invalid update type provided, must be one of 'daily' or 'full'")
 	return "", err
 }

src/cif/types.go

@@ -0,0 +1,6 @@
+package cif
+
+type CIFUpdate struct {
+	Timestamp int64
+	Sequence  int64
+}

src/cif/update.go

@@ -3,20 +3,23 @@ package cif
 import (
 	"errors"
 
+	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
 	"git.fjla.uk/owlboard/timetable-mgr/log"
 	"go.uber.org/zap"
 )
 
-func runUpdate(updateType string) error {
+func runUpdate(updateType string, metadata *dbAccess.CifMetadata) (*dbAccess.CifMetadata, error) {
 	url, err := getUpdateUrl(updateType)
 	if err != nil {
 		log.Msg.Error("Unable to get the update URL", zap.Error(err))
-		return err
+		return nil, err
 	}
-	return errors.New("This function is not yet defined")
+	log.Msg.Debug("", zap.String("URL", url))
+	return nil, errors.New("function is not yet defined")
 
 	// Fetch Data
-	// Check that the data is not too old.  Maybe aim for less than two days?
+	// Use the values in metadata to determine which day to attempt to update.
+	// Before running any actions on the data, check the sequence number and timestamp againse previous updates
 	// Write a parsing function that can handle VSTP as well as SCHEDULE data
 	// Handle database management
 }

src/corpus/check.go

@@ -10,7 +10,7 @@ import (
 )
 
 func CheckCorpus(cfg *helpers.Configuration) {
-	log.Msg.Info("Checking age of CORPUS Data")
+	log.Msg.Debug("Checking age of CORPUS Data")
 	utime, err := dbAccess.CheckUpdateTime(dbAccess.CorpusCollection)
 	if err != nil {
 		log.Msg.Error("Error checking last CORPUS update", zap.Error(err))

src/dbAccess/cif.go

@@ -0,0 +1,60 @@
+package dbAccess
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.uber.org/zap"
+)
+
+const Doctype = "CifMetadata"
+
+type CifMetadata struct {
+	Doctype       string    `json:"type"`
+	LastUpdate    time.Time `json:"lastUpdate"`
+	LastTimestamp int64     `json:"lastTimestamp"`
+	LastSequence  int64     `json:"lastSequence"`
+}
+
+func GetCifMetadata() (*CifMetadata, error) {
+	database := MongoClient.Database(databaseName)
+	collection := database.Collection(metaCollection)
+	filter := bson.M{"type": Doctype}
+	var result CifMetadata
+	err := collection.FindOne(context.Background(), filter).Decode(&result)
+	if err != nil {
+		if errors.Is(err, mongo.ErrNoDocuments) {
+			return nil, nil
+		}
+		log.Msg.Error("Error fetching CIF Metadata")
+		return nil, err
+	}
+
+	return &result, nil
+}
+
+func PutCifMetadata(metadata CifMetadata) bool {
+	database := MongoClient.Database(databaseName)
+	collection := database.Collection(metaCollection)
+	options := options.Update().SetUpsert(true)
+	filter := bson.M{"type": Doctype}
+	update := bson.M{
+		"type":          Doctype,
+		"LastUpdate":    metadata.LastUpdate,
+		"LastTimestamp": metadata.LastTimestamp,
+		"LastSequence":  metadata.LastSequence,
+	}
+
+	_, err := collection.UpdateOne(context.Background(), filter, update, options)
+
+	if err != nil {
+		log.Msg.Error("Error updating CIF Metadata", zap.Error(err))
+		return false
+	}
+	return true
+}

src/dbAccess/client.go

@@ -16,6 +16,7 @@ const databaseName string = "owlboard"
 const CorpusCollection string = "corpus"
 const StationsCollection string = "stations"
 const metaCollection string = "meta"
+const cifMetaCollection string = "cifMeta"
 const TimetableCollection string = "timetable"
 
 // Provide the DB Connection to other functions

src/helpers/config_loader.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 )
 
 type ConfigParameter struct {
@@ -92,6 +93,7 @@ func LoadConfig() (*Configuration, error) {
 }
 
 func (c *Configuration) setConfigValue(key, value string) {
+	value = strings.TrimSpace(value)
 	switch key {
 	case "nrod_user":
 		c.NrodUser = value