OwlBoard/timetable-mgr
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add CIF checking logic

Browse Source
This commit is contained in:
Fred Boniface 2024-03-28 22:47:08 +00:00
parent e76956bea2
commit d16634f07f
9 changed files with 127 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
src/cif/check.go
View File

@ -9,37 +9,55 @@ import (
"go.uber.org/zap"
)
func CifCheck(cfg *helpers.Configuration) {
log.Msg.Info("Checking age of CIF Data")
utime, err := dbAccess.CheckUpdateTime(dbAccess.TimetableCollection)
// Break this down in to smaller, simpler functions
func CifCheck(cfg *helpers.Configuration) error {
log.Msg.Debug("Checking age of CIF Data")
metadata, err := dbAccess.GetCifMetadata()
if err != nil {
log.Msg.Error("Error checking last timetable update", zap.Error(err))
log.Msg.Error("Unable to fetch CifMetadata", zap.Error(err))
return err
}
if metadata == nil {
log.Msg.Info("No metadata found for last CIF Update, recreating timetable")
newMeta, err := runUpdate("full", nil)
if err != nil {
log.Msg.Error("CIF Update failed", zap.Error(err))
return err
}
ok := dbAccess.PutCifMetadata(*newMeta)
if !ok {
log.Msg.Warn("CIF Update Successful but metadata update failed")
return nil
}
}
lastUpdate := time.Unix(utime, 0)
currentTime := time.Now().In(time.UTC)
london, _ := time.LoadLocation("Europe/London")
londonTime := currentTime.In(london)
dataAge := currentTime.Sub(lastUpdate)
maxAge := 22 * time.Hour
londonTimeNow := time.Now().In(london)
day := 12 * time.Hour
updateThreshold := londonTimeNow.Add(-day)
availableHour := 6
log.Msg.Debug("CIF Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", maxAge))
if dataAge >= maxAge {
log.Msg.Warn("Timetable data is more than 24 hours old")
if londonTime.Hour() >= updateHour {
runUpdate("daily")
} else {
log.Msg.Debug("Timetable update is required but data is not yet available")
if londonTimeNow.Hour() >= availableHour {
if metadata.LastUpdate.Before(updateThreshold) || metadata.LastUpdate.Equal(updateThreshold) {
newMeta, err := runUpdate("full", metadata)
if err != nil {
log.Msg.Error("CIF Update failed", zap.Error(err))
return err
}
if newMeta == nil {
log.Msg.Info("CIF Update requirements not met, will retry")
return nil
}
ok := dbAccess.PutCifMetadata(*newMeta)
if !ok {
log.Msg.Warn("CIF Update Successful but metadata update failed")
return nil
}
return nil
}
} else if dataAge > 48 {
log.Msg.Warn("Timetable data is more than 48 hours old")
if londonTime.Hour() >= updateHour {
runUpdate("full")
} else {
log.Msg.Debug("Waiting until todays data is available before updating")
}
} else {
log.Msg.Info("Timetable data is less than 24 hours old")
}
log.Msg.Info("CIF Data does not require updating at this time", zap.Time("Last Update", metadata.LastUpdate))
return nil
}

5
src/cif/constants.go Normal file
View File

@ -0,0 +1,5 @@
package cif
const updateHour = 7
const dailyUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-"
const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full"

6
src/cif/helpers.go
View File

@ -8,10 +8,6 @@ import (
"go.uber.org/zap"
)
const updateHour = 7
const dailyUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-"
const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full"
func getDayString() string {
london, err := time.LoadLocation("Europe/London")
if err != nil {
@ -32,6 +28,6 @@ func getUpdateUrl(updateType string) (string, error) {
} else if updateType == "full" {
return fullUpdateUrl, nil
}
err := errors.New("Invalid update type provided, must be one of 'daily' or 'full'")
err := errors.New("invalid update type provided, must be one of 'daily' or 'full'")
return "", err
}

6
src/cif/types.go Normal file
View File

@ -0,0 +1,6 @@
package cif
type CIFUpdate struct {
Timestamp int64
Sequence int64
}

11
src/cif/update.go
View File

@ -3,20 +3,23 @@ package cif
import (
"errors"
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
"git.fjla.uk/owlboard/timetable-mgr/log"
"go.uber.org/zap"
)
func runUpdate(updateType string) error {
func runUpdate(updateType string, metadata *dbAccess.CifMetadata) (*dbAccess.CifMetadata, error) {
url, err := getUpdateUrl(updateType)
if err != nil {
log.Msg.Error("Unable to get the update URL", zap.Error(err))
return err
return nil, err
}
return errors.New("This function is not yet defined")
log.Msg.Debug("", zap.String("URL", url))
return nil, errors.New("function is not yet defined")
// Fetch Data
// Check that the data is not too old. Maybe aim for less than two days?
// Use the values in metadata to determine which day to attempt to update.
// Before running any actions on the data, check the sequence number and timestamp againse previous updates
// Write a parsing function that can handle VSTP as well as SCHEDULE data
// Handle database management
}

2
src/corpus/check.go
View File

@ -10,7 +10,7 @@ import (
)
func CheckCorpus(cfg *helpers.Configuration) {
log.Msg.Info("Checking age of CORPUS Data")
log.Msg.Debug("Checking age of CORPUS Data")
utime, err := dbAccess.CheckUpdateTime(dbAccess.CorpusCollection)
if err != nil {
log.Msg.Error("Error checking last CORPUS update", zap.Error(err))

60
src/dbAccess/cif.go Normal file
View File

@ -0,0 +1,60 @@
package dbAccess
import (
"context"
"errors"
"time"
"git.fjla.uk/owlboard/timetable-mgr/log"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.uber.org/zap"
)
const Doctype = "CifMetadata"
type CifMetadata struct {
Doctype string `json:"type"`
LastUpdate time.Time `json:"lastUpdate"`
LastTimestamp int64 `json:"lastTimestamp"`
LastSequence int64 `json:"lastSequence"`
}
func GetCifMetadata() (*CifMetadata, error) {
database := MongoClient.Database(databaseName)
collection := database.Collection(metaCollection)
filter := bson.M{"type": Doctype}
var result CifMetadata
err := collection.FindOne(context.Background(), filter).Decode(&result)
if err != nil {
if errors.Is(err, mongo.ErrNoDocuments) {
return nil, nil
}
log.Msg.Error("Error fetching CIF Metadata")
return nil, err
}
return &result, nil
}
func PutCifMetadata(metadata CifMetadata) bool {
database := MongoClient.Database(databaseName)
collection := database.Collection(metaCollection)
options := options.Update().SetUpsert(true)
filter := bson.M{"type": Doctype}
update := bson.M{
"type": Doctype,
"LastUpdate": metadata.LastUpdate,
"LastTimestamp": metadata.LastTimestamp,
"LastSequence": metadata.LastSequence,
}
_, err := collection.UpdateOne(context.Background(), filter, update, options)
if err != nil {
log.Msg.Error("Error updating CIF Metadata", zap.Error(err))
return false
}
return true
}

1
src/dbAccess/client.go
View File

@ -16,6 +16,7 @@ const databaseName string = "owlboard"
const CorpusCollection string = "corpus"
const StationsCollection string = "stations"
const metaCollection string = "meta"
const cifMetaCollection string = "cifMeta"
const TimetableCollection string = "timetable"
// Provide the DB Connection to other functions

2
src/helpers/config_loader.go
View File

@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"os"
"strings"
)
type ConfigParameter struct {
@ -92,6 +93,7 @@ func LoadConfig() (*Configuration, error) {
}
func (c *Configuration) setConfigValue(key, value string) {
value = strings.TrimSpace(value)
switch key {
case "nrod_user":
c.NrodUser = value