Add CIF checking logic
This commit is contained in:
parent
e76956bea2
commit
d16634f07f
@ -9,37 +9,55 @@ import (
|
|||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
func CifCheck(cfg *helpers.Configuration) {
|
// Break this down in to smaller, simpler functions
|
||||||
log.Msg.Info("Checking age of CIF Data")
|
func CifCheck(cfg *helpers.Configuration) error {
|
||||||
utime, err := dbAccess.CheckUpdateTime(dbAccess.TimetableCollection)
|
log.Msg.Debug("Checking age of CIF Data")
|
||||||
|
|
||||||
|
metadata, err := dbAccess.GetCifMetadata()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Msg.Error("Error checking last timetable update", zap.Error(err))
|
log.Msg.Error("Unable to fetch CifMetadata", zap.Error(err))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if metadata == nil {
|
||||||
|
log.Msg.Info("No metadata found for last CIF Update, recreating timetable")
|
||||||
|
newMeta, err := runUpdate("full", nil)
|
||||||
|
if err != nil {
|
||||||
|
log.Msg.Error("CIF Update failed", zap.Error(err))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ok := dbAccess.PutCifMetadata(*newMeta)
|
||||||
|
if !ok {
|
||||||
|
log.Msg.Warn("CIF Update Successful but metadata update failed")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
lastUpdate := time.Unix(utime, 0)
|
|
||||||
currentTime := time.Now().In(time.UTC)
|
|
||||||
london, _ := time.LoadLocation("Europe/London")
|
london, _ := time.LoadLocation("Europe/London")
|
||||||
londonTime := currentTime.In(london)
|
londonTimeNow := time.Now().In(london)
|
||||||
dataAge := currentTime.Sub(lastUpdate)
|
day := 12 * time.Hour
|
||||||
maxAge := 22 * time.Hour
|
updateThreshold := londonTimeNow.Add(-day)
|
||||||
|
availableHour := 6
|
||||||
|
|
||||||
log.Msg.Debug("CIF Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", maxAge))
|
if londonTimeNow.Hour() >= availableHour {
|
||||||
|
if metadata.LastUpdate.Before(updateThreshold) || metadata.LastUpdate.Equal(updateThreshold) {
|
||||||
if dataAge >= maxAge {
|
newMeta, err := runUpdate("full", metadata)
|
||||||
log.Msg.Warn("Timetable data is more than 24 hours old")
|
if err != nil {
|
||||||
if londonTime.Hour() >= updateHour {
|
log.Msg.Error("CIF Update failed", zap.Error(err))
|
||||||
runUpdate("daily")
|
return err
|
||||||
} else {
|
}
|
||||||
log.Msg.Debug("Timetable update is required but data is not yet available")
|
if newMeta == nil {
|
||||||
|
log.Msg.Info("CIF Update requirements not met, will retry")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ok := dbAccess.PutCifMetadata(*newMeta)
|
||||||
|
if !ok {
|
||||||
|
log.Msg.Warn("CIF Update Successful but metadata update failed")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
} else if dataAge > 48 {
|
|
||||||
log.Msg.Warn("Timetable data is more than 48 hours old")
|
|
||||||
if londonTime.Hour() >= updateHour {
|
|
||||||
runUpdate("full")
|
|
||||||
} else {
|
|
||||||
log.Msg.Debug("Waiting until todays data is available before updating")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
log.Msg.Info("Timetable data is less than 24 hours old")
|
|
||||||
}
|
}
|
||||||
|
log.Msg.Info("CIF Data does not require updating at this time", zap.Time("Last Update", metadata.LastUpdate))
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
5
src/cif/constants.go
Normal file
5
src/cif/constants.go
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
package cif
|
||||||
|
|
||||||
|
const updateHour = 7
|
||||||
|
const dailyUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-"
|
||||||
|
const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full"
|
@ -8,10 +8,6 @@ import (
|
|||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
const updateHour = 7
|
|
||||||
const dailyUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-"
|
|
||||||
const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full"
|
|
||||||
|
|
||||||
func getDayString() string {
|
func getDayString() string {
|
||||||
london, err := time.LoadLocation("Europe/London")
|
london, err := time.LoadLocation("Europe/London")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -32,6 +28,6 @@ func getUpdateUrl(updateType string) (string, error) {
|
|||||||
} else if updateType == "full" {
|
} else if updateType == "full" {
|
||||||
return fullUpdateUrl, nil
|
return fullUpdateUrl, nil
|
||||||
}
|
}
|
||||||
err := errors.New("Invalid update type provided, must be one of 'daily' or 'full'")
|
err := errors.New("invalid update type provided, must be one of 'daily' or 'full'")
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
6
src/cif/types.go
Normal file
6
src/cif/types.go
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
package cif
|
||||||
|
|
||||||
|
type CIFUpdate struct {
|
||||||
|
Timestamp int64
|
||||||
|
Sequence int64
|
||||||
|
}
|
@ -3,20 +3,23 @@ package cif
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
|
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
func runUpdate(updateType string) error {
|
func runUpdate(updateType string, metadata *dbAccess.CifMetadata) (*dbAccess.CifMetadata, error) {
|
||||||
url, err := getUpdateUrl(updateType)
|
url, err := getUpdateUrl(updateType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Msg.Error("Unable to get the update URL", zap.Error(err))
|
log.Msg.Error("Unable to get the update URL", zap.Error(err))
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
return errors.New("This function is not yet defined")
|
log.Msg.Debug("", zap.String("URL", url))
|
||||||
|
return nil, errors.New("function is not yet defined")
|
||||||
|
|
||||||
// Fetch Data
|
// Fetch Data
|
||||||
// Check that the data is not too old. Maybe aim for less than two days?
|
// Use the values in metadata to determine which day to attempt to update.
|
||||||
|
// Before running any actions on the data, check the sequence number and timestamp againse previous updates
|
||||||
// Write a parsing function that can handle VSTP as well as SCHEDULE data
|
// Write a parsing function that can handle VSTP as well as SCHEDULE data
|
||||||
// Handle database management
|
// Handle database management
|
||||||
}
|
}
|
||||||
|
@ -10,7 +10,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func CheckCorpus(cfg *helpers.Configuration) {
|
func CheckCorpus(cfg *helpers.Configuration) {
|
||||||
log.Msg.Info("Checking age of CORPUS Data")
|
log.Msg.Debug("Checking age of CORPUS Data")
|
||||||
utime, err := dbAccess.CheckUpdateTime(dbAccess.CorpusCollection)
|
utime, err := dbAccess.CheckUpdateTime(dbAccess.CorpusCollection)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Msg.Error("Error checking last CORPUS update", zap.Error(err))
|
log.Msg.Error("Error checking last CORPUS update", zap.Error(err))
|
||||||
|
60
src/dbAccess/cif.go
Normal file
60
src/dbAccess/cif.go
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
package dbAccess
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||||
|
"go.mongodb.org/mongo-driver/bson"
|
||||||
|
"go.mongodb.org/mongo-driver/mongo"
|
||||||
|
"go.mongodb.org/mongo-driver/mongo/options"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Doctype = "CifMetadata"
|
||||||
|
|
||||||
|
type CifMetadata struct {
|
||||||
|
Doctype string `json:"type"`
|
||||||
|
LastUpdate time.Time `json:"lastUpdate"`
|
||||||
|
LastTimestamp int64 `json:"lastTimestamp"`
|
||||||
|
LastSequence int64 `json:"lastSequence"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetCifMetadata() (*CifMetadata, error) {
|
||||||
|
database := MongoClient.Database(databaseName)
|
||||||
|
collection := database.Collection(metaCollection)
|
||||||
|
filter := bson.M{"type": Doctype}
|
||||||
|
var result CifMetadata
|
||||||
|
err := collection.FindOne(context.Background(), filter).Decode(&result)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
log.Msg.Error("Error fetching CIF Metadata")
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func PutCifMetadata(metadata CifMetadata) bool {
|
||||||
|
database := MongoClient.Database(databaseName)
|
||||||
|
collection := database.Collection(metaCollection)
|
||||||
|
options := options.Update().SetUpsert(true)
|
||||||
|
filter := bson.M{"type": Doctype}
|
||||||
|
update := bson.M{
|
||||||
|
"type": Doctype,
|
||||||
|
"LastUpdate": metadata.LastUpdate,
|
||||||
|
"LastTimestamp": metadata.LastTimestamp,
|
||||||
|
"LastSequence": metadata.LastSequence,
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := collection.UpdateOne(context.Background(), filter, update, options)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Msg.Error("Error updating CIF Metadata", zap.Error(err))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
@ -16,6 +16,7 @@ const databaseName string = "owlboard"
|
|||||||
const CorpusCollection string = "corpus"
|
const CorpusCollection string = "corpus"
|
||||||
const StationsCollection string = "stations"
|
const StationsCollection string = "stations"
|
||||||
const metaCollection string = "meta"
|
const metaCollection string = "meta"
|
||||||
|
const cifMetaCollection string = "cifMeta"
|
||||||
const TimetableCollection string = "timetable"
|
const TimetableCollection string = "timetable"
|
||||||
|
|
||||||
// Provide the DB Connection to other functions
|
// Provide the DB Connection to other functions
|
||||||
|
@ -4,6 +4,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ConfigParameter struct {
|
type ConfigParameter struct {
|
||||||
@ -92,6 +93,7 @@ func LoadConfig() (*Configuration, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *Configuration) setConfigValue(key, value string) {
|
func (c *Configuration) setConfigValue(key, value string) {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
switch key {
|
switch key {
|
||||||
case "nrod_user":
|
case "nrod_user":
|
||||||
c.NrodUser = value
|
c.NrodUser = value
|
||||||
|
Loading…
Reference in New Issue
Block a user