timetable-extension #1
@ -9,37 +9,55 @@ import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func CifCheck(cfg *helpers.Configuration) {
|
||||
log.Msg.Info("Checking age of CIF Data")
|
||||
utime, err := dbAccess.CheckUpdateTime(dbAccess.TimetableCollection)
|
||||
// Break this down in to smaller, simpler functions
|
||||
func CifCheck(cfg *helpers.Configuration) error {
|
||||
log.Msg.Debug("Checking age of CIF Data")
|
||||
|
||||
metadata, err := dbAccess.GetCifMetadata()
|
||||
if err != nil {
|
||||
log.Msg.Error("Error checking last timetable update", zap.Error(err))
|
||||
log.Msg.Error("Unable to fetch CifMetadata", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
if metadata == nil {
|
||||
log.Msg.Info("No metadata found for last CIF Update, recreating timetable")
|
||||
newMeta, err := runUpdate("full", nil)
|
||||
if err != nil {
|
||||
log.Msg.Error("CIF Update failed", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
ok := dbAccess.PutCifMetadata(*newMeta)
|
||||
if !ok {
|
||||
log.Msg.Warn("CIF Update Successful but metadata update failed")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
lastUpdate := time.Unix(utime, 0)
|
||||
currentTime := time.Now().In(time.UTC)
|
||||
london, _ := time.LoadLocation("Europe/London")
|
||||
londonTime := currentTime.In(london)
|
||||
dataAge := currentTime.Sub(lastUpdate)
|
||||
maxAge := 22 * time.Hour
|
||||
londonTimeNow := time.Now().In(london)
|
||||
day := 12 * time.Hour
|
||||
updateThreshold := londonTimeNow.Add(-day)
|
||||
availableHour := 6
|
||||
|
||||
log.Msg.Debug("CIF Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", maxAge))
|
||||
|
||||
if dataAge >= maxAge {
|
||||
log.Msg.Warn("Timetable data is more than 24 hours old")
|
||||
if londonTime.Hour() >= updateHour {
|
||||
runUpdate("daily")
|
||||
} else {
|
||||
log.Msg.Debug("Timetable update is required but data is not yet available")
|
||||
if londonTimeNow.Hour() >= availableHour {
|
||||
if metadata.LastUpdate.Before(updateThreshold) || metadata.LastUpdate.Equal(updateThreshold) {
|
||||
newMeta, err := runUpdate("full", metadata)
|
||||
if err != nil {
|
||||
log.Msg.Error("CIF Update failed", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
if newMeta == nil {
|
||||
log.Msg.Info("CIF Update requirements not met, will retry")
|
||||
return nil
|
||||
}
|
||||
ok := dbAccess.PutCifMetadata(*newMeta)
|
||||
if !ok {
|
||||
log.Msg.Warn("CIF Update Successful but metadata update failed")
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
} else if dataAge > 48 {
|
||||
log.Msg.Warn("Timetable data is more than 48 hours old")
|
||||
if londonTime.Hour() >= updateHour {
|
||||
runUpdate("full")
|
||||
} else {
|
||||
log.Msg.Debug("Waiting until todays data is available before updating")
|
||||
}
|
||||
} else {
|
||||
log.Msg.Info("Timetable data is less than 24 hours old")
|
||||
}
|
||||
log.Msg.Info("CIF Data does not require updating at this time", zap.Time("Last Update", metadata.LastUpdate))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
5
src/cif/constants.go
Normal file
5
src/cif/constants.go
Normal file
@ -0,0 +1,5 @@
|
||||
package cif
|
||||
|
||||
const updateHour = 7
|
||||
const dailyUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-"
|
||||
const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full"
|
@ -8,10 +8,6 @@ import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const updateHour = 7
|
||||
const dailyUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-"
|
||||
const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full"
|
||||
|
||||
func getDayString() string {
|
||||
london, err := time.LoadLocation("Europe/London")
|
||||
if err != nil {
|
||||
@ -32,6 +28,6 @@ func getUpdateUrl(updateType string) (string, error) {
|
||||
} else if updateType == "full" {
|
||||
return fullUpdateUrl, nil
|
||||
}
|
||||
err := errors.New("Invalid update type provided, must be one of 'daily' or 'full'")
|
||||
err := errors.New("invalid update type provided, must be one of 'daily' or 'full'")
|
||||
return "", err
|
||||
}
|
||||
|
6
src/cif/types.go
Normal file
6
src/cif/types.go
Normal file
@ -0,0 +1,6 @@
|
||||
package cif
|
||||
|
||||
type CIFUpdate struct {
|
||||
Timestamp int64
|
||||
Sequence int64
|
||||
}
|
@ -3,20 +3,23 @@ package cif
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func runUpdate(updateType string) error {
|
||||
func runUpdate(updateType string, metadata *dbAccess.CifMetadata) (*dbAccess.CifMetadata, error) {
|
||||
url, err := getUpdateUrl(updateType)
|
||||
if err != nil {
|
||||
log.Msg.Error("Unable to get the update URL", zap.Error(err))
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
return errors.New("This function is not yet defined")
|
||||
log.Msg.Debug("", zap.String("URL", url))
|
||||
return nil, errors.New("function is not yet defined")
|
||||
|
||||
// Fetch Data
|
||||
// Check that the data is not too old. Maybe aim for less than two days?
|
||||
// Use the values in metadata to determine which day to attempt to update.
|
||||
// Before running any actions on the data, check the sequence number and timestamp againse previous updates
|
||||
// Write a parsing function that can handle VSTP as well as SCHEDULE data
|
||||
// Handle database management
|
||||
}
|
||||
|
@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
func CheckCorpus(cfg *helpers.Configuration) {
|
||||
log.Msg.Info("Checking age of CORPUS Data")
|
||||
log.Msg.Debug("Checking age of CORPUS Data")
|
||||
utime, err := dbAccess.CheckUpdateTime(dbAccess.CorpusCollection)
|
||||
if err != nil {
|
||||
log.Msg.Error("Error checking last CORPUS update", zap.Error(err))
|
||||
|
60
src/dbAccess/cif.go
Normal file
60
src/dbAccess/cif.go
Normal file
@ -0,0 +1,60 @@
|
||||
package dbAccess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const Doctype = "CifMetadata"
|
||||
|
||||
type CifMetadata struct {
|
||||
Doctype string `json:"type"`
|
||||
LastUpdate time.Time `json:"lastUpdate"`
|
||||
LastTimestamp int64 `json:"lastTimestamp"`
|
||||
LastSequence int64 `json:"lastSequence"`
|
||||
}
|
||||
|
||||
func GetCifMetadata() (*CifMetadata, error) {
|
||||
database := MongoClient.Database(databaseName)
|
||||
collection := database.Collection(metaCollection)
|
||||
filter := bson.M{"type": Doctype}
|
||||
var result CifMetadata
|
||||
err := collection.FindOne(context.Background(), filter).Decode(&result)
|
||||
if err != nil {
|
||||
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return nil, nil
|
||||
}
|
||||
log.Msg.Error("Error fetching CIF Metadata")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func PutCifMetadata(metadata CifMetadata) bool {
|
||||
database := MongoClient.Database(databaseName)
|
||||
collection := database.Collection(metaCollection)
|
||||
options := options.Update().SetUpsert(true)
|
||||
filter := bson.M{"type": Doctype}
|
||||
update := bson.M{
|
||||
"type": Doctype,
|
||||
"LastUpdate": metadata.LastUpdate,
|
||||
"LastTimestamp": metadata.LastTimestamp,
|
||||
"LastSequence": metadata.LastSequence,
|
||||
}
|
||||
|
||||
_, err := collection.UpdateOne(context.Background(), filter, update, options)
|
||||
|
||||
if err != nil {
|
||||
log.Msg.Error("Error updating CIF Metadata", zap.Error(err))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
@ -16,6 +16,7 @@ const databaseName string = "owlboard"
|
||||
const CorpusCollection string = "corpus"
|
||||
const StationsCollection string = "stations"
|
||||
const metaCollection string = "meta"
|
||||
const cifMetaCollection string = "cifMeta"
|
||||
const TimetableCollection string = "timetable"
|
||||
|
||||
// Provide the DB Connection to other functions
|
||||
|
@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type ConfigParameter struct {
|
||||
@ -92,6 +93,7 @@ func LoadConfig() (*Configuration, error) {
|
||||
}
|
||||
|
||||
func (c *Configuration) setConfigValue(key, value string) {
|
||||
value = strings.TrimSpace(value)
|
||||
switch key {
|
||||
case "nrod_user":
|
||||
c.NrodUser = value
|
||||
|
Loading…
Reference in New Issue
Block a user