timetable-extension #1

Open
fred.boniface wants to merge 154 commits from timetable-extension into main
3 changed files with 24 additions and 20 deletions
Showing only changes of commit 76ae25fab6 - Show all commits

View File

@ -11,5 +11,9 @@ const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAu
// The time at which CIF Data is expected to be available for download (full hour)
const dataAvailable = 6
// Define update type strings to pass into metadata
const fullUpdateType = "full"
const dailyUpdateType = "daily"
// An object representing the Europe/London timezone
var londonTimezone, _ = time.LoadLocation("Europe/London")

View File

@ -43,7 +43,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
}
newMeta := generateMetadata(&parsed.header)
ok := dbAccess.PutCifMetadata(newMeta)
ok := dbAccess.PutCifMetadata(newMeta, fullUpdateType)
if !ok {
log.Msg.Warn("CIF Data updated, but metadata write failed")
}
@ -66,12 +66,6 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
return err
}
// DOES NOT WORK WITH NEW NROD STREAMER
// If debug mode is on, call debugWriteDownload
// if helpers.Runtime == "debug" {
// debugWriteDownload(data)
// }
// Parse CIF file
parsed, err := parseCifDataStream(data)
if err != nil {
@ -79,10 +73,14 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
return err
}
// Make `data` a nil pointer as it is no longer required
data = nil
log.Msg.Debug("Starting metadata checks")
// If last update type is == fullUpdateType and LastUpdate less than twodays? ago then run the update regardless of sequence
// this is because the sequence number of full and daily updates to not appear to match up although I will need to check this!
//if metadata.LastUpdateType == fullUpdateType && metadata.LastUpdate //after_yesterday_at_0600 {
// Somehow skip the below checks! Maybe I need a checkMetadata function!
//}
// Check CIF Sequence
// Skip if LastSequence is >= to this sequence
if metadata.LastSequence >= parsed.header.Metadata.Sequence {
@ -107,7 +105,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
// metadata = generateMetadata(&parsed.header)
}
ok := dbAccess.PutCifMetadata(metadata)
ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType)
if !ok {
log.Msg.Warn("CIF Data updated, but metadata write failed.")
}

View File

@ -18,10 +18,11 @@ const Doctype = "CifMetadata"
// The type describing the CifMetadata 'type' in the database.
// This type will be moved to owlboard/go-types
type CifMetadata struct {
Doctype string `bson:"type"`
LastUpdate time.Time `bson:"lastUpdate"`
LastTimestamp int64 `bson:"lastTimestamp"`
LastSequence int64 `bson:"lastSequence"`
Doctype string `bson:"type"`
LastUpdate time.Time `bson:"lastUpdate"`
LastTimestamp int64 `bson:"lastTimestamp"`
LastSequence int64 `bson:"lastSequence"`
LastUpdateType string `bson:"lastUpdateType"`
}
// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
@ -45,17 +46,18 @@ func GetCifMetadata() (*CifMetadata, error) {
}
// Uses upsert to Insert/Update the CifMetadata in the database
func PutCifMetadata(metadata *CifMetadata) bool {
func PutCifMetadata(metadata *CifMetadata, lastUpdateType string) bool {
database := MongoClient.Database(databaseName)
collection := database.Collection(metaCollection)
options := options.Update().SetUpsert(true)
filter := bson.M{"type": Doctype}
update := bson.M{
"$set": bson.M{
"type": Doctype,
"lastUpdate": metadata.LastUpdate,
"lastTimestamp": metadata.LastTimestamp,
"lastSequence": metadata.LastSequence,
"type": Doctype,
"lastUpdate": metadata.LastUpdate,
"lastTimestamp": metadata.LastTimestamp,
"lastSequence": metadata.LastSequence,
"lastUpdateType": lastUpdateType,
},
}