Add LastUpdateType to CIF Metadata
All checks were successful
Go Test / test (push) Successful in 37s
All checks were successful
Go Test / test (push) Successful in 37s
This commit is contained in:
parent
4459d4d316
commit
76ae25fab6
@ -11,5 +11,9 @@ const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAu
|
|||||||
// The time at which CIF Data is expected to be available for download (full hour)
|
// The time at which CIF Data is expected to be available for download (full hour)
|
||||||
const dataAvailable = 6
|
const dataAvailable = 6
|
||||||
|
|
||||||
|
// Define update type strings to pass into metadata
|
||||||
|
const fullUpdateType = "full"
|
||||||
|
const dailyUpdateType = "daily"
|
||||||
|
|
||||||
// An object representing the Europe/London timezone
|
// An object representing the Europe/London timezone
|
||||||
var londonTimezone, _ = time.LoadLocation("Europe/London")
|
var londonTimezone, _ = time.LoadLocation("Europe/London")
|
||||||
|
@ -43,7 +43,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
newMeta := generateMetadata(&parsed.header)
|
newMeta := generateMetadata(&parsed.header)
|
||||||
ok := dbAccess.PutCifMetadata(newMeta)
|
ok := dbAccess.PutCifMetadata(newMeta, fullUpdateType)
|
||||||
if !ok {
|
if !ok {
|
||||||
log.Msg.Warn("CIF Data updated, but metadata write failed")
|
log.Msg.Warn("CIF Data updated, but metadata write failed")
|
||||||
}
|
}
|
||||||
@ -66,12 +66,6 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// DOES NOT WORK WITH NEW NROD STREAMER
|
|
||||||
// If debug mode is on, call debugWriteDownload
|
|
||||||
// if helpers.Runtime == "debug" {
|
|
||||||
// debugWriteDownload(data)
|
|
||||||
// }
|
|
||||||
|
|
||||||
// Parse CIF file
|
// Parse CIF file
|
||||||
parsed, err := parseCifDataStream(data)
|
parsed, err := parseCifDataStream(data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -79,10 +73,14 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Make `data` a nil pointer as it is no longer required
|
|
||||||
data = nil
|
|
||||||
|
|
||||||
log.Msg.Debug("Starting metadata checks")
|
log.Msg.Debug("Starting metadata checks")
|
||||||
|
|
||||||
|
// If last update type is == fullUpdateType and LastUpdate less than twodays? ago then run the update regardless of sequence
|
||||||
|
// this is because the sequence number of full and daily updates to not appear to match up although I will need to check this!
|
||||||
|
//if metadata.LastUpdateType == fullUpdateType && metadata.LastUpdate //after_yesterday_at_0600 {
|
||||||
|
// Somehow skip the below checks! Maybe I need a checkMetadata function!
|
||||||
|
//}
|
||||||
|
|
||||||
// Check CIF Sequence
|
// Check CIF Sequence
|
||||||
// Skip if LastSequence is >= to this sequence
|
// Skip if LastSequence is >= to this sequence
|
||||||
if metadata.LastSequence >= parsed.header.Metadata.Sequence {
|
if metadata.LastSequence >= parsed.header.Metadata.Sequence {
|
||||||
@ -107,7 +105,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
|||||||
// metadata = generateMetadata(&parsed.header)
|
// metadata = generateMetadata(&parsed.header)
|
||||||
}
|
}
|
||||||
|
|
||||||
ok := dbAccess.PutCifMetadata(metadata)
|
ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType)
|
||||||
if !ok {
|
if !ok {
|
||||||
log.Msg.Warn("CIF Data updated, but metadata write failed.")
|
log.Msg.Warn("CIF Data updated, but metadata write failed.")
|
||||||
}
|
}
|
||||||
|
@ -18,10 +18,11 @@ const Doctype = "CifMetadata"
|
|||||||
// The type describing the CifMetadata 'type' in the database.
|
// The type describing the CifMetadata 'type' in the database.
|
||||||
// This type will be moved to owlboard/go-types
|
// This type will be moved to owlboard/go-types
|
||||||
type CifMetadata struct {
|
type CifMetadata struct {
|
||||||
Doctype string `bson:"type"`
|
Doctype string `bson:"type"`
|
||||||
LastUpdate time.Time `bson:"lastUpdate"`
|
LastUpdate time.Time `bson:"lastUpdate"`
|
||||||
LastTimestamp int64 `bson:"lastTimestamp"`
|
LastTimestamp int64 `bson:"lastTimestamp"`
|
||||||
LastSequence int64 `bson:"lastSequence"`
|
LastSequence int64 `bson:"lastSequence"`
|
||||||
|
LastUpdateType string `bson:"lastUpdateType"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
|
// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
|
||||||
@ -45,17 +46,18 @@ func GetCifMetadata() (*CifMetadata, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Uses upsert to Insert/Update the CifMetadata in the database
|
// Uses upsert to Insert/Update the CifMetadata in the database
|
||||||
func PutCifMetadata(metadata *CifMetadata) bool {
|
func PutCifMetadata(metadata *CifMetadata, lastUpdateType string) bool {
|
||||||
database := MongoClient.Database(databaseName)
|
database := MongoClient.Database(databaseName)
|
||||||
collection := database.Collection(metaCollection)
|
collection := database.Collection(metaCollection)
|
||||||
options := options.Update().SetUpsert(true)
|
options := options.Update().SetUpsert(true)
|
||||||
filter := bson.M{"type": Doctype}
|
filter := bson.M{"type": Doctype}
|
||||||
update := bson.M{
|
update := bson.M{
|
||||||
"$set": bson.M{
|
"$set": bson.M{
|
||||||
"type": Doctype,
|
"type": Doctype,
|
||||||
"lastUpdate": metadata.LastUpdate,
|
"lastUpdate": metadata.LastUpdate,
|
||||||
"lastTimestamp": metadata.LastTimestamp,
|
"lastTimestamp": metadata.LastTimestamp,
|
||||||
"lastSequence": metadata.LastSequence,
|
"lastSequence": metadata.LastSequence,
|
||||||
|
"lastUpdateType": lastUpdateType,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user