diff --git a/cif/check.go b/cif/check.go index d044079..ca4d7d3 100644 --- a/cif/check.go +++ b/cif/check.go @@ -46,6 +46,7 @@ func CheckCif(cfg *helpers.Configuration) { // Check how many days since last update, if more than 5, run full update, else run update daysSinceLastUpdate := howManyDaysAgo(metadata.LastUpdate) if daysSinceLastUpdate > 5 { + log.Msg.Debug("Full Update Requested due to time since last update", zap.Int("daysSinceLastUpdate", daysSinceLastUpdate)) log.Msg.Info("Full CIF download required") err := runCifFullDownload(cfg) if err != nil { @@ -62,6 +63,4 @@ func CheckCif(cfg *helpers.Configuration) { if err != nil { log.Msg.Error("Unable to run CIF update", zap.Error(err)) } - - return } diff --git a/cif/constants.go b/cif/constants.go index 286dd44..782c33d 100644 --- a/cif/constants.go +++ b/cif/constants.go @@ -12,4 +12,4 @@ const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAu const dataAvailable = 6 // An object representing the Europe/London timezone -var londonTimezone, err = time.LoadLocation("Europe/London") +var londonTimezone, _ = time.LoadLocation("Europe/London") diff --git a/cif/convert.go b/cif/convert.go index 9261d20..5bf791d 100644 --- a/cif/convert.go +++ b/cif/convert.go @@ -21,7 +21,7 @@ func ConvertServiceType(input *upstreamApi.JsonScheduleV1, vstp bool) (*database PlanSpeed: parseSpeed(&input.ScheduleSegment.CifSpeed), ScheduleStartDate: ParseCifDate(&input.ScheduleStartDate, "start"), ScheduleEndDate: ParseCifDate(&input.ScheduleEndDate, "end"), - FirstClass: hasFirstClass(&input.ScheduleSegment.CifTrainClass), + FirstClass: hasFirstClass(&input.ScheduleSegment.CifTrainClass, &input.ScheduleSegment.SignallingId), Catering: hasCatering(&input.ScheduleSegment.CifCateringCode), Sleeper: hasSleeper(&input.ScheduleSegment.CifSleepers), DaysRun: parseDaysRun(&input.ScheduleDaysRun), @@ -37,14 +37,12 @@ func parseSpeed(CIFSpeed *string) int32 { return 0 } if *CIFSpeed == "" { - log.Msg.Debug("Speed data not provided") return int32(0) } actualSpeed, exists := helpers.SpeedMap[*CIFSpeed] if !exists { actualSpeed = *CIFSpeed } - log.Msg.Debug("Corrected Speed: " + actualSpeed) speed, err := strconv.ParseInt(actualSpeed, 10, 32) if err != nil { @@ -87,10 +85,17 @@ func isPublic(input *upstreamApi.CifScheduleLocation) bool { } // Ascertains whether the service offers first class -func hasFirstClass(input *string) bool { - if input == nil { +func hasFirstClass(input, signallingId *string) bool { + if input == nil || signallingId == nil { return false } + + // Handle non passenger headcodes and ensure first class is not shown as available + firstChar := (*signallingId)[0] + if firstChar == '3' || firstChar == '4' || firstChar == '5' || firstChar == '6' || firstChar == '7' || firstChar == '8' || firstChar == '0' { + return false + } + return *input != "S" } diff --git a/cif/convert_test.go b/cif/convert_test.go index 0d78908..75132ed 100644 --- a/cif/convert_test.go +++ b/cif/convert_test.go @@ -87,23 +87,34 @@ func TestIsPublic(t *testing.T) { func TestHasFirstClass(t *testing.T) { testCases := []struct { - input string - expect bool + input string + headcode string + expect bool }{ - {"", true}, - {"B", true}, - {"S", false}, + {"", "1A00", true}, + {"B", "2A05", true}, + {"S", "1C99", false}, + {"", "3C23", false}, + {"", "5Q21", false}, + {"", "5D32", false}, + {"", "9O12", true}, + {"B", "9D32", true}, + {"", "7R43", false}, + {"B", "6Y77", false}, + {"", "8P98", false}, + {"S", "4O89", false}, + {"", "4E43", false}, } for _, tc := range testCases { - result := hasFirstClass(&tc.input) + result := hasFirstClass(&tc.input, &tc.headcode) if result != tc.expect { - t.Errorf("For %s, expected %t, but got %t", tc.input, tc.expect, result) + t.Errorf("For %s & headcode %s, expected %t, but got %t", tc.input, tc.headcode, tc.expect, result) } } - nilResult := hasFirstClass(nil) + nilResult := hasFirstClass(nil, nil) if nilResult { t.Errorf("hasFirstClass failed to handle nil pointer, expected %t, got %t", false, nilResult) } diff --git a/cif/helpers.go b/cif/helpers.go index 7f89ebf..b3e1e9a 100644 --- a/cif/helpers.go +++ b/cif/helpers.go @@ -44,11 +44,13 @@ func isSameToday(t time.Time) bool { // Returns how many days ago `t` was compared to today func howManyDaysAgo(t time.Time) int { - today := time.Now().In(time.UTC).Truncate(24 * time.Hour) - input := t.In(time.UTC).Truncate(24 * time.Hour) + log.Msg.Debug("Calculating how many days ago", zap.Time("Input time", t)) + // Truncate both times to midnight in UTC timezone + today := time.Now().UTC().Truncate(24 * time.Hour) + input := t.UTC().Truncate(24 * time.Hour) diff := today.Sub(input) - days := int(diff.Hours() / 24) + days := int(diff / (24 * time.Hour)) return days } diff --git a/cif/helpers_test.go b/cif/helpers_test.go index 47ad779..4cf734a 100644 --- a/cif/helpers_test.go +++ b/cif/helpers_test.go @@ -24,11 +24,11 @@ func TestHowManyDaysAgo(t *testing.T) { input time.Time expected int }{ - {time.Now(), 0}, // Today - {time.Now().Add(-24 * time.Hour), 1}, // Yesterday - {time.Now().Add(-48 * time.Hour), 2}, // Ereyesterday - {time.Now().Add(24 * time.Hour), -1}, // Tomorrow - {time.Now().Add(48 * time.Hour), -2}, // Overmorrow + {time.Now().In(time.UTC), 0}, // Today + {time.Now().In(time.UTC).Add(-24 * time.Hour), 1}, // Yesterday + {time.Now().In(time.UTC).Add(-48 * time.Hour), 2}, // Ereyesterday + {time.Now().In(time.UTC).Add(24 * time.Hour), -1}, // Tomorrow + {time.Now().In(time.UTC).Add(48 * time.Hour), -2}, // Overmorrow } for _, tc := range testCases { diff --git a/cif/process.go b/cif/process.go index a46f1a0..d644fad 100644 --- a/cif/process.go +++ b/cif/process.go @@ -18,9 +18,11 @@ func processParsedCif(data *parsedData) error { for _, item := range data.sched { switch item.TransactionType { case "Delete": - deleteTasks = append(deleteTasks, &item) + deleteItem := item // Create new variable to ensure repetition of pointers + deleteTasks = append(deleteTasks, &deleteItem) case "Create": - createTasks = append(createTasks, &item) + createItem := item // Create new variable to ensure repetition of pointers + createTasks = append(createTasks, &createItem) default: log.Msg.Error("Unknown transaction type in CIF Schedule", zap.String("TransactionType", item.TransactionType)) } diff --git a/cif/process_test.go b/cif/process_test.go index 6117003..4beb9d7 100644 --- a/cif/process_test.go +++ b/cif/process_test.go @@ -35,6 +35,7 @@ func TestGenerateMetadata(t *testing.T) { if result == nil { t.Errorf("generateMetadata returned nil pointer") + return // Static type checking likes this return to be here, even if it is redundant in reality. } if result.Doctype != expected.Doctype { diff --git a/cif/update.go b/cif/update.go index 65eb196..990fe8b 100644 --- a/cif/update.go +++ b/cif/update.go @@ -40,7 +40,11 @@ func runCifFullDownload(cfg *helpers.Configuration) error { log.Msg.Error("Error processing CIF data", zap.Error(err)) } - // Generate & Write metadata + newMeta := generateMetadata(&parsed.header) + ok := dbAccess.PutCifMetadata(newMeta) + if !ok { + log.Msg.Warn("CIF Data updated, but metadata write failed") + } return nil } diff --git a/dbAccess/cif.go b/dbAccess/cif.go index ef8668e..dabeb62 100644 --- a/dbAccess/cif.go +++ b/dbAccess/cif.go @@ -18,10 +18,10 @@ const Doctype = "CifMetadata" // The type describing the CifMetadata 'type' in the database. // This type will be moved to owlboard/go-types type CifMetadata struct { - Doctype string `json:"type"` - LastUpdate time.Time `json:"lastUpdate"` - LastTimestamp int64 `json:"lastTimestamp"` - LastSequence int64 `json:"lastSequence"` + Doctype string `bson:"type"` + LastUpdate time.Time `bson:"lastUpdate"` + LastTimestamp int64 `bson:"lastTimestamp"` + LastSequence int64 `bson:"lastSequence"` } // Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example. @@ -39,20 +39,24 @@ func GetCifMetadata() (*CifMetadata, error) { return nil, err } + log.Msg.Debug("Fetched CIF Metadata from database", zap.Any("Metadata", result)) + return &result, nil } // Uses upsert to Insert/Update the CifMetadata in the database -func PutCifMetadata(metadata CifMetadata) bool { +func PutCifMetadata(metadata *CifMetadata) bool { database := MongoClient.Database(databaseName) collection := database.Collection(metaCollection) options := options.Update().SetUpsert(true) filter := bson.M{"type": Doctype} update := bson.M{ - "type": Doctype, - "LastUpdate": metadata.LastUpdate, - "LastTimestamp": metadata.LastTimestamp, - "LastSequence": metadata.LastSequence, + "$set": bson.M{ + "type": Doctype, + "lastUpdate": metadata.LastUpdate, + "lastTimestamp": metadata.LastTimestamp, + "lastSequence": metadata.LastSequence, + }, } _, err := collection.UpdateOne(context.Background(), filter, update, options) @@ -61,14 +65,21 @@ func PutCifMetadata(metadata CifMetadata) bool { log.Msg.Error("Error updating CIF Metadata", zap.Error(err)) return false } + + log.Msg.Info("New CIF Metadata written", zap.Time("Update time", metadata.LastUpdate)) return true } // Handles 'Delete' tasks from CIF Schedule updates, accepts DeleteQuery types and batches deletions. func DeleteCifEntries(deletions []database.DeleteQuery) error { - collection := MongoClient.Database(databaseName).Collection(timetableCollection) + // Skip if deletions is empty + if len(deletions) == 0 { + log.Msg.Info("No deletions required") + return nil + } // Prepare deletion tasks + collection := MongoClient.Database(databaseName).Collection(timetableCollection) bulkDeletions := make([]mongo.WriteModel, 0, len(deletions)) for _, deleteQuery := range deletions { @@ -95,6 +106,12 @@ func DeleteCifEntries(deletions []database.DeleteQuery) error { // Handles 'Create' tasks for CIF Schedule updates, accepts Service structs and batches their creation. func CreateCifEntries(schedules []database.Service) error { + // Skip if deletions is empty + if len(schedules) == 0 { + log.Msg.Info("No creations required") + return nil + } + collection := MongoClient.Database(databaseName).Collection(timetableCollection) models := make([]mongo.WriteModel, 0, len(schedules)) diff --git a/go.mod b/go.mod index 60d90ec..c3ba9ef 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.fjla.uk/owlboard/timetable-mgr go 1.21 require ( - git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580 + git.fjla.uk/owlboard/go-types v0.0.0-20240408193146-4719be9c13eb github.com/go-stomp/stomp/v3 v3.0.5 go.mongodb.org/mongo-driver v1.12.0 go.uber.org/zap v1.24.0 diff --git a/go.sum b/go.sum index 614f768..2dbfbf5 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ git.fjla.uk/owlboard/go-types v0.0.0-20240407202712-e58d7d1d9aa9 h1:aNxMYEsbBkFx git.fjla.uk/owlboard/go-types v0.0.0-20240407202712-e58d7d1d9aa9/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY= git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580 h1:bEaC1JfqiSSJH65iP/NXMyBo85JMB41VBkiJdWbnHYM= git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY= +git.fjla.uk/owlboard/go-types v0.0.0-20240408193146-4719be9c13eb h1:aLd0nzuU13hxycz9F4Z4PVq5dp/TxuzywPGZTJXbnq0= +git.fjla.uk/owlboard/go-types v0.0.0-20240408193146-4719be9c13eb/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/main.go b/main.go index 8aac05c..77f1e8b 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,7 @@ import ( _ "time/tzdata" "git.fjla.uk/owlboard/timetable-mgr/background" - "git.fjla.uk/owlboard/timetable-mgr/corpus" + "git.fjla.uk/owlboard/timetable-mgr/cif" "git.fjla.uk/owlboard/timetable-mgr/dbAccess" "git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/log" @@ -41,7 +41,7 @@ func main() { background.InitTicker(cfg, stop) // Test CORPUS Fetching - go corpus.CheckCorpus(cfg) + go cif.CheckCif(cfg) if cfg.VstpOn { messaging.StompInit(cfg)