diff --git a/src/cif/check.go b/src/cif/check.go index be375f4..d044079 100644 --- a/src/cif/check.go +++ b/src/cif/check.go @@ -9,54 +9,6 @@ import ( "go.uber.org/zap" ) -// REWRITING CifCheck into CheckCif (at bottom of file), CifCheck and parseMetadata become redundant - -// Loads CifMetadata and passes it to parseMetadata, this function is what you should call to initiate the CifUpdate process. -func CifCheck(cfg *helpers.Configuration) error { - log.Msg.Debug("Checking age of CIF Data") - metadata, err := dbAccess.GetCifMetadata() - if err != nil { - return err - } - - err = parseMetadata(metadata, cfg) - if err != nil { - log.Msg.Error("Error updating CIF Data", zap.Error(err)) - return err - } - - return nil -} - -// Requests a full update if no metadata exists, or a daily update if metadata does exist. -// The daily update function does further metadata parsing to determine what exactly needs downloading. -func parseMetadata(metadata *dbAccess.CifMetadata, cfg *helpers.Configuration) error { - if metadata == nil { - log.Msg.Info("No metadata, creating Timetable data") - newMeta, err := runFullUpdate(cfg) - if err != nil { - return err - } - ok := dbAccess.PutCifMetadata(*newMeta) - if !ok { - log.Msg.Error("CIF Data updated but Metadata Update failed") - } - return nil - } - - log.Msg.Debug("Requesting CIF Data Update") - // When testing done, this function returns newMetadata which needs putting into DB - _, err := runUpdate(metadata, cfg) - if err != nil { - return err - } - //ok := dbAccess.PutCifMetadata(*newMeta) - //if !ok { - // log.Msg.Error("CIF Data updated but Metadata Update failed") - //} - return nil -} - // Checks if the CIF Data needs updating, and what type of update is needed (Full/Partial) and if partial // what days data needs updating, then calls an update function to handle the update. func CheckCif(cfg *helpers.Configuration) { @@ -75,12 +27,12 @@ func CheckCif(cfg *helpers.Configuration) { return } + // If no metadata is found in DB, presume no CIF data exists if metadata == nil { log.Msg.Info("Full CIF download required") err := runCifFullDownload(cfg) if err != nil { - log.Msg.Error("Unable to run full CIF Update") - return + log.Msg.Error("Unable to run full CIF Update", zap.Error(err)) } return } @@ -91,8 +43,25 @@ func CheckCif(cfg *helpers.Configuration) { return } - // + // Check how many days since last update, if more than 5, run full update, else run update + daysSinceLastUpdate := howManyDaysAgo(metadata.LastUpdate) + if daysSinceLastUpdate > 5 { + log.Msg.Info("Full CIF download required") + err := runCifFullDownload(cfg) + if err != nil { + log.Msg.Error("Unable to run full CIF Update", zap.Error(err)) + } + return + } - // Here I need to determine which days I need to update CIF data for, then pass to an appropriate function: - // newMetadata, err := runCifUpdate(days []time.Time, cfg) + daysToUpdate := generateUpdateDays(daysSinceLastUpdate) + + // Run the update + log.Msg.Info("CIF Update required", zap.Any("days to update", daysToUpdate)) + err = runCifUpdateDownload(cfg, metadata, daysToUpdate) + if err != nil { + log.Msg.Error("Unable to run CIF update", zap.Error(err)) + } + + return } diff --git a/src/cif/helpers.go b/src/cif/helpers.go index 8ead8ae..49463a2 100644 --- a/src/cif/helpers.go +++ b/src/cif/helpers.go @@ -51,3 +51,20 @@ func howManyDaysAgo(t time.Time) int { days := int(diff.Hours() / 24) return days } + +// Generates a slice of time.Time values representing which days files need downloading +func generateUpdateDays(days int) []time.Time { + var updateDays []time.Time + + for i := 0; i < days; i++ { + day := time.Now().Add(-time.Duration(i) * 24 * time.Hour) + updateDays = append(updateDays, day) + } + + // Reverse slice to ensure chronological order + for i, j := 0, len(updateDays)-1; i < j; i, j = i+1, j-1 { + updateDays[i], updateDays[j] = updateDays[j], updateDays[i] + } + + return updateDays +} diff --git a/src/cif/helpers_test.go b/src/cif/helpers_test.go index 49cff57..69c66fe 100644 --- a/src/cif/helpers_test.go +++ b/src/cif/helpers_test.go @@ -17,3 +17,76 @@ func TestIsSameDay(t *testing.T) { t.Errorf("Error in isSameDay(notToday). Expected false, got true.") } } + +func TestHowManyDaysAgo(t *testing.T) { + testCases := []struct { + input time.Time + expected int + }{ + {time.Now(), 0}, // Today + {time.Now().Add(-24 * time.Hour), 1}, // Yesterday + {time.Now().Add(-48 * time.Hour), 2}, // Ereyesterday + {time.Now().Add(24 * time.Hour), -1}, // Tomorrow + {time.Now().Add(48 * time.Hour), -2}, // Overmorrow + } + + for _, tc := range testCases { + result := howManyDaysAgo(tc.input) + if result != tc.expected { + t.Errorf("For input %v, expected %d but got %d", tc.input, tc.expected, result) + } + } +} + +func TestGetDayString(t *testing.T) { + testCases := []struct { + input time.Time + expected string + }{ // Note that the test times are in UTC, but days are checked in Europe/London + {time.Date(2024, time.April, 7, 0, 0, 0, 0, time.UTC), "sun"}, + {time.Date(2024, time.April, 4, 21, 0, 0, 0, time.UTC), "thu"}, + {time.Date(2001, time.September, 11, 12, 46, 0, 0, time.UTC), "tue"}, + } + + for _, tc := range testCases { + result := getDayString(tc.input) + if result != tc.expected { + t.Errorf("For input %v, expected %s, but got %s", tc.input, tc.expected, result) + } + } +} + +func TestGenerateUpdateDays(t *testing.T) { + testCases := []struct { + days int + expected []time.Time + }{ + {1, []time.Time{time.Now()}}, + {2, []time.Time{time.Now().Add(-24 * time.Hour), time.Now()}}, + {4, []time.Time{time.Now().Add(-72 * time.Hour), + time.Now().Add(-48 * time.Hour), + time.Now().Add(-24 * time.Hour), + time.Now(), + }}, + } + + for _, tc := range testCases { + result := generateUpdateDays(tc.days) + + if len(result) != len(tc.expected) { + t.Errorf("For %d days, expected %v, but got %v", tc.days, tc.expected, result) + continue + } + + for i := range result { + if !isSameDate(result[i], tc.expected[i]) { + t.Errorf("For %d days, expected %v, but got %v", tc.days, tc.expected, result) + } + } + } +} + +// Checks if two time values have the same year, month and day. +func isSameDate(t1, t2 time.Time) bool { + return t1.Year() == t2.Year() && t1.Month() == t2.Month() && t1.Day() == t2.Day() +} diff --git a/src/cif/parse_test.go b/src/cif/parse_test.go new file mode 100644 index 0000000..337dba3 --- /dev/null +++ b/src/cif/parse_test.go @@ -0,0 +1,105 @@ +package cif + +import ( + "testing" + "time" + + "git.fjla.uk/owlboard/go-types/pkg/upstreamApi" +) + +func TestParseCifData(t *testing.T) { + testCases := []struct { + name string + input []byte + expected *parsedData + } +} + +func returnParsedData() *parsedData { + var parsed = { + name: "Valid JSON Data 1", + input: []byte(` + {"JsonTimetableV1":{"classification":"public","timestamp":1711227636,"owner":"Network Rail","Sender":{"organisation":"Rockshore","application":"NTROD","component":"SCHEDULE"},"Metadata":{"type":"update","sequence":4307}}} + {"JsonAssociationV1":{"transaction_type":"Create","main_train_uid":"L65283","assoc_train_uid":"L65245","assoc_start_date":"2023-12-11T00:00:00Z","assoc_end_date":"2024-03-28T00:00:00Z","assoc_days":"1111000","category":" ","date_indicator":" ","location":"RDNGSTN","base_location_suffix":null,"assoc_location_suffix":null,"diagram_type":"T","CIF_stp_indicator":"C"}} + {"JsonAssociationV1":{"transaction_type":"Delete","main_train_uid":"L65283","assoc_train_uid":"L65245","assoc_start_date":"2024-03-18T00:00:00Z","location":"RDNGSTN","base_location_suffix":null,"diagram_type":"T","CIF_stp_indicator":"C"}} + {"JsonScheduleV1":{"CIF_stp_indicator":"O","CIF_train_uid":"C25513","schedule_start_date":"2024-03-23","transaction_type":"Delete"}} + {"JsonScheduleV1":{"CIF_bank_holiday_running":null,"CIF_stp_indicator":"O","CIF_train_uid":"Y04345","applicable_timetable":"Y","atoc_code":"SW","new_schedule_segment":{"traction_class":"","uic_code":""},"schedule_days_runs":"0000001","schedule_end_date":"2024-05-19","schedule_segment":{"signalling_id":"5A20","CIF_train_category":"EE","CIF_headcode":"00","CIF_course_indicator":1,"CIF_train_service_code":"24676004","CIF_business_sector":"??","CIF_power_type":"EMU","CIF_timing_load":null,"CIF_speed":"100","CIF_operating_characteristics":"D","CIF_train_class":null,"CIF_sleepers":null,"CIF_reservations":null,"CIF_connection_indicator":null,"CIF_catering_code":null,"CIF_service_branding":"","schedule_location":[{"location_type":"LO","record_identity":"LO","tiploc_code":"FARNHMD","tiploc_instance":null,"departure":"0721","public_departure":null,"platform":null,"line":null,"engineering_allowance":null,"pathing_allowance":null,"performance_allowance":null},{"location_type":"LT","record_identity":"LT","tiploc_code":"FARNHAM","tiploc_instance":null,"platform":"1","arrival":"0729","public_arrival":null,"path":null}]},"schedule_start_date":"2024-05-19","train_status":"T","transaction_type":"Create"}} + {"EOF":true} + `), + expected: &parsedData{ + header: upstreamApi.JsonTimetableV1{ + Classification: "public", + Timestamp: 1711227636, + Owner: "Network Rail", + Sender: upstreamApi.TimetableSender{ + Organisation: "Rockshore", + Application: "NTROD", + Component: "SCHEDULE", + }, + Metadata: upstreamApi.TimetableMetadata{ + Type: "update", + Sequence: 4307, + }, + }, + assoc: []upstreamApi.JsonAssociationV1{ + TransactionType: "Create", + MainTrainUid: "L65283", + AssocTrainUid: "L65245", + AssocStartDate: time.Date(2023, time.December, 11, 0, 0, 0, 0, time.UTC), + AssocEndDate: time.Date(2024, time.March, 28, 0, 0, 0, 0, time.UTC), + AssocDays: "1111000", + Location: "RDNGSTN", + DiagramType: "T", + CifStpIndicator: "C", + }, { + TransactionType: "Delete", + MainTrainUid: "L65283", + AssocTrainUid: "L65245", + AssocStartDate: time.Date(2024, time.March, 18, 0, 0, 0, 0, time.UTC), + Location: "RDNGSTN", + DiagramType: "T", + CifStpIndicator: "C", + }, + sched: []upstreamApi.JsonScheduleV1{ + CifStpIndicator: "O", + CifTrainUid: "C25513", + ScheduleStartDate: "2024-03-23", + TransactionType: "Delete", + }, { + CifStpIndicator: "O", + CifTrainUid: "Y04345", + ApplicableTimetable: "Y", + AtocCode: "SW", + NewScheduleSegment: upstreamApi.CifNewScheduleSegment{}, + ScheduleDaysRun: "0000001", + ScheduleEndDate: "2024-05-19", + ScheduleSegment: upstreamApi.CifScheduleSegment{ + SignallingId: "5A20", + CifTrainCategory: "EE", + CifHeadcode: "OO", + CifCourseIndicator: 1, + CifTrainServiceCode: "24676004", + CifBusinessSector: "??", + CifPowerType: "EMU", + CifSpeed: "100", + CifOperatingCharacteristics: "D", + ScheduleLocation: []upstreamApi.CifScheduleLocation{ + LocationType: "LO", + RecordIdentity: "LO", + TiplocCode: "FARNHMD", + Departure: "0721", + }, { + LocationType: "LT", + RecordIdentity: "LT", + TiplocCode: "FARNHAM", + Platform: "1", + Arrival: "0729", + }, + }, + ScheduleStartDate: "2024-05-19", + TrainStatus: "T", + TransactionType: "Create", + }, + }, + } +} \ No newline at end of file diff --git a/src/cif/process.go b/src/cif/process.go index 6189e5a..db8c643 100644 --- a/src/cif/process.go +++ b/src/cif/process.go @@ -1,21 +1,6 @@ package cif -import ( - "errors" - - "git.fjla.uk/owlboard/timetable-mgr/dbAccess" -) - -// Processes all data in the schedule segment of 'data', interacts with the database and returns new metadata -func processCifData(metadata *dbAccess.CifMetadata, data *parsedData) (*dbAccess.CifMetadata, error) { - if data == nil { - err := errors.New("data is not defined") - return nil, err - } - if metadata == nil { - err := errors.New("metadata is not defined") - return nil, err - } - - return nil, nil +// Processes parsed CIF data and applies the data to the database +func processParsedCif(data *parsedData) error { + return nil } diff --git a/src/cif/update.go b/src/cif/update.go index 939fc52..a096e99 100644 --- a/src/cif/update.go +++ b/src/cif/update.go @@ -1,7 +1,6 @@ package cif import ( - "sort" "time" "git.fjla.uk/owlboard/timetable-mgr/dbAccess" @@ -11,105 +10,79 @@ import ( "go.uber.org/zap" ) -// Runs a full update of the CIF Data, discarding any existing data and returns a new metadata struct -func runFullUpdate(cfg *helpers.Configuration) (*dbAccess.CifMetadata, error) { - log.Msg.Warn("All existing timetable data will be deleted") +// Replaces all existing CIF Data with a new download +func runCifFullDownload(cfg *helpers.Configuration) error { + log.Msg.Info("Downloading all CIF Data") + + // Download CIF Data file url, err := getUpdateUrl("full") if err != nil { - log.Msg.Error("Unable to get update URL", zap.Error(err)) - return nil, err + log.Msg.Error("Error getting download URL", zap.Error(err)) } - - fullCifData, err := nrod.NrodDownload(url, cfg) + data, err := nrod.NrodDownload(url, cfg) if err != nil { - log.Msg.Error("Unable to get CIF Data", zap.Error(err)) - return nil, err + log.Msg.Error("Error downloading CIF data", zap.Error(err)) } - log.Msg.Debug("CIF Data Downloaded", zap.ByteString("CIF Data", fullCifData)) + // Parse CIF file + parsed, err := parseCifData(data) + if err != nil { + log.Msg.Error("Error parsing CIF data", zap.Error(err)) + return err + } - // I now need to define a processing function and ensure a valid type exists, then I can pass that type to a CIF Put Full function - // which will handle placing the data into the database + // Drop timetable collection + dbAccess.DropCollection(dbAccess.TimetableCollection) - return nil, nil + // Process CIF file + err = processParsedCif(parsed) + if err != nil { + log.Msg.Error("Error processing CIF data", zap.Error(err)) + } + + // Generate & Write metadata + + return nil } -// Runs the daily update for CIF Data, can handle up to five days updates at once. -func runUpdate(metadata *dbAccess.CifMetadata, cfg *helpers.Configuration) (*dbAccess.CifMetadata, error) { - // Do not run update if last update was on same day - if isSameToday(metadata.LastUpdate) { - log.Msg.Info("No CIF Update Required", zap.Time("Last update", metadata.LastUpdate)) - return nil, nil - } +// Runs a CIF Update for up to five days +func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMetadata, days []time.Time) error { + log.Msg.Info("Downloading CIF Updates") - // Do not run update before 0600 as todays data will not be available - if time.Now().Hour() < 6 { - log.Msg.Info("Too early to update CIF Data") - return nil, nil - } + // Loop over dates + for _, time := range days { + log.Msg.Info("Downloading CIF File", zap.Time("CIF Data from", time)) - // Check how many days ago last update was - lastUpateDays := howManyDaysAgo(metadata.LastUpdate) - if lastUpateDays > 5 { - log.Msg.Warn("CIF Data is more than five days old. Running Full Update") - newMeta, err := runFullUpdate(cfg) + // Download CIF data file + data, err := fetchUpdate(time, cfg) if err != nil { - log.Msg.Error("CIF Update failed", zap.Error(err)) - return nil, err + log.Msg.Error("Error fetching CIF update", zap.Error(err)) + return err } - return newMeta, nil - } - - // Create a slice containing which dates need updating - firstUpdate := time.Now().In(time.UTC).AddDate(0, 0, -lastUpateDays) - finalUpdate := time.Now().In(time.UTC) - var dates []time.Time - - for d := firstUpdate; d.Before(finalUpdate) || d.Equal(finalUpdate); d = d.AddDate(0, 0, 1) { - dates = append(dates, d) - } - - sort.Slice(dates, func(i, j int) bool { - return dates[i].Before(dates[j]) - }) - - log.Msg.Info("Updating CIF Data", zap.Any("dates to update", dates)) - - // Iterate over each date, fetching then parsing the data - for _, date := range dates { - data, err := fetchUpdate(date, cfg) + // Parse CIF file + parsed, err := parseCifData(data) if err != nil { - log.Msg.Error("Error fetching data", zap.Time("date", date)) - continue - } // parseCifData function needs writing - parsedData, err := parseCifData(data) - if err != nil { - log.Msg.Error("Error parsing data", zap.Time("date", date)) + log.Msg.Error("Error parsing CIF data", zap.Error(err)) + return err } - if parsedData == nil { // TEMPORARY DEVELOPMENT CHECK - log.Msg.Error("No data parsed") - return nil, nil - } - - // Process data (manages the database interactions and >>returns the new metadata) - _, err = processCifData(metadata, parsedData) - if err != nil { - log.Msg.Error("Error processing CIF Data", zap.Error(err)) - } - - log.Msg.Info("CIF Data updated", zap.Time("date", date)) + // Check metadata sequence - Handle a metadata sequence error. Probably by deleting metadata so next update triggers full download + //// I need to check what the sequence looks like in a full download first. + // Process CIF file + // Generate & Write metadata } - return nil, nil //TEMPORARY + + return nil } -// Fetches CIF Updates for a given day +// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) { url, err := getUpdateUrl("daily") if err != nil { return nil, err } + // Append day string to URL url = url + getDayString(t) downloadedData, err := nrod.NrodDownload(url, cfg) diff --git a/src/dbAccess/client.go b/src/dbAccess/client.go index 72586f7..ea591a6 100644 --- a/src/dbAccess/client.go +++ b/src/dbAccess/client.go @@ -12,12 +12,6 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) -const databaseName string = "owlboard" -const CorpusCollection string = "corpus" -const StationsCollection string = "stations" -const metaCollection string = "meta" -const TimetableCollection string = "timetable" - // Provide the DB Connection to other functions var MongoClient (*mongo.Client) diff --git a/src/dbAccess/contants.go b/src/dbAccess/contants.go new file mode 100644 index 0000000..612f334 --- /dev/null +++ b/src/dbAccess/contants.go @@ -0,0 +1,7 @@ +package dbAccess + +const databaseName string = "owlboard" +const CorpusCollection string = "corpus" +const StationsCollection string = "stations" +const metaCollection string = "meta" +const TimetableCollection string = "timetable"