Lots more work on CIF processing

src/cif/check.go

@@ -9,54 +9,6 @@ import (
 	"go.uber.org/zap"
 )
 
-// REWRITING CifCheck into CheckCif (at bottom of file), CifCheck and parseMetadata become redundant
-
-// Loads CifMetadata and passes it to parseMetadata, this function is what you should call to initiate the CifUpdate process.
-func CifCheck(cfg *helpers.Configuration) error {
-	log.Msg.Debug("Checking age of CIF Data")
-	metadata, err := dbAccess.GetCifMetadata()
-	if err != nil {
-		return err
-	}
-
-	err = parseMetadata(metadata, cfg)
-	if err != nil {
-		log.Msg.Error("Error updating CIF Data", zap.Error(err))
-		return err
-	}
-
-	return nil
-}
-
-// Requests a full update if no metadata exists, or a daily update if metadata does exist.
-// The daily update function does further metadata parsing to determine what exactly needs downloading.
-func parseMetadata(metadata *dbAccess.CifMetadata, cfg *helpers.Configuration) error {
-	if metadata == nil {
-		log.Msg.Info("No metadata, creating Timetable data")
-		newMeta, err := runFullUpdate(cfg)
-		if err != nil {
-			return err
-		}
-		ok := dbAccess.PutCifMetadata(*newMeta)
-		if !ok {
-			log.Msg.Error("CIF Data updated but Metadata Update failed")
-		}
-		return nil
-	}
-
-	log.Msg.Debug("Requesting CIF Data Update")
-	// When testing done, this function returns newMetadata which needs putting into DB
-	_, err := runUpdate(metadata, cfg)
-	if err != nil {
-		return err
-	}
-	//ok := dbAccess.PutCifMetadata(*newMeta)
-	//if !ok {
-	//	log.Msg.Error("CIF Data updated but Metadata Update failed")
-	//}
-	return nil
-}
-
 // Checks if the CIF Data needs updating, and what type of update is needed (Full/Partial) and if partial
 // what days data needs updating, then calls an update function to handle the update.
 func CheckCif(cfg *helpers.Configuration) {
@@ -75,12 +27,12 @@ func CheckCif(cfg *helpers.Configuration) {
 		return
 	}
 
+	// If no metadata is found in DB, presume no CIF data exists
 	if metadata == nil {
 		log.Msg.Info("Full CIF download required")
 		err := runCifFullDownload(cfg)
 		if err != nil {
-			log.Msg.Error("Unable to run full CIF Update")
+			log.Msg.Error("Unable to run full CIF Update", zap.Error(err))
-			return
 		}
 		return
 	}
@@ -91,8 +43,25 @@ func CheckCif(cfg *helpers.Configuration) {
 		return
 	}
 
-	//
+	// Check how many days since last update, if more than 5, run full update, else run update
+	daysSinceLastUpdate := howManyDaysAgo(metadata.LastUpdate)
+	if daysSinceLastUpdate > 5 {
+		log.Msg.Info("Full CIF download required")
+		err := runCifFullDownload(cfg)
+		if err != nil {
+			log.Msg.Error("Unable to run full CIF Update", zap.Error(err))
+		}
+		return
+	}
 
-	// Here I need to determine which days I need to update CIF data for, then pass to an appropriate function:
+	daysToUpdate := generateUpdateDays(daysSinceLastUpdate)
-	// newMetadata, err := runCifUpdate(days []time.Time, cfg)
+
+	// Run the update
+	log.Msg.Info("CIF Update required", zap.Any("days to update", daysToUpdate))
+	err = runCifUpdateDownload(cfg, metadata, daysToUpdate)
+	if err != nil {
+		log.Msg.Error("Unable to run CIF update", zap.Error(err))
+	}
+
+	return
 }

src/cif/helpers.go

@@ -51,3 +51,20 @@ func howManyDaysAgo(t time.Time) int {
 	days := int(diff.Hours() / 24)
 	return days
 }
+
+// Generates a slice of time.Time values representing which days files need downloading
+func generateUpdateDays(days int) []time.Time {
+	var updateDays []time.Time
+
+	for i := 0; i < days; i++ {
+		day := time.Now().Add(-time.Duration(i) * 24 * time.Hour)
+		updateDays = append(updateDays, day)
+	}
+
+	// Reverse slice to ensure chronological order
+	for i, j := 0, len(updateDays)-1; i < j; i, j = i+1, j-1 {
+		updateDays[i], updateDays[j] = updateDays[j], updateDays[i]
+	}
+
+	return updateDays
+}

src/cif/helpers_test.go

@@ -17,3 +17,76 @@ func TestIsSameDay(t *testing.T) {
 		t.Errorf("Error in isSameDay(notToday).  Expected false, got true.")
 	}
 }
+
+func TestHowManyDaysAgo(t *testing.T) {
+	testCases := []struct {
+		input    time.Time
+		expected int
+	}{
+		{time.Now(), 0},                      // Today
+		{time.Now().Add(-24 * time.Hour), 1}, // Yesterday
+		{time.Now().Add(-48 * time.Hour), 2}, // Ereyesterday
+		{time.Now().Add(24 * time.Hour), -1}, // Tomorrow
+		{time.Now().Add(48 * time.Hour), -2}, // Overmorrow
+	}
+
+	for _, tc := range testCases {
+		result := howManyDaysAgo(tc.input)
+		if result != tc.expected {
+			t.Errorf("For input %v, expected %d but got %d", tc.input, tc.expected, result)
+		}
+	}
+}
+
+func TestGetDayString(t *testing.T) {
+	testCases := []struct {
+		input    time.Time
+		expected string
+	}{ // Note that the test times are in UTC, but days are checked in Europe/London
+		{time.Date(2024, time.April, 7, 0, 0, 0, 0, time.UTC), "sun"},
+		{time.Date(2024, time.April, 4, 21, 0, 0, 0, time.UTC), "thu"},
+		{time.Date(2001, time.September, 11, 12, 46, 0, 0, time.UTC), "tue"},
+	}
+
+	for _, tc := range testCases {
+		result := getDayString(tc.input)
+		if result != tc.expected {
+			t.Errorf("For input %v, expected %s, but got %s", tc.input, tc.expected, result)
+		}
+	}
+}
+
+func TestGenerateUpdateDays(t *testing.T) {
+	testCases := []struct {
+		days     int
+		expected []time.Time
+	}{
+		{1, []time.Time{time.Now()}},
+		{2, []time.Time{time.Now().Add(-24 * time.Hour), time.Now()}},
+		{4, []time.Time{time.Now().Add(-72 * time.Hour),
+			time.Now().Add(-48 * time.Hour),
+			time.Now().Add(-24 * time.Hour),
+			time.Now(),
+		}},
+	}
+
+	for _, tc := range testCases {
+		result := generateUpdateDays(tc.days)
+
+		if len(result) != len(tc.expected) {
+			t.Errorf("For %d days, expected %v, but got %v", tc.days, tc.expected, result)
+			continue
+		}
+
+		for i := range result {
+			if !isSameDate(result[i], tc.expected[i]) {
+				t.Errorf("For %d days, expected %v, but got %v", tc.days, tc.expected, result)
+			}
+		}
+	}
+}
+
+// Checks if two time values have the same year, month and day.
+func isSameDate(t1, t2 time.Time) bool {
+	return t1.Year() == t2.Year() && t1.Month() == t2.Month() && t1.Day() == t2.Day()
+}

src/cif/parse_test.go

@@ -0,0 +1,105 @@
+package cif
+
+import (
+	"testing"
+	"time"
+
+	"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
+)
+
+func TestParseCifData(t *testing.T) {
+	testCases := []struct {
+		name     string
+		input    []byte
+		expected *parsedData
+	}
+}
+
+func returnParsedData() *parsedData {
+	var parsed = {
+		name: "Valid JSON Data 1",
+		input: []byte(`
+		    {"JsonTimetableV1":{"classification":"public","timestamp":1711227636,"owner":"Network Rail","Sender":{"organisation":"Rockshore","application":"NTROD","component":"SCHEDULE"},"Metadata":{"type":"update","sequence":4307}}}
+			{"JsonAssociationV1":{"transaction_type":"Create","main_train_uid":"L65283","assoc_train_uid":"L65245","assoc_start_date":"2023-12-11T00:00:00Z","assoc_end_date":"2024-03-28T00:00:00Z","assoc_days":"1111000","category":"  ","date_indicator":" ","location":"RDNGSTN","base_location_suffix":null,"assoc_location_suffix":null,"diagram_type":"T","CIF_stp_indicator":"C"}}
+            {"JsonAssociationV1":{"transaction_type":"Delete","main_train_uid":"L65283","assoc_train_uid":"L65245","assoc_start_date":"2024-03-18T00:00:00Z","location":"RDNGSTN","base_location_suffix":null,"diagram_type":"T","CIF_stp_indicator":"C"}}
+			{"JsonScheduleV1":{"CIF_stp_indicator":"O","CIF_train_uid":"C25513","schedule_start_date":"2024-03-23","transaction_type":"Delete"}}
+			{"JsonScheduleV1":{"CIF_bank_holiday_running":null,"CIF_stp_indicator":"O","CIF_train_uid":"Y04345","applicable_timetable":"Y","atoc_code":"SW","new_schedule_segment":{"traction_class":"","uic_code":""},"schedule_days_runs":"0000001","schedule_end_date":"2024-05-19","schedule_segment":{"signalling_id":"5A20","CIF_train_category":"EE","CIF_headcode":"00","CIF_course_indicator":1,"CIF_train_service_code":"24676004","CIF_business_sector":"??","CIF_power_type":"EMU","CIF_timing_load":null,"CIF_speed":"100","CIF_operating_characteristics":"D","CIF_train_class":null,"CIF_sleepers":null,"CIF_reservations":null,"CIF_connection_indicator":null,"CIF_catering_code":null,"CIF_service_branding":"","schedule_location":[{"location_type":"LO","record_identity":"LO","tiploc_code":"FARNHMD","tiploc_instance":null,"departure":"0721","public_departure":null,"platform":null,"line":null,"engineering_allowance":null,"pathing_allowance":null,"performance_allowance":null},{"location_type":"LT","record_identity":"LT","tiploc_code":"FARNHAM","tiploc_instance":null,"platform":"1","arrival":"0729","public_arrival":null,"path":null}]},"schedule_start_date":"2024-05-19","train_status":"T","transaction_type":"Create"}}		
+			{"EOF":true}
+		`),
+		expected: &parsedData{
+			header: upstreamApi.JsonTimetableV1{
+				Classification: "public",
+				Timestamp:      1711227636,
+				Owner:          "Network Rail",
+				Sender: upstreamApi.TimetableSender{
+					Organisation: "Rockshore",
+					Application:  "NTROD",
+					Component:    "SCHEDULE",
+				},
+				Metadata: upstreamApi.TimetableMetadata{
+					Type:     "update",
+					Sequence: 4307,
+				},
+			},
+			assoc: []upstreamApi.JsonAssociationV1{
+				TransactionType: "Create",
+				MainTrainUid:    "L65283",
+				AssocTrainUid:   "L65245",
+				AssocStartDate:  time.Date(2023, time.December, 11, 0, 0, 0, 0, time.UTC),
+				AssocEndDate:    time.Date(2024, time.March, 28, 0, 0, 0, 0, time.UTC),
+				AssocDays:       "1111000",
+				Location:        "RDNGSTN",
+				DiagramType:     "T",
+				CifStpIndicator: "C",
+			}, {
+				TransactionType: "Delete",
+				MainTrainUid:    "L65283",
+				AssocTrainUid:   "L65245",
+				AssocStartDate:  time.Date(2024, time.March, 18, 0, 0, 0, 0, time.UTC),
+				Location:        "RDNGSTN",
+				DiagramType:     "T",
+				CifStpIndicator: "C",
+			},
+			sched: []upstreamApi.JsonScheduleV1{
+				CifStpIndicator:   "O",
+				CifTrainUid:       "C25513",
+				ScheduleStartDate: "2024-03-23",
+				TransactionType:   "Delete",
+			}, {
+				CifStpIndicator:     "O",
+				CifTrainUid:         "Y04345",
+				ApplicableTimetable: "Y",
+				AtocCode:            "SW",
+				NewScheduleSegment:  upstreamApi.CifNewScheduleSegment{},
+				ScheduleDaysRun:     "0000001",
+				ScheduleEndDate:     "2024-05-19",
+				ScheduleSegment: upstreamApi.CifScheduleSegment{
+					SignallingId:                "5A20",
+					CifTrainCategory:            "EE",
+					CifHeadcode:                 "OO",
+					CifCourseIndicator:          1,
+					CifTrainServiceCode:         "24676004",
+					CifBusinessSector:           "??",
+					CifPowerType:                "EMU",
+					CifSpeed:                    "100",
+					CifOperatingCharacteristics: "D",
+					ScheduleLocation: []upstreamApi.CifScheduleLocation{
+						LocationType:   "LO",
+						RecordIdentity: "LO",
+						TiplocCode:     "FARNHMD",
+						Departure:      "0721",
+					}, {
+						LocationType:   "LT",
+						RecordIdentity: "LT",
+						TiplocCode:     "FARNHAM",
+						Platform:       "1",
+						Arrival:        "0729",
+					},
+				},
+				ScheduleStartDate: "2024-05-19",
+				TrainStatus:       "T",
+				TransactionType:   "Create",
+			},
+		},
+	}
+}

src/cif/process.go

@@ -1,21 +1,6 @@
 package cif
 
-import (
+// Processes parsed CIF data and applies the data to the database
-	"errors"
+func processParsedCif(data *parsedData) error {
-
+	return nil
-	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
-)
-
-// Processes all data in the schedule segment of 'data', interacts with the database and returns new metadata
-func processCifData(metadata *dbAccess.CifMetadata, data *parsedData) (*dbAccess.CifMetadata, error) {
-	if data == nil {
-		err := errors.New("data is not defined")
-		return nil, err
-	}
-	if metadata == nil {
-		err := errors.New("metadata is not defined")
-		return nil, err
-	}
-
-	return nil, nil
 }

src/cif/update.go

@@ -1,7 +1,6 @@
 package cif
 
 import (
-	"sort"
 	"time"
 
 	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
@@ -11,105 +10,79 @@ import (
 	"go.uber.org/zap"
 )
 
-// Runs a full update of the CIF Data, discarding any existing data and returns a new metadata struct
+// Replaces all existing CIF Data with a new download
-func runFullUpdate(cfg *helpers.Configuration) (*dbAccess.CifMetadata, error) {
+func runCifFullDownload(cfg *helpers.Configuration) error {
-	log.Msg.Warn("All existing timetable data will be deleted")
+	log.Msg.Info("Downloading all CIF Data")
+
+	// Download CIF Data file
 	url, err := getUpdateUrl("full")
 	if err != nil {
-		log.Msg.Error("Unable to get update URL", zap.Error(err))
+		log.Msg.Error("Error getting download URL", zap.Error(err))
-		return nil, err
 	}
-
+	data, err := nrod.NrodDownload(url, cfg)
-	fullCifData, err := nrod.NrodDownload(url, cfg)
 	if err != nil {
-		log.Msg.Error("Unable to get CIF Data", zap.Error(err))
+		log.Msg.Error("Error downloading CIF data", zap.Error(err))
-		return nil, err
 	}
 
-	log.Msg.Debug("CIF Data Downloaded", zap.ByteString("CIF Data", fullCifData))
+	// Parse CIF file
+	parsed, err := parseCifData(data)
+	if err != nil {
+		log.Msg.Error("Error parsing CIF data", zap.Error(err))
+		return err
+	}
 
-	// I now need to define a processing function and ensure a valid type exists, then I can pass that type to a CIF Put Full function
+	// Drop timetable collection
-	// which will handle placing the data into the database
+	dbAccess.DropCollection(dbAccess.TimetableCollection)
 
-	return nil, nil
+	// Process CIF file
+	err = processParsedCif(parsed)
+	if err != nil {
+		log.Msg.Error("Error processing CIF data", zap.Error(err))
+	}
+
+	// Generate & Write metadata
+
+	return nil
 }
 
-// Runs the daily update for CIF Data, can handle up to five days updates at once.
+// Runs a CIF Update for up to five days
-func runUpdate(metadata *dbAccess.CifMetadata, cfg *helpers.Configuration) (*dbAccess.CifMetadata, error) {
+func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMetadata, days []time.Time) error {
-	// Do not run update if last update was on same day
+	log.Msg.Info("Downloading CIF Updates")
-	if isSameToday(metadata.LastUpdate) {
-		log.Msg.Info("No CIF Update Required", zap.Time("Last update", metadata.LastUpdate))
-		return nil, nil
-	}
 
-	// Do not run update before 0600 as todays data will not be available
+	// Loop over dates
-	if time.Now().Hour() < 6 {
+	for _, time := range days {
-		log.Msg.Info("Too early to update CIF Data")
+		log.Msg.Info("Downloading CIF File", zap.Time("CIF Data from", time))
-		return nil, nil
-	}
 
-	// Check how many days ago last update was
+		// Download CIF data file
-	lastUpateDays := howManyDaysAgo(metadata.LastUpdate)
+		data, err := fetchUpdate(time, cfg)
-	if lastUpateDays > 5 {
-		log.Msg.Warn("CIF Data is more than five days old.  Running Full Update")
-		newMeta, err := runFullUpdate(cfg)
 		if err != nil {
-			log.Msg.Error("CIF Update failed", zap.Error(err))
+			log.Msg.Error("Error fetching CIF update", zap.Error(err))
-			return nil, err
+			return err
 		}
-		return newMeta, nil
+		// Parse CIF file
-	}
+		parsed, err := parseCifData(data)
-
-	// Create a slice containing which dates need updating
-	firstUpdate := time.Now().In(time.UTC).AddDate(0, 0, -lastUpateDays)
-	finalUpdate := time.Now().In(time.UTC)
-	var dates []time.Time
-
-	for d := firstUpdate; d.Before(finalUpdate) || d.Equal(finalUpdate); d = d.AddDate(0, 0, 1) {
-		dates = append(dates, d)
-	}
-
-	sort.Slice(dates, func(i, j int) bool {
-		return dates[i].Before(dates[j])
-	})
-
-	log.Msg.Info("Updating CIF Data", zap.Any("dates to update", dates))
-
-	// Iterate over each date, fetching then parsing the data
-	for _, date := range dates {
-		data, err := fetchUpdate(date, cfg)
 		if err != nil {
-			log.Msg.Error("Error fetching data", zap.Time("date", date))
+			log.Msg.Error("Error parsing CIF data", zap.Error(err))
-			continue
+			return err
-		} // parseCifData function needs writing
-		parsedData, err := parseCifData(data)
-		if err != nil {
-			log.Msg.Error("Error parsing data", zap.Time("date", date))
 		}
 
-		if parsedData == nil { // TEMPORARY DEVELOPMENT CHECK
+		// Check metadata sequence - Handle a metadata sequence error. Probably by deleting metadata so next update triggers full download
-			log.Msg.Error("No data parsed")
+		//// I need to check what the sequence looks like in a full download first.
-			return nil, nil
+		// Process CIF file
-		}
+		// Generate & Write metadata
-
-		// Process data (manages the database interactions and >>returns the new metadata)
-		_, err = processCifData(metadata, parsedData)
-		if err != nil {
-			log.Msg.Error("Error processing CIF Data", zap.Error(err))
-		}
-
-		log.Msg.Info("CIF Data updated", zap.Time("date", date))
 	}
-	return nil, nil //TEMPORARY
+
+	return nil
 }
 
-// Fetches CIF Updates for a given day
+// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
 func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) {
 	url, err := getUpdateUrl("daily")
 	if err != nil {
 		return nil, err
 	}
 
+	// Append day string to URL
 	url = url + getDayString(t)
 
 	downloadedData, err := nrod.NrodDownload(url, cfg)

src/dbAccess/client.go

@@ -12,12 +12,6 @@ import (
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
-const databaseName string = "owlboard"
-const CorpusCollection string = "corpus"
-const StationsCollection string = "stations"
-const metaCollection string = "meta"
-const TimetableCollection string = "timetable"
-
 // Provide the DB Connection to other functions
 var MongoClient (*mongo.Client)
 

src/dbAccess/contants.go

@@ -0,0 +1,7 @@
+package dbAccess
+
+const databaseName string = "owlboard"
+const CorpusCollection string = "corpus"
+const StationsCollection string = "stations"
+const metaCollection string = "meta"
+const TimetableCollection string = "timetable"