Reorganise repo

cif/check.go

@@ -0,0 +1,67 @@
+package cif
+
+import (
+	"time"
+
+	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
+	"git.fjla.uk/owlboard/timetable-mgr/helpers"
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.uber.org/zap"
+)
+
+// Checks if the CIF Data needs updating, and what type of update is needed (Full/Partial) and if partial
+// what days data needs updating, then calls an update function to handle the update.
+func CheckCif(cfg *helpers.Configuration) {
+	// Check that it is after 0600, if not then skip update
+	if time.Now().In(londonTimezone).Hour() <= dataAvailable {
+		log.Msg.Info("Too early to update CIF data, not published until 0600")
+		return
+	}
+
+	log.Msg.Info("Checking age of CIF Data")
+
+	// Load and read metadata from database
+	metadata, err := dbAccess.GetCifMetadata()
+	if err != nil {
+		log.Msg.Error("Unable to read last update time")
+		return
+	}
+
+	// If no metadata is found in DB, presume no CIF data exists
+	if metadata == nil {
+		log.Msg.Info("Full CIF download required")
+		err := runCifFullDownload(cfg)
+		if err != nil {
+			log.Msg.Error("Unable to run full CIF Update", zap.Error(err))
+		}
+		return
+	}
+
+	// Check if last update was today
+	if isSameToday(metadata.LastUpdate) {
+		log.Msg.Info("CIF Data has already been updated today, skipping")
+		return
+	}
+
+	// Check how many days since last update, if more than 5, run full update, else run update
+	daysSinceLastUpdate := howManyDaysAgo(metadata.LastUpdate)
+	if daysSinceLastUpdate > 5 {
+		log.Msg.Info("Full CIF download required")
+		err := runCifFullDownload(cfg)
+		if err != nil {
+			log.Msg.Error("Unable to run full CIF Update", zap.Error(err))
+		}
+		return
+	}
+
+	daysToUpdate := generateUpdateDays(daysSinceLastUpdate)
+
+	// Run the update
+	log.Msg.Info("CIF Update required", zap.Any("days to update", daysToUpdate))
+	err = runCifUpdateDownload(cfg, metadata, daysToUpdate)
+	if err != nil {
+		log.Msg.Error("Unable to run CIF update", zap.Error(err))
+	}
+
+	return
+}

cif/constants.go

@@ -0,0 +1,15 @@
+package cif
+
+import "time"
+
+// The URL required for a daily update of the CIF Data - The 'day string' must be appended
+const dailyUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-"
+
+// The URL required for a full fetch of the CIF Data
+const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full"
+
+// The time at which CIF Data is expected to be available for download (full hour)
+const dataAvailable = 6
+
+// An object representing the Europe/London timezone
+var londonTimezone, err = time.LoadLocation("Europe/London")

cif/helpers.go

@@ -0,0 +1,103 @@
+package cif
+
+import (
+	"errors"
+	"time"
+
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.uber.org/zap"
+)
+
+// Fetches the day string for the provided date.
+func getDayString(t time.Time) string {
+	london, err := time.LoadLocation("Europe/London")
+	if err != nil {
+		log.Msg.Error("Unable to load time zone info", zap.Error(err))
+	}
+
+	timeNow := t.In(london)
+	day := timeNow.Weekday()
+
+	dayStrings := [...]string{"sun", "mon", "tue", "wed", "thu", "fri", "sat"}
+
+	return dayStrings[day]
+}
+
+// Simply returns the correct URL for either a 'daily' or 'full' update.
+func getUpdateUrl(updateType string) (string, error) {
+	if updateType == "daily" {
+		return dailyUpdateUrl, nil
+	} else if updateType == "full" {
+		return fullUpdateUrl, nil
+	}
+	err := errors.New("invalid update type provided, must be one of 'daily' or 'full'")
+	return "", err
+}
+
+// Takes a time.Time as input and returns True if it is
+// the same day as now, or false if it is not the same day as now
+func isSameToday(t time.Time) bool {
+	test := t.In(time.UTC)
+	today := time.Now().In(time.UTC)
+	return test.Year() == today.Year() && test.Month() == today.Month() && test.Day() == today.Day()
+}
+
+// Returns how many days ago `t` was compared to today
+func howManyDaysAgo(t time.Time) int {
+	today := time.Now().In(time.UTC).Truncate(24 * time.Hour)
+	input := t.In(time.UTC).Truncate(24 * time.Hour)
+
+	diff := today.Sub(input)
+	days := int(diff.Hours() / 24)
+	return days
+}
+
+// Generates a slice of time.Time values representing which days files need downloading
+func generateUpdateDays(days int) []time.Time {
+	var updateDays []time.Time
+
+	for i := 0; i < days; i++ {
+		day := time.Now().Add(-time.Duration(i) * 24 * time.Hour)
+		updateDays = append(updateDays, day)
+	}
+
+	// Reverse slice to ensure chronological order
+	for i, j := 0, len(updateDays)-1; i < j; i, j = i+1, j-1 {
+		updateDays[i], updateDays[j] = updateDays[j], updateDays[i]
+	}
+
+	return updateDays
+}
+
+// Parses CIF Schedule Start/End Dates (YYYY-MM-DD) into time.Time types (00:00:00 for start, 23:59:59 for end)
+func ParseCifDate(input, startOrEnd string) time.Time {
+	layout := "2006-01-02" // Layout of input
+	t, err := time.ParseInLocation(layout, input, londonTimezone)
+	if err != nil {
+		log.Msg.Error("Error parsing date string", zap.String("date string", input), zap.Error(err))
+		return time.Time{}
+	}
+
+	if startOrEnd == "start" {
+		t = time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, londonTimezone)
+	} else if startOrEnd == "end" {
+		t = time.Date(t.Year(), t.Month(), t.Day(), 23, 59, 59, 0, londonTimezone)
+	} else {
+		log.Msg.Error("Error parsing date string", zap.String("date string", input), zap.Error(err))
+		return time.Time{}
+	}
+
+	return t
+}
+
+// Parses CIF days_run field and converts to array of day strings
+func parseDaysRun(daysBinary string) []string {
+	shortDays := []string{"m", "t", "w", "th", "f", "s", "su"}
+	var result []string
+	for i, digit := range daysBinary {
+		if digit == '1' {
+			result = append(result, shortDays[i])
+		}
+	}
+	return result
+}

cif/helpers_test.go

@@ -0,0 +1,135 @@
+package cif
+
+import (
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestIsSameDay(t *testing.T) {
+	today := time.Now()
+	notToday := time.Date(2024, 01, 23, 23, 01, 3, 421, time.Local)
+
+	if !isSameToday(today) {
+		t.Errorf("Error in isSameDay(today).  Expected true, got false.")
+	}
+
+	if isSameToday(notToday) {
+		t.Errorf("Error in isSameDay(notToday).  Expected false, got true.")
+	}
+}
+
+func TestHowManyDaysAgo(t *testing.T) {
+	testCases := []struct {
+		input    time.Time
+		expected int
+	}{
+		{time.Now(), 0},                      // Today
+		{time.Now().Add(-24 * time.Hour), 1}, // Yesterday
+		{time.Now().Add(-48 * time.Hour), 2}, // Ereyesterday
+		{time.Now().Add(24 * time.Hour), -1}, // Tomorrow
+		{time.Now().Add(48 * time.Hour), -2}, // Overmorrow
+	}
+
+	for _, tc := range testCases {
+		result := howManyDaysAgo(tc.input)
+		if result != tc.expected {
+			t.Errorf("For input %v, expected %d but got %d", tc.input, tc.expected, result)
+		}
+	}
+}
+
+func TestGetDayString(t *testing.T) {
+	testCases := []struct {
+		input    time.Time
+		expected string
+	}{ // Note that the test times are in UTC, but days are checked in Europe/London
+		{time.Date(2024, time.April, 7, 0, 0, 0, 0, time.UTC), "sun"},
+		{time.Date(2024, time.April, 4, 21, 0, 0, 0, time.UTC), "thu"},
+		{time.Date(2001, time.September, 11, 12, 46, 0, 0, time.UTC), "tue"},
+	}
+
+	for _, tc := range testCases {
+		result := getDayString(tc.input)
+		if result != tc.expected {
+			t.Errorf("For input %v, expected %s, but got %s", tc.input, tc.expected, result)
+		}
+	}
+}
+
+func TestGenerateUpdateDays(t *testing.T) {
+	testCases := []struct {
+		days     int
+		expected []time.Time
+	}{
+		{1, []time.Time{time.Now()}},
+		{2, []time.Time{time.Now().Add(-24 * time.Hour), time.Now()}},
+		{4, []time.Time{time.Now().Add(-72 * time.Hour),
+			time.Now().Add(-48 * time.Hour),
+			time.Now().Add(-24 * time.Hour),
+			time.Now(),
+		}},
+	}
+
+	for _, tc := range testCases {
+		result := generateUpdateDays(tc.days)
+
+		if len(result) != len(tc.expected) {
+			t.Errorf("For %d days, expected %v, but got %v", tc.days, tc.expected, result)
+			continue
+		}
+
+		for i := range result {
+			if !isSameDate(result[i], tc.expected[i]) {
+				t.Errorf("For %d days, expected %v, but got %v", tc.days, tc.expected, result)
+			}
+		}
+	}
+}
+
+func TestParseCifDate(t *testing.T) {
+	testCases := []struct {
+		dateString string
+		startOrEnd string
+		expect     time.Time
+	}{
+		{"2024-04-05", "start", time.Date(2024, time.April, 5, 0, 0, 0, 0, londonTimezone)},
+		{"2022-01-01", "start", time.Date(2022, time.January, 1, 0, 0, 0, 0, londonTimezone)},
+		{"2015-09-26", "end", time.Date(2015, time.September, 26, 23, 59, 59, 0, londonTimezone)},
+		{"2018-03-13", "end", time.Date(2018, time.March, 13, 23, 59, 59, 0, londonTimezone)},
+	}
+
+	layout := "2006-01-02 15:04:05" // Layout for printing times in error cases.
+
+	for _, tc := range testCases {
+		result := ParseCifDate(tc.dateString, tc.startOrEnd)
+		if result != tc.expect {
+			t.Errorf("For datestring %s, startOrEnd %s, expected %s, but got %s", tc.dateString, tc.startOrEnd, tc.expect.Format(layout), result.Format(layout))
+		}
+	}
+}
+
+func TestParseDaysRun(t *testing.T) {
+	testCases := []struct {
+		input  string
+		expect []string
+	}{
+		{"1111111", []string{"m", "t", "w", "th", "f", "s", "su"}},
+		{"0000001", []string{"su"}},
+		{"1000000", []string{"m"}},
+		{"0000100", []string{"f"}},
+		{"0111000", []string{"t", "w", "th"}},
+	}
+
+	for _, tc := range testCases {
+		result := parseDaysRun(tc.input)
+		if !reflect.DeepEqual(result, tc.expect) {
+			t.Errorf("For input %s, expected %v, but got %v", tc.input, tc.expect, result)
+		}
+	}
+}
+
+// Checks if two time values have the same year, month and day.
+func isSameDate(t1, t2 time.Time) bool {
+	return t1.Year() == t2.Year() && t1.Month() == t2.Month() && t1.Day() == t2.Day()
+}

cif/parse.go

@@ -0,0 +1,64 @@
+package cif
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.uber.org/zap"
+)
+
+// Unmarshalls data into the correct types for processing
+func parseCifData(data []byte) (*parsedData, error) {
+	// Split the data into lines
+	lines := bytes.Split(data, []byte("\n"))
+
+	// Initialise variable for the parsed data
+	var parsed parsedData
+	parsed.assoc = make([]upstreamApi.JsonAssociationV1, 0)
+	parsed.sched = make([]upstreamApi.JsonScheduleV1, 0)
+
+	for _, line := range lines {
+
+		// Skip empty lines to avoid logging errors when there is no error
+		if len(bytes.TrimSpace(line)) == 0 {
+			continue
+		}
+
+		// Map each line for processing
+		var obj map[string]json.RawMessage
+		if err := json.Unmarshal(line, &obj); err != nil {
+			log.Msg.Error("Error decoding line", zap.String("line", string(line)), zap.Error(err))
+			continue
+		}
+
+		// Loop through the mapped data and unmarshal to the correct type
+		for key, value := range obj {
+			switch key {
+			case "JsonTimetableV1":
+				var timetable upstreamApi.JsonTimetableV1
+				if err := json.Unmarshal(value, &timetable); err != nil {
+					log.Msg.Error("Unable to parse JSON Timetable", zap.Error(err), zap.String("line", string(value)))
+					continue
+				}
+				parsed.header = timetable
+			case "JsonAssociationV1":
+				var association upstreamApi.JsonAssociationV1
+				if err := json.Unmarshal(value, &association); err != nil {
+					log.Msg.Error("Error decoding JSON Association", zap.Error(err))
+					continue
+				}
+				parsed.assoc = append(parsed.assoc, association)
+			case "JsonScheduleV1":
+				var schedule upstreamApi.JsonScheduleV1
+				if err := json.Unmarshal(value, &schedule); err != nil {
+					log.Msg.Error("Error decoding JSON Schedule", zap.Error(err))
+					continue
+				}
+				parsed.sched = append(parsed.sched, schedule)
+			}
+		}
+	}
+	return &parsed, nil
+}

cif/process.go

@@ -0,0 +1,68 @@
+package cif
+
+import (
+	"git.fjla.uk/owlboard/go-types/pkg/database"
+	"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.uber.org/zap"
+)
+
+// Processes parsed CIF data and applies the data to the database
+func processParsedCif(data *parsedData) error {
+	createTasks := make([]*upstreamApi.JsonScheduleV1, 0)
+	deleteTasks := make([]*upstreamApi.JsonScheduleV1, 0)
+
+	for _, item := range data.sched {
+		switch item.TransactionType {
+		case "Delete":
+			deleteTasks = append(deleteTasks, &item)
+		case "Create":
+			createTasks = append(createTasks, &item)
+		default:
+			log.Msg.Error("Unknown transaction type in CIF Schedule", zap.String("TransactionType", item.TransactionType))
+		}
+	}
+
+	err := doDeletions(deleteTasks)
+	if err != nil {
+		log.Msg.Error("Error deleting CIF Entries", zap.Error(err))
+		return err
+	}
+	err = doCreations(createTasks)
+	if err != nil {
+		log.Msg.Error("Error creating CIF Entries", zap.Error(err))
+		return err
+	}
+
+	return nil
+}
+
+// Create delete query types and pass to the function which batches the deletions
+func doDeletions(deletions []*upstreamApi.JsonScheduleV1) error {
+	log.Msg.Info("Preparing CIF update Delete tasks", zap.Int("Delete task count", len(deletions)))
+
+	deleteQueries := make([]database.DeleteQuery, 0)
+	for _, item := range deletions {
+		query := database.DeleteQuery{
+			ScheduleStartDate: ParseCifDate(item.ScheduleStartDate, "start"),
+			StpIndicator:      item.CifStpIndicator,
+			TrainUid:          item.CifTrainUid,
+		}
+
+		deleteQueries = append(deleteQueries, query)
+	}
+	return nil
+}
+
+// Convert to the correct struct for the database and pass to the function which batches insertions
+func doCreations(creations []*upstreamApi.JsonScheduleV1) error {
+	log.Msg.Info("Preparing CIF update Create tasks", zap.Int("Create task count", len(creations)))
+
+	createDocuments := make([]database.Service, 0)
+	for _, item := range creations {
+		document := database.Service{}
+
+		createDocuments = append(createDocuments, document)
+	}
+	return nil
+}

cif/types.go

@@ -0,0 +1,10 @@
+package cif
+
+import "git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
+
+// Holds parsed data for processing
+type parsedData struct {
+	header upstreamApi.JsonTimetableV1
+	assoc  []upstreamApi.JsonAssociationV1
+	sched  []upstreamApi.JsonScheduleV1
+}

cif/update.go

@@ -0,0 +1,93 @@
+package cif
+
+import (
+	"time"
+
+	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
+	"git.fjla.uk/owlboard/timetable-mgr/helpers"
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"git.fjla.uk/owlboard/timetable-mgr/nrod"
+	"go.uber.org/zap"
+)
+
+// Replaces all existing CIF Data with a new download
+func runCifFullDownload(cfg *helpers.Configuration) error {
+	log.Msg.Info("Downloading all CIF Data")
+
+	// Download CIF Data file
+	url, err := getUpdateUrl("full")
+	if err != nil {
+		log.Msg.Error("Error getting download URL", zap.Error(err))
+	}
+	data, err := nrod.NrodDownload(url, cfg)
+	if err != nil {
+		log.Msg.Error("Error downloading CIF data", zap.Error(err))
+	}
+
+	// Parse CIF file
+	parsed, err := parseCifData(data)
+	if err != nil {
+		log.Msg.Error("Error parsing CIF data", zap.Error(err))
+		return err
+	}
+
+	// Drop timetable collection
+	dbAccess.DropCollection(dbAccess.TimetableCollection)
+
+	// Process CIF file
+	err = processParsedCif(parsed)
+	if err != nil {
+		log.Msg.Error("Error processing CIF data", zap.Error(err))
+	}
+
+	// Generate & Write metadata
+
+	return nil
+}
+
+// Runs a CIF Update for up to five days
+func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMetadata, days []time.Time) error {
+	log.Msg.Info("Downloading CIF Updates")
+
+	// Loop over dates
+	for _, time := range days {
+		log.Msg.Info("Downloading CIF File", zap.Time("CIF Data from", time))
+
+		// Download CIF data file
+		data, err := fetchUpdate(time, cfg)
+		if err != nil {
+			log.Msg.Error("Error fetching CIF update", zap.Error(err))
+			return err
+		}
+		// Parse CIF file
+		parsed, err = parseCifData(data)
+		if err != nil {
+			log.Msg.Error("Error parsing CIF data", zap.Error(err))
+			return err
+		}
+
+		// Check metadata sequence - Handle a metadata sequence error. Probably by deleting metadata so next update triggers full download
+		//// I need to check what the sequence looks like in a full download first.
+		// Process CIF file
+		// Generate & Write metadata
+	}
+
+	return nil
+}
+
+// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
+func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) {
+	url, err := getUpdateUrl("daily")
+	if err != nil {
+		return nil, err
+	}
+
+	// Append day string to URL
+	url = url + getDayString(t)
+
+	downloadedData, err := nrod.NrodDownload(url, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return downloadedData, nil
+}