From 12c37fe86d8e3bbd9b69eee992775c9873c95dc6 Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Sat, 30 Mar 2024 01:09:12 +0000 Subject: [PATCH] Further work on CIF Data downloads --- src/cif/helpers.go | 25 +++++++++++--- src/cif/update.go | 79 ++++++++++++++++++++++++++++++++++++++------ src/nrod/download.go | 1 + 3 files changed, 91 insertions(+), 14 deletions(-) diff --git a/src/cif/helpers.go b/src/cif/helpers.go index efe1faf..e093b16 100644 --- a/src/cif/helpers.go +++ b/src/cif/helpers.go @@ -8,14 +8,14 @@ import ( "go.uber.org/zap" ) -// Fetches the day string for TODAYs update. Needs adjusting to be able to accept a time.Time type and return the day string for that day -func getDayString() string { +// Fetches the day string for the provided date. +func getDayString(t time.Time) string { london, err := time.LoadLocation("Europe/London") if err != nil { log.Msg.Error("Unable to load time zone info", zap.Error(err)) } - timeNow := time.Now().In(london) + timeNow := t.In(london) day := timeNow.Weekday() dayStrings := [...]string{"sun", "mon", "tue", "wed", "thu", "fri", "sat"} @@ -26,10 +26,27 @@ func getDayString() string { // Simply returns the correct URL for either a 'daily' or 'full' update. func getUpdateUrl(updateType string) (string, error) { if updateType == "daily" { - return dailyUpdateUrl + getDayString(), nil + return dailyUpdateUrl, nil } else if updateType == "full" { return fullUpdateUrl, nil } err := errors.New("invalid update type provided, must be one of 'daily' or 'full'") return "", err } + +// Takes a time.Time as input and returns True if it is +// the same day as now, or false if it is not the same day as now +func isSameToday(t time.Time) bool { + today := time.Now().In(time.UTC) + return t.Year() == today.Year() && t.Month() == today.Month() && t.Day() == today.Day() +} + +// Returns how many days ago `t` was compared to today +func howManyDaysAgo(t time.Time) int { + today := time.Now().In(time.UTC).Truncate(24 * time.Hour) + input := t.In(time.UTC).Truncate(24 * time.Hour) + + diff := today.Sub(input) + days := int(diff.Hours() / 24) + return days +} diff --git a/src/cif/update.go b/src/cif/update.go index 4c2f5a3..06c3b38 100644 --- a/src/cif/update.go +++ b/src/cif/update.go @@ -1,7 +1,8 @@ package cif import ( - "errors" + "sort" + "time" "git.fjla.uk/owlboard/timetable-mgr/dbAccess" "git.fjla.uk/owlboard/timetable-mgr/helpers" @@ -33,16 +34,58 @@ func runFullUpdate(cfg *helpers.Configuration) (*dbAccess.CifMetadata, error) { return nil, nil } -// Run the specified update type. Update type must be one of 'daily' or 'full' -// In the case of daily update, things get complicated as it needs to handle cases where up to five days have been missed. -func runUpdate(updateType string, metadata *dbAccess.CifMetadata) (*dbAccess.CifMetadata, error) { - url, err := getUpdateUrl(updateType) - if err != nil { - log.Msg.Error("Unable to get the update URL", zap.Error(err)) - return nil, err +// Runs the daily update for CIF Data, can handle up to five days updates at once. +func runUpdate(metadata *dbAccess.CifMetadata, cfg *helpers.Configuration) (*dbAccess.CifMetadata, error) { + // Do not run update if last update was on same day + if isSameToday(metadata.LastUpdate) { + log.Msg.Info("No CIF Update Required", zap.Time("Last update", metadata.LastUpdate)) + return nil, nil + } + + // Do not run update before 0600 as todays data will not be available + if time.Now().Hour() < 6 { + log.Msg.Info("Too early to update CIF Data") + return nil, nil + } + + // Check how many days ago last update was + lastUpateDays := howManyDaysAgo(metadata.LastUpdate) + if lastUpateDays > 5 { + log.Msg.Warn("CIF Data is more than five days old. Running Full Update") + newMeta, err := runFullUpdate(cfg) + if err != nil { + log.Msg.Error("CIF Update failed", zap.Error(err)) + } + return newMeta, nil + } + + // Create a slice containing which dates need updating + firstUpdate := time.Now().In(time.UTC).AddDate(0, 0, -lastUpateDays) + finalUpdate := time.Now().In(time.UTC) + var dates []time.Time + + for d := firstUpdate; d.Before(finalUpdate) || d.Equal(finalUpdate); d = d.AddDate(0, 0, 1) { + dates = append(dates, d) + } + + sort.Slice(dates, func(i, j int) bool { + return dates[i].Before(dates[j]) + }) + + // Iterate over each date, fetching then parsing the data + for _, date := range dates { + data, err := fetchUpdate(date, cfg) + if err != nil { + log.Msg.Error("Error fetching data", zap.Time("date", date)) + continue + } // parseCifData function needs writing + parsedData, err := parseCifData(data, metadata) + if err != nil { + log.Msg.Error("Error parsing data", zap.Time("date", date)) + } + // Apply data to Database + log.Msg.Info("CIF Data updated", zap.Time("date", date)) } - log.Msg.Debug("", zap.String("URL", url)) - return nil, errors.New("function is not yet defined") // Use the values in metadata to determine which day to attempt to update. // First check if the last update was today, if so, I can return nil, nil - No update required @@ -52,3 +95,19 @@ func runUpdate(updateType string, metadata *dbAccess.CifMetadata) (*dbAccess.Cif // Write a parsing function that can handle VSTP as well as SCHEDULE data // Handle database management } + +// Fetches CIF Updates for a given day +func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) { + url, err := getUpdateUrl("daily") + if err != nil { + return nil, err + } + + url = url + getDayString(t) + + downloadedData, err := nrod.NrodDownload(url, cfg) + if err != nil { + return nil, err + } + return downloadedData, nil +} diff --git a/src/nrod/download.go b/src/nrod/download.go index 35b21f7..bdecc73 100644 --- a/src/nrod/download.go +++ b/src/nrod/download.go @@ -38,6 +38,7 @@ func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) { return nil, err } + // Yes, I know `readedData` is not proper English. But readData reads more like a verb action. readedData, err := nrodExtract(*resp) if err != nil { log.Msg.Error("Unable to read response data")