Further work on CIF Data downloads

This commit is contained in:
Fred Boniface 2024-03-30 01:09:12 +00:00
parent 8a94e51437
commit 12c37fe86d
3 changed files with 91 additions and 14 deletions

View File

@ -8,14 +8,14 @@ import (
"go.uber.org/zap"
)
// Fetches the day string for TODAYs update. Needs adjusting to be able to accept a time.Time type and return the day string for that day
func getDayString() string {
// Fetches the day string for the provided date.
func getDayString(t time.Time) string {
london, err := time.LoadLocation("Europe/London")
if err != nil {
log.Msg.Error("Unable to load time zone info", zap.Error(err))
}
timeNow := time.Now().In(london)
timeNow := t.In(london)
day := timeNow.Weekday()
dayStrings := [...]string{"sun", "mon", "tue", "wed", "thu", "fri", "sat"}
@ -26,10 +26,27 @@ func getDayString() string {
// Simply returns the correct URL for either a 'daily' or 'full' update.
func getUpdateUrl(updateType string) (string, error) {
if updateType == "daily" {
return dailyUpdateUrl + getDayString(), nil
return dailyUpdateUrl, nil
} else if updateType == "full" {
return fullUpdateUrl, nil
}
err := errors.New("invalid update type provided, must be one of 'daily' or 'full'")
return "", err
}
// Takes a time.Time as input and returns True if it is
// the same day as now, or false if it is not the same day as now
func isSameToday(t time.Time) bool {
today := time.Now().In(time.UTC)
return t.Year() == today.Year() && t.Month() == today.Month() && t.Day() == today.Day()
}
// Returns how many days ago `t` was compared to today
func howManyDaysAgo(t time.Time) int {
today := time.Now().In(time.UTC).Truncate(24 * time.Hour)
input := t.In(time.UTC).Truncate(24 * time.Hour)
diff := today.Sub(input)
days := int(diff.Hours() / 24)
return days
}

View File

@ -1,7 +1,8 @@
package cif
import (
"errors"
"sort"
"time"
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
"git.fjla.uk/owlboard/timetable-mgr/helpers"
@ -33,16 +34,58 @@ func runFullUpdate(cfg *helpers.Configuration) (*dbAccess.CifMetadata, error) {
return nil, nil
}
// Run the specified update type. Update type must be one of 'daily' or 'full'
// In the case of daily update, things get complicated as it needs to handle cases where up to five days have been missed.
func runUpdate(updateType string, metadata *dbAccess.CifMetadata) (*dbAccess.CifMetadata, error) {
url, err := getUpdateUrl(updateType)
if err != nil {
log.Msg.Error("Unable to get the update URL", zap.Error(err))
return nil, err
// Runs the daily update for CIF Data, can handle up to five days updates at once.
func runUpdate(metadata *dbAccess.CifMetadata, cfg *helpers.Configuration) (*dbAccess.CifMetadata, error) {
// Do not run update if last update was on same day
if isSameToday(metadata.LastUpdate) {
log.Msg.Info("No CIF Update Required", zap.Time("Last update", metadata.LastUpdate))
return nil, nil
}
// Do not run update before 0600 as todays data will not be available
if time.Now().Hour() < 6 {
log.Msg.Info("Too early to update CIF Data")
return nil, nil
}
// Check how many days ago last update was
lastUpateDays := howManyDaysAgo(metadata.LastUpdate)
if lastUpateDays > 5 {
log.Msg.Warn("CIF Data is more than five days old. Running Full Update")
newMeta, err := runFullUpdate(cfg)
if err != nil {
log.Msg.Error("CIF Update failed", zap.Error(err))
}
return newMeta, nil
}
// Create a slice containing which dates need updating
firstUpdate := time.Now().In(time.UTC).AddDate(0, 0, -lastUpateDays)
finalUpdate := time.Now().In(time.UTC)
var dates []time.Time
for d := firstUpdate; d.Before(finalUpdate) || d.Equal(finalUpdate); d = d.AddDate(0, 0, 1) {
dates = append(dates, d)
}
sort.Slice(dates, func(i, j int) bool {
return dates[i].Before(dates[j])
})
// Iterate over each date, fetching then parsing the data
for _, date := range dates {
data, err := fetchUpdate(date, cfg)
if err != nil {
log.Msg.Error("Error fetching data", zap.Time("date", date))
continue
} // parseCifData function needs writing
parsedData, err := parseCifData(data, metadata)
if err != nil {
log.Msg.Error("Error parsing data", zap.Time("date", date))
}
// Apply data to Database
log.Msg.Info("CIF Data updated", zap.Time("date", date))
}
log.Msg.Debug("", zap.String("URL", url))
return nil, errors.New("function is not yet defined")
// Use the values in metadata to determine which day to attempt to update.
// First check if the last update was today, if so, I can return nil, nil - No update required
@ -52,3 +95,19 @@ func runUpdate(updateType string, metadata *dbAccess.CifMetadata) (*dbAccess.Cif
// Write a parsing function that can handle VSTP as well as SCHEDULE data
// Handle database management
}
// Fetches CIF Updates for a given day
func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) {
url, err := getUpdateUrl("daily")
if err != nil {
return nil, err
}
url = url + getDayString(t)
downloadedData, err := nrod.NrodDownload(url, cfg)
if err != nil {
return nil, err
}
return downloadedData, nil
}

View File

@ -38,6 +38,7 @@ func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) {
return nil, err
}
// Yes, I know `readedData` is not proper English. But readData reads more like a verb action.
readedData, err := nrodExtract(*resp)
if err != nil {
log.Msg.Error("Unable to read response data")