Further work on CIF Data downloads

src/cif/helpers.go

@@ -8,14 +8,14 @@ import (
 	"go.uber.org/zap"
 )
 
-// Fetches the day string for TODAYs update.  Needs adjusting to be able to accept a time.Time type and return the day string for that day
+// Fetches the day string for the provided date.
-func getDayString() string {
+func getDayString(t time.Time) string {
 	london, err := time.LoadLocation("Europe/London")
 	if err != nil {
 		log.Msg.Error("Unable to load time zone info", zap.Error(err))
 	}
 
-	timeNow := time.Now().In(london)
+	timeNow := t.In(london)
 	day := timeNow.Weekday()
 
 	dayStrings := [...]string{"sun", "mon", "tue", "wed", "thu", "fri", "sat"}
@@ -26,10 +26,27 @@ func getDayString() string {
 // Simply returns the correct URL for either a 'daily' or 'full' update.
 func getUpdateUrl(updateType string) (string, error) {
 	if updateType == "daily" {
-		return dailyUpdateUrl + getDayString(), nil
+		return dailyUpdateUrl, nil
 	} else if updateType == "full" {
 		return fullUpdateUrl, nil
 	}
 	err := errors.New("invalid update type provided, must be one of 'daily' or 'full'")
 	return "", err
 }
+
+// Takes a time.Time as input and returns True if it is
+// the same day as now, or false if it is not the same day as now
+func isSameToday(t time.Time) bool {
+	today := time.Now().In(time.UTC)
+	return t.Year() == today.Year() && t.Month() == today.Month() && t.Day() == today.Day()
+}
+
+// Returns how many days ago `t` was compared to today
+func howManyDaysAgo(t time.Time) int {
+	today := time.Now().In(time.UTC).Truncate(24 * time.Hour)
+	input := t.In(time.UTC).Truncate(24 * time.Hour)
+
+	diff := today.Sub(input)
+	days := int(diff.Hours() / 24)
+	return days
+}

src/cif/update.go

@@ -1,7 +1,8 @@
 package cif
 
 import (
-	"errors"
+	"sort"
+	"time"
 
 	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
 	"git.fjla.uk/owlboard/timetable-mgr/helpers"
@@ -33,16 +34,58 @@ func runFullUpdate(cfg *helpers.Configuration) (*dbAccess.CifMetadata, error) {
 	return nil, nil
 }
 
-// Run the specified update type.  Update type must be one of 'daily' or 'full'
+// Runs the daily update for CIF Data, can handle up to five days updates at once.
-// In the case of daily update, things get complicated as it needs to handle cases where up to five days have been missed.
+func runUpdate(metadata *dbAccess.CifMetadata, cfg *helpers.Configuration) (*dbAccess.CifMetadata, error) {
-func runUpdate(updateType string, metadata *dbAccess.CifMetadata) (*dbAccess.CifMetadata, error) {
+	// Do not run update if last update was on same day
-	url, err := getUpdateUrl(updateType)
+	if isSameToday(metadata.LastUpdate) {
-	if err != nil {
+		log.Msg.Info("No CIF Update Required", zap.Time("Last update", metadata.LastUpdate))
-		log.Msg.Error("Unable to get the update URL", zap.Error(err))
+		return nil, nil
-		return nil, err
+	}
+
+	// Do not run update before 0600 as todays data will not be available
+	if time.Now().Hour() < 6 {
+		log.Msg.Info("Too early to update CIF Data")
+		return nil, nil
+	}
+
+	// Check how many days ago last update was
+	lastUpateDays := howManyDaysAgo(metadata.LastUpdate)
+	if lastUpateDays > 5 {
+		log.Msg.Warn("CIF Data is more than five days old.  Running Full Update")
+		newMeta, err := runFullUpdate(cfg)
+		if err != nil {
+			log.Msg.Error("CIF Update failed", zap.Error(err))
+		}
+		return newMeta, nil
+	}
+
+	// Create a slice containing which dates need updating
+	firstUpdate := time.Now().In(time.UTC).AddDate(0, 0, -lastUpateDays)
+	finalUpdate := time.Now().In(time.UTC)
+	var dates []time.Time
+
+	for d := firstUpdate; d.Before(finalUpdate) || d.Equal(finalUpdate); d = d.AddDate(0, 0, 1) {
+		dates = append(dates, d)
+	}
+
+	sort.Slice(dates, func(i, j int) bool {
+		return dates[i].Before(dates[j])
+	})
+
+	// Iterate over each date, fetching then parsing the data
+	for _, date := range dates {
+		data, err := fetchUpdate(date, cfg)
+		if err != nil {
+			log.Msg.Error("Error fetching data", zap.Time("date", date))
+			continue
+		} // parseCifData function needs writing
+		parsedData, err := parseCifData(data, metadata)
+		if err != nil {
+			log.Msg.Error("Error parsing data", zap.Time("date", date))
+		}
+		// Apply data to Database
+		log.Msg.Info("CIF Data updated", zap.Time("date", date))
 	}
-	log.Msg.Debug("", zap.String("URL", url))
-	return nil, errors.New("function is not yet defined")
 
 	// Use the values in metadata to determine which day to attempt to update.
 	// First check if the last update was today, if so, I can return nil, nil - No update required
@@ -52,3 +95,19 @@ func runUpdate(updateType string, metadata *dbAccess.CifMetadata) (*dbAccess.Cif
 	// Write a parsing function that can handle VSTP as well as SCHEDULE data
 	// Handle database management
 }
+
+// Fetches CIF Updates for a given day
+func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) {
+	url, err := getUpdateUrl("daily")
+	if err != nil {
+		return nil, err
+	}
+
+	url = url + getDayString(t)
+
+	downloadedData, err := nrod.NrodDownload(url, cfg)
+	if err != nil {
+		return nil, err
+	}
+	return downloadedData, nil
+}

src/nrod/download.go

@@ -38,6 +38,7 @@ func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) {
 		return nil, err
 	}
 
+	// Yes, I know `readedData` is not proper English.  But readData reads more like a verb action.
 	readedData, err := nrodExtract(*resp)
 	if err != nil {
 		log.Msg.Error("Unable to read response data")