Complete CIF Metadata parsing logic

cif/check.go

@@ -23,7 +23,7 @@ func CheckCif(cfg *helpers.Configuration) {
 	// Load and read metadata from database
 	metadata, err := dbAccess.GetCifMetadata()
 	if err != nil {
-		log.Msg.Error("Unable to read last update time")
+		log.Msg.Error("Unable to read last update time", zap.Error(err))
 		return
 	}
 

cif/metadata.go

@@ -0,0 +1,39 @@
+package cif
+
+import (
+	"fmt"
+	"time"
+
+	"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
+	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
+)
+
+// Evaluates last and new metadata and determines whether an update is required.
+// // If last update == "full" && lastupdate after yesterday at 0600, run update.
+// // Beyond that condition, comparisons are made between old and new metadata.
+func checkMetadata(oldMeta *dbAccess.CifMetadata, newMeta *upstreamApi.JsonTimetableV1) (reason string, updateRequired bool) {
+	// Handle nil pointer - although this should be resolved in the calling function in ideal situations
+	if oldMeta == nil || newMeta == nil {
+		return "nil-pointer", false
+	}
+
+	// Handle non-matching sequence numbers between full and daily download types
+	if isAfterYesterdayAt0600(oldMeta.LastUpdate) {
+		return "last-update-full", true
+	}
+
+	// Check that the new metadata sequence is as expected.
+	if newMeta.Metadata.Sequence == oldMeta.LastSequence+1 {
+		return "correct-sequence", true
+	} else {
+		s := fmt.Sprintf("incorrect sequence, Old: %d, New: %d, Expected New: %d", oldMeta.LastSequence, newMeta.Metadata.Sequence, oldMeta.LastSequence+1)
+		return s, false
+	}
+}
+
+// Evaluates whether the given time is after yesterday at 0600
+func isAfterYesterdayAt0600(t time.Time) bool {
+	yesterday0600 := time.Now().In(time.UTC).AddDate(0, 0, -1)
+	yesterday0600 = time.Date(yesterday0600.Year(), yesterday0600.Month(), yesterday0600.Day(), 6, 0, 0, 0, time.UTC)
+	return t.After(yesterday0600)
+}

cif/process.go

@@ -13,6 +13,7 @@ import (
 // Processes parsed CIF data and applies the data to the database
 func processParsedCif(data *parsedData) error {
 	log.Msg.Debug("Starting CIF Processing")
+	log.Msg.Info("Processing CIF Data", zap.Int("schedule-count", len(data.sched)))
 
 	// Batch size for processing
 	batchSize := 250 // Needs optimising for better RAM use.  1000 ~ 5.7GB, 500 ~ 5.5GB, 750 ~ 5.2GB

cif/update.go

@@ -1,7 +1,6 @@
 package cif
 
 import (
-	"errors"
 	"io"
 	"time"
 
@@ -73,36 +72,24 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 			return err
 		}
 
+		// Check CIF Metadata
 		log.Msg.Debug("Starting metadata checks")
 
-		// If last update type is == fullUpdateType and LastUpdate less than twodays? ago then run the update regardless of sequence
+		reason, update := checkMetadata(metadata, &parsed.header)
-		// this is because the sequence number of full and daily updates to not appear to match up although I will need to check this!
+		if !update {
-		//if metadata.LastUpdateType == fullUpdateType && metadata.LastUpdate //after_yesterday_at_0600 {
+			log.Msg.Warn("Update file not processed", zap.String("reason", reason))
-		// Somehow skip the below checks!  Maybe I need a checkMetadata function!
-		//}
-
-		// Check CIF Sequence
-		// Skip if LastSequence is >= to this sequence
-		if metadata.LastSequence >= parsed.header.Metadata.Sequence {
-			log.Msg.Info("Skipping CIF file, already processed", zap.Int64("LastSequence", metadata.LastSequence), zap.Int64("New sequence", parsed.header.Metadata.Sequence))
 			continue
 		}
 
-		// Fail if sequence number is not as expected
+		log.Msg.Info("CIF Data is suitable for processing", zap.String("reason", reason))
-		if parsed.header.Metadata.Sequence != metadata.LastSequence+1 {
-			err := errors.New("out of sequence CIF data")
-			log.Msg.Error("CIF sequence not as expected", zap.Error(err), zap.Int64("LastSequence", metadata.LastSequence), zap.Int64("New Sequence", parsed.header.Metadata.Sequence))
-			return err
-		}
-		log.Msg.Debug("Metadata checks complete")
 
-		// Do further sequence checks - parsed.header.Metadata.Sequence MUST = metadata.LastSequence + 1
-
-		log.Msg.Debug("CIF Data has passed checks and should now be processed <<<<<<")
 		// Process CIF file
+		err = processParsedCif(parsed)
+		if err != nil {
+			log.Msg.Error("Error processing CIF data", zap.Error(err))
+		}
 
-		// Temporarily disable METADATA GENERATION AND WRITING
+		metadata = generateMetadata(&parsed.header)
-		//		metadata = generateMetadata(&parsed.header)
 	}
 
 	ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType)