cif/constants.go

@@ -11,5 +11,9 @@ const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAu
 // The time at which CIF Data is expected to be available for download (full hour)
 const dataAvailable = 6
 
+// Define update type strings to pass into metadata
+const fullUpdateType = "full"
+const dailyUpdateType = "daily"
+
 // An object representing the Europe/London timezone
 var londonTimezone, _ = time.LoadLocation("Europe/London")

cif/update.go

@@ -43,7 +43,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
 	}
 
 	newMeta := generateMetadata(&parsed.header)
-	ok := dbAccess.PutCifMetadata(newMeta)
+	ok := dbAccess.PutCifMetadata(newMeta, fullUpdateType)
 	if !ok {
 		log.Msg.Warn("CIF Data updated, but metadata write failed")
 	}
@@ -66,12 +66,6 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 			return err
 		}
 
-		// DOES NOT WORK WITH NEW NROD STREAMER
-		// If debug mode is on, call debugWriteDownload
-		//		if helpers.Runtime == "debug" {
-		//			debugWriteDownload(data)
-		//		}
-
 		// Parse CIF file
 		parsed, err := parseCifDataStream(data)
 		if err != nil {
@@ -79,10 +73,14 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 			return err
 		}
 
-		// Make `data` a nil pointer as it is no longer required
-		data = nil
-
 		log.Msg.Debug("Starting metadata checks")
+
+		// If last update type is == fullUpdateType and LastUpdate less than twodays? ago then run the update regardless of sequence
+		// this is because the sequence number of full and daily updates to not appear to match up although I will need to check this!
+		//if metadata.LastUpdateType == fullUpdateType && metadata.LastUpdate //after_yesterday_at_0600 {
+		// Somehow skip the below checks!  Maybe I need a checkMetadata function!
+		//}
+
 		// Check CIF Sequence
 		// Skip if LastSequence is >= to this sequence
 		if metadata.LastSequence >= parsed.header.Metadata.Sequence {
@@ -107,7 +105,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 		//		metadata = generateMetadata(&parsed.header)
 	}
 
-	ok := dbAccess.PutCifMetadata(metadata)
+	ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType)
 	if !ok {
 		log.Msg.Warn("CIF Data updated, but metadata write failed.")
 	}

dbAccess/cif.go

@@ -22,6 +22,7 @@ type CifMetadata struct {
 	LastUpdate     time.Time `bson:"lastUpdate"`
 	LastTimestamp  int64     `bson:"lastTimestamp"`
 	LastSequence   int64     `bson:"lastSequence"`
+	LastUpdateType string    `bson:"lastUpdateType"`
 }
 
 // Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
@@ -45,7 +46,7 @@ func GetCifMetadata() (*CifMetadata, error) {
 }
 
 // Uses upsert to Insert/Update the CifMetadata in the database
-func PutCifMetadata(metadata *CifMetadata) bool {
+func PutCifMetadata(metadata *CifMetadata, lastUpdateType string) bool {
 	database := MongoClient.Database(databaseName)
 	collection := database.Collection(metaCollection)
 	options := options.Update().SetUpsert(true)
@@ -56,6 +57,7 @@ func PutCifMetadata(metadata *CifMetadata) bool {
 			"lastUpdate":     metadata.LastUpdate,
 			"lastTimestamp":  metadata.LastTimestamp,
 			"lastSequence":   metadata.LastSequence,
+			"lastUpdateType": lastUpdateType,
 		},
 	}