cif/constants.go

@@ -11,5 +11,9 @@ const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAu
 // The time at which CIF Data is expected to be available for download (full hour)
 const dataAvailable = 6
 
+// Define update type strings to pass into metadata
+const fullUpdateType = "full"
+const dailyUpdateType = "daily"
+
 // An object representing the Europe/London timezone
 var londonTimezone, _ = time.LoadLocation("Europe/London")

cif/update.go

@@ -43,7 +43,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
 	}
 
 	newMeta := generateMetadata(&parsed.header)
-	ok := dbAccess.PutCifMetadata(newMeta)
+	ok := dbAccess.PutCifMetadata(newMeta, fullUpdateType)
 	if !ok {
 		log.Msg.Warn("CIF Data updated, but metadata write failed")
 	}
@@ -66,12 +66,6 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 			return err
 		}
 
-		// DOES NOT WORK WITH NEW NROD STREAMER
-		// If debug mode is on, call debugWriteDownload
-		//		if helpers.Runtime == "debug" {
-		//			debugWriteDownload(data)
-		//		}
-
 		// Parse CIF file
 		parsed, err := parseCifDataStream(data)
 		if err != nil {
@@ -79,10 +73,14 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 			return err
 		}
 
-		// Make `data` a nil pointer as it is no longer required
-		data = nil
-
 		log.Msg.Debug("Starting metadata checks")
+
+		// If last update type is == fullUpdateType and LastUpdate less than twodays? ago then run the update regardless of sequence
+		// this is because the sequence number of full and daily updates to not appear to match up although I will need to check this!
+		//if metadata.LastUpdateType == fullUpdateType && metadata.LastUpdate //after_yesterday_at_0600 {
+		// Somehow skip the below checks!  Maybe I need a checkMetadata function!
+		//}
+
 		// Check CIF Sequence
 		// Skip if LastSequence is >= to this sequence
 		if metadata.LastSequence >= parsed.header.Metadata.Sequence {
@@ -107,7 +105,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 		//		metadata = generateMetadata(&parsed.header)
 	}
 
-	ok := dbAccess.PutCifMetadata(metadata)
+	ok := dbAccess.PutCifMetadata(metadata, dailyUpdateType)
 	if !ok {
 		log.Msg.Warn("CIF Data updated, but metadata write failed.")
 	}

dbAccess/cif.go

@@ -18,10 +18,11 @@ const Doctype = "CifMetadata"
 // The type describing the CifMetadata 'type' in the database.
 // This type will be moved to owlboard/go-types
 type CifMetadata struct {
-	Doctype       string    `bson:"type"`
-	LastUpdate    time.Time `bson:"lastUpdate"`
-	LastTimestamp int64     `bson:"lastTimestamp"`
-	LastSequence  int64     `bson:"lastSequence"`
+	Doctype        string    `bson:"type"`
+	LastUpdate     time.Time `bson:"lastUpdate"`
+	LastTimestamp  int64     `bson:"lastTimestamp"`
+	LastSequence   int64     `bson:"lastSequence"`
+	LastUpdateType string    `bson:"lastUpdateType"`
 }
 
 // Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
@@ -45,17 +46,18 @@ func GetCifMetadata() (*CifMetadata, error) {
 }
 
 // Uses upsert to Insert/Update the CifMetadata in the database
-func PutCifMetadata(metadata *CifMetadata) bool {
+func PutCifMetadata(metadata *CifMetadata, lastUpdateType string) bool {
 	database := MongoClient.Database(databaseName)
 	collection := database.Collection(metaCollection)
 	options := options.Update().SetUpsert(true)
 	filter := bson.M{"type": Doctype}
 	update := bson.M{
 		"$set": bson.M{
-			"type":          Doctype,
-			"lastUpdate":    metadata.LastUpdate,
-			"lastTimestamp": metadata.LastTimestamp,
-			"lastSequence":  metadata.LastSequence,
+			"type":           Doctype,
+			"lastUpdate":     metadata.LastUpdate,
+			"lastTimestamp":  metadata.LastTimestamp,
+			"lastSequence":   metadata.LastSequence,
+			"lastUpdateType": lastUpdateType,
 		},
 	}