Add comments

cif/parse.go

@@ -12,6 +12,7 @@ import (
 )
 
 // Unmarshalls data into the correct types for processing
+// - Currently not used
 func parseCifData(data *[]byte) (*parsedData, error) {
 	log.Msg.Debug("Starting CIF Data parsing")
 	if data == nil {

cif/update.go

@@ -2,6 +2,7 @@ package cif
 
 import (
 	"errors"
+	"io"
 	"time"
 
 	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
@@ -25,6 +26,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
 		log.Msg.Error("Error downloading CIF data", zap.Error(err))
 	}
 
+	// DOES NOT WORK WITH NEW DOWNLOAD STREAMING
 	// If debug mode is on, call debugWriteDownload
 	//	if helpers.Runtime == "debug" {
 	//		debugWriteDownload(dataStream)
@@ -37,6 +39,8 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
 		return err
 	}
 
+	// Look to stream data onwards to the parsing function
+
 	// Make `data` a nil pointer as it is no longer required
 	dataStream = nil
 
@@ -73,13 +77,14 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 			return err
 		}
 
+		// DOES NOT WORK WITH NEW NROD STREAMER
 		// If debug mode is on, call debugWriteDownload
-		if helpers.Runtime == "debug" {
-			debugWriteDownload(data)
-		}
+		//		if helpers.Runtime == "debug" {
+		//			debugWriteDownload(data)
+		//		}
 
 		// Parse CIF file
-		parsed, err := parseCifData(data)
+		parsed, err := parseCifDataStream(data)
 		if err != nil {
 			log.Msg.Error("Error parsing CIF data", zap.Error(err))
 			return err
@@ -109,7 +114,8 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 		log.Msg.Debug("CIF Data has passed checks and should now be processed <<<<<<")
 		// Process CIF file
 
-		metadata = generateMetadata(&parsed.header)
+		// Temporarily disable METADATA GENERATION AND WRITING
+		//		metadata = generateMetadata(&parsed.header)
 	}
 
 	ok := dbAccess.PutCifMetadata(metadata)
@@ -121,7 +127,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 }
 
 // Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
-func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
+func fetchUpdate(t time.Time, cfg *helpers.Configuration) (io.ReadCloser, error) {
 	url, err := getUpdateUrl("daily")
 	if err != nil {
 		return nil, err
@@ -130,9 +136,9 @@ func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
 	// Append day string to URL
 	url = url + getDayString(t)
 
-	downloadedData, err := nrod.NrodDownload(url, cfg)
+	dataStream, err := nrod.NrodStream(url, cfg)
 	if err != nil {
 		return nil, err
 	}
-	return downloadedData, nil
+	return dataStream, nil
 }

corpus/check.go

@@ -36,6 +36,3 @@ func CheckCorpus(cfg *helpers.Configuration) {
 		log.Msg.Info("CORPUS Data is less than two weeks old, not updating")
 	}
 }
-
-// Check if corpus data needs updating.
-// Run update if needed.

dbAccess/access.go

@@ -61,6 +61,7 @@ func SetUpdateTime(collectionName string) error {
 	update := bson.M{
 		"$set": bson.M{
 			"updated":      updateTime,
+			"updated_time": time.Now().In(time.UTC),
 			"target":       collectionName,
 			"type":         "collection",
 		},
@@ -78,13 +79,13 @@ func SetUpdateTime(collectionName string) error {
 // Currently uses the old name of mq-client
 func PushVersionToDb() {
 	version := database.Version{
-		Target:    "mq-client",
-		Component: "mq-client",
+		Target:    "timetable-mgr",
+		Component: "timetable-mgr",
 		Version:   helpers.Version,
 	}
 	versionSelector := database.VersionSelector{
-		Target:    "mq-client",
-		Component: "mq-client",
+		Target:    "timetable-mgr",
+		Component: "timetable-mgr",
 	}
 	opts := options.Update().SetUpsert(true)
 	coll := MongoClient.Database("owlboard").Collection("versions")

nrod/streams.go

@@ -41,7 +41,7 @@ func NrodStream(url string, cfg *helpers.Configuration) (io.ReadCloser, error) {
 	}
 
 	// Run the data through the extractor function and return io.ReadCloser, error from
-	// directly
+	// that function directly
 	return NrodStreamExtract(resp)
 }