Complete CORPUS Update implementation and begin CIF update implementation

src/background/ticker.go

@@ -4,11 +4,12 @@ import (
 	"time"
 
 	"git.fjla.uk/owlboard/timetable-mgr/cif"
+	"git.fjla.uk/owlboard/timetable-mgr/corpus"
 	"git.fjla.uk/owlboard/timetable-mgr/helpers"
 	"git.fjla.uk/owlboard/timetable-mgr/log"
 )
 
-const frequency = 20000 * time.Millisecond // Figure out a sensible frequency!
+const frequency = 3 * time.Second // Figure out a sensible frequency!
 
 func InitTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
 	go runTicker(cfg, stop)
@@ -24,7 +25,8 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
 		case <-stop:
 			return
 		case <-ticker.C:
-			cif.CifCheck(cfg)
+			go cif.CifCheck(cfg)
+			go corpus.CheckCorpus(cfg)
 		}
 	}
 }

src/cif/check.go

@@ -1,4 +1,32 @@
 package cif
 
-// Check if CIF Data needs updating
+import (
-// Run update if required
+	"time"
+
+	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
+	"git.fjla.uk/owlboard/timetable-mgr/helpers"
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.uber.org/zap"
+)
+
+func CifCheck(cfg *helpers.Configuration) {
+	log.Msg.Info("Checking age of CIF Data")
+	utime, err := dbAccess.CheckUpdateTime(dbAccess.TimetableCollection)
+	if err != nil {
+		log.Msg.Error("Error checking last timetable update", zap.Error(err))
+	}
+
+	lastUpdate := time.Unix(utime, 0)
+	currentTime := time.Now()
+	dataAge := currentTime.Sub(lastUpdate)
+	day := 24 * time.Hour
+
+	log.Msg.Debug("CIF Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", day))
+
+	if dataAge >= day {
+		log.Msg.Warn("Timetable data is more than 24 hours old")
+		// Call Update Here
+	} else {
+		log.Msg.Info("Timetable data is less than 24 hours old")
+	}
+}

src/cif/runner.go

@@ -1,15 +1 @@
 package cif
-
-import (
-	"fmt"
-	"runtime"
-
-	"git.fjla.uk/owlboard/timetable-mgr/helpers"
-	"git.fjla.uk/owlboard/timetable-mgr/log"
-)
-
-func CifCheck(cfg *helpers.Configuration) {
-	log.Msg.Debug("CIF Task Started")
-	numGoroutines := runtime.NumGoroutine()
-	fmt.Println("Number of goroutines running: ", numGoroutines)
-}

src/corpus/check.go

@@ -1,4 +1,38 @@
 package corpus
 
+import (
+	"time"
+
+	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
+	"git.fjla.uk/owlboard/timetable-mgr/helpers"
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.uber.org/zap"
+)
+
+func CheckCorpus(cfg *helpers.Configuration) {
+	log.Msg.Info("Checking age of CORPUS Data")
+	utime, err := dbAccess.CheckUpdateTime(dbAccess.CorpusCollection)
+	if err != nil {
+		log.Msg.Error("Error checking last CORPUS update", zap.Error(err))
+	}
+
+	lastUpdate := time.Unix(utime, 0)
+	currentTime := time.Now()
+	dataAge := currentTime.Sub(lastUpdate)
+	fortnight := 14 * 24 * time.Hour
+
+	log.Msg.Debug("CORPUS Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", 14*24*time.Hour))
+
+	if dataAge >= fortnight {
+		log.Msg.Info("CORPUS Data is more than two weeks old")
+		err := RunCorpusUpdate(cfg)
+		if err != nil {
+			log.Msg.Warn("CORPUS Update did not run")
+		}
+	} else {
+		log.Msg.Info("CORPUS Data is less than two weeks old")
+	}
+}
+
 // Check if corpus data needs updating.
 // Run update if needed.

src/corpus/update.go

@@ -1,39 +1,51 @@
 package corpus
 
 import (
-	"fmt"
+	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
-
 	"git.fjla.uk/owlboard/timetable-mgr/helpers"
 	"git.fjla.uk/owlboard/timetable-mgr/log"
 	"go.uber.org/zap"
 )
 
-func RunCorpusUpdate(cfg *helpers.Configuration) {
+func RunCorpusUpdate(cfg *helpers.Configuration) error {
 	resp, err := fetchCorpus(cfg)
 	if err != nil {
 		log.Msg.Error("Failed to update Corpus data", zap.Error(err))
+		return err
 	}
 
 	datastring, err := extractCorpusResponse(resp)
 	if err != nil {
 		log.Msg.Error("Error extracting Corpus data", zap.Error(err))
+		return err
 	}
 
 	unsortedCorpusData, err := parseCorpusData(datastring)
 	if err != nil {
 		log.Msg.Error("Error parsing Corpus data", zap.Error(err))
+		return err
 	}
 
 	corpusData := pruneCorpusEntries(unsortedCorpusData)
 	stationData := createStationEntries(corpusData)
 
-	for _, entry := range corpusData {
+	if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
-		fmt.Printf("DESC: %s, NLC: %d, CRS: '%s'\n",
+		log.Msg.Warn("CORPUS data may be incomplete")
-			entry.NLCDESC, entry.NLC, entry.CRS)
+		log.Msg.Error("Error dropping CORPUS Data", zap.Error(err))
+	}
+	if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
+		log.Msg.Warn("Stations data may be incomplete")
+		log.Msg.Error("Error dropping stations Data", zap.Error(err))
 	}
 
-	for _, entry := range stationData {
+	if err := dbAccess.PutManyCorpus(corpusData); err != nil {
-		fmt.Printf("NAME: %s, CRS: %s, TIPLOC: %s\n",
+		log.Msg.Warn("CORPUS data may be incomplete")
-			entry.NLCDESC, entry.CRS, entry.TIPLOC)
+		log.Msg.Error("Error inserting CORPUS Data", zap.Error(err))
 	}
+	if err := dbAccess.PutManyStations(stationData); err != nil {
+		log.Msg.Warn("Stations data may be incomplete")
+		log.Msg.Error("Error inserting stations data", zap.Error(err))
+	}
+
+	return nil
 }

src/dbAccess/access.go

@@ -2,6 +2,7 @@ package dbAccess
 
 import (
 	"context"
+	"time"
 
 	"git.fjla.uk/owlboard/go-types/pkg/database"
 	"git.fjla.uk/owlboard/timetable-mgr/helpers"
@@ -33,7 +34,6 @@ func CheckUpdateTime(collectionName string) (int64, error) {
 		{Key: "target", Value: collectionName},
 		{Key: "type", Value: "collection"},
 	}
-
 	var result struct {
 		Updated int64 `bson:"updated"`
 	}
@@ -41,10 +41,35 @@ func CheckUpdateTime(collectionName string) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-
 	return result.Updated, nil
 }
 
+func SetUpdateTime(collectionName string) error {
+	log.Msg.Info("Setting update time", zap.String("collection", collectionName))
+	database := MongoClient.Database(databaseName)
+	collection := database.Collection("meta")
+	options := options.Update().SetUpsert(true)
+	updateTime := time.Now().Unix()
+	filter := bson.M{
+		"target": collectionName,
+		"type":   "collection",
+	}
+	update := bson.M{
+		"$set": bson.M{
+			"updated": updateTime,
+			"target":  collectionName,
+			"type":    "collection",
+		},
+	}
+	_, err := collection.UpdateOne(context.Background(), filter, update, options)
+
+	if err != nil {
+		log.Msg.Error("Error setting update time", zap.String("collection", collectionName), zap.Error(err))
+		return err
+	}
+	return nil
+}
+
 func PushVersionToDb() {
 	version := database.Version{
 		Target:    "mq-client",

src/dbAccess/client.go

@@ -16,6 +16,7 @@ const databaseName string = "owlboard"
 const CorpusCollection string = "corpus"
 const StationsCollection string = "stations"
 const metaCollection string = "meta"
+const TimetableCollection string = "timetable"
 
 // Provide the DB Connection to other functions
 var MongoClient (*mongo.Client)

src/dbAccess/common.go

@@ -0,0 +1 @@
+package dbAccess

src/dbAccess/corpus.go

@@ -2,24 +2,50 @@ package dbAccess
 
 import (
 	"context"
+
+	"git.fjla.uk/owlboard/go-types/pkg/database"
 )
 
-const corpusCollection = "corpus"
+func PutManyCorpus(corpusData []database.CorpusEntry) error {
-const stationsCollection = "stations"
+	collection := MongoClient.Database(databaseName).Collection(CorpusCollection)
 
-func dropExistingCorpus() error {
+	documents := convertCorpusToInterfaceSlice(corpusData)
-	database := MongoClient.Database(databaseName)
+
-	collection := database.Collection(corpusCollection)
+	_, err := collection.InsertMany(context.Background(), documents)
-	err := collection.Drop(context.Background())
-	if err != nil {
-		return err
-	}
-
-	collection = database.Collection(stationsCollection)
-	err = collection.Drop(context.Background())
 	if err != nil {
 		return err
 	}
 
+	SetUpdateTime(CorpusCollection)
 	return nil
 }
+
+func PutManyStations(stationsData []database.StationEntry) error {
+	collection := MongoClient.Database(databaseName).Collection(StationsCollection)
+
+	documents := convertStationsToInterfaceSlice(stationsData)
+
+	_, err := collection.InsertMany(context.Background(), documents)
+	if err != nil {
+		return err
+	}
+
+	SetUpdateTime(StationsCollection)
+	return nil
+}
+
+func convertCorpusToInterfaceSlice(corpusData []database.CorpusEntry) []interface{} {
+	interfaceSlice := make([]interface{}, len(corpusData))
+	for i, doc := range corpusData {
+		interfaceSlice[i] = doc
+	}
+	return interfaceSlice
+}
+
+func convertStationsToInterfaceSlice(stationsData []database.StationEntry) []interface{} {
+	interfaceSlice := make([]interface{}, len(stationsData))
+	for i, doc := range stationsData {
+		interfaceSlice[i] = doc
+	}
+	return interfaceSlice
+}

src/go.sum

@@ -1,9 +1,3 @@
-git.fjla.uk/owlboard/go-types v0.0.0-20230727192011-171bd3eafd83 h1:q+I66M4YVRnKwdyYqcwou7TTniM1uwUSh3Bpa8SDLuM=
-git.fjla.uk/owlboard/go-types v0.0.0-20230727192011-171bd3eafd83/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
-git.fjla.uk/owlboard/go-types v0.0.0-20240326142657-0fc57cb903e6 h1:p3m0gpFqsyBRMm1eW2p/awTVR2LXE1QEE6XbIhCSpws=
-git.fjla.uk/owlboard/go-types v0.0.0-20240326142657-0fc57cb903e6/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
-git.fjla.uk/owlboard/go-types v0.0.0-20240326151548-5491671b8bb4 h1:FqDR2uRWVpZjwyEUHrsHSBXcnRgUmoRlF/lGojL7KcE=
-git.fjla.uk/owlboard/go-types v0.0.0-20240326151548-5491671b8bb4/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
 git.fjla.uk/owlboard/go-types v0.0.0-20240326154559-f85646ac1a58 h1:8r1oGpD1yG4Jk9B9vKe2KNbngmbCxP+S3ISNxL/P3xc=
 git.fjla.uk/owlboard/go-types v0.0.0-20240326154559-f85646ac1a58/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=

src/main.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"git.fjla.uk/owlboard/timetable-mgr/background"
-	"git.fjla.uk/owlboard/timetable-mgr/corpus"
 	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
 	"git.fjla.uk/owlboard/timetable-mgr/helpers"
 	"git.fjla.uk/owlboard/timetable-mgr/log"
@@ -40,7 +39,7 @@ func main() {
 	background.InitTicker(cfg, stop)
 
 	// Test Corpus Fetching
-	corpus.RunCorpusUpdate(cfg)
+	//corpus.RunCorpusUpdate(cfg)
 
 	if cfg.VstpOn {
 		messaging.StompInit(cfg)