From 592675a24b42637d0a251d58f1621ed00357b945 Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Tue, 26 Mar 2024 22:33:11 +0000 Subject: [PATCH] Complete CORPUS Update implementation and begin CIF update implementation --- src/background/ticker.go | 6 +++-- src/cif/check.go | 32 +++++++++++++++++++++++-- src/cif/runner.go | 14 ----------- src/corpus/check.go | 34 +++++++++++++++++++++++++++ src/corpus/update.go | 30 ++++++++++++++++-------- src/dbAccess/access.go | 29 +++++++++++++++++++++-- src/dbAccess/client.go | 1 + src/dbAccess/common.go | 1 + src/dbAccess/corpus.go | 50 ++++++++++++++++++++++++++++++---------- src/go.sum | 6 ----- src/main.go | 3 +-- 11 files changed, 157 insertions(+), 49 deletions(-) create mode 100644 src/dbAccess/common.go diff --git a/src/background/ticker.go b/src/background/ticker.go index d84b108..bd2d954 100644 --- a/src/background/ticker.go +++ b/src/background/ticker.go @@ -4,11 +4,12 @@ import ( "time" "git.fjla.uk/owlboard/timetable-mgr/cif" + "git.fjla.uk/owlboard/timetable-mgr/corpus" "git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/log" ) -const frequency = 20000 * time.Millisecond // Figure out a sensible frequency! +const frequency = 3 * time.Second // Figure out a sensible frequency! func InitTicker(cfg *helpers.Configuration, stop <-chan struct{}) { go runTicker(cfg, stop) @@ -24,7 +25,8 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) { case <-stop: return case <-ticker.C: - cif.CifCheck(cfg) + go cif.CifCheck(cfg) + go corpus.CheckCorpus(cfg) } } } diff --git a/src/cif/check.go b/src/cif/check.go index 68a197d..72bfdf1 100644 --- a/src/cif/check.go +++ b/src/cif/check.go @@ -1,4 +1,32 @@ package cif -// Check if CIF Data needs updating -// Run update if required +import ( + "time" + + "git.fjla.uk/owlboard/timetable-mgr/dbAccess" + "git.fjla.uk/owlboard/timetable-mgr/helpers" + "git.fjla.uk/owlboard/timetable-mgr/log" + "go.uber.org/zap" +) + +func CifCheck(cfg *helpers.Configuration) { + log.Msg.Info("Checking age of CIF Data") + utime, err := dbAccess.CheckUpdateTime(dbAccess.TimetableCollection) + if err != nil { + log.Msg.Error("Error checking last timetable update", zap.Error(err)) + } + + lastUpdate := time.Unix(utime, 0) + currentTime := time.Now() + dataAge := currentTime.Sub(lastUpdate) + day := 24 * time.Hour + + log.Msg.Debug("CIF Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", day)) + + if dataAge >= day { + log.Msg.Warn("Timetable data is more than 24 hours old") + // Call Update Here + } else { + log.Msg.Info("Timetable data is less than 24 hours old") + } +} diff --git a/src/cif/runner.go b/src/cif/runner.go index 15c0f45..c881d66 100644 --- a/src/cif/runner.go +++ b/src/cif/runner.go @@ -1,15 +1 @@ package cif - -import ( - "fmt" - "runtime" - - "git.fjla.uk/owlboard/timetable-mgr/helpers" - "git.fjla.uk/owlboard/timetable-mgr/log" -) - -func CifCheck(cfg *helpers.Configuration) { - log.Msg.Debug("CIF Task Started") - numGoroutines := runtime.NumGoroutine() - fmt.Println("Number of goroutines running: ", numGoroutines) -} diff --git a/src/corpus/check.go b/src/corpus/check.go index 4087156..992882d 100644 --- a/src/corpus/check.go +++ b/src/corpus/check.go @@ -1,4 +1,38 @@ package corpus +import ( + "time" + + "git.fjla.uk/owlboard/timetable-mgr/dbAccess" + "git.fjla.uk/owlboard/timetable-mgr/helpers" + "git.fjla.uk/owlboard/timetable-mgr/log" + "go.uber.org/zap" +) + +func CheckCorpus(cfg *helpers.Configuration) { + log.Msg.Info("Checking age of CORPUS Data") + utime, err := dbAccess.CheckUpdateTime(dbAccess.CorpusCollection) + if err != nil { + log.Msg.Error("Error checking last CORPUS update", zap.Error(err)) + } + + lastUpdate := time.Unix(utime, 0) + currentTime := time.Now() + dataAge := currentTime.Sub(lastUpdate) + fortnight := 14 * 24 * time.Hour + + log.Msg.Debug("CORPUS Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", 14*24*time.Hour)) + + if dataAge >= fortnight { + log.Msg.Info("CORPUS Data is more than two weeks old") + err := RunCorpusUpdate(cfg) + if err != nil { + log.Msg.Warn("CORPUS Update did not run") + } + } else { + log.Msg.Info("CORPUS Data is less than two weeks old") + } +} + // Check if corpus data needs updating. // Run update if needed. diff --git a/src/corpus/update.go b/src/corpus/update.go index 2607ed4..b6bdbde 100644 --- a/src/corpus/update.go +++ b/src/corpus/update.go @@ -1,39 +1,51 @@ package corpus import ( - "fmt" - + "git.fjla.uk/owlboard/timetable-mgr/dbAccess" "git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/log" "go.uber.org/zap" ) -func RunCorpusUpdate(cfg *helpers.Configuration) { +func RunCorpusUpdate(cfg *helpers.Configuration) error { resp, err := fetchCorpus(cfg) if err != nil { log.Msg.Error("Failed to update Corpus data", zap.Error(err)) + return err } datastring, err := extractCorpusResponse(resp) if err != nil { log.Msg.Error("Error extracting Corpus data", zap.Error(err)) + return err } unsortedCorpusData, err := parseCorpusData(datastring) if err != nil { log.Msg.Error("Error parsing Corpus data", zap.Error(err)) + return err } corpusData := pruneCorpusEntries(unsortedCorpusData) stationData := createStationEntries(corpusData) - for _, entry := range corpusData { - fmt.Printf("DESC: %s, NLC: %d, CRS: '%s'\n", - entry.NLCDESC, entry.NLC, entry.CRS) + if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil { + log.Msg.Warn("CORPUS data may be incomplete") + log.Msg.Error("Error dropping CORPUS Data", zap.Error(err)) + } + if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil { + log.Msg.Warn("Stations data may be incomplete") + log.Msg.Error("Error dropping stations Data", zap.Error(err)) } - for _, entry := range stationData { - fmt.Printf("NAME: %s, CRS: %s, TIPLOC: %s\n", - entry.NLCDESC, entry.CRS, entry.TIPLOC) + if err := dbAccess.PutManyCorpus(corpusData); err != nil { + log.Msg.Warn("CORPUS data may be incomplete") + log.Msg.Error("Error inserting CORPUS Data", zap.Error(err)) } + if err := dbAccess.PutManyStations(stationData); err != nil { + log.Msg.Warn("Stations data may be incomplete") + log.Msg.Error("Error inserting stations data", zap.Error(err)) + } + + return nil } diff --git a/src/dbAccess/access.go b/src/dbAccess/access.go index e565848..0a93a9a 100644 --- a/src/dbAccess/access.go +++ b/src/dbAccess/access.go @@ -2,6 +2,7 @@ package dbAccess import ( "context" + "time" "git.fjla.uk/owlboard/go-types/pkg/database" "git.fjla.uk/owlboard/timetable-mgr/helpers" @@ -33,7 +34,6 @@ func CheckUpdateTime(collectionName string) (int64, error) { {Key: "target", Value: collectionName}, {Key: "type", Value: "collection"}, } - var result struct { Updated int64 `bson:"updated"` } @@ -41,10 +41,35 @@ func CheckUpdateTime(collectionName string) (int64, error) { if err != nil { return 0, err } - return result.Updated, nil } +func SetUpdateTime(collectionName string) error { + log.Msg.Info("Setting update time", zap.String("collection", collectionName)) + database := MongoClient.Database(databaseName) + collection := database.Collection("meta") + options := options.Update().SetUpsert(true) + updateTime := time.Now().Unix() + filter := bson.M{ + "target": collectionName, + "type": "collection", + } + update := bson.M{ + "$set": bson.M{ + "updated": updateTime, + "target": collectionName, + "type": "collection", + }, + } + _, err := collection.UpdateOne(context.Background(), filter, update, options) + + if err != nil { + log.Msg.Error("Error setting update time", zap.String("collection", collectionName), zap.Error(err)) + return err + } + return nil +} + func PushVersionToDb() { version := database.Version{ Target: "mq-client", diff --git a/src/dbAccess/client.go b/src/dbAccess/client.go index 8f87d0d..fbf6d89 100644 --- a/src/dbAccess/client.go +++ b/src/dbAccess/client.go @@ -16,6 +16,7 @@ const databaseName string = "owlboard" const CorpusCollection string = "corpus" const StationsCollection string = "stations" const metaCollection string = "meta" +const TimetableCollection string = "timetable" // Provide the DB Connection to other functions var MongoClient (*mongo.Client) diff --git a/src/dbAccess/common.go b/src/dbAccess/common.go new file mode 100644 index 0000000..bb922db --- /dev/null +++ b/src/dbAccess/common.go @@ -0,0 +1 @@ +package dbAccess diff --git a/src/dbAccess/corpus.go b/src/dbAccess/corpus.go index 2511310..2b9bbbb 100644 --- a/src/dbAccess/corpus.go +++ b/src/dbAccess/corpus.go @@ -2,24 +2,50 @@ package dbAccess import ( "context" + + "git.fjla.uk/owlboard/go-types/pkg/database" ) -const corpusCollection = "corpus" -const stationsCollection = "stations" +func PutManyCorpus(corpusData []database.CorpusEntry) error { + collection := MongoClient.Database(databaseName).Collection(CorpusCollection) -func dropExistingCorpus() error { - database := MongoClient.Database(databaseName) - collection := database.Collection(corpusCollection) - err := collection.Drop(context.Background()) - if err != nil { - return err - } - - collection = database.Collection(stationsCollection) - err = collection.Drop(context.Background()) + documents := convertCorpusToInterfaceSlice(corpusData) + + _, err := collection.InsertMany(context.Background(), documents) if err != nil { return err } + SetUpdateTime(CorpusCollection) return nil } + +func PutManyStations(stationsData []database.StationEntry) error { + collection := MongoClient.Database(databaseName).Collection(StationsCollection) + + documents := convertStationsToInterfaceSlice(stationsData) + + _, err := collection.InsertMany(context.Background(), documents) + if err != nil { + return err + } + + SetUpdateTime(StationsCollection) + return nil +} + +func convertCorpusToInterfaceSlice(corpusData []database.CorpusEntry) []interface{} { + interfaceSlice := make([]interface{}, len(corpusData)) + for i, doc := range corpusData { + interfaceSlice[i] = doc + } + return interfaceSlice +} + +func convertStationsToInterfaceSlice(stationsData []database.StationEntry) []interface{} { + interfaceSlice := make([]interface{}, len(stationsData)) + for i, doc := range stationsData { + interfaceSlice[i] = doc + } + return interfaceSlice +} diff --git a/src/go.sum b/src/go.sum index 50c54e1..ea35936 100644 --- a/src/go.sum +++ b/src/go.sum @@ -1,9 +1,3 @@ -git.fjla.uk/owlboard/go-types v0.0.0-20230727192011-171bd3eafd83 h1:q+I66M4YVRnKwdyYqcwou7TTniM1uwUSh3Bpa8SDLuM= -git.fjla.uk/owlboard/go-types v0.0.0-20230727192011-171bd3eafd83/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY= -git.fjla.uk/owlboard/go-types v0.0.0-20240326142657-0fc57cb903e6 h1:p3m0gpFqsyBRMm1eW2p/awTVR2LXE1QEE6XbIhCSpws= -git.fjla.uk/owlboard/go-types v0.0.0-20240326142657-0fc57cb903e6/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY= -git.fjla.uk/owlboard/go-types v0.0.0-20240326151548-5491671b8bb4 h1:FqDR2uRWVpZjwyEUHrsHSBXcnRgUmoRlF/lGojL7KcE= -git.fjla.uk/owlboard/go-types v0.0.0-20240326151548-5491671b8bb4/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY= git.fjla.uk/owlboard/go-types v0.0.0-20240326154559-f85646ac1a58 h1:8r1oGpD1yG4Jk9B9vKe2KNbngmbCxP+S3ISNxL/P3xc= git.fjla.uk/owlboard/go-types v0.0.0-20240326154559-f85646ac1a58/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= diff --git a/src/main.go b/src/main.go index 53c778d..3319e1d 100644 --- a/src/main.go +++ b/src/main.go @@ -8,7 +8,6 @@ import ( "time" "git.fjla.uk/owlboard/timetable-mgr/background" - "git.fjla.uk/owlboard/timetable-mgr/corpus" "git.fjla.uk/owlboard/timetable-mgr/dbAccess" "git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/log" @@ -40,7 +39,7 @@ func main() { background.InitTicker(cfg, stop) // Test Corpus Fetching - corpus.RunCorpusUpdate(cfg) + //corpus.RunCorpusUpdate(cfg) if cfg.VstpOn { messaging.StompInit(cfg)