OwlBoard/timetable-mgr
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Complete CORPUS Update implementation and begin CIF update implementation

Browse Source
This commit is contained in:
Fred Boniface 2024-03-26 22:33:11 +00:00
parent c144a98fd1
commit 592675a24b
11 changed files with 157 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/background/ticker.go
View File

@ -4,11 +4,12 @@ import (
"time"
"git.fjla.uk/owlboard/timetable-mgr/cif"
"git.fjla.uk/owlboard/timetable-mgr/corpus"
"git.fjla.uk/owlboard/timetable-mgr/helpers"
"git.fjla.uk/owlboard/timetable-mgr/log"
)
const frequency = 20000 * time.Millisecond // Figure out a sensible frequency!
const frequency = 3 * time.Second // Figure out a sensible frequency!
func InitTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
go runTicker(cfg, stop)
@ -24,7 +25,8 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
case <-stop:
return
case <-ticker.C:
cif.CifCheck(cfg)
go cif.CifCheck(cfg)
go corpus.CheckCorpus(cfg)
}
}
}

32
src/cif/check.go
View File

@ -1,4 +1,32 @@
package cif
// Check if CIF Data needs updating
// Run update if required
import (
"time"
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
"git.fjla.uk/owlboard/timetable-mgr/helpers"
"git.fjla.uk/owlboard/timetable-mgr/log"
"go.uber.org/zap"
)
func CifCheck(cfg *helpers.Configuration) {
log.Msg.Info("Checking age of CIF Data")
utime, err := dbAccess.CheckUpdateTime(dbAccess.TimetableCollection)
if err != nil {
log.Msg.Error("Error checking last timetable update", zap.Error(err))
}
lastUpdate := time.Unix(utime, 0)
currentTime := time.Now()
dataAge := currentTime.Sub(lastUpdate)
day := 24 * time.Hour
log.Msg.Debug("CIF Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", day))
if dataAge >= day {
log.Msg.Warn("Timetable data is more than 24 hours old")
// Call Update Here
} else {
log.Msg.Info("Timetable data is less than 24 hours old")
}
}

14
src/cif/runner.go
View File

@ -1,15 +1 @@
package cif
import (
"fmt"
"runtime"
"git.fjla.uk/owlboard/timetable-mgr/helpers"
"git.fjla.uk/owlboard/timetable-mgr/log"
)
func CifCheck(cfg *helpers.Configuration) {
log.Msg.Debug("CIF Task Started")
numGoroutines := runtime.NumGoroutine()
fmt.Println("Number of goroutines running: ", numGoroutines)
}

34
src/corpus/check.go
View File

@ -1,4 +1,38 @@
package corpus
import (
"time"
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
"git.fjla.uk/owlboard/timetable-mgr/helpers"
"git.fjla.uk/owlboard/timetable-mgr/log"
"go.uber.org/zap"
)
func CheckCorpus(cfg *helpers.Configuration) {
log.Msg.Info("Checking age of CORPUS Data")
utime, err := dbAccess.CheckUpdateTime(dbAccess.CorpusCollection)
if err != nil {
log.Msg.Error("Error checking last CORPUS update", zap.Error(err))
}
lastUpdate := time.Unix(utime, 0)
currentTime := time.Now()
dataAge := currentTime.Sub(lastUpdate)
fortnight := 14 * 24 * time.Hour
log.Msg.Debug("CORPUS Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", 14*24*time.Hour))
if dataAge >= fortnight {
log.Msg.Info("CORPUS Data is more than two weeks old")
err := RunCorpusUpdate(cfg)
if err != nil {
log.Msg.Warn("CORPUS Update did not run")
}
} else {
log.Msg.Info("CORPUS Data is less than two weeks old")
}
}
// Check if corpus data needs updating.
// Run update if needed.

30
src/corpus/update.go
View File

@ -1,39 +1,51 @@
package corpus
import (
"fmt"
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
"git.fjla.uk/owlboard/timetable-mgr/helpers"
"git.fjla.uk/owlboard/timetable-mgr/log"
"go.uber.org/zap"
)
func RunCorpusUpdate(cfg *helpers.Configuration) {
func RunCorpusUpdate(cfg *helpers.Configuration) error {
resp, err := fetchCorpus(cfg)
if err != nil {
log.Msg.Error("Failed to update Corpus data", zap.Error(err))
return err
}
datastring, err := extractCorpusResponse(resp)
if err != nil {
log.Msg.Error("Error extracting Corpus data", zap.Error(err))
return err
}
unsortedCorpusData, err := parseCorpusData(datastring)
if err != nil {
log.Msg.Error("Error parsing Corpus data", zap.Error(err))
return err
}
corpusData := pruneCorpusEntries(unsortedCorpusData)
stationData := createStationEntries(corpusData)
for _, entry := range corpusData {
fmt.Printf("DESC: %s, NLC: %d, CRS: '%s'\n",
entry.NLCDESC, entry.NLC, entry.CRS)
if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
log.Msg.Warn("CORPUS data may be incomplete")
log.Msg.Error("Error dropping CORPUS Data", zap.Error(err))
}
if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
log.Msg.Warn("Stations data may be incomplete")
log.Msg.Error("Error dropping stations Data", zap.Error(err))
}
for _, entry := range stationData {
fmt.Printf("NAME: %s, CRS: %s, TIPLOC: %s\n",
entry.NLCDESC, entry.CRS, entry.TIPLOC)
if err := dbAccess.PutManyCorpus(corpusData); err != nil {
log.Msg.Warn("CORPUS data may be incomplete")
log.Msg.Error("Error inserting CORPUS Data", zap.Error(err))
}
if err := dbAccess.PutManyStations(stationData); err != nil {
log.Msg.Warn("Stations data may be incomplete")
log.Msg.Error("Error inserting stations data", zap.Error(err))
}
return nil
}

29
src/dbAccess/access.go
View File

@ -2,6 +2,7 @@ package dbAccess
import (
"context"
"time"
"git.fjla.uk/owlboard/go-types/pkg/database"
"git.fjla.uk/owlboard/timetable-mgr/helpers"
@ -33,7 +34,6 @@ func CheckUpdateTime(collectionName string) (int64, error) {
{Key: "target", Value: collectionName},
{Key: "type", Value: "collection"},
}
var result struct {
Updated int64 `bson:"updated"`
}
@ -41,10 +41,35 @@ func CheckUpdateTime(collectionName string) (int64, error) {
if err != nil {
return 0, err
}
return result.Updated, nil
}
func SetUpdateTime(collectionName string) error {
log.Msg.Info("Setting update time", zap.String("collection", collectionName))
database := MongoClient.Database(databaseName)
collection := database.Collection("meta")
options := options.Update().SetUpsert(true)
updateTime := time.Now().Unix()
filter := bson.M{
"target": collectionName,
"type": "collection",
}
update := bson.M{
"$set": bson.M{
"updated": updateTime,
"target": collectionName,
"type": "collection",
},
}
_, err := collection.UpdateOne(context.Background(), filter, update, options)
if err != nil {
log.Msg.Error("Error setting update time", zap.String("collection", collectionName), zap.Error(err))
return err
}
return nil
}
func PushVersionToDb() {
version := database.Version{
Target: "mq-client",

1
src/dbAccess/client.go
View File

@ -16,6 +16,7 @@ const databaseName string = "owlboard"
const CorpusCollection string = "corpus"
const StationsCollection string = "stations"
const metaCollection string = "meta"
const TimetableCollection string = "timetable"
// Provide the DB Connection to other functions
var MongoClient (*mongo.Client)

1
src/dbAccess/common.go Normal file
View File

@ -0,0 +1 @@
package dbAccess

50
src/dbAccess/corpus.go
View File

@ -2,24 +2,50 @@ package dbAccess
import (
"context"
"git.fjla.uk/owlboard/go-types/pkg/database"
)
const corpusCollection = "corpus"
const stationsCollection = "stations"
func PutManyCorpus(corpusData []database.CorpusEntry) error {
collection := MongoClient.Database(databaseName).Collection(CorpusCollection)
func dropExistingCorpus() error {
database := MongoClient.Database(databaseName)
collection := database.Collection(corpusCollection)
err := collection.Drop(context.Background())
if err != nil {
return err
}
collection = database.Collection(stationsCollection)
err = collection.Drop(context.Background())
documents := convertCorpusToInterfaceSlice(corpusData)
_, err := collection.InsertMany(context.Background(), documents)
if err != nil {
return err
}
SetUpdateTime(CorpusCollection)
return nil
}
func PutManyStations(stationsData []database.StationEntry) error {
collection := MongoClient.Database(databaseName).Collection(StationsCollection)
documents := convertStationsToInterfaceSlice(stationsData)
_, err := collection.InsertMany(context.Background(), documents)
if err != nil {
return err
}
SetUpdateTime(StationsCollection)
return nil
}
func convertCorpusToInterfaceSlice(corpusData []database.CorpusEntry) []interface{} {
interfaceSlice := make([]interface{}, len(corpusData))
for i, doc := range corpusData {
interfaceSlice[i] = doc
}
return interfaceSlice
}
func convertStationsToInterfaceSlice(stationsData []database.StationEntry) []interface{} {
interfaceSlice := make([]interface{}, len(stationsData))
for i, doc := range stationsData {
interfaceSlice[i] = doc
}
return interfaceSlice
}

6
src/go.sum
View File

@ -1,9 +1,3 @@
git.fjla.uk/owlboard/go-types v0.0.0-20230727192011-171bd3eafd83 h1:q+I66M4YVRnKwdyYqcwou7TTniM1uwUSh3Bpa8SDLuM=
git.fjla.uk/owlboard/go-types v0.0.0-20230727192011-171bd3eafd83/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
git.fjla.uk/owlboard/go-types v0.0.0-20240326142657-0fc57cb903e6 h1:p3m0gpFqsyBRMm1eW2p/awTVR2LXE1QEE6XbIhCSpws=
git.fjla.uk/owlboard/go-types v0.0.0-20240326142657-0fc57cb903e6/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
git.fjla.uk/owlboard/go-types v0.0.0-20240326151548-5491671b8bb4 h1:FqDR2uRWVpZjwyEUHrsHSBXcnRgUmoRlF/lGojL7KcE=
git.fjla.uk/owlboard/go-types v0.0.0-20240326151548-5491671b8bb4/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
git.fjla.uk/owlboard/go-types v0.0.0-20240326154559-f85646ac1a58 h1:8r1oGpD1yG4Jk9B9vKe2KNbngmbCxP+S3ISNxL/P3xc=
git.fjla.uk/owlboard/go-types v0.0.0-20240326154559-f85646ac1a58/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=

3
src/main.go
View File

@ -8,7 +8,6 @@ import (
"time"
"git.fjla.uk/owlboard/timetable-mgr/background"
"git.fjla.uk/owlboard/timetable-mgr/corpus"
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
"git.fjla.uk/owlboard/timetable-mgr/helpers"
"git.fjla.uk/owlboard/timetable-mgr/log"
@ -40,7 +39,7 @@ func main() {
background.InitTicker(cfg, stop)
// Test Corpus Fetching
corpus.RunCorpusUpdate(cfg)
//corpus.RunCorpusUpdate(cfg)
if cfg.VstpOn {
messaging.StompInit(cfg)