Complete CORPUS Update implementation and begin CIF update implementation
This commit is contained in:
parent
c144a98fd1
commit
592675a24b
@ -4,11 +4,12 @@ import (
|
||||
"time"
|
||||
|
||||
"git.fjla.uk/owlboard/timetable-mgr/cif"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
)
|
||||
|
||||
const frequency = 20000 * time.Millisecond // Figure out a sensible frequency!
|
||||
const frequency = 3 * time.Second // Figure out a sensible frequency!
|
||||
|
||||
func InitTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
|
||||
go runTicker(cfg, stop)
|
||||
@ -24,7 +25,8 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
|
||||
case <-stop:
|
||||
return
|
||||
case <-ticker.C:
|
||||
cif.CifCheck(cfg)
|
||||
go cif.CifCheck(cfg)
|
||||
go corpus.CheckCorpus(cfg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1,4 +1,32 @@
|
||||
package cif
|
||||
|
||||
// Check if CIF Data needs updating
|
||||
// Run update if required
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func CifCheck(cfg *helpers.Configuration) {
|
||||
log.Msg.Info("Checking age of CIF Data")
|
||||
utime, err := dbAccess.CheckUpdateTime(dbAccess.TimetableCollection)
|
||||
if err != nil {
|
||||
log.Msg.Error("Error checking last timetable update", zap.Error(err))
|
||||
}
|
||||
|
||||
lastUpdate := time.Unix(utime, 0)
|
||||
currentTime := time.Now()
|
||||
dataAge := currentTime.Sub(lastUpdate)
|
||||
day := 24 * time.Hour
|
||||
|
||||
log.Msg.Debug("CIF Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", day))
|
||||
|
||||
if dataAge >= day {
|
||||
log.Msg.Warn("Timetable data is more than 24 hours old")
|
||||
// Call Update Here
|
||||
} else {
|
||||
log.Msg.Info("Timetable data is less than 24 hours old")
|
||||
}
|
||||
}
|
||||
|
@ -1,15 +1 @@
|
||||
package cif
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
)
|
||||
|
||||
func CifCheck(cfg *helpers.Configuration) {
|
||||
log.Msg.Debug("CIF Task Started")
|
||||
numGoroutines := runtime.NumGoroutine()
|
||||
fmt.Println("Number of goroutines running: ", numGoroutines)
|
||||
}
|
||||
|
@ -1,4 +1,38 @@
|
||||
package corpus
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func CheckCorpus(cfg *helpers.Configuration) {
|
||||
log.Msg.Info("Checking age of CORPUS Data")
|
||||
utime, err := dbAccess.CheckUpdateTime(dbAccess.CorpusCollection)
|
||||
if err != nil {
|
||||
log.Msg.Error("Error checking last CORPUS update", zap.Error(err))
|
||||
}
|
||||
|
||||
lastUpdate := time.Unix(utime, 0)
|
||||
currentTime := time.Now()
|
||||
dataAge := currentTime.Sub(lastUpdate)
|
||||
fortnight := 14 * 24 * time.Hour
|
||||
|
||||
log.Msg.Debug("CORPUS Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", 14*24*time.Hour))
|
||||
|
||||
if dataAge >= fortnight {
|
||||
log.Msg.Info("CORPUS Data is more than two weeks old")
|
||||
err := RunCorpusUpdate(cfg)
|
||||
if err != nil {
|
||||
log.Msg.Warn("CORPUS Update did not run")
|
||||
}
|
||||
} else {
|
||||
log.Msg.Info("CORPUS Data is less than two weeks old")
|
||||
}
|
||||
}
|
||||
|
||||
// Check if corpus data needs updating.
|
||||
// Run update if needed.
|
||||
|
@ -1,39 +1,51 @@
|
||||
package corpus
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func RunCorpusUpdate(cfg *helpers.Configuration) {
|
||||
func RunCorpusUpdate(cfg *helpers.Configuration) error {
|
||||
resp, err := fetchCorpus(cfg)
|
||||
if err != nil {
|
||||
log.Msg.Error("Failed to update Corpus data", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
datastring, err := extractCorpusResponse(resp)
|
||||
if err != nil {
|
||||
log.Msg.Error("Error extracting Corpus data", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
unsortedCorpusData, err := parseCorpusData(datastring)
|
||||
if err != nil {
|
||||
log.Msg.Error("Error parsing Corpus data", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
corpusData := pruneCorpusEntries(unsortedCorpusData)
|
||||
stationData := createStationEntries(corpusData)
|
||||
|
||||
for _, entry := range corpusData {
|
||||
fmt.Printf("DESC: %s, NLC: %d, CRS: '%s'\n",
|
||||
entry.NLCDESC, entry.NLC, entry.CRS)
|
||||
if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
|
||||
log.Msg.Warn("CORPUS data may be incomplete")
|
||||
log.Msg.Error("Error dropping CORPUS Data", zap.Error(err))
|
||||
}
|
||||
if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
|
||||
log.Msg.Warn("Stations data may be incomplete")
|
||||
log.Msg.Error("Error dropping stations Data", zap.Error(err))
|
||||
}
|
||||
|
||||
for _, entry := range stationData {
|
||||
fmt.Printf("NAME: %s, CRS: %s, TIPLOC: %s\n",
|
||||
entry.NLCDESC, entry.CRS, entry.TIPLOC)
|
||||
if err := dbAccess.PutManyCorpus(corpusData); err != nil {
|
||||
log.Msg.Warn("CORPUS data may be incomplete")
|
||||
log.Msg.Error("Error inserting CORPUS Data", zap.Error(err))
|
||||
}
|
||||
if err := dbAccess.PutManyStations(stationData); err != nil {
|
||||
log.Msg.Warn("Stations data may be incomplete")
|
||||
log.Msg.Error("Error inserting stations data", zap.Error(err))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -2,6 +2,7 @@ package dbAccess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.fjla.uk/owlboard/go-types/pkg/database"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||
@ -33,7 +34,6 @@ func CheckUpdateTime(collectionName string) (int64, error) {
|
||||
{Key: "target", Value: collectionName},
|
||||
{Key: "type", Value: "collection"},
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Updated int64 `bson:"updated"`
|
||||
}
|
||||
@ -41,10 +41,35 @@ func CheckUpdateTime(collectionName string) (int64, error) {
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return result.Updated, nil
|
||||
}
|
||||
|
||||
func SetUpdateTime(collectionName string) error {
|
||||
log.Msg.Info("Setting update time", zap.String("collection", collectionName))
|
||||
database := MongoClient.Database(databaseName)
|
||||
collection := database.Collection("meta")
|
||||
options := options.Update().SetUpsert(true)
|
||||
updateTime := time.Now().Unix()
|
||||
filter := bson.M{
|
||||
"target": collectionName,
|
||||
"type": "collection",
|
||||
}
|
||||
update := bson.M{
|
||||
"$set": bson.M{
|
||||
"updated": updateTime,
|
||||
"target": collectionName,
|
||||
"type": "collection",
|
||||
},
|
||||
}
|
||||
_, err := collection.UpdateOne(context.Background(), filter, update, options)
|
||||
|
||||
if err != nil {
|
||||
log.Msg.Error("Error setting update time", zap.String("collection", collectionName), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func PushVersionToDb() {
|
||||
version := database.Version{
|
||||
Target: "mq-client",
|
||||
|
@ -16,6 +16,7 @@ const databaseName string = "owlboard"
|
||||
const CorpusCollection string = "corpus"
|
||||
const StationsCollection string = "stations"
|
||||
const metaCollection string = "meta"
|
||||
const TimetableCollection string = "timetable"
|
||||
|
||||
// Provide the DB Connection to other functions
|
||||
var MongoClient (*mongo.Client)
|
||||
|
1
src/dbAccess/common.go
Normal file
1
src/dbAccess/common.go
Normal file
@ -0,0 +1 @@
|
||||
package dbAccess
|
@ -2,24 +2,50 @@ package dbAccess
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.fjla.uk/owlboard/go-types/pkg/database"
|
||||
)
|
||||
|
||||
const corpusCollection = "corpus"
|
||||
const stationsCollection = "stations"
|
||||
func PutManyCorpus(corpusData []database.CorpusEntry) error {
|
||||
collection := MongoClient.Database(databaseName).Collection(CorpusCollection)
|
||||
|
||||
func dropExistingCorpus() error {
|
||||
database := MongoClient.Database(databaseName)
|
||||
collection := database.Collection(corpusCollection)
|
||||
err := collection.Drop(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
collection = database.Collection(stationsCollection)
|
||||
err = collection.Drop(context.Background())
|
||||
documents := convertCorpusToInterfaceSlice(corpusData)
|
||||
|
||||
_, err := collection.InsertMany(context.Background(), documents)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
SetUpdateTime(CorpusCollection)
|
||||
return nil
|
||||
}
|
||||
|
||||
func PutManyStations(stationsData []database.StationEntry) error {
|
||||
collection := MongoClient.Database(databaseName).Collection(StationsCollection)
|
||||
|
||||
documents := convertStationsToInterfaceSlice(stationsData)
|
||||
|
||||
_, err := collection.InsertMany(context.Background(), documents)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
SetUpdateTime(StationsCollection)
|
||||
return nil
|
||||
}
|
||||
|
||||
func convertCorpusToInterfaceSlice(corpusData []database.CorpusEntry) []interface{} {
|
||||
interfaceSlice := make([]interface{}, len(corpusData))
|
||||
for i, doc := range corpusData {
|
||||
interfaceSlice[i] = doc
|
||||
}
|
||||
return interfaceSlice
|
||||
}
|
||||
|
||||
func convertStationsToInterfaceSlice(stationsData []database.StationEntry) []interface{} {
|
||||
interfaceSlice := make([]interface{}, len(stationsData))
|
||||
for i, doc := range stationsData {
|
||||
interfaceSlice[i] = doc
|
||||
}
|
||||
return interfaceSlice
|
||||
}
|
||||
|
@ -1,9 +1,3 @@
|
||||
git.fjla.uk/owlboard/go-types v0.0.0-20230727192011-171bd3eafd83 h1:q+I66M4YVRnKwdyYqcwou7TTniM1uwUSh3Bpa8SDLuM=
|
||||
git.fjla.uk/owlboard/go-types v0.0.0-20230727192011-171bd3eafd83/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
|
||||
git.fjla.uk/owlboard/go-types v0.0.0-20240326142657-0fc57cb903e6 h1:p3m0gpFqsyBRMm1eW2p/awTVR2LXE1QEE6XbIhCSpws=
|
||||
git.fjla.uk/owlboard/go-types v0.0.0-20240326142657-0fc57cb903e6/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
|
||||
git.fjla.uk/owlboard/go-types v0.0.0-20240326151548-5491671b8bb4 h1:FqDR2uRWVpZjwyEUHrsHSBXcnRgUmoRlF/lGojL7KcE=
|
||||
git.fjla.uk/owlboard/go-types v0.0.0-20240326151548-5491671b8bb4/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
|
||||
git.fjla.uk/owlboard/go-types v0.0.0-20240326154559-f85646ac1a58 h1:8r1oGpD1yG4Jk9B9vKe2KNbngmbCxP+S3ISNxL/P3xc=
|
||||
git.fjla.uk/owlboard/go-types v0.0.0-20240326154559-f85646ac1a58/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
|
||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||
|
@ -8,7 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
"git.fjla.uk/owlboard/timetable-mgr/background"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
@ -40,7 +39,7 @@ func main() {
|
||||
background.InitTicker(cfg, stop)
|
||||
|
||||
// Test Corpus Fetching
|
||||
corpus.RunCorpusUpdate(cfg)
|
||||
//corpus.RunCorpusUpdate(cfg)
|
||||
|
||||
if cfg.VstpOn {
|
||||
messaging.StompInit(cfg)
|
||||
|
Loading…
Reference in New Issue
Block a user