Complete alpha newStations data, deactivate CORPUS Stations data

This commit is contained in:
Fred Boniface 2024-06-30 19:29:41 +01:00
parent 358e69bec7
commit fed0208291
5 changed files with 145 additions and 18 deletions

View File

@ -10,6 +10,7 @@ import (
"git.fjla.uk/owlboard/timetable-mgr/corpus"
"git.fjla.uk/owlboard/timetable-mgr/helpers"
"git.fjla.uk/owlboard/timetable-mgr/log"
"git.fjla.uk/owlboard/timetable-mgr/stations"
"go.uber.org/zap"
)
@ -40,7 +41,7 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
log.Debug("Running background tasks")
go cif.CheckCif(cfg)
go corpus.CheckCorpus(cfg)
//go stations.Check()
go stations.Check()
}
}
}

View File

@ -23,29 +23,29 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error {
}
corpusData := pruneCorpusEntries(*unsortedCorpusData)
stationData := createStationEntries(corpusData)
//stationData := createStationEntries(corpusData)
if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
log.Warn("CORPUS data may be incomplete")
log.Error("Error dropping CORPUS Data", zap.Error(err))
return err
}
if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
log.Warn("Stations data may be incomplete")
log.Error("Error dropping stations Data", zap.Error(err))
return err
}
//if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
// log.Warn("Stations data may be incomplete")
// log.Error("Error dropping stations Data", zap.Error(err))
// return err
//}
if err := dbAccess.PutManyCorpus(corpusData); err != nil {
log.Warn("CORPUS data may be incomplete")
log.Error("Error inserting CORPUS Data", zap.Error(err))
return err
}
if err := dbAccess.PutManyStations(stationData); err != nil {
log.Warn("Stations data may be incomplete")
log.Error("Error inserting stations data", zap.Error(err))
return err
}
//if err := dbAccess.PutManyStations(stationData); err != nil {
// log.Warn("Stations data may be incomplete")
// log.Error("Error inserting stations data", zap.Error(err))
// return err
//}
if err := dbAccess.CreateCorpusIndexes(); err != nil {
log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err))

91
dbAccess/stations.go Normal file
View File

@ -0,0 +1,91 @@
package dbAccess
import (
"context"
"errors"
"time"
"git.fjla.uk/owlboard/go-types/pkg/database"
"git.fjla.uk/owlboard/timetable-mgr/log"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.uber.org/zap"
)
const StationsMetaDoctype string = "StationsMetadata"
type StationsMetadata struct {
Doctype string
LastUpdate time.Time
}
// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
func GetStationsMetadata() (*StationsMetadata, error) {
database := MongoClient.Database(DatabaseName)
collection := database.Collection(MetaCollection)
filter := bson.M{"type": StationsMetaDoctype}
var result StationsMetadata
err := collection.FindOne(context.Background(), filter).Decode(&result)
if err != nil {
if errors.Is(err, mongo.ErrNoDocuments) {
return nil, nil
}
return nil, err
}
log.Debug("Fetched Stations Metadata from database", zap.Any("Metadata", result))
return &result, nil
}
func SetStationsMetadata(time time.Time) bool {
database := MongoClient.Database(DatabaseName)
collection := database.Collection(MetaCollection)
options := options.Update().SetUpsert(true)
filter := bson.M{"type": Doctype}
update := bson.M{
"$set": bson.M{
"type": StationsMetaDoctype,
"lastUpdate": time,
},
}
_, err := collection.UpdateOne(context.Background(), filter, update, options)
if err != nil {
log.Error("Error updating Stations Metadata", zap.Error(err))
return false
}
log.Info("New Stations Metadata written", zap.Time("Update time", time))
return true
}
// Puts an array of Stations documents into the database
func PutManyNewStations(stationsData *[]database.Station) error {
collection := MongoClient.Database(DatabaseName).Collection(StationsCollection)
documents := convertNewStationsToInterfaceSlice(stationsData)
_, err := collection.InsertMany(context.Background(), *documents)
if err != nil {
return err
}
SetUpdateTime(StationsCollection)
return nil
}
// Converts []database.Station types into interface slices required to put them into the database
func convertNewStationsToInterfaceSlice(stationsData *[]database.Station) *[]interface{} {
interfaceSlice := make([]interface{}, len(*stationsData))
for i, doc := range *stationsData {
interfaceSlice[i] = doc
}
return &interfaceSlice
}
func CreateStationIndeces() bool {
return false
}

View File

@ -65,8 +65,7 @@ func main() {
// not delayed until the first ticker event.
go cif.CheckCif(cfg)
go corpus.CheckCorpus(cfg)
stations.Check()
go stations.Check()
if cfg.VstpOn {
messaging.StompInit(cfg)

View File

@ -1,19 +1,55 @@
package stations
import "fmt"
import (
"fmt"
"time"
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
"git.fjla.uk/owlboard/timetable-mgr/log"
"go.uber.org/zap"
)
func Check() {
run()
oldMetadata, err := dbAccess.GetStationsMetadata()
if err != nil {
log.Error("Error reading Stations metadata", zap.Error(err))
}
if oldMetadata == nil {
log.Info("No old metadata for stations, rebuilding")
} else {
timeSinceLastUpdate := time.Since(oldMetadata.LastUpdate)
if timeSinceLastUpdate <= time.Hour*24*7 {
log.Info("Stations update not required")
return
}
}
ok := run()
if !ok {
log.Error("Error updating Stations data")
}
}
func run() {
func run() bool {
// Download
data, data2, err := download()
if err != nil {
fmt.Println(err)
return false
}
_, err = parseData(data, data2)
// Parse
stations, err := parseData(data, data2)
if err != nil {
fmt.Println(err)
return false
}
// Drop
dbAccess.DropCollection("stations")
// Push
dbAccess.PutManyNewStations(&stations)
return true
}