Complete alpha newStations data, deactivate CORPUS Stations data

This commit is contained in:
Fred Boniface 2024-06-30 19:29:41 +01:00
parent 358e69bec7
commit fed0208291
5 changed files with 145 additions and 18 deletions

View File

@ -10,6 +10,7 @@ import (
"git.fjla.uk/owlboard/timetable-mgr/corpus" "git.fjla.uk/owlboard/timetable-mgr/corpus"
"git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/helpers"
"git.fjla.uk/owlboard/timetable-mgr/log" "git.fjla.uk/owlboard/timetable-mgr/log"
"git.fjla.uk/owlboard/timetable-mgr/stations"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -40,7 +41,7 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
log.Debug("Running background tasks") log.Debug("Running background tasks")
go cif.CheckCif(cfg) go cif.CheckCif(cfg)
go corpus.CheckCorpus(cfg) go corpus.CheckCorpus(cfg)
//go stations.Check() go stations.Check()
} }
} }
} }

View File

@ -23,29 +23,29 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error {
} }
corpusData := pruneCorpusEntries(*unsortedCorpusData) corpusData := pruneCorpusEntries(*unsortedCorpusData)
stationData := createStationEntries(corpusData) //stationData := createStationEntries(corpusData)
if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil { if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
log.Warn("CORPUS data may be incomplete") log.Warn("CORPUS data may be incomplete")
log.Error("Error dropping CORPUS Data", zap.Error(err)) log.Error("Error dropping CORPUS Data", zap.Error(err))
return err return err
} }
if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil { //if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
log.Warn("Stations data may be incomplete") // log.Warn("Stations data may be incomplete")
log.Error("Error dropping stations Data", zap.Error(err)) // log.Error("Error dropping stations Data", zap.Error(err))
return err // return err
} //}
if err := dbAccess.PutManyCorpus(corpusData); err != nil { if err := dbAccess.PutManyCorpus(corpusData); err != nil {
log.Warn("CORPUS data may be incomplete") log.Warn("CORPUS data may be incomplete")
log.Error("Error inserting CORPUS Data", zap.Error(err)) log.Error("Error inserting CORPUS Data", zap.Error(err))
return err return err
} }
if err := dbAccess.PutManyStations(stationData); err != nil { //if err := dbAccess.PutManyStations(stationData); err != nil {
log.Warn("Stations data may be incomplete") // log.Warn("Stations data may be incomplete")
log.Error("Error inserting stations data", zap.Error(err)) // log.Error("Error inserting stations data", zap.Error(err))
return err // return err
} //}
if err := dbAccess.CreateCorpusIndexes(); err != nil { if err := dbAccess.CreateCorpusIndexes(); err != nil {
log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err)) log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err))

91
dbAccess/stations.go Normal file
View File

@ -0,0 +1,91 @@
package dbAccess
import (
"context"
"errors"
"time"
"git.fjla.uk/owlboard/go-types/pkg/database"
"git.fjla.uk/owlboard/timetable-mgr/log"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.uber.org/zap"
)
const StationsMetaDoctype string = "StationsMetadata"
type StationsMetadata struct {
Doctype string
LastUpdate time.Time
}
// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
func GetStationsMetadata() (*StationsMetadata, error) {
database := MongoClient.Database(DatabaseName)
collection := database.Collection(MetaCollection)
filter := bson.M{"type": StationsMetaDoctype}
var result StationsMetadata
err := collection.FindOne(context.Background(), filter).Decode(&result)
if err != nil {
if errors.Is(err, mongo.ErrNoDocuments) {
return nil, nil
}
return nil, err
}
log.Debug("Fetched Stations Metadata from database", zap.Any("Metadata", result))
return &result, nil
}
func SetStationsMetadata(time time.Time) bool {
database := MongoClient.Database(DatabaseName)
collection := database.Collection(MetaCollection)
options := options.Update().SetUpsert(true)
filter := bson.M{"type": Doctype}
update := bson.M{
"$set": bson.M{
"type": StationsMetaDoctype,
"lastUpdate": time,
},
}
_, err := collection.UpdateOne(context.Background(), filter, update, options)
if err != nil {
log.Error("Error updating Stations Metadata", zap.Error(err))
return false
}
log.Info("New Stations Metadata written", zap.Time("Update time", time))
return true
}
// Puts an array of Stations documents into the database
func PutManyNewStations(stationsData *[]database.Station) error {
collection := MongoClient.Database(DatabaseName).Collection(StationsCollection)
documents := convertNewStationsToInterfaceSlice(stationsData)
_, err := collection.InsertMany(context.Background(), *documents)
if err != nil {
return err
}
SetUpdateTime(StationsCollection)
return nil
}
// Converts []database.Station types into interface slices required to put them into the database
func convertNewStationsToInterfaceSlice(stationsData *[]database.Station) *[]interface{} {
interfaceSlice := make([]interface{}, len(*stationsData))
for i, doc := range *stationsData {
interfaceSlice[i] = doc
}
return &interfaceSlice
}
func CreateStationIndeces() bool {
return false
}

View File

@ -65,8 +65,7 @@ func main() {
// not delayed until the first ticker event. // not delayed until the first ticker event.
go cif.CheckCif(cfg) go cif.CheckCif(cfg)
go corpus.CheckCorpus(cfg) go corpus.CheckCorpus(cfg)
go stations.Check()
stations.Check()
if cfg.VstpOn { if cfg.VstpOn {
messaging.StompInit(cfg) messaging.StompInit(cfg)

View File

@ -1,19 +1,55 @@
package stations package stations
import "fmt" import (
"fmt"
"time"
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
"git.fjla.uk/owlboard/timetable-mgr/log"
"go.uber.org/zap"
)
func Check() { func Check() {
run() oldMetadata, err := dbAccess.GetStationsMetadata()
if err != nil {
log.Error("Error reading Stations metadata", zap.Error(err))
}
if oldMetadata == nil {
log.Info("No old metadata for stations, rebuilding")
} else {
timeSinceLastUpdate := time.Since(oldMetadata.LastUpdate)
if timeSinceLastUpdate <= time.Hour*24*7 {
log.Info("Stations update not required")
return
}
}
ok := run()
if !ok {
log.Error("Error updating Stations data")
}
} }
func run() { func run() bool {
// Download
data, data2, err := download() data, data2, err := download()
if err != nil { if err != nil {
fmt.Println(err) fmt.Println(err)
return false
} }
_, err = parseData(data, data2) // Parse
stations, err := parseData(data, data2)
if err != nil { if err != nil {
fmt.Println(err) fmt.Println(err)
return false
} }
// Drop
dbAccess.DropCollection("stations")
// Push
dbAccess.PutManyNewStations(&stations)
return true
} }