From fed02082915c0a631964968a7c1639f98638ae86 Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Sun, 30 Jun 2024 19:29:41 +0100 Subject: [PATCH] Complete alpha newStations data, deactivate CORPUS Stations data --- background/ticker.go | 3 +- corpus/update.go | 22 +++++------ dbAccess/stations.go | 91 ++++++++++++++++++++++++++++++++++++++++++++ main.go | 3 +- stations/check.go | 44 +++++++++++++++++++-- 5 files changed, 145 insertions(+), 18 deletions(-) create mode 100644 dbAccess/stations.go diff --git a/background/ticker.go b/background/ticker.go index 5c1a9e7..e47bf74 100644 --- a/background/ticker.go +++ b/background/ticker.go @@ -10,6 +10,7 @@ import ( "git.fjla.uk/owlboard/timetable-mgr/corpus" "git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/log" + "git.fjla.uk/owlboard/timetable-mgr/stations" "go.uber.org/zap" ) @@ -40,7 +41,7 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) { log.Debug("Running background tasks") go cif.CheckCif(cfg) go corpus.CheckCorpus(cfg) - //go stations.Check() + go stations.Check() } } } diff --git a/corpus/update.go b/corpus/update.go index ee6d6eb..06a819e 100644 --- a/corpus/update.go +++ b/corpus/update.go @@ -23,29 +23,29 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error { } corpusData := pruneCorpusEntries(*unsortedCorpusData) - stationData := createStationEntries(corpusData) + //stationData := createStationEntries(corpusData) if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil { log.Warn("CORPUS data may be incomplete") log.Error("Error dropping CORPUS Data", zap.Error(err)) return err } - if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil { - log.Warn("Stations data may be incomplete") - log.Error("Error dropping stations Data", zap.Error(err)) - return err - } + //if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil { + // log.Warn("Stations data may be incomplete") + // log.Error("Error dropping stations Data", zap.Error(err)) + // return err + //} if err := dbAccess.PutManyCorpus(corpusData); err != nil { log.Warn("CORPUS data may be incomplete") log.Error("Error inserting CORPUS Data", zap.Error(err)) return err } - if err := dbAccess.PutManyStations(stationData); err != nil { - log.Warn("Stations data may be incomplete") - log.Error("Error inserting stations data", zap.Error(err)) - return err - } + //if err := dbAccess.PutManyStations(stationData); err != nil { + // log.Warn("Stations data may be incomplete") + // log.Error("Error inserting stations data", zap.Error(err)) + // return err + //} if err := dbAccess.CreateCorpusIndexes(); err != nil { log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err)) diff --git a/dbAccess/stations.go b/dbAccess/stations.go new file mode 100644 index 0000000..a9e3e19 --- /dev/null +++ b/dbAccess/stations.go @@ -0,0 +1,91 @@ +package dbAccess + +import ( + "context" + "errors" + "time" + + "git.fjla.uk/owlboard/go-types/pkg/database" + "git.fjla.uk/owlboard/timetable-mgr/log" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.uber.org/zap" +) + +const StationsMetaDoctype string = "StationsMetadata" + +type StationsMetadata struct { + Doctype string + LastUpdate time.Time +} + +// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example. +func GetStationsMetadata() (*StationsMetadata, error) { + database := MongoClient.Database(DatabaseName) + collection := database.Collection(MetaCollection) + filter := bson.M{"type": StationsMetaDoctype} + var result StationsMetadata + err := collection.FindOne(context.Background(), filter).Decode(&result) + if err != nil { + if errors.Is(err, mongo.ErrNoDocuments) { + return nil, nil + } + return nil, err + } + + log.Debug("Fetched Stations Metadata from database", zap.Any("Metadata", result)) + + return &result, nil +} + +func SetStationsMetadata(time time.Time) bool { + database := MongoClient.Database(DatabaseName) + collection := database.Collection(MetaCollection) + options := options.Update().SetUpsert(true) + filter := bson.M{"type": Doctype} + update := bson.M{ + "$set": bson.M{ + "type": StationsMetaDoctype, + "lastUpdate": time, + }, + } + + _, err := collection.UpdateOne(context.Background(), filter, update, options) + + if err != nil { + log.Error("Error updating Stations Metadata", zap.Error(err)) + return false + } + + log.Info("New Stations Metadata written", zap.Time("Update time", time)) + return true +} + +// Puts an array of Stations documents into the database +func PutManyNewStations(stationsData *[]database.Station) error { + collection := MongoClient.Database(DatabaseName).Collection(StationsCollection) + + documents := convertNewStationsToInterfaceSlice(stationsData) + + _, err := collection.InsertMany(context.Background(), *documents) + if err != nil { + return err + } + + SetUpdateTime(StationsCollection) + return nil +} + +// Converts []database.Station types into interface slices required to put them into the database +func convertNewStationsToInterfaceSlice(stationsData *[]database.Station) *[]interface{} { + interfaceSlice := make([]interface{}, len(*stationsData)) + for i, doc := range *stationsData { + interfaceSlice[i] = doc + } + return &interfaceSlice +} + +func CreateStationIndeces() bool { + return false +} diff --git a/main.go b/main.go index 4b620ad..1110724 100644 --- a/main.go +++ b/main.go @@ -65,8 +65,7 @@ func main() { // not delayed until the first ticker event. go cif.CheckCif(cfg) go corpus.CheckCorpus(cfg) - - stations.Check() + go stations.Check() if cfg.VstpOn { messaging.StompInit(cfg) diff --git a/stations/check.go b/stations/check.go index 59a3c36..5fa6c7e 100644 --- a/stations/check.go +++ b/stations/check.go @@ -1,19 +1,55 @@ package stations -import "fmt" +import ( + "fmt" + "time" + + "git.fjla.uk/owlboard/timetable-mgr/dbAccess" + "git.fjla.uk/owlboard/timetable-mgr/log" + "go.uber.org/zap" +) func Check() { - run() + oldMetadata, err := dbAccess.GetStationsMetadata() + if err != nil { + log.Error("Error reading Stations metadata", zap.Error(err)) + } + if oldMetadata == nil { + log.Info("No old metadata for stations, rebuilding") + } else { + timeSinceLastUpdate := time.Since(oldMetadata.LastUpdate) + if timeSinceLastUpdate <= time.Hour*24*7 { + log.Info("Stations update not required") + return + } + } + + ok := run() + if !ok { + log.Error("Error updating Stations data") + } } -func run() { +func run() bool { + // Download data, data2, err := download() if err != nil { fmt.Println(err) + return false } - _, err = parseData(data, data2) + // Parse + stations, err := parseData(data, data2) if err != nil { fmt.Println(err) + return false } + + // Drop + dbAccess.DropCollection("stations") + + // Push + dbAccess.PutManyNewStations(&stations) + + return true }