Complete alpha newStations data, deactivate CORPUS Stations data
This commit is contained in:
parent
358e69bec7
commit
fed0208291
@ -10,6 +10,7 @@ import (
|
|||||||
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||||
|
"git.fjla.uk/owlboard/timetable-mgr/stations"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -40,7 +41,7 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
|
|||||||
log.Debug("Running background tasks")
|
log.Debug("Running background tasks")
|
||||||
go cif.CheckCif(cfg)
|
go cif.CheckCif(cfg)
|
||||||
go corpus.CheckCorpus(cfg)
|
go corpus.CheckCorpus(cfg)
|
||||||
//go stations.Check()
|
go stations.Check()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -23,29 +23,29 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
corpusData := pruneCorpusEntries(*unsortedCorpusData)
|
corpusData := pruneCorpusEntries(*unsortedCorpusData)
|
||||||
stationData := createStationEntries(corpusData)
|
//stationData := createStationEntries(corpusData)
|
||||||
|
|
||||||
if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
|
if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
|
||||||
log.Warn("CORPUS data may be incomplete")
|
log.Warn("CORPUS data may be incomplete")
|
||||||
log.Error("Error dropping CORPUS Data", zap.Error(err))
|
log.Error("Error dropping CORPUS Data", zap.Error(err))
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
|
//if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
|
||||||
log.Warn("Stations data may be incomplete")
|
// log.Warn("Stations data may be incomplete")
|
||||||
log.Error("Error dropping stations Data", zap.Error(err))
|
// log.Error("Error dropping stations Data", zap.Error(err))
|
||||||
return err
|
// return err
|
||||||
}
|
//}
|
||||||
|
|
||||||
if err := dbAccess.PutManyCorpus(corpusData); err != nil {
|
if err := dbAccess.PutManyCorpus(corpusData); err != nil {
|
||||||
log.Warn("CORPUS data may be incomplete")
|
log.Warn("CORPUS data may be incomplete")
|
||||||
log.Error("Error inserting CORPUS Data", zap.Error(err))
|
log.Error("Error inserting CORPUS Data", zap.Error(err))
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := dbAccess.PutManyStations(stationData); err != nil {
|
//if err := dbAccess.PutManyStations(stationData); err != nil {
|
||||||
log.Warn("Stations data may be incomplete")
|
// log.Warn("Stations data may be incomplete")
|
||||||
log.Error("Error inserting stations data", zap.Error(err))
|
// log.Error("Error inserting stations data", zap.Error(err))
|
||||||
return err
|
// return err
|
||||||
}
|
//}
|
||||||
|
|
||||||
if err := dbAccess.CreateCorpusIndexes(); err != nil {
|
if err := dbAccess.CreateCorpusIndexes(); err != nil {
|
||||||
log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err))
|
log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err))
|
||||||
|
91
dbAccess/stations.go
Normal file
91
dbAccess/stations.go
Normal file
@ -0,0 +1,91 @@
|
|||||||
|
package dbAccess
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.fjla.uk/owlboard/go-types/pkg/database"
|
||||||
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||||
|
"go.mongodb.org/mongo-driver/bson"
|
||||||
|
"go.mongodb.org/mongo-driver/mongo"
|
||||||
|
"go.mongodb.org/mongo-driver/mongo/options"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
const StationsMetaDoctype string = "StationsMetadata"
|
||||||
|
|
||||||
|
type StationsMetadata struct {
|
||||||
|
Doctype string
|
||||||
|
LastUpdate time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
|
||||||
|
func GetStationsMetadata() (*StationsMetadata, error) {
|
||||||
|
database := MongoClient.Database(DatabaseName)
|
||||||
|
collection := database.Collection(MetaCollection)
|
||||||
|
filter := bson.M{"type": StationsMetaDoctype}
|
||||||
|
var result StationsMetadata
|
||||||
|
err := collection.FindOne(context.Background(), filter).Decode(&result)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debug("Fetched Stations Metadata from database", zap.Any("Metadata", result))
|
||||||
|
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func SetStationsMetadata(time time.Time) bool {
|
||||||
|
database := MongoClient.Database(DatabaseName)
|
||||||
|
collection := database.Collection(MetaCollection)
|
||||||
|
options := options.Update().SetUpsert(true)
|
||||||
|
filter := bson.M{"type": Doctype}
|
||||||
|
update := bson.M{
|
||||||
|
"$set": bson.M{
|
||||||
|
"type": StationsMetaDoctype,
|
||||||
|
"lastUpdate": time,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := collection.UpdateOne(context.Background(), filter, update, options)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error updating Stations Metadata", zap.Error(err))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("New Stations Metadata written", zap.Time("Update time", time))
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Puts an array of Stations documents into the database
|
||||||
|
func PutManyNewStations(stationsData *[]database.Station) error {
|
||||||
|
collection := MongoClient.Database(DatabaseName).Collection(StationsCollection)
|
||||||
|
|
||||||
|
documents := convertNewStationsToInterfaceSlice(stationsData)
|
||||||
|
|
||||||
|
_, err := collection.InsertMany(context.Background(), *documents)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
SetUpdateTime(StationsCollection)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Converts []database.Station types into interface slices required to put them into the database
|
||||||
|
func convertNewStationsToInterfaceSlice(stationsData *[]database.Station) *[]interface{} {
|
||||||
|
interfaceSlice := make([]interface{}, len(*stationsData))
|
||||||
|
for i, doc := range *stationsData {
|
||||||
|
interfaceSlice[i] = doc
|
||||||
|
}
|
||||||
|
return &interfaceSlice
|
||||||
|
}
|
||||||
|
|
||||||
|
func CreateStationIndeces() bool {
|
||||||
|
return false
|
||||||
|
}
|
3
main.go
3
main.go
@ -65,8 +65,7 @@ func main() {
|
|||||||
// not delayed until the first ticker event.
|
// not delayed until the first ticker event.
|
||||||
go cif.CheckCif(cfg)
|
go cif.CheckCif(cfg)
|
||||||
go corpus.CheckCorpus(cfg)
|
go corpus.CheckCorpus(cfg)
|
||||||
|
go stations.Check()
|
||||||
stations.Check()
|
|
||||||
|
|
||||||
if cfg.VstpOn {
|
if cfg.VstpOn {
|
||||||
messaging.StompInit(cfg)
|
messaging.StompInit(cfg)
|
||||||
|
@ -1,19 +1,55 @@
|
|||||||
package stations
|
package stations
|
||||||
|
|
||||||
import "fmt"
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||||
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
func Check() {
|
func Check() {
|
||||||
run()
|
oldMetadata, err := dbAccess.GetStationsMetadata()
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error reading Stations metadata", zap.Error(err))
|
||||||
|
}
|
||||||
|
if oldMetadata == nil {
|
||||||
|
log.Info("No old metadata for stations, rebuilding")
|
||||||
|
} else {
|
||||||
|
timeSinceLastUpdate := time.Since(oldMetadata.LastUpdate)
|
||||||
|
if timeSinceLastUpdate <= time.Hour*24*7 {
|
||||||
|
log.Info("Stations update not required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ok := run()
|
||||||
|
if !ok {
|
||||||
|
log.Error("Error updating Stations data")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func run() {
|
func run() bool {
|
||||||
|
// Download
|
||||||
data, data2, err := download()
|
data, data2, err := download()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println(err)
|
fmt.Println(err)
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = parseData(data, data2)
|
// Parse
|
||||||
|
stations, err := parseData(data, data2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println(err)
|
fmt.Println(err)
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Drop
|
||||||
|
dbAccess.DropCollection("stations")
|
||||||
|
|
||||||
|
// Push
|
||||||
|
dbAccess.PutManyNewStations(&stations)
|
||||||
|
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user