timetable-extension #1
@ -10,6 +10,7 @@ import (
|
||||
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/stations"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
@ -40,7 +41,7 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
|
||||
log.Debug("Running background tasks")
|
||||
go cif.CheckCif(cfg)
|
||||
go corpus.CheckCorpus(cfg)
|
||||
//go stations.Check()
|
||||
go stations.Check()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -23,29 +23,29 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error {
|
||||
}
|
||||
|
||||
corpusData := pruneCorpusEntries(*unsortedCorpusData)
|
||||
stationData := createStationEntries(corpusData)
|
||||
//stationData := createStationEntries(corpusData)
|
||||
|
||||
if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
|
||||
log.Warn("CORPUS data may be incomplete")
|
||||
log.Error("Error dropping CORPUS Data", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
|
||||
log.Warn("Stations data may be incomplete")
|
||||
log.Error("Error dropping stations Data", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
//if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
|
||||
// log.Warn("Stations data may be incomplete")
|
||||
// log.Error("Error dropping stations Data", zap.Error(err))
|
||||
// return err
|
||||
//}
|
||||
|
||||
if err := dbAccess.PutManyCorpus(corpusData); err != nil {
|
||||
log.Warn("CORPUS data may be incomplete")
|
||||
log.Error("Error inserting CORPUS Data", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
if err := dbAccess.PutManyStations(stationData); err != nil {
|
||||
log.Warn("Stations data may be incomplete")
|
||||
log.Error("Error inserting stations data", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
//if err := dbAccess.PutManyStations(stationData); err != nil {
|
||||
// log.Warn("Stations data may be incomplete")
|
||||
// log.Error("Error inserting stations data", zap.Error(err))
|
||||
// return err
|
||||
//}
|
||||
|
||||
if err := dbAccess.CreateCorpusIndexes(); err != nil {
|
||||
log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err))
|
||||
|
91
dbAccess/stations.go
Normal file
91
dbAccess/stations.go
Normal file
@ -0,0 +1,91 @@
|
||||
package dbAccess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.fjla.uk/owlboard/go-types/pkg/database"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const StationsMetaDoctype string = "StationsMetadata"
|
||||
|
||||
type StationsMetadata struct {
|
||||
Doctype string
|
||||
LastUpdate time.Time
|
||||
}
|
||||
|
||||
// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
|
||||
func GetStationsMetadata() (*StationsMetadata, error) {
|
||||
database := MongoClient.Database(DatabaseName)
|
||||
collection := database.Collection(MetaCollection)
|
||||
filter := bson.M{"type": StationsMetaDoctype}
|
||||
var result StationsMetadata
|
||||
err := collection.FindOne(context.Background(), filter).Decode(&result)
|
||||
if err != nil {
|
||||
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Debug("Fetched Stations Metadata from database", zap.Any("Metadata", result))
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func SetStationsMetadata(time time.Time) bool {
|
||||
database := MongoClient.Database(DatabaseName)
|
||||
collection := database.Collection(MetaCollection)
|
||||
options := options.Update().SetUpsert(true)
|
||||
filter := bson.M{"type": Doctype}
|
||||
update := bson.M{
|
||||
"$set": bson.M{
|
||||
"type": StationsMetaDoctype,
|
||||
"lastUpdate": time,
|
||||
},
|
||||
}
|
||||
|
||||
_, err := collection.UpdateOne(context.Background(), filter, update, options)
|
||||
|
||||
if err != nil {
|
||||
log.Error("Error updating Stations Metadata", zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
log.Info("New Stations Metadata written", zap.Time("Update time", time))
|
||||
return true
|
||||
}
|
||||
|
||||
// Puts an array of Stations documents into the database
|
||||
func PutManyNewStations(stationsData *[]database.Station) error {
|
||||
collection := MongoClient.Database(DatabaseName).Collection(StationsCollection)
|
||||
|
||||
documents := convertNewStationsToInterfaceSlice(stationsData)
|
||||
|
||||
_, err := collection.InsertMany(context.Background(), *documents)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
SetUpdateTime(StationsCollection)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Converts []database.Station types into interface slices required to put them into the database
|
||||
func convertNewStationsToInterfaceSlice(stationsData *[]database.Station) *[]interface{} {
|
||||
interfaceSlice := make([]interface{}, len(*stationsData))
|
||||
for i, doc := range *stationsData {
|
||||
interfaceSlice[i] = doc
|
||||
}
|
||||
return &interfaceSlice
|
||||
}
|
||||
|
||||
func CreateStationIndeces() bool {
|
||||
return false
|
||||
}
|
3
main.go
3
main.go
@ -65,8 +65,7 @@ func main() {
|
||||
// not delayed until the first ticker event.
|
||||
go cif.CheckCif(cfg)
|
||||
go corpus.CheckCorpus(cfg)
|
||||
|
||||
stations.Check()
|
||||
go stations.Check()
|
||||
|
||||
if cfg.VstpOn {
|
||||
messaging.StompInit(cfg)
|
||||
|
@ -1,19 +1,55 @@
|
||||
package stations
|
||||
|
||||
import "fmt"
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func Check() {
|
||||
run()
|
||||
oldMetadata, err := dbAccess.GetStationsMetadata()
|
||||
if err != nil {
|
||||
log.Error("Error reading Stations metadata", zap.Error(err))
|
||||
}
|
||||
if oldMetadata == nil {
|
||||
log.Info("No old metadata for stations, rebuilding")
|
||||
} else {
|
||||
timeSinceLastUpdate := time.Since(oldMetadata.LastUpdate)
|
||||
if timeSinceLastUpdate <= time.Hour*24*7 {
|
||||
log.Info("Stations update not required")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func run() {
|
||||
ok := run()
|
||||
if !ok {
|
||||
log.Error("Error updating Stations data")
|
||||
}
|
||||
}
|
||||
|
||||
func run() bool {
|
||||
// Download
|
||||
data, data2, err := download()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return false
|
||||
}
|
||||
|
||||
_, err = parseData(data, data2)
|
||||
// Parse
|
||||
stations, err := parseData(data, data2)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Drop
|
||||
dbAccess.DropCollection("stations")
|
||||
|
||||
// Push
|
||||
dbAccess.PutManyNewStations(&stations)
|
||||
|
||||
return true
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user