Compare commits
2 Commits
358e69bec7
...
9496c9aae0
Author | SHA1 | Date |
---|---|---|
Fred Boniface | 9496c9aae0 | |
Fred Boniface | fed0208291 |
|
@ -10,6 +10,7 @@ import (
|
|||
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/stations"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
|
@ -40,7 +41,7 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
|
|||
log.Debug("Running background tasks")
|
||||
go cif.CheckCif(cfg)
|
||||
go corpus.CheckCorpus(cfg)
|
||||
//go stations.Check()
|
||||
go stations.Check()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -23,29 +23,29 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error {
|
|||
}
|
||||
|
||||
corpusData := pruneCorpusEntries(*unsortedCorpusData)
|
||||
stationData := createStationEntries(corpusData)
|
||||
//stationData := createStationEntries(corpusData)
|
||||
|
||||
if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
|
||||
log.Warn("CORPUS data may be incomplete")
|
||||
log.Error("Error dropping CORPUS Data", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
|
||||
log.Warn("Stations data may be incomplete")
|
||||
log.Error("Error dropping stations Data", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
//if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
|
||||
// log.Warn("Stations data may be incomplete")
|
||||
// log.Error("Error dropping stations Data", zap.Error(err))
|
||||
// return err
|
||||
//}
|
||||
|
||||
if err := dbAccess.PutManyCorpus(corpusData); err != nil {
|
||||
log.Warn("CORPUS data may be incomplete")
|
||||
log.Error("Error inserting CORPUS Data", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
if err := dbAccess.PutManyStations(stationData); err != nil {
|
||||
log.Warn("Stations data may be incomplete")
|
||||
log.Error("Error inserting stations data", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
//if err := dbAccess.PutManyStations(stationData); err != nil {
|
||||
// log.Warn("Stations data may be incomplete")
|
||||
// log.Error("Error inserting stations data", zap.Error(err))
|
||||
// return err
|
||||
//}
|
||||
|
||||
if err := dbAccess.CreateCorpusIndexes(); err != nil {
|
||||
log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err))
|
||||
|
|
|
@ -84,6 +84,22 @@ func CreateCorpusIndexes() error {
|
|||
|
||||
func GetTiplocFromCrs(crs string) (tiploc string, err error) {
|
||||
// Return TIPLOC from CRS code
|
||||
|
||||
// PIPELINE:
|
||||
/*
|
||||
bson.A{
|
||||
bson.D{{"$match", bson.D{{"3ALPHA", "BTH"}}}},
|
||||
bson.D{
|
||||
{"$project",
|
||||
bson.D{
|
||||
{"TIPLOC", 1},
|
||||
{"STANOX", 1},
|
||||
{"_id", 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
*/
|
||||
err = fmt.Errorf("not yet written")
|
||||
return "", err
|
||||
}
|
||||
|
|
|
@ -0,0 +1,91 @@
|
|||
package dbAccess
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.fjla.uk/owlboard/go-types/pkg/database"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const StationsMetaDoctype string = "StationsMetadata"
|
||||
|
||||
type StationsMetadata struct {
|
||||
Doctype string
|
||||
LastUpdate time.Time
|
||||
}
|
||||
|
||||
// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
|
||||
func GetStationsMetadata() (*StationsMetadata, error) {
|
||||
database := MongoClient.Database(DatabaseName)
|
||||
collection := database.Collection(MetaCollection)
|
||||
filter := bson.M{"type": StationsMetaDoctype}
|
||||
var result StationsMetadata
|
||||
err := collection.FindOne(context.Background(), filter).Decode(&result)
|
||||
if err != nil {
|
||||
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Debug("Fetched Stations Metadata from database", zap.Any("Metadata", result))
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func SetStationsMetadata(time time.Time) bool {
|
||||
database := MongoClient.Database(DatabaseName)
|
||||
collection := database.Collection(MetaCollection)
|
||||
options := options.Update().SetUpsert(true)
|
||||
filter := bson.M{"type": Doctype}
|
||||
update := bson.M{
|
||||
"$set": bson.M{
|
||||
"type": StationsMetaDoctype,
|
||||
"lastUpdate": time,
|
||||
},
|
||||
}
|
||||
|
||||
_, err := collection.UpdateOne(context.Background(), filter, update, options)
|
||||
|
||||
if err != nil {
|
||||
log.Error("Error updating Stations Metadata", zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
log.Info("New Stations Metadata written", zap.Time("Update time", time))
|
||||
return true
|
||||
}
|
||||
|
||||
// Puts an array of Stations documents into the database
|
||||
func PutManyNewStations(stationsData *[]database.Station) error {
|
||||
collection := MongoClient.Database(DatabaseName).Collection(StationsCollection)
|
||||
|
||||
documents := convertNewStationsToInterfaceSlice(stationsData)
|
||||
|
||||
_, err := collection.InsertMany(context.Background(), *documents)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
SetUpdateTime(StationsCollection)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Converts []database.Station types into interface slices required to put them into the database
|
||||
func convertNewStationsToInterfaceSlice(stationsData *[]database.Station) *[]interface{} {
|
||||
interfaceSlice := make([]interface{}, len(*stationsData))
|
||||
for i, doc := range *stationsData {
|
||||
interfaceSlice[i] = doc
|
||||
}
|
||||
return &interfaceSlice
|
||||
}
|
||||
|
||||
func CreateStationIndeces() bool {
|
||||
return false
|
||||
}
|
3
main.go
3
main.go
|
@ -65,8 +65,7 @@ func main() {
|
|||
// not delayed until the first ticker event.
|
||||
go cif.CheckCif(cfg)
|
||||
go corpus.CheckCorpus(cfg)
|
||||
|
||||
stations.Check()
|
||||
go stations.Check()
|
||||
|
||||
if cfg.VstpOn {
|
||||
messaging.StompInit(cfg)
|
||||
|
|
|
@ -1,19 +1,55 @@
|
|||
package stations
|
||||
|
||||
import "fmt"
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func Check() {
|
||||
run()
|
||||
oldMetadata, err := dbAccess.GetStationsMetadata()
|
||||
if err != nil {
|
||||
log.Error("Error reading Stations metadata", zap.Error(err))
|
||||
}
|
||||
if oldMetadata == nil {
|
||||
log.Info("No old metadata for stations, rebuilding")
|
||||
} else {
|
||||
timeSinceLastUpdate := time.Since(oldMetadata.LastUpdate)
|
||||
if timeSinceLastUpdate <= time.Hour*24*7 {
|
||||
log.Info("Stations update not required")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func run() {
|
||||
ok := run()
|
||||
if !ok {
|
||||
log.Error("Error updating Stations data")
|
||||
}
|
||||
}
|
||||
|
||||
func run() bool {
|
||||
// Download
|
||||
data, data2, err := download()
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return false
|
||||
}
|
||||
|
||||
_, err = parseData(data, data2)
|
||||
// Parse
|
||||
stations, err := parseData(data, data2)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return false
|
||||
}
|
||||
|
||||
// Drop
|
||||
dbAccess.DropCollection("stations")
|
||||
|
||||
// Push
|
||||
dbAccess.PutManyNewStations(&stations)
|
||||
|
||||
return true
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue