Compare commits
No commits in common. "9496c9aae0ce380783cc714b5c89fb9077b24fb1" and "358e69bec75e493e133d19f0fee28a8291942114" have entirely different histories.
9496c9aae0
...
358e69bec7
|
@ -10,7 +10,6 @@ import (
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/stations"
|
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -41,7 +40,7 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
|
||||||
log.Debug("Running background tasks")
|
log.Debug("Running background tasks")
|
||||||
go cif.CheckCif(cfg)
|
go cif.CheckCif(cfg)
|
||||||
go corpus.CheckCorpus(cfg)
|
go corpus.CheckCorpus(cfg)
|
||||||
go stations.Check()
|
//go stations.Check()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,29 +23,29 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
corpusData := pruneCorpusEntries(*unsortedCorpusData)
|
corpusData := pruneCorpusEntries(*unsortedCorpusData)
|
||||||
//stationData := createStationEntries(corpusData)
|
stationData := createStationEntries(corpusData)
|
||||||
|
|
||||||
if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
|
if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
|
||||||
log.Warn("CORPUS data may be incomplete")
|
log.Warn("CORPUS data may be incomplete")
|
||||||
log.Error("Error dropping CORPUS Data", zap.Error(err))
|
log.Error("Error dropping CORPUS Data", zap.Error(err))
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
//if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
|
if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
|
||||||
// log.Warn("Stations data may be incomplete")
|
log.Warn("Stations data may be incomplete")
|
||||||
// log.Error("Error dropping stations Data", zap.Error(err))
|
log.Error("Error dropping stations Data", zap.Error(err))
|
||||||
// return err
|
return err
|
||||||
//}
|
}
|
||||||
|
|
||||||
if err := dbAccess.PutManyCorpus(corpusData); err != nil {
|
if err := dbAccess.PutManyCorpus(corpusData); err != nil {
|
||||||
log.Warn("CORPUS data may be incomplete")
|
log.Warn("CORPUS data may be incomplete")
|
||||||
log.Error("Error inserting CORPUS Data", zap.Error(err))
|
log.Error("Error inserting CORPUS Data", zap.Error(err))
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
//if err := dbAccess.PutManyStations(stationData); err != nil {
|
if err := dbAccess.PutManyStations(stationData); err != nil {
|
||||||
// log.Warn("Stations data may be incomplete")
|
log.Warn("Stations data may be incomplete")
|
||||||
// log.Error("Error inserting stations data", zap.Error(err))
|
log.Error("Error inserting stations data", zap.Error(err))
|
||||||
// return err
|
return err
|
||||||
//}
|
}
|
||||||
|
|
||||||
if err := dbAccess.CreateCorpusIndexes(); err != nil {
|
if err := dbAccess.CreateCorpusIndexes(); err != nil {
|
||||||
log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err))
|
log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err))
|
||||||
|
|
|
@ -84,22 +84,6 @@ func CreateCorpusIndexes() error {
|
||||||
|
|
||||||
func GetTiplocFromCrs(crs string) (tiploc string, err error) {
|
func GetTiplocFromCrs(crs string) (tiploc string, err error) {
|
||||||
// Return TIPLOC from CRS code
|
// Return TIPLOC from CRS code
|
||||||
|
|
||||||
// PIPELINE:
|
|
||||||
/*
|
|
||||||
bson.A{
|
|
||||||
bson.D{{"$match", bson.D{{"3ALPHA", "BTH"}}}},
|
|
||||||
bson.D{
|
|
||||||
{"$project",
|
|
||||||
bson.D{
|
|
||||||
{"TIPLOC", 1},
|
|
||||||
{"STANOX", 1},
|
|
||||||
{"_id", 0},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
err = fmt.Errorf("not yet written")
|
err = fmt.Errorf("not yet written")
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,91 +0,0 @@
|
||||||
package dbAccess
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.fjla.uk/owlboard/go-types/pkg/database"
|
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
|
||||||
"go.mongodb.org/mongo-driver/bson"
|
|
||||||
"go.mongodb.org/mongo-driver/mongo"
|
|
||||||
"go.mongodb.org/mongo-driver/mongo/options"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
)
|
|
||||||
|
|
||||||
const StationsMetaDoctype string = "StationsMetadata"
|
|
||||||
|
|
||||||
type StationsMetadata struct {
|
|
||||||
Doctype string
|
|
||||||
LastUpdate time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
|
|
||||||
func GetStationsMetadata() (*StationsMetadata, error) {
|
|
||||||
database := MongoClient.Database(DatabaseName)
|
|
||||||
collection := database.Collection(MetaCollection)
|
|
||||||
filter := bson.M{"type": StationsMetaDoctype}
|
|
||||||
var result StationsMetadata
|
|
||||||
err := collection.FindOne(context.Background(), filter).Decode(&result)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, mongo.ErrNoDocuments) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Debug("Fetched Stations Metadata from database", zap.Any("Metadata", result))
|
|
||||||
|
|
||||||
return &result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func SetStationsMetadata(time time.Time) bool {
|
|
||||||
database := MongoClient.Database(DatabaseName)
|
|
||||||
collection := database.Collection(MetaCollection)
|
|
||||||
options := options.Update().SetUpsert(true)
|
|
||||||
filter := bson.M{"type": Doctype}
|
|
||||||
update := bson.M{
|
|
||||||
"$set": bson.M{
|
|
||||||
"type": StationsMetaDoctype,
|
|
||||||
"lastUpdate": time,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := collection.UpdateOne(context.Background(), filter, update, options)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Error updating Stations Metadata", zap.Error(err))
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info("New Stations Metadata written", zap.Time("Update time", time))
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Puts an array of Stations documents into the database
|
|
||||||
func PutManyNewStations(stationsData *[]database.Station) error {
|
|
||||||
collection := MongoClient.Database(DatabaseName).Collection(StationsCollection)
|
|
||||||
|
|
||||||
documents := convertNewStationsToInterfaceSlice(stationsData)
|
|
||||||
|
|
||||||
_, err := collection.InsertMany(context.Background(), *documents)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
SetUpdateTime(StationsCollection)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Converts []database.Station types into interface slices required to put them into the database
|
|
||||||
func convertNewStationsToInterfaceSlice(stationsData *[]database.Station) *[]interface{} {
|
|
||||||
interfaceSlice := make([]interface{}, len(*stationsData))
|
|
||||||
for i, doc := range *stationsData {
|
|
||||||
interfaceSlice[i] = doc
|
|
||||||
}
|
|
||||||
return &interfaceSlice
|
|
||||||
}
|
|
||||||
|
|
||||||
func CreateStationIndeces() bool {
|
|
||||||
return false
|
|
||||||
}
|
|
3
main.go
3
main.go
|
@ -65,7 +65,8 @@ func main() {
|
||||||
// not delayed until the first ticker event.
|
// not delayed until the first ticker event.
|
||||||
go cif.CheckCif(cfg)
|
go cif.CheckCif(cfg)
|
||||||
go corpus.CheckCorpus(cfg)
|
go corpus.CheckCorpus(cfg)
|
||||||
go stations.Check()
|
|
||||||
|
stations.Check()
|
||||||
|
|
||||||
if cfg.VstpOn {
|
if cfg.VstpOn {
|
||||||
messaging.StompInit(cfg)
|
messaging.StompInit(cfg)
|
||||||
|
|
|
@ -1,55 +1,19 @@
|
||||||
package stations
|
package stations
|
||||||
|
|
||||||
import (
|
import "fmt"
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Check() {
|
func Check() {
|
||||||
oldMetadata, err := dbAccess.GetStationsMetadata()
|
run()
|
||||||
if err != nil {
|
|
||||||
log.Error("Error reading Stations metadata", zap.Error(err))
|
|
||||||
}
|
|
||||||
if oldMetadata == nil {
|
|
||||||
log.Info("No old metadata for stations, rebuilding")
|
|
||||||
} else {
|
|
||||||
timeSinceLastUpdate := time.Since(oldMetadata.LastUpdate)
|
|
||||||
if timeSinceLastUpdate <= time.Hour*24*7 {
|
|
||||||
log.Info("Stations update not required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ok := run()
|
func run() {
|
||||||
if !ok {
|
|
||||||
log.Error("Error updating Stations data")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func run() bool {
|
|
||||||
// Download
|
|
||||||
data, data2, err := download()
|
data, data2, err := download()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println(err)
|
fmt.Println(err)
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse
|
_, err = parseData(data, data2)
|
||||||
stations, err := parseData(data, data2)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println(err)
|
fmt.Println(err)
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Drop
|
|
||||||
dbAccess.DropCollection("stations")
|
|
||||||
|
|
||||||
// Push
|
|
||||||
dbAccess.PutManyNewStations(&stations)
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue