Complete alpha newStations data, deactivate CORPUS Stations data

background/ticker.go

@@ -10,6 +10,7 @@ import (
 	"git.fjla.uk/owlboard/timetable-mgr/corpus"
 	"git.fjla.uk/owlboard/timetable-mgr/helpers"
 	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"git.fjla.uk/owlboard/timetable-mgr/stations"
 	"go.uber.org/zap"
 )
 
@@ -40,7 +41,7 @@ func runTicker(cfg *helpers.Configuration, stop <-chan struct{}) {
 			log.Debug("Running background tasks")
 			go cif.CheckCif(cfg)
 			go corpus.CheckCorpus(cfg)
-			//go stations.Check()
+			go stations.Check()
 		}
 	}
 }

corpus/update.go

@@ -23,29 +23,29 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error {
 	}
 
 	corpusData := pruneCorpusEntries(*unsortedCorpusData)
-	stationData := createStationEntries(corpusData)
+	//stationData := createStationEntries(corpusData)
 
 	if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
 		log.Warn("CORPUS data may be incomplete")
 		log.Error("Error dropping CORPUS Data", zap.Error(err))
 		return err
 	}
-	if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
-		log.Warn("Stations data may be incomplete")
-		log.Error("Error dropping stations Data", zap.Error(err))
-		return err
-	}
+	//if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
+	//	log.Warn("Stations data may be incomplete")
+	//	log.Error("Error dropping stations Data", zap.Error(err))
+	//	return err
+	//}
 
 	if err := dbAccess.PutManyCorpus(corpusData); err != nil {
 		log.Warn("CORPUS data may be incomplete")
 		log.Error("Error inserting CORPUS Data", zap.Error(err))
 		return err
 	}
-	if err := dbAccess.PutManyStations(stationData); err != nil {
-		log.Warn("Stations data may be incomplete")
-		log.Error("Error inserting stations data", zap.Error(err))
-		return err
-	}
+	//if err := dbAccess.PutManyStations(stationData); err != nil {
+	//	log.Warn("Stations data may be incomplete")
+	//	log.Error("Error inserting stations data", zap.Error(err))
+	//	return err
+	//}
 
 	if err := dbAccess.CreateCorpusIndexes(); err != nil {
 		log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err))

dbAccess/stations.go

@@ -0,0 +1,91 @@
+package dbAccess
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"git.fjla.uk/owlboard/go-types/pkg/database"
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.uber.org/zap"
+)
+
+const StationsMetaDoctype string = "StationsMetadata"
+
+type StationsMetadata struct {
+	Doctype    string
+	LastUpdate time.Time
+}
+
+// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
+func GetStationsMetadata() (*StationsMetadata, error) {
+	database := MongoClient.Database(DatabaseName)
+	collection := database.Collection(MetaCollection)
+	filter := bson.M{"type": StationsMetaDoctype}
+	var result StationsMetadata
+	err := collection.FindOne(context.Background(), filter).Decode(&result)
+	if err != nil {
+		if errors.Is(err, mongo.ErrNoDocuments) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	log.Debug("Fetched Stations Metadata from database", zap.Any("Metadata", result))
+
+	return &result, nil
+}
+
+func SetStationsMetadata(time time.Time) bool {
+	database := MongoClient.Database(DatabaseName)
+	collection := database.Collection(MetaCollection)
+	options := options.Update().SetUpsert(true)
+	filter := bson.M{"type": Doctype}
+	update := bson.M{
+		"$set": bson.M{
+			"type":       StationsMetaDoctype,
+			"lastUpdate": time,
+		},
+	}
+
+	_, err := collection.UpdateOne(context.Background(), filter, update, options)
+
+	if err != nil {
+		log.Error("Error updating Stations Metadata", zap.Error(err))
+		return false
+	}
+
+	log.Info("New Stations Metadata written", zap.Time("Update time", time))
+	return true
+}
+
+// Puts an array of Stations documents into the database
+func PutManyNewStations(stationsData *[]database.Station) error {
+	collection := MongoClient.Database(DatabaseName).Collection(StationsCollection)
+
+	documents := convertNewStationsToInterfaceSlice(stationsData)
+
+	_, err := collection.InsertMany(context.Background(), *documents)
+	if err != nil {
+		return err
+	}
+
+	SetUpdateTime(StationsCollection)
+	return nil
+}
+
+// Converts []database.Station types into interface slices required to put them into the database
+func convertNewStationsToInterfaceSlice(stationsData *[]database.Station) *[]interface{} {
+	interfaceSlice := make([]interface{}, len(*stationsData))
+	for i, doc := range *stationsData {
+		interfaceSlice[i] = doc
+	}
+	return &interfaceSlice
+}
+
+func CreateStationIndeces() bool {
+	return false
+}

main.go

@@ -65,8 +65,7 @@ func main() {
 	// not delayed until the first ticker event.
 	go cif.CheckCif(cfg)
 	go corpus.CheckCorpus(cfg)
-
-	stations.Check()
+	go stations.Check()
 
 	if cfg.VstpOn {
 		messaging.StompInit(cfg)

stations/check.go

@@ -1,19 +1,55 @@
 package stations
 
-import "fmt"
+import (
+	"fmt"
+	"time"
+
+	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.uber.org/zap"
+)
 
 func Check() {
-	run()
+	oldMetadata, err := dbAccess.GetStationsMetadata()
+	if err != nil {
+		log.Error("Error reading Stations metadata", zap.Error(err))
+	}
+	if oldMetadata == nil {
+		log.Info("No old metadata for stations, rebuilding")
+	} else {
+		timeSinceLastUpdate := time.Since(oldMetadata.LastUpdate)
+		if timeSinceLastUpdate <= time.Hour*24*7 {
+			log.Info("Stations update not required")
+			return
+		}
 	}
 
-func run() {
+	ok := run()
+	if !ok {
+		log.Error("Error updating Stations data")
+	}
+}
+
+func run() bool {
+	// Download
 	data, data2, err := download()
 	if err != nil {
 		fmt.Println(err)
+		return false
 	}
 
-	_, err = parseData(data, data2)
+	// Parse
+	stations, err := parseData(data, data2)
 	if err != nil {
 		fmt.Println(err)
+		return false
 	}
+
+	// Drop
+	dbAccess.DropCollection("stations")
+
+	// Push
+	dbAccess.PutManyNewStations(&stations)
+
+	return true
 }