Reorganise repo

corpus/check.go

@@ -0,0 +1,41 @@
+package corpus
+
+import (
+	"time"
+
+	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
+	"git.fjla.uk/owlboard/timetable-mgr/helpers"
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.uber.org/zap"
+)
+
+// Checks if the CORPUS Data needs updating, and calls an updater function if needed.
+func CheckCorpus(cfg *helpers.Configuration) {
+	log.Msg.Debug("Checking age of CORPUS Data")
+	utime, err := dbAccess.CheckUpdateTime(dbAccess.CorpusCollection)
+	if err != nil {
+		log.Msg.Error("Error checking last CORPUS update", zap.Error(err))
+	}
+
+	lastUpdate := time.Unix(utime, 0)
+	currentTime := time.Now()
+	dataAge := currentTime.Sub(lastUpdate)
+	fortnight := 14 * 24 * time.Hour
+
+	log.Msg.Debug("CORPUS Data", zap.Duration("Data Age", dataAge), zap.Duration("Max Age", 14*24*time.Hour))
+
+	if dataAge >= fortnight {
+		log.Msg.Info("CORPUS Data is more than two weeks old")
+		err := RunCorpusUpdate(cfg)
+		if err != nil {
+			log.Msg.Warn("CORPUS Update did not run")
+		} else {
+			log.Msg.Info("CORPUS data has been updated")
+		}
+	} else {
+		log.Msg.Info("CORPUS Data is less than two weeks old, not updating")
+	}
+}
+
+// Check if corpus data needs updating.
+// Run update if needed.

corpus/constants.go

@@ -0,0 +1,4 @@
+package corpus
+
+// The download URL for CORPUS data
+const url string = "https://publicdatafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS"

corpus/parse.go

@@ -0,0 +1,78 @@
+package corpus
+
+import (
+	"encoding/json"
+	"errors"
+
+	"git.fjla.uk/owlboard/go-types/pkg/database"
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"go.uber.org/zap"
+)
+
+// Accepts CORPUS data as a byte array and formats it ready for database insertion
+func parseCorpusData(jsonData *[]byte) ([]database.CorpusEntry, error) {
+	log.Msg.Info("Unmarshalling CORPUS Data")
+
+	var dataMap map[string]interface{}
+	err := json.Unmarshal(*jsonData, &dataMap)
+	if err != nil {
+		log.Msg.Error("Unable to unmarshal CORPUS data", zap.Error(err))
+	}
+
+	corpusDataArrayInterface, ok := dataMap["TIPLOCDATA"]
+	if !ok {
+		err := errors.New("corpus Data not in expected format")
+		log.Msg.Error("Error parsing CORPUS Data", zap.Error(err))
+		return nil, err
+	}
+
+	corpusDataArray, ok := corpusDataArrayInterface.([]interface{})
+	if !ok {
+		err := errors.New("corpus data missing the data array")
+		log.Msg.Error("Error parsing CORPUS Data", zap.Error(err))
+		return nil, err
+	}
+
+	var corpusEntries []database.CorpusEntry
+	for _, item := range corpusDataArray {
+		jsonItem, err := json.Marshal(item)
+		if err != nil {
+			log.Msg.Error("Error parsing CORPUS Data", zap.Error(err))
+			return nil, err
+		}
+
+		var corpusEntry database.CorpusEntry
+		err = json.Unmarshal(jsonItem, &corpusEntry)
+		if err != nil {
+			log.Msg.Error("Error parsing CORPUS Data", zap.Error(err))
+			return nil, err
+		}
+
+		corpusEntries = append(corpusEntries, corpusEntry)
+	}
+
+	return corpusEntries, nil
+}
+
+// Removes empty fields from CORPUS entries
+func pruneCorpusEntries(corpusEntries []database.CorpusEntry) []database.CorpusEntry {
+	for i := range corpusEntries {
+		if corpusEntries[i].CRS == " " {
+			corpusEntries[i].CRS = ""
+		}
+		if corpusEntries[i].TIPLOC == " " {
+			corpusEntries[i].TIPLOC = ""
+		}
+		if corpusEntries[i].NLCDESC16 == " " {
+			corpusEntries[i].NLCDESC16 = ""
+		}
+		if corpusEntries[i].STANOX == " " {
+			corpusEntries[i].STANOX = ""
+		}
+		if corpusEntries[i].UIC == " " {
+			corpusEntries[i].UIC = ""
+		}
+	}
+
+	return corpusEntries
+}

corpus/stations.go

@@ -0,0 +1,23 @@
+package corpus
+
+import "git.fjla.uk/owlboard/go-types/pkg/database"
+
+// Removes non-station entities from the CORPUS Data, ready for insertion to the database (stations collection)
+func createStationEntries(corpusData []database.CorpusEntry) []database.StationEntry {
+	var stationEntries []database.StationEntry
+
+	for _, entry := range corpusData {
+		if entry.CRS != "" {
+			stationEntry := database.StationEntry{
+				CRS:     entry.CRS,
+				TIPLOC:  entry.TIPLOC,
+				NLCDESC: entry.NLCDESC,
+				STANOX:  entry.STANOX,
+			}
+
+			stationEntries = append(stationEntries, stationEntry)
+		}
+	}
+
+	return stationEntries
+}

corpus/update.go

@@ -0,0 +1,51 @@
+package corpus
+
+import (
+	"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
+	"git.fjla.uk/owlboard/timetable-mgr/helpers"
+	"git.fjla.uk/owlboard/timetable-mgr/log"
+	"git.fjla.uk/owlboard/timetable-mgr/nrod"
+	"go.uber.org/zap"
+)
+
+// Runs all stages of the CORPUS Update process
+func RunCorpusUpdate(cfg *helpers.Configuration) error {
+	resp, err := nrod.NrodDownload(url, cfg)
+	if err != nil {
+		log.Msg.Error("Failed to fetch CORPUS data", zap.Error(err))
+		return err
+	}
+
+	unsortedCorpusData, err := parseCorpusData(&resp)
+	if err != nil {
+		log.Msg.Error("Error parsing Corpus data", zap.Error(err))
+		return err
+	}
+
+	corpusData := pruneCorpusEntries(unsortedCorpusData)
+	stationData := createStationEntries(corpusData)
+
+	if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
+		log.Msg.Warn("CORPUS data may be incomplete")
+		log.Msg.Error("Error dropping CORPUS Data", zap.Error(err))
+		return err
+	}
+	if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
+		log.Msg.Warn("Stations data may be incomplete")
+		log.Msg.Error("Error dropping stations Data", zap.Error(err))
+		return err
+	}
+
+	if err := dbAccess.PutManyCorpus(corpusData); err != nil {
+		log.Msg.Warn("CORPUS data may be incomplete")
+		log.Msg.Error("Error inserting CORPUS Data", zap.Error(err))
+		return err
+	}
+	if err := dbAccess.PutManyStations(stationData); err != nil {
+		log.Msg.Warn("Stations data may be incomplete")
+		log.Msg.Error("Error inserting stations data", zap.Error(err))
+		return err
+	}
+
+	return nil
+}