timetable-extension #1

Open
fred.boniface wants to merge 154 commits from timetable-extension into main
5 changed files with 24 additions and 19 deletions
Showing only changes of commit 39405e6d6a - Show all commits

View File

@ -12,6 +12,7 @@ import (
)
// Unmarshalls data into the correct types for processing
// - Currently not used
func parseCifData(data *[]byte) (*parsedData, error) {
log.Msg.Debug("Starting CIF Data parsing")
if data == nil {

View File

@ -2,6 +2,7 @@ package cif
import (
"errors"
"io"
"time"
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
@ -25,6 +26,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
log.Msg.Error("Error downloading CIF data", zap.Error(err))
}
// DOES NOT WORK WITH NEW DOWNLOAD STREAMING
// If debug mode is on, call debugWriteDownload
// if helpers.Runtime == "debug" {
// debugWriteDownload(dataStream)
@ -37,6 +39,8 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
return err
}
// Look to stream data onwards to the parsing function
// Make `data` a nil pointer as it is no longer required
dataStream = nil
@ -73,13 +77,14 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
return err
}
// DOES NOT WORK WITH NEW NROD STREAMER
// If debug mode is on, call debugWriteDownload
if helpers.Runtime == "debug" {
debugWriteDownload(data)
}
// if helpers.Runtime == "debug" {
// debugWriteDownload(data)
// }
// Parse CIF file
parsed, err := parseCifData(data)
parsed, err := parseCifDataStream(data)
if err != nil {
log.Msg.Error("Error parsing CIF data", zap.Error(err))
return err
@ -109,7 +114,8 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
log.Msg.Debug("CIF Data has passed checks and should now be processed <<<<<<")
// Process CIF file
metadata = generateMetadata(&parsed.header)
// Temporarily disable METADATA GENERATION AND WRITING
// metadata = generateMetadata(&parsed.header)
}
ok := dbAccess.PutCifMetadata(metadata)
@ -121,7 +127,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
}
// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
func fetchUpdate(t time.Time, cfg *helpers.Configuration) (io.ReadCloser, error) {
url, err := getUpdateUrl("daily")
if err != nil {
return nil, err
@ -130,9 +136,9 @@ func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
// Append day string to URL
url = url + getDayString(t)
downloadedData, err := nrod.NrodDownload(url, cfg)
dataStream, err := nrod.NrodStream(url, cfg)
if err != nil {
return nil, err
}
return downloadedData, nil
return dataStream, nil
}

View File

@ -36,6 +36,3 @@ func CheckCorpus(cfg *helpers.Configuration) {
log.Msg.Info("CORPUS Data is less than two weeks old, not updating")
}
}
// Check if corpus data needs updating.
// Run update if needed.

View File

@ -60,9 +60,10 @@ func SetUpdateTime(collectionName string) error {
}
update := bson.M{
"$set": bson.M{
"updated": updateTime,
"target": collectionName,
"type": "collection",
"updated": updateTime,
"updated_time": time.Now().In(time.UTC),
"target": collectionName,
"type": "collection",
},
}
_, err := collection.UpdateOne(context.Background(), filter, update, options)
@ -78,13 +79,13 @@ func SetUpdateTime(collectionName string) error {
// Currently uses the old name of mq-client
func PushVersionToDb() {
version := database.Version{
Target: "mq-client",
Component: "mq-client",
Target: "timetable-mgr",
Component: "timetable-mgr",
Version: helpers.Version,
}
versionSelector := database.VersionSelector{
Target: "mq-client",
Component: "mq-client",
Target: "timetable-mgr",
Component: "timetable-mgr",
}
opts := options.Update().SetUpsert(true)
coll := MongoClient.Database("owlboard").Collection("versions")

View File

@ -41,7 +41,7 @@ func NrodStream(url string, cfg *helpers.Configuration) (io.ReadCloser, error) {
}
// Run the data through the extractor function and return io.ReadCloser, error from
// directly
// that function directly
return NrodStreamExtract(resp)
}