timetable-extension #1

Open
fred.boniface wants to merge 154 commits from timetable-extension into main
5 changed files with 24 additions and 19 deletions
Showing only changes of commit 39405e6d6a - Show all commits

View File

@ -12,6 +12,7 @@ import (
) )
// Unmarshalls data into the correct types for processing // Unmarshalls data into the correct types for processing
// - Currently not used
func parseCifData(data *[]byte) (*parsedData, error) { func parseCifData(data *[]byte) (*parsedData, error) {
log.Msg.Debug("Starting CIF Data parsing") log.Msg.Debug("Starting CIF Data parsing")
if data == nil { if data == nil {

View File

@ -2,6 +2,7 @@ package cif
import ( import (
"errors" "errors"
"io"
"time" "time"
"git.fjla.uk/owlboard/timetable-mgr/dbAccess" "git.fjla.uk/owlboard/timetable-mgr/dbAccess"
@ -25,6 +26,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
log.Msg.Error("Error downloading CIF data", zap.Error(err)) log.Msg.Error("Error downloading CIF data", zap.Error(err))
} }
// DOES NOT WORK WITH NEW DOWNLOAD STREAMING
// If debug mode is on, call debugWriteDownload // If debug mode is on, call debugWriteDownload
// if helpers.Runtime == "debug" { // if helpers.Runtime == "debug" {
// debugWriteDownload(dataStream) // debugWriteDownload(dataStream)
@ -37,6 +39,8 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
return err return err
} }
// Look to stream data onwards to the parsing function
// Make `data` a nil pointer as it is no longer required // Make `data` a nil pointer as it is no longer required
dataStream = nil dataStream = nil
@ -73,13 +77,14 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
return err return err
} }
// DOES NOT WORK WITH NEW NROD STREAMER
// If debug mode is on, call debugWriteDownload // If debug mode is on, call debugWriteDownload
if helpers.Runtime == "debug" { // if helpers.Runtime == "debug" {
debugWriteDownload(data) // debugWriteDownload(data)
} // }
// Parse CIF file // Parse CIF file
parsed, err := parseCifData(data) parsed, err := parseCifDataStream(data)
if err != nil { if err != nil {
log.Msg.Error("Error parsing CIF data", zap.Error(err)) log.Msg.Error("Error parsing CIF data", zap.Error(err))
return err return err
@ -109,7 +114,8 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
log.Msg.Debug("CIF Data has passed checks and should now be processed <<<<<<") log.Msg.Debug("CIF Data has passed checks and should now be processed <<<<<<")
// Process CIF file // Process CIF file
metadata = generateMetadata(&parsed.header) // Temporarily disable METADATA GENERATION AND WRITING
// metadata = generateMetadata(&parsed.header)
} }
ok := dbAccess.PutCifMetadata(metadata) ok := dbAccess.PutCifMetadata(metadata)
@ -121,7 +127,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
} }
// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day // Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) { func fetchUpdate(t time.Time, cfg *helpers.Configuration) (io.ReadCloser, error) {
url, err := getUpdateUrl("daily") url, err := getUpdateUrl("daily")
if err != nil { if err != nil {
return nil, err return nil, err
@ -130,9 +136,9 @@ func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
// Append day string to URL // Append day string to URL
url = url + getDayString(t) url = url + getDayString(t)
downloadedData, err := nrod.NrodDownload(url, cfg) dataStream, err := nrod.NrodStream(url, cfg)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return downloadedData, nil return dataStream, nil
} }

View File

@ -36,6 +36,3 @@ func CheckCorpus(cfg *helpers.Configuration) {
log.Msg.Info("CORPUS Data is less than two weeks old, not updating") log.Msg.Info("CORPUS Data is less than two weeks old, not updating")
} }
} }
// Check if corpus data needs updating.
// Run update if needed.

View File

@ -61,6 +61,7 @@ func SetUpdateTime(collectionName string) error {
update := bson.M{ update := bson.M{
"$set": bson.M{ "$set": bson.M{
"updated": updateTime, "updated": updateTime,
"updated_time": time.Now().In(time.UTC),
"target": collectionName, "target": collectionName,
"type": "collection", "type": "collection",
}, },
@ -78,13 +79,13 @@ func SetUpdateTime(collectionName string) error {
// Currently uses the old name of mq-client // Currently uses the old name of mq-client
func PushVersionToDb() { func PushVersionToDb() {
version := database.Version{ version := database.Version{
Target: "mq-client", Target: "timetable-mgr",
Component: "mq-client", Component: "timetable-mgr",
Version: helpers.Version, Version: helpers.Version,
} }
versionSelector := database.VersionSelector{ versionSelector := database.VersionSelector{
Target: "mq-client", Target: "timetable-mgr",
Component: "mq-client", Component: "timetable-mgr",
} }
opts := options.Update().SetUpsert(true) opts := options.Update().SetUpsert(true)
coll := MongoClient.Database("owlboard").Collection("versions") coll := MongoClient.Database("owlboard").Collection("versions")

View File

@ -41,7 +41,7 @@ func NrodStream(url string, cfg *helpers.Configuration) (io.ReadCloser, error) {
} }
// Run the data through the extractor function and return io.ReadCloser, error from // Run the data through the extractor function and return io.ReadCloser, error from
// directly // that function directly
return NrodStreamExtract(resp) return NrodStreamExtract(resp)
} }