timetable-extension #1
@ -12,6 +12,7 @@ import (
|
||||
)
|
||||
|
||||
// Unmarshalls data into the correct types for processing
|
||||
// - Currently not used
|
||||
func parseCifData(data *[]byte) (*parsedData, error) {
|
||||
log.Msg.Debug("Starting CIF Data parsing")
|
||||
if data == nil {
|
||||
|
@ -2,6 +2,7 @@ package cif
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||
@ -25,6 +26,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
|
||||
log.Msg.Error("Error downloading CIF data", zap.Error(err))
|
||||
}
|
||||
|
||||
// DOES NOT WORK WITH NEW DOWNLOAD STREAMING
|
||||
// If debug mode is on, call debugWriteDownload
|
||||
// if helpers.Runtime == "debug" {
|
||||
// debugWriteDownload(dataStream)
|
||||
@ -37,6 +39,8 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Look to stream data onwards to the parsing function
|
||||
|
||||
// Make `data` a nil pointer as it is no longer required
|
||||
dataStream = nil
|
||||
|
||||
@ -73,13 +77,14 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
||||
return err
|
||||
}
|
||||
|
||||
// DOES NOT WORK WITH NEW NROD STREAMER
|
||||
// If debug mode is on, call debugWriteDownload
|
||||
if helpers.Runtime == "debug" {
|
||||
debugWriteDownload(data)
|
||||
}
|
||||
// if helpers.Runtime == "debug" {
|
||||
// debugWriteDownload(data)
|
||||
// }
|
||||
|
||||
// Parse CIF file
|
||||
parsed, err := parseCifData(data)
|
||||
parsed, err := parseCifDataStream(data)
|
||||
if err != nil {
|
||||
log.Msg.Error("Error parsing CIF data", zap.Error(err))
|
||||
return err
|
||||
@ -109,7 +114,8 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
||||
log.Msg.Debug("CIF Data has passed checks and should now be processed <<<<<<")
|
||||
// Process CIF file
|
||||
|
||||
metadata = generateMetadata(&parsed.header)
|
||||
// Temporarily disable METADATA GENERATION AND WRITING
|
||||
// metadata = generateMetadata(&parsed.header)
|
||||
}
|
||||
|
||||
ok := dbAccess.PutCifMetadata(metadata)
|
||||
@ -121,7 +127,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
||||
}
|
||||
|
||||
// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
|
||||
func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
|
||||
func fetchUpdate(t time.Time, cfg *helpers.Configuration) (io.ReadCloser, error) {
|
||||
url, err := getUpdateUrl("daily")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -130,9 +136,9 @@ func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
|
||||
// Append day string to URL
|
||||
url = url + getDayString(t)
|
||||
|
||||
downloadedData, err := nrod.NrodDownload(url, cfg)
|
||||
dataStream, err := nrod.NrodStream(url, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return downloadedData, nil
|
||||
return dataStream, nil
|
||||
}
|
||||
|
@ -36,6 +36,3 @@ func CheckCorpus(cfg *helpers.Configuration) {
|
||||
log.Msg.Info("CORPUS Data is less than two weeks old, not updating")
|
||||
}
|
||||
}
|
||||
|
||||
// Check if corpus data needs updating.
|
||||
// Run update if needed.
|
||||
|
@ -60,9 +60,10 @@ func SetUpdateTime(collectionName string) error {
|
||||
}
|
||||
update := bson.M{
|
||||
"$set": bson.M{
|
||||
"updated": updateTime,
|
||||
"target": collectionName,
|
||||
"type": "collection",
|
||||
"updated": updateTime,
|
||||
"updated_time": time.Now().In(time.UTC),
|
||||
"target": collectionName,
|
||||
"type": "collection",
|
||||
},
|
||||
}
|
||||
_, err := collection.UpdateOne(context.Background(), filter, update, options)
|
||||
@ -78,13 +79,13 @@ func SetUpdateTime(collectionName string) error {
|
||||
// Currently uses the old name of mq-client
|
||||
func PushVersionToDb() {
|
||||
version := database.Version{
|
||||
Target: "mq-client",
|
||||
Component: "mq-client",
|
||||
Target: "timetable-mgr",
|
||||
Component: "timetable-mgr",
|
||||
Version: helpers.Version,
|
||||
}
|
||||
versionSelector := database.VersionSelector{
|
||||
Target: "mq-client",
|
||||
Component: "mq-client",
|
||||
Target: "timetable-mgr",
|
||||
Component: "timetable-mgr",
|
||||
}
|
||||
opts := options.Update().SetUpsert(true)
|
||||
coll := MongoClient.Database("owlboard").Collection("versions")
|
||||
|
@ -41,7 +41,7 @@ func NrodStream(url string, cfg *helpers.Configuration) (io.ReadCloser, error) {
|
||||
}
|
||||
|
||||
// Run the data through the extractor function and return io.ReadCloser, error from
|
||||
// directly
|
||||
// that function directly
|
||||
return NrodStreamExtract(resp)
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user