timetable-extension #1
@ -12,6 +12,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Unmarshalls data into the correct types for processing
|
// Unmarshalls data into the correct types for processing
|
||||||
|
// - Currently not used
|
||||||
func parseCifData(data *[]byte) (*parsedData, error) {
|
func parseCifData(data *[]byte) (*parsedData, error) {
|
||||||
log.Msg.Debug("Starting CIF Data parsing")
|
log.Msg.Debug("Starting CIF Data parsing")
|
||||||
if data == nil {
|
if data == nil {
|
||||||
|
@ -2,6 +2,7 @@ package cif
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"io"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||||
@ -25,6 +26,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
|
|||||||
log.Msg.Error("Error downloading CIF data", zap.Error(err))
|
log.Msg.Error("Error downloading CIF data", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DOES NOT WORK WITH NEW DOWNLOAD STREAMING
|
||||||
// If debug mode is on, call debugWriteDownload
|
// If debug mode is on, call debugWriteDownload
|
||||||
// if helpers.Runtime == "debug" {
|
// if helpers.Runtime == "debug" {
|
||||||
// debugWriteDownload(dataStream)
|
// debugWriteDownload(dataStream)
|
||||||
@ -37,6 +39,8 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Look to stream data onwards to the parsing function
|
||||||
|
|
||||||
// Make `data` a nil pointer as it is no longer required
|
// Make `data` a nil pointer as it is no longer required
|
||||||
dataStream = nil
|
dataStream = nil
|
||||||
|
|
||||||
@ -73,13 +77,14 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DOES NOT WORK WITH NEW NROD STREAMER
|
||||||
// If debug mode is on, call debugWriteDownload
|
// If debug mode is on, call debugWriteDownload
|
||||||
if helpers.Runtime == "debug" {
|
// if helpers.Runtime == "debug" {
|
||||||
debugWriteDownload(data)
|
// debugWriteDownload(data)
|
||||||
}
|
// }
|
||||||
|
|
||||||
// Parse CIF file
|
// Parse CIF file
|
||||||
parsed, err := parseCifData(data)
|
parsed, err := parseCifDataStream(data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Msg.Error("Error parsing CIF data", zap.Error(err))
|
log.Msg.Error("Error parsing CIF data", zap.Error(err))
|
||||||
return err
|
return err
|
||||||
@ -109,7 +114,8 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
|||||||
log.Msg.Debug("CIF Data has passed checks and should now be processed <<<<<<")
|
log.Msg.Debug("CIF Data has passed checks and should now be processed <<<<<<")
|
||||||
// Process CIF file
|
// Process CIF file
|
||||||
|
|
||||||
metadata = generateMetadata(&parsed.header)
|
// Temporarily disable METADATA GENERATION AND WRITING
|
||||||
|
// metadata = generateMetadata(&parsed.header)
|
||||||
}
|
}
|
||||||
|
|
||||||
ok := dbAccess.PutCifMetadata(metadata)
|
ok := dbAccess.PutCifMetadata(metadata)
|
||||||
@ -121,7 +127,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
|
// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
|
||||||
func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
|
func fetchUpdate(t time.Time, cfg *helpers.Configuration) (io.ReadCloser, error) {
|
||||||
url, err := getUpdateUrl("daily")
|
url, err := getUpdateUrl("daily")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -130,9 +136,9 @@ func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
|
|||||||
// Append day string to URL
|
// Append day string to URL
|
||||||
url = url + getDayString(t)
|
url = url + getDayString(t)
|
||||||
|
|
||||||
downloadedData, err := nrod.NrodDownload(url, cfg)
|
dataStream, err := nrod.NrodStream(url, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return downloadedData, nil
|
return dataStream, nil
|
||||||
}
|
}
|
||||||
|
@ -36,6 +36,3 @@ func CheckCorpus(cfg *helpers.Configuration) {
|
|||||||
log.Msg.Info("CORPUS Data is less than two weeks old, not updating")
|
log.Msg.Info("CORPUS Data is less than two weeks old, not updating")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if corpus data needs updating.
|
|
||||||
// Run update if needed.
|
|
||||||
|
@ -61,6 +61,7 @@ func SetUpdateTime(collectionName string) error {
|
|||||||
update := bson.M{
|
update := bson.M{
|
||||||
"$set": bson.M{
|
"$set": bson.M{
|
||||||
"updated": updateTime,
|
"updated": updateTime,
|
||||||
|
"updated_time": time.Now().In(time.UTC),
|
||||||
"target": collectionName,
|
"target": collectionName,
|
||||||
"type": "collection",
|
"type": "collection",
|
||||||
},
|
},
|
||||||
@ -78,13 +79,13 @@ func SetUpdateTime(collectionName string) error {
|
|||||||
// Currently uses the old name of mq-client
|
// Currently uses the old name of mq-client
|
||||||
func PushVersionToDb() {
|
func PushVersionToDb() {
|
||||||
version := database.Version{
|
version := database.Version{
|
||||||
Target: "mq-client",
|
Target: "timetable-mgr",
|
||||||
Component: "mq-client",
|
Component: "timetable-mgr",
|
||||||
Version: helpers.Version,
|
Version: helpers.Version,
|
||||||
}
|
}
|
||||||
versionSelector := database.VersionSelector{
|
versionSelector := database.VersionSelector{
|
||||||
Target: "mq-client",
|
Target: "timetable-mgr",
|
||||||
Component: "mq-client",
|
Component: "timetable-mgr",
|
||||||
}
|
}
|
||||||
opts := options.Update().SetUpsert(true)
|
opts := options.Update().SetUpsert(true)
|
||||||
coll := MongoClient.Database("owlboard").Collection("versions")
|
coll := MongoClient.Database("owlboard").Collection("versions")
|
||||||
|
@ -41,7 +41,7 @@ func NrodStream(url string, cfg *helpers.Configuration) (io.ReadCloser, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Run the data through the extractor function and return io.ReadCloser, error from
|
// Run the data through the extractor function and return io.ReadCloser, error from
|
||||||
// directly
|
// that function directly
|
||||||
return NrodStreamExtract(resp)
|
return NrodStreamExtract(resp)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user