Lots more work on CIF processing
This commit is contained in:
parent
2fdb840644
commit
adf745aa61
@ -9,54 +9,6 @@ import (
|
|||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
// REWRITING CifCheck into CheckCif (at bottom of file), CifCheck and parseMetadata become redundant
|
|
||||||
|
|
||||||
// Loads CifMetadata and passes it to parseMetadata, this function is what you should call to initiate the CifUpdate process.
|
|
||||||
func CifCheck(cfg *helpers.Configuration) error {
|
|
||||||
log.Msg.Debug("Checking age of CIF Data")
|
|
||||||
metadata, err := dbAccess.GetCifMetadata()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = parseMetadata(metadata, cfg)
|
|
||||||
if err != nil {
|
|
||||||
log.Msg.Error("Error updating CIF Data", zap.Error(err))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Requests a full update if no metadata exists, or a daily update if metadata does exist.
|
|
||||||
// The daily update function does further metadata parsing to determine what exactly needs downloading.
|
|
||||||
func parseMetadata(metadata *dbAccess.CifMetadata, cfg *helpers.Configuration) error {
|
|
||||||
if metadata == nil {
|
|
||||||
log.Msg.Info("No metadata, creating Timetable data")
|
|
||||||
newMeta, err := runFullUpdate(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
ok := dbAccess.PutCifMetadata(*newMeta)
|
|
||||||
if !ok {
|
|
||||||
log.Msg.Error("CIF Data updated but Metadata Update failed")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Msg.Debug("Requesting CIF Data Update")
|
|
||||||
// When testing done, this function returns newMetadata which needs putting into DB
|
|
||||||
_, err := runUpdate(metadata, cfg)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
//ok := dbAccess.PutCifMetadata(*newMeta)
|
|
||||||
//if !ok {
|
|
||||||
// log.Msg.Error("CIF Data updated but Metadata Update failed")
|
|
||||||
//}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checks if the CIF Data needs updating, and what type of update is needed (Full/Partial) and if partial
|
// Checks if the CIF Data needs updating, and what type of update is needed (Full/Partial) and if partial
|
||||||
// what days data needs updating, then calls an update function to handle the update.
|
// what days data needs updating, then calls an update function to handle the update.
|
||||||
func CheckCif(cfg *helpers.Configuration) {
|
func CheckCif(cfg *helpers.Configuration) {
|
||||||
@ -75,12 +27,12 @@ func CheckCif(cfg *helpers.Configuration) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If no metadata is found in DB, presume no CIF data exists
|
||||||
if metadata == nil {
|
if metadata == nil {
|
||||||
log.Msg.Info("Full CIF download required")
|
log.Msg.Info("Full CIF download required")
|
||||||
err := runCifFullDownload(cfg)
|
err := runCifFullDownload(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Msg.Error("Unable to run full CIF Update")
|
log.Msg.Error("Unable to run full CIF Update", zap.Error(err))
|
||||||
return
|
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -91,8 +43,25 @@ func CheckCif(cfg *helpers.Configuration) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
// Check how many days since last update, if more than 5, run full update, else run update
|
||||||
|
daysSinceLastUpdate := howManyDaysAgo(metadata.LastUpdate)
|
||||||
|
if daysSinceLastUpdate > 5 {
|
||||||
|
log.Msg.Info("Full CIF download required")
|
||||||
|
err := runCifFullDownload(cfg)
|
||||||
|
if err != nil {
|
||||||
|
log.Msg.Error("Unable to run full CIF Update", zap.Error(err))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Here I need to determine which days I need to update CIF data for, then pass to an appropriate function:
|
daysToUpdate := generateUpdateDays(daysSinceLastUpdate)
|
||||||
// newMetadata, err := runCifUpdate(days []time.Time, cfg)
|
|
||||||
|
// Run the update
|
||||||
|
log.Msg.Info("CIF Update required", zap.Any("days to update", daysToUpdate))
|
||||||
|
err = runCifUpdateDownload(cfg, metadata, daysToUpdate)
|
||||||
|
if err != nil {
|
||||||
|
log.Msg.Error("Unable to run CIF update", zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
@ -51,3 +51,20 @@ func howManyDaysAgo(t time.Time) int {
|
|||||||
days := int(diff.Hours() / 24)
|
days := int(diff.Hours() / 24)
|
||||||
return days
|
return days
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Generates a slice of time.Time values representing which days files need downloading
|
||||||
|
func generateUpdateDays(days int) []time.Time {
|
||||||
|
var updateDays []time.Time
|
||||||
|
|
||||||
|
for i := 0; i < days; i++ {
|
||||||
|
day := time.Now().Add(-time.Duration(i) * 24 * time.Hour)
|
||||||
|
updateDays = append(updateDays, day)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reverse slice to ensure chronological order
|
||||||
|
for i, j := 0, len(updateDays)-1; i < j; i, j = i+1, j-1 {
|
||||||
|
updateDays[i], updateDays[j] = updateDays[j], updateDays[i]
|
||||||
|
}
|
||||||
|
|
||||||
|
return updateDays
|
||||||
|
}
|
||||||
|
@ -17,3 +17,76 @@ func TestIsSameDay(t *testing.T) {
|
|||||||
t.Errorf("Error in isSameDay(notToday). Expected false, got true.")
|
t.Errorf("Error in isSameDay(notToday). Expected false, got true.")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHowManyDaysAgo(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
input time.Time
|
||||||
|
expected int
|
||||||
|
}{
|
||||||
|
{time.Now(), 0}, // Today
|
||||||
|
{time.Now().Add(-24 * time.Hour), 1}, // Yesterday
|
||||||
|
{time.Now().Add(-48 * time.Hour), 2}, // Ereyesterday
|
||||||
|
{time.Now().Add(24 * time.Hour), -1}, // Tomorrow
|
||||||
|
{time.Now().Add(48 * time.Hour), -2}, // Overmorrow
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
result := howManyDaysAgo(tc.input)
|
||||||
|
if result != tc.expected {
|
||||||
|
t.Errorf("For input %v, expected %d but got %d", tc.input, tc.expected, result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetDayString(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
input time.Time
|
||||||
|
expected string
|
||||||
|
}{ // Note that the test times are in UTC, but days are checked in Europe/London
|
||||||
|
{time.Date(2024, time.April, 7, 0, 0, 0, 0, time.UTC), "sun"},
|
||||||
|
{time.Date(2024, time.April, 4, 21, 0, 0, 0, time.UTC), "thu"},
|
||||||
|
{time.Date(2001, time.September, 11, 12, 46, 0, 0, time.UTC), "tue"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
result := getDayString(tc.input)
|
||||||
|
if result != tc.expected {
|
||||||
|
t.Errorf("For input %v, expected %s, but got %s", tc.input, tc.expected, result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateUpdateDays(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
days int
|
||||||
|
expected []time.Time
|
||||||
|
}{
|
||||||
|
{1, []time.Time{time.Now()}},
|
||||||
|
{2, []time.Time{time.Now().Add(-24 * time.Hour), time.Now()}},
|
||||||
|
{4, []time.Time{time.Now().Add(-72 * time.Hour),
|
||||||
|
time.Now().Add(-48 * time.Hour),
|
||||||
|
time.Now().Add(-24 * time.Hour),
|
||||||
|
time.Now(),
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
result := generateUpdateDays(tc.days)
|
||||||
|
|
||||||
|
if len(result) != len(tc.expected) {
|
||||||
|
t.Errorf("For %d days, expected %v, but got %v", tc.days, tc.expected, result)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range result {
|
||||||
|
if !isSameDate(result[i], tc.expected[i]) {
|
||||||
|
t.Errorf("For %d days, expected %v, but got %v", tc.days, tc.expected, result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Checks if two time values have the same year, month and day.
|
||||||
|
func isSameDate(t1, t2 time.Time) bool {
|
||||||
|
return t1.Year() == t2.Year() && t1.Month() == t2.Month() && t1.Day() == t2.Day()
|
||||||
|
}
|
||||||
|
105
src/cif/parse_test.go
Normal file
105
src/cif/parse_test.go
Normal file
@ -0,0 +1,105 @@
|
|||||||
|
package cif
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseCifData(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
input []byte
|
||||||
|
expected *parsedData
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func returnParsedData() *parsedData {
|
||||||
|
var parsed = {
|
||||||
|
name: "Valid JSON Data 1",
|
||||||
|
input: []byte(`
|
||||||
|
{"JsonTimetableV1":{"classification":"public","timestamp":1711227636,"owner":"Network Rail","Sender":{"organisation":"Rockshore","application":"NTROD","component":"SCHEDULE"},"Metadata":{"type":"update","sequence":4307}}}
|
||||||
|
{"JsonAssociationV1":{"transaction_type":"Create","main_train_uid":"L65283","assoc_train_uid":"L65245","assoc_start_date":"2023-12-11T00:00:00Z","assoc_end_date":"2024-03-28T00:00:00Z","assoc_days":"1111000","category":" ","date_indicator":" ","location":"RDNGSTN","base_location_suffix":null,"assoc_location_suffix":null,"diagram_type":"T","CIF_stp_indicator":"C"}}
|
||||||
|
{"JsonAssociationV1":{"transaction_type":"Delete","main_train_uid":"L65283","assoc_train_uid":"L65245","assoc_start_date":"2024-03-18T00:00:00Z","location":"RDNGSTN","base_location_suffix":null,"diagram_type":"T","CIF_stp_indicator":"C"}}
|
||||||
|
{"JsonScheduleV1":{"CIF_stp_indicator":"O","CIF_train_uid":"C25513","schedule_start_date":"2024-03-23","transaction_type":"Delete"}}
|
||||||
|
{"JsonScheduleV1":{"CIF_bank_holiday_running":null,"CIF_stp_indicator":"O","CIF_train_uid":"Y04345","applicable_timetable":"Y","atoc_code":"SW","new_schedule_segment":{"traction_class":"","uic_code":""},"schedule_days_runs":"0000001","schedule_end_date":"2024-05-19","schedule_segment":{"signalling_id":"5A20","CIF_train_category":"EE","CIF_headcode":"00","CIF_course_indicator":1,"CIF_train_service_code":"24676004","CIF_business_sector":"??","CIF_power_type":"EMU","CIF_timing_load":null,"CIF_speed":"100","CIF_operating_characteristics":"D","CIF_train_class":null,"CIF_sleepers":null,"CIF_reservations":null,"CIF_connection_indicator":null,"CIF_catering_code":null,"CIF_service_branding":"","schedule_location":[{"location_type":"LO","record_identity":"LO","tiploc_code":"FARNHMD","tiploc_instance":null,"departure":"0721","public_departure":null,"platform":null,"line":null,"engineering_allowance":null,"pathing_allowance":null,"performance_allowance":null},{"location_type":"LT","record_identity":"LT","tiploc_code":"FARNHAM","tiploc_instance":null,"platform":"1","arrival":"0729","public_arrival":null,"path":null}]},"schedule_start_date":"2024-05-19","train_status":"T","transaction_type":"Create"}}
|
||||||
|
{"EOF":true}
|
||||||
|
`),
|
||||||
|
expected: &parsedData{
|
||||||
|
header: upstreamApi.JsonTimetableV1{
|
||||||
|
Classification: "public",
|
||||||
|
Timestamp: 1711227636,
|
||||||
|
Owner: "Network Rail",
|
||||||
|
Sender: upstreamApi.TimetableSender{
|
||||||
|
Organisation: "Rockshore",
|
||||||
|
Application: "NTROD",
|
||||||
|
Component: "SCHEDULE",
|
||||||
|
},
|
||||||
|
Metadata: upstreamApi.TimetableMetadata{
|
||||||
|
Type: "update",
|
||||||
|
Sequence: 4307,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
assoc: []upstreamApi.JsonAssociationV1{
|
||||||
|
TransactionType: "Create",
|
||||||
|
MainTrainUid: "L65283",
|
||||||
|
AssocTrainUid: "L65245",
|
||||||
|
AssocStartDate: time.Date(2023, time.December, 11, 0, 0, 0, 0, time.UTC),
|
||||||
|
AssocEndDate: time.Date(2024, time.March, 28, 0, 0, 0, 0, time.UTC),
|
||||||
|
AssocDays: "1111000",
|
||||||
|
Location: "RDNGSTN",
|
||||||
|
DiagramType: "T",
|
||||||
|
CifStpIndicator: "C",
|
||||||
|
}, {
|
||||||
|
TransactionType: "Delete",
|
||||||
|
MainTrainUid: "L65283",
|
||||||
|
AssocTrainUid: "L65245",
|
||||||
|
AssocStartDate: time.Date(2024, time.March, 18, 0, 0, 0, 0, time.UTC),
|
||||||
|
Location: "RDNGSTN",
|
||||||
|
DiagramType: "T",
|
||||||
|
CifStpIndicator: "C",
|
||||||
|
},
|
||||||
|
sched: []upstreamApi.JsonScheduleV1{
|
||||||
|
CifStpIndicator: "O",
|
||||||
|
CifTrainUid: "C25513",
|
||||||
|
ScheduleStartDate: "2024-03-23",
|
||||||
|
TransactionType: "Delete",
|
||||||
|
}, {
|
||||||
|
CifStpIndicator: "O",
|
||||||
|
CifTrainUid: "Y04345",
|
||||||
|
ApplicableTimetable: "Y",
|
||||||
|
AtocCode: "SW",
|
||||||
|
NewScheduleSegment: upstreamApi.CifNewScheduleSegment{},
|
||||||
|
ScheduleDaysRun: "0000001",
|
||||||
|
ScheduleEndDate: "2024-05-19",
|
||||||
|
ScheduleSegment: upstreamApi.CifScheduleSegment{
|
||||||
|
SignallingId: "5A20",
|
||||||
|
CifTrainCategory: "EE",
|
||||||
|
CifHeadcode: "OO",
|
||||||
|
CifCourseIndicator: 1,
|
||||||
|
CifTrainServiceCode: "24676004",
|
||||||
|
CifBusinessSector: "??",
|
||||||
|
CifPowerType: "EMU",
|
||||||
|
CifSpeed: "100",
|
||||||
|
CifOperatingCharacteristics: "D",
|
||||||
|
ScheduleLocation: []upstreamApi.CifScheduleLocation{
|
||||||
|
LocationType: "LO",
|
||||||
|
RecordIdentity: "LO",
|
||||||
|
TiplocCode: "FARNHMD",
|
||||||
|
Departure: "0721",
|
||||||
|
}, {
|
||||||
|
LocationType: "LT",
|
||||||
|
RecordIdentity: "LT",
|
||||||
|
TiplocCode: "FARNHAM",
|
||||||
|
Platform: "1",
|
||||||
|
Arrival: "0729",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
ScheduleStartDate: "2024-05-19",
|
||||||
|
TrainStatus: "T",
|
||||||
|
TransactionType: "Create",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
@ -1,21 +1,6 @@
|
|||||||
package cif
|
package cif
|
||||||
|
|
||||||
import (
|
// Processes parsed CIF data and applies the data to the database
|
||||||
"errors"
|
func processParsedCif(data *parsedData) error {
|
||||||
|
return nil
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Processes all data in the schedule segment of 'data', interacts with the database and returns new metadata
|
|
||||||
func processCifData(metadata *dbAccess.CifMetadata, data *parsedData) (*dbAccess.CifMetadata, error) {
|
|
||||||
if data == nil {
|
|
||||||
err := errors.New("data is not defined")
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if metadata == nil {
|
|
||||||
err := errors.New("metadata is not defined")
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, nil
|
|
||||||
}
|
}
|
||||||
|
@ -1,7 +1,6 @@
|
|||||||
package cif
|
package cif
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"sort"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||||
@ -11,105 +10,79 @@ import (
|
|||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Runs a full update of the CIF Data, discarding any existing data and returns a new metadata struct
|
// Replaces all existing CIF Data with a new download
|
||||||
func runFullUpdate(cfg *helpers.Configuration) (*dbAccess.CifMetadata, error) {
|
func runCifFullDownload(cfg *helpers.Configuration) error {
|
||||||
log.Msg.Warn("All existing timetable data will be deleted")
|
log.Msg.Info("Downloading all CIF Data")
|
||||||
|
|
||||||
|
// Download CIF Data file
|
||||||
url, err := getUpdateUrl("full")
|
url, err := getUpdateUrl("full")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Msg.Error("Unable to get update URL", zap.Error(err))
|
log.Msg.Error("Error getting download URL", zap.Error(err))
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
data, err := nrod.NrodDownload(url, cfg)
|
||||||
fullCifData, err := nrod.NrodDownload(url, cfg)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Msg.Error("Unable to get CIF Data", zap.Error(err))
|
log.Msg.Error("Error downloading CIF data", zap.Error(err))
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Msg.Debug("CIF Data Downloaded", zap.ByteString("CIF Data", fullCifData))
|
// Parse CIF file
|
||||||
|
parsed, err := parseCifData(data)
|
||||||
|
if err != nil {
|
||||||
|
log.Msg.Error("Error parsing CIF data", zap.Error(err))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// I now need to define a processing function and ensure a valid type exists, then I can pass that type to a CIF Put Full function
|
// Drop timetable collection
|
||||||
// which will handle placing the data into the database
|
dbAccess.DropCollection(dbAccess.TimetableCollection)
|
||||||
|
|
||||||
return nil, nil
|
// Process CIF file
|
||||||
|
err = processParsedCif(parsed)
|
||||||
|
if err != nil {
|
||||||
|
log.Msg.Error("Error processing CIF data", zap.Error(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate & Write metadata
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Runs the daily update for CIF Data, can handle up to five days updates at once.
|
// Runs a CIF Update for up to five days
|
||||||
func runUpdate(metadata *dbAccess.CifMetadata, cfg *helpers.Configuration) (*dbAccess.CifMetadata, error) {
|
func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMetadata, days []time.Time) error {
|
||||||
// Do not run update if last update was on same day
|
log.Msg.Info("Downloading CIF Updates")
|
||||||
if isSameToday(metadata.LastUpdate) {
|
|
||||||
log.Msg.Info("No CIF Update Required", zap.Time("Last update", metadata.LastUpdate))
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Do not run update before 0600 as todays data will not be available
|
// Loop over dates
|
||||||
if time.Now().Hour() < 6 {
|
for _, time := range days {
|
||||||
log.Msg.Info("Too early to update CIF Data")
|
log.Msg.Info("Downloading CIF File", zap.Time("CIF Data from", time))
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check how many days ago last update was
|
// Download CIF data file
|
||||||
lastUpateDays := howManyDaysAgo(metadata.LastUpdate)
|
data, err := fetchUpdate(time, cfg)
|
||||||
if lastUpateDays > 5 {
|
|
||||||
log.Msg.Warn("CIF Data is more than five days old. Running Full Update")
|
|
||||||
newMeta, err := runFullUpdate(cfg)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Msg.Error("CIF Update failed", zap.Error(err))
|
log.Msg.Error("Error fetching CIF update", zap.Error(err))
|
||||||
return nil, err
|
return err
|
||||||
}
|
}
|
||||||
return newMeta, nil
|
// Parse CIF file
|
||||||
}
|
parsed, err := parseCifData(data)
|
||||||
|
|
||||||
// Create a slice containing which dates need updating
|
|
||||||
firstUpdate := time.Now().In(time.UTC).AddDate(0, 0, -lastUpateDays)
|
|
||||||
finalUpdate := time.Now().In(time.UTC)
|
|
||||||
var dates []time.Time
|
|
||||||
|
|
||||||
for d := firstUpdate; d.Before(finalUpdate) || d.Equal(finalUpdate); d = d.AddDate(0, 0, 1) {
|
|
||||||
dates = append(dates, d)
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Slice(dates, func(i, j int) bool {
|
|
||||||
return dates[i].Before(dates[j])
|
|
||||||
})
|
|
||||||
|
|
||||||
log.Msg.Info("Updating CIF Data", zap.Any("dates to update", dates))
|
|
||||||
|
|
||||||
// Iterate over each date, fetching then parsing the data
|
|
||||||
for _, date := range dates {
|
|
||||||
data, err := fetchUpdate(date, cfg)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Msg.Error("Error fetching data", zap.Time("date", date))
|
log.Msg.Error("Error parsing CIF data", zap.Error(err))
|
||||||
continue
|
return err
|
||||||
} // parseCifData function needs writing
|
|
||||||
parsedData, err := parseCifData(data)
|
|
||||||
if err != nil {
|
|
||||||
log.Msg.Error("Error parsing data", zap.Time("date", date))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if parsedData == nil { // TEMPORARY DEVELOPMENT CHECK
|
// Check metadata sequence - Handle a metadata sequence error. Probably by deleting metadata so next update triggers full download
|
||||||
log.Msg.Error("No data parsed")
|
//// I need to check what the sequence looks like in a full download first.
|
||||||
return nil, nil
|
// Process CIF file
|
||||||
}
|
// Generate & Write metadata
|
||||||
|
|
||||||
// Process data (manages the database interactions and >>returns the new metadata)
|
|
||||||
_, err = processCifData(metadata, parsedData)
|
|
||||||
if err != nil {
|
|
||||||
log.Msg.Error("Error processing CIF Data", zap.Error(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Msg.Info("CIF Data updated", zap.Time("date", date))
|
|
||||||
}
|
}
|
||||||
return nil, nil //TEMPORARY
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetches CIF Updates for a given day
|
// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
|
||||||
func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) {
|
func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) {
|
||||||
url, err := getUpdateUrl("daily")
|
url, err := getUpdateUrl("daily")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Append day string to URL
|
||||||
url = url + getDayString(t)
|
url = url + getDayString(t)
|
||||||
|
|
||||||
downloadedData, err := nrod.NrodDownload(url, cfg)
|
downloadedData, err := nrod.NrodDownload(url, cfg)
|
||||||
|
@ -12,12 +12,6 @@ import (
|
|||||||
"go.mongodb.org/mongo-driver/mongo/options"
|
"go.mongodb.org/mongo-driver/mongo/options"
|
||||||
)
|
)
|
||||||
|
|
||||||
const databaseName string = "owlboard"
|
|
||||||
const CorpusCollection string = "corpus"
|
|
||||||
const StationsCollection string = "stations"
|
|
||||||
const metaCollection string = "meta"
|
|
||||||
const TimetableCollection string = "timetable"
|
|
||||||
|
|
||||||
// Provide the DB Connection to other functions
|
// Provide the DB Connection to other functions
|
||||||
var MongoClient (*mongo.Client)
|
var MongoClient (*mongo.Client)
|
||||||
|
|
||||||
|
7
src/dbAccess/contants.go
Normal file
7
src/dbAccess/contants.go
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
package dbAccess
|
||||||
|
|
||||||
|
const databaseName string = "owlboard"
|
||||||
|
const CorpusCollection string = "corpus"
|
||||||
|
const StationsCollection string = "stations"
|
||||||
|
const metaCollection string = "meta"
|
||||||
|
const TimetableCollection string = "timetable"
|
Loading…
Reference in New Issue
Block a user