Writing full CIF data now works. Still need to work on updating CIF data.

This commit is contained in:
Fred Boniface 2024-04-08 21:08:07 +01:00
parent d9c249a47a
commit 3e5ed2c10a
13 changed files with 83 additions and 40 deletions

View File

@ -46,6 +46,7 @@ func CheckCif(cfg *helpers.Configuration) {
// Check how many days since last update, if more than 5, run full update, else run update // Check how many days since last update, if more than 5, run full update, else run update
daysSinceLastUpdate := howManyDaysAgo(metadata.LastUpdate) daysSinceLastUpdate := howManyDaysAgo(metadata.LastUpdate)
if daysSinceLastUpdate > 5 { if daysSinceLastUpdate > 5 {
log.Msg.Debug("Full Update Requested due to time since last update", zap.Int("daysSinceLastUpdate", daysSinceLastUpdate))
log.Msg.Info("Full CIF download required") log.Msg.Info("Full CIF download required")
err := runCifFullDownload(cfg) err := runCifFullDownload(cfg)
if err != nil { if err != nil {
@ -62,6 +63,4 @@ func CheckCif(cfg *helpers.Configuration) {
if err != nil { if err != nil {
log.Msg.Error("Unable to run CIF update", zap.Error(err)) log.Msg.Error("Unable to run CIF update", zap.Error(err))
} }
return
} }

View File

@ -12,4 +12,4 @@ const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAu
const dataAvailable = 6 const dataAvailable = 6
// An object representing the Europe/London timezone // An object representing the Europe/London timezone
var londonTimezone, err = time.LoadLocation("Europe/London") var londonTimezone, _ = time.LoadLocation("Europe/London")

View File

@ -21,7 +21,7 @@ func ConvertServiceType(input *upstreamApi.JsonScheduleV1, vstp bool) (*database
PlanSpeed: parseSpeed(&input.ScheduleSegment.CifSpeed), PlanSpeed: parseSpeed(&input.ScheduleSegment.CifSpeed),
ScheduleStartDate: ParseCifDate(&input.ScheduleStartDate, "start"), ScheduleStartDate: ParseCifDate(&input.ScheduleStartDate, "start"),
ScheduleEndDate: ParseCifDate(&input.ScheduleEndDate, "end"), ScheduleEndDate: ParseCifDate(&input.ScheduleEndDate, "end"),
FirstClass: hasFirstClass(&input.ScheduleSegment.CifTrainClass), FirstClass: hasFirstClass(&input.ScheduleSegment.CifTrainClass, &input.ScheduleSegment.SignallingId),
Catering: hasCatering(&input.ScheduleSegment.CifCateringCode), Catering: hasCatering(&input.ScheduleSegment.CifCateringCode),
Sleeper: hasSleeper(&input.ScheduleSegment.CifSleepers), Sleeper: hasSleeper(&input.ScheduleSegment.CifSleepers),
DaysRun: parseDaysRun(&input.ScheduleDaysRun), DaysRun: parseDaysRun(&input.ScheduleDaysRun),
@ -37,14 +37,12 @@ func parseSpeed(CIFSpeed *string) int32 {
return 0 return 0
} }
if *CIFSpeed == "" { if *CIFSpeed == "" {
log.Msg.Debug("Speed data not provided")
return int32(0) return int32(0)
} }
actualSpeed, exists := helpers.SpeedMap[*CIFSpeed] actualSpeed, exists := helpers.SpeedMap[*CIFSpeed]
if !exists { if !exists {
actualSpeed = *CIFSpeed actualSpeed = *CIFSpeed
} }
log.Msg.Debug("Corrected Speed: " + actualSpeed)
speed, err := strconv.ParseInt(actualSpeed, 10, 32) speed, err := strconv.ParseInt(actualSpeed, 10, 32)
if err != nil { if err != nil {
@ -87,10 +85,17 @@ func isPublic(input *upstreamApi.CifScheduleLocation) bool {
} }
// Ascertains whether the service offers first class // Ascertains whether the service offers first class
func hasFirstClass(input *string) bool { func hasFirstClass(input, signallingId *string) bool {
if input == nil { if input == nil || signallingId == nil {
return false return false
} }
// Handle non passenger headcodes and ensure first class is not shown as available
firstChar := (*signallingId)[0]
if firstChar == '3' || firstChar == '4' || firstChar == '5' || firstChar == '6' || firstChar == '7' || firstChar == '8' || firstChar == '0' {
return false
}
return *input != "S" return *input != "S"
} }

View File

@ -87,23 +87,34 @@ func TestIsPublic(t *testing.T) {
func TestHasFirstClass(t *testing.T) { func TestHasFirstClass(t *testing.T) {
testCases := []struct { testCases := []struct {
input string input string
expect bool headcode string
expect bool
}{ }{
{"", true}, {"", "1A00", true},
{"B", true}, {"B", "2A05", true},
{"S", false}, {"S", "1C99", false},
{"", "3C23", false},
{"", "5Q21", false},
{"", "5D32", false},
{"", "9O12", true},
{"B", "9D32", true},
{"", "7R43", false},
{"B", "6Y77", false},
{"", "8P98", false},
{"S", "4O89", false},
{"", "4E43", false},
} }
for _, tc := range testCases { for _, tc := range testCases {
result := hasFirstClass(&tc.input) result := hasFirstClass(&tc.input, &tc.headcode)
if result != tc.expect { if result != tc.expect {
t.Errorf("For %s, expected %t, but got %t", tc.input, tc.expect, result) t.Errorf("For %s & headcode %s, expected %t, but got %t", tc.input, tc.headcode, tc.expect, result)
} }
} }
nilResult := hasFirstClass(nil) nilResult := hasFirstClass(nil, nil)
if nilResult { if nilResult {
t.Errorf("hasFirstClass failed to handle nil pointer, expected %t, got %t", false, nilResult) t.Errorf("hasFirstClass failed to handle nil pointer, expected %t, got %t", false, nilResult)
} }

View File

@ -44,11 +44,13 @@ func isSameToday(t time.Time) bool {
// Returns how many days ago `t` was compared to today // Returns how many days ago `t` was compared to today
func howManyDaysAgo(t time.Time) int { func howManyDaysAgo(t time.Time) int {
today := time.Now().In(time.UTC).Truncate(24 * time.Hour) log.Msg.Debug("Calculating how many days ago", zap.Time("Input time", t))
input := t.In(time.UTC).Truncate(24 * time.Hour) // Truncate both times to midnight in UTC timezone
today := time.Now().UTC().Truncate(24 * time.Hour)
input := t.UTC().Truncate(24 * time.Hour)
diff := today.Sub(input) diff := today.Sub(input)
days := int(diff.Hours() / 24) days := int(diff / (24 * time.Hour))
return days return days
} }

View File

@ -24,11 +24,11 @@ func TestHowManyDaysAgo(t *testing.T) {
input time.Time input time.Time
expected int expected int
}{ }{
{time.Now(), 0}, // Today {time.Now().In(time.UTC), 0}, // Today
{time.Now().Add(-24 * time.Hour), 1}, // Yesterday {time.Now().In(time.UTC).Add(-24 * time.Hour), 1}, // Yesterday
{time.Now().Add(-48 * time.Hour), 2}, // Ereyesterday {time.Now().In(time.UTC).Add(-48 * time.Hour), 2}, // Ereyesterday
{time.Now().Add(24 * time.Hour), -1}, // Tomorrow {time.Now().In(time.UTC).Add(24 * time.Hour), -1}, // Tomorrow
{time.Now().Add(48 * time.Hour), -2}, // Overmorrow {time.Now().In(time.UTC).Add(48 * time.Hour), -2}, // Overmorrow
} }
for _, tc := range testCases { for _, tc := range testCases {

View File

@ -18,9 +18,11 @@ func processParsedCif(data *parsedData) error {
for _, item := range data.sched { for _, item := range data.sched {
switch item.TransactionType { switch item.TransactionType {
case "Delete": case "Delete":
deleteTasks = append(deleteTasks, &item) deleteItem := item // Create new variable to ensure repetition of pointers
deleteTasks = append(deleteTasks, &deleteItem)
case "Create": case "Create":
createTasks = append(createTasks, &item) createItem := item // Create new variable to ensure repetition of pointers
createTasks = append(createTasks, &createItem)
default: default:
log.Msg.Error("Unknown transaction type in CIF Schedule", zap.String("TransactionType", item.TransactionType)) log.Msg.Error("Unknown transaction type in CIF Schedule", zap.String("TransactionType", item.TransactionType))
} }

View File

@ -35,6 +35,7 @@ func TestGenerateMetadata(t *testing.T) {
if result == nil { if result == nil {
t.Errorf("generateMetadata returned nil pointer") t.Errorf("generateMetadata returned nil pointer")
return // Static type checking likes this return to be here, even if it is redundant in reality.
} }
if result.Doctype != expected.Doctype { if result.Doctype != expected.Doctype {

View File

@ -40,7 +40,11 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
log.Msg.Error("Error processing CIF data", zap.Error(err)) log.Msg.Error("Error processing CIF data", zap.Error(err))
} }
// Generate & Write metadata newMeta := generateMetadata(&parsed.header)
ok := dbAccess.PutCifMetadata(newMeta)
if !ok {
log.Msg.Warn("CIF Data updated, but metadata write failed")
}
return nil return nil
} }

View File

@ -18,10 +18,10 @@ const Doctype = "CifMetadata"
// The type describing the CifMetadata 'type' in the database. // The type describing the CifMetadata 'type' in the database.
// This type will be moved to owlboard/go-types // This type will be moved to owlboard/go-types
type CifMetadata struct { type CifMetadata struct {
Doctype string `json:"type"` Doctype string `bson:"type"`
LastUpdate time.Time `json:"lastUpdate"` LastUpdate time.Time `bson:"lastUpdate"`
LastTimestamp int64 `json:"lastTimestamp"` LastTimestamp int64 `bson:"lastTimestamp"`
LastSequence int64 `json:"lastSequence"` LastSequence int64 `bson:"lastSequence"`
} }
// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example. // Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
@ -39,20 +39,24 @@ func GetCifMetadata() (*CifMetadata, error) {
return nil, err return nil, err
} }
log.Msg.Debug("Fetched CIF Metadata from database", zap.Any("Metadata", result))
return &result, nil return &result, nil
} }
// Uses upsert to Insert/Update the CifMetadata in the database // Uses upsert to Insert/Update the CifMetadata in the database
func PutCifMetadata(metadata CifMetadata) bool { func PutCifMetadata(metadata *CifMetadata) bool {
database := MongoClient.Database(databaseName) database := MongoClient.Database(databaseName)
collection := database.Collection(metaCollection) collection := database.Collection(metaCollection)
options := options.Update().SetUpsert(true) options := options.Update().SetUpsert(true)
filter := bson.M{"type": Doctype} filter := bson.M{"type": Doctype}
update := bson.M{ update := bson.M{
"type": Doctype, "$set": bson.M{
"LastUpdate": metadata.LastUpdate, "type": Doctype,
"LastTimestamp": metadata.LastTimestamp, "lastUpdate": metadata.LastUpdate,
"LastSequence": metadata.LastSequence, "lastTimestamp": metadata.LastTimestamp,
"lastSequence": metadata.LastSequence,
},
} }
_, err := collection.UpdateOne(context.Background(), filter, update, options) _, err := collection.UpdateOne(context.Background(), filter, update, options)
@ -61,14 +65,21 @@ func PutCifMetadata(metadata CifMetadata) bool {
log.Msg.Error("Error updating CIF Metadata", zap.Error(err)) log.Msg.Error("Error updating CIF Metadata", zap.Error(err))
return false return false
} }
log.Msg.Info("New CIF Metadata written", zap.Time("Update time", metadata.LastUpdate))
return true return true
} }
// Handles 'Delete' tasks from CIF Schedule updates, accepts DeleteQuery types and batches deletions. // Handles 'Delete' tasks from CIF Schedule updates, accepts DeleteQuery types and batches deletions.
func DeleteCifEntries(deletions []database.DeleteQuery) error { func DeleteCifEntries(deletions []database.DeleteQuery) error {
collection := MongoClient.Database(databaseName).Collection(timetableCollection) // Skip if deletions is empty
if len(deletions) == 0 {
log.Msg.Info("No deletions required")
return nil
}
// Prepare deletion tasks // Prepare deletion tasks
collection := MongoClient.Database(databaseName).Collection(timetableCollection)
bulkDeletions := make([]mongo.WriteModel, 0, len(deletions)) bulkDeletions := make([]mongo.WriteModel, 0, len(deletions))
for _, deleteQuery := range deletions { for _, deleteQuery := range deletions {
@ -95,6 +106,12 @@ func DeleteCifEntries(deletions []database.DeleteQuery) error {
// Handles 'Create' tasks for CIF Schedule updates, accepts Service structs and batches their creation. // Handles 'Create' tasks for CIF Schedule updates, accepts Service structs and batches their creation.
func CreateCifEntries(schedules []database.Service) error { func CreateCifEntries(schedules []database.Service) error {
// Skip if deletions is empty
if len(schedules) == 0 {
log.Msg.Info("No creations required")
return nil
}
collection := MongoClient.Database(databaseName).Collection(timetableCollection) collection := MongoClient.Database(databaseName).Collection(timetableCollection)
models := make([]mongo.WriteModel, 0, len(schedules)) models := make([]mongo.WriteModel, 0, len(schedules))

2
go.mod
View File

@ -3,7 +3,7 @@ module git.fjla.uk/owlboard/timetable-mgr
go 1.21 go 1.21
require ( require (
git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580 git.fjla.uk/owlboard/go-types v0.0.0-20240408193146-4719be9c13eb
github.com/go-stomp/stomp/v3 v3.0.5 github.com/go-stomp/stomp/v3 v3.0.5
go.mongodb.org/mongo-driver v1.12.0 go.mongodb.org/mongo-driver v1.12.0
go.uber.org/zap v1.24.0 go.uber.org/zap v1.24.0

2
go.sum
View File

@ -10,6 +10,8 @@ git.fjla.uk/owlboard/go-types v0.0.0-20240407202712-e58d7d1d9aa9 h1:aNxMYEsbBkFx
git.fjla.uk/owlboard/go-types v0.0.0-20240407202712-e58d7d1d9aa9/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY= git.fjla.uk/owlboard/go-types v0.0.0-20240407202712-e58d7d1d9aa9/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580 h1:bEaC1JfqiSSJH65iP/NXMyBo85JMB41VBkiJdWbnHYM= git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580 h1:bEaC1JfqiSSJH65iP/NXMyBo85JMB41VBkiJdWbnHYM=
git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY= git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
git.fjla.uk/owlboard/go-types v0.0.0-20240408193146-4719be9c13eb h1:aLd0nzuU13hxycz9F4Z4PVq5dp/TxuzywPGZTJXbnq0=
git.fjla.uk/owlboard/go-types v0.0.0-20240408193146-4719be9c13eb/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

View File

@ -9,7 +9,7 @@ import (
_ "time/tzdata" _ "time/tzdata"
"git.fjla.uk/owlboard/timetable-mgr/background" "git.fjla.uk/owlboard/timetable-mgr/background"
"git.fjla.uk/owlboard/timetable-mgr/corpus" "git.fjla.uk/owlboard/timetable-mgr/cif"
"git.fjla.uk/owlboard/timetable-mgr/dbAccess" "git.fjla.uk/owlboard/timetable-mgr/dbAccess"
"git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/helpers"
"git.fjla.uk/owlboard/timetable-mgr/log" "git.fjla.uk/owlboard/timetable-mgr/log"
@ -41,7 +41,7 @@ func main() {
background.InitTicker(cfg, stop) background.InitTicker(cfg, stop)
// Test CORPUS Fetching // Test CORPUS Fetching
go corpus.CheckCorpus(cfg) go cif.CheckCif(cfg)
if cfg.VstpOn { if cfg.VstpOn {
messaging.StompInit(cfg) messaging.StompInit(cfg)