timetable-extension #1
@ -46,6 +46,7 @@ func CheckCif(cfg *helpers.Configuration) {
|
|||||||
// Check how many days since last update, if more than 5, run full update, else run update
|
// Check how many days since last update, if more than 5, run full update, else run update
|
||||||
daysSinceLastUpdate := howManyDaysAgo(metadata.LastUpdate)
|
daysSinceLastUpdate := howManyDaysAgo(metadata.LastUpdate)
|
||||||
if daysSinceLastUpdate > 5 {
|
if daysSinceLastUpdate > 5 {
|
||||||
|
log.Msg.Debug("Full Update Requested due to time since last update", zap.Int("daysSinceLastUpdate", daysSinceLastUpdate))
|
||||||
log.Msg.Info("Full CIF download required")
|
log.Msg.Info("Full CIF download required")
|
||||||
err := runCifFullDownload(cfg)
|
err := runCifFullDownload(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -62,6 +63,4 @@ func CheckCif(cfg *helpers.Configuration) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
log.Msg.Error("Unable to run CIF update", zap.Error(err))
|
log.Msg.Error("Unable to run CIF update", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
@ -12,4 +12,4 @@ const fullUpdateUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAu
|
|||||||
const dataAvailable = 6
|
const dataAvailable = 6
|
||||||
|
|
||||||
// An object representing the Europe/London timezone
|
// An object representing the Europe/London timezone
|
||||||
var londonTimezone, err = time.LoadLocation("Europe/London")
|
var londonTimezone, _ = time.LoadLocation("Europe/London")
|
||||||
|
@ -21,7 +21,7 @@ func ConvertServiceType(input *upstreamApi.JsonScheduleV1, vstp bool) (*database
|
|||||||
PlanSpeed: parseSpeed(&input.ScheduleSegment.CifSpeed),
|
PlanSpeed: parseSpeed(&input.ScheduleSegment.CifSpeed),
|
||||||
ScheduleStartDate: ParseCifDate(&input.ScheduleStartDate, "start"),
|
ScheduleStartDate: ParseCifDate(&input.ScheduleStartDate, "start"),
|
||||||
ScheduleEndDate: ParseCifDate(&input.ScheduleEndDate, "end"),
|
ScheduleEndDate: ParseCifDate(&input.ScheduleEndDate, "end"),
|
||||||
FirstClass: hasFirstClass(&input.ScheduleSegment.CifTrainClass),
|
FirstClass: hasFirstClass(&input.ScheduleSegment.CifTrainClass, &input.ScheduleSegment.SignallingId),
|
||||||
Catering: hasCatering(&input.ScheduleSegment.CifCateringCode),
|
Catering: hasCatering(&input.ScheduleSegment.CifCateringCode),
|
||||||
Sleeper: hasSleeper(&input.ScheduleSegment.CifSleepers),
|
Sleeper: hasSleeper(&input.ScheduleSegment.CifSleepers),
|
||||||
DaysRun: parseDaysRun(&input.ScheduleDaysRun),
|
DaysRun: parseDaysRun(&input.ScheduleDaysRun),
|
||||||
@ -37,14 +37,12 @@ func parseSpeed(CIFSpeed *string) int32 {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
if *CIFSpeed == "" {
|
if *CIFSpeed == "" {
|
||||||
log.Msg.Debug("Speed data not provided")
|
|
||||||
return int32(0)
|
return int32(0)
|
||||||
}
|
}
|
||||||
actualSpeed, exists := helpers.SpeedMap[*CIFSpeed]
|
actualSpeed, exists := helpers.SpeedMap[*CIFSpeed]
|
||||||
if !exists {
|
if !exists {
|
||||||
actualSpeed = *CIFSpeed
|
actualSpeed = *CIFSpeed
|
||||||
}
|
}
|
||||||
log.Msg.Debug("Corrected Speed: " + actualSpeed)
|
|
||||||
|
|
||||||
speed, err := strconv.ParseInt(actualSpeed, 10, 32)
|
speed, err := strconv.ParseInt(actualSpeed, 10, 32)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -87,10 +85,17 @@ func isPublic(input *upstreamApi.CifScheduleLocation) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Ascertains whether the service offers first class
|
// Ascertains whether the service offers first class
|
||||||
func hasFirstClass(input *string) bool {
|
func hasFirstClass(input, signallingId *string) bool {
|
||||||
if input == nil {
|
if input == nil || signallingId == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Handle non passenger headcodes and ensure first class is not shown as available
|
||||||
|
firstChar := (*signallingId)[0]
|
||||||
|
if firstChar == '3' || firstChar == '4' || firstChar == '5' || firstChar == '6' || firstChar == '7' || firstChar == '8' || firstChar == '0' {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
return *input != "S"
|
return *input != "S"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -88,22 +88,33 @@ func TestIsPublic(t *testing.T) {
|
|||||||
func TestHasFirstClass(t *testing.T) {
|
func TestHasFirstClass(t *testing.T) {
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
input string
|
input string
|
||||||
|
headcode string
|
||||||
expect bool
|
expect bool
|
||||||
}{
|
}{
|
||||||
{"", true},
|
{"", "1A00", true},
|
||||||
{"B", true},
|
{"B", "2A05", true},
|
||||||
{"S", false},
|
{"S", "1C99", false},
|
||||||
|
{"", "3C23", false},
|
||||||
|
{"", "5Q21", false},
|
||||||
|
{"", "5D32", false},
|
||||||
|
{"", "9O12", true},
|
||||||
|
{"B", "9D32", true},
|
||||||
|
{"", "7R43", false},
|
||||||
|
{"B", "6Y77", false},
|
||||||
|
{"", "8P98", false},
|
||||||
|
{"S", "4O89", false},
|
||||||
|
{"", "4E43", false},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
result := hasFirstClass(&tc.input)
|
result := hasFirstClass(&tc.input, &tc.headcode)
|
||||||
|
|
||||||
if result != tc.expect {
|
if result != tc.expect {
|
||||||
t.Errorf("For %s, expected %t, but got %t", tc.input, tc.expect, result)
|
t.Errorf("For %s & headcode %s, expected %t, but got %t", tc.input, tc.headcode, tc.expect, result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
nilResult := hasFirstClass(nil)
|
nilResult := hasFirstClass(nil, nil)
|
||||||
if nilResult {
|
if nilResult {
|
||||||
t.Errorf("hasFirstClass failed to handle nil pointer, expected %t, got %t", false, nilResult)
|
t.Errorf("hasFirstClass failed to handle nil pointer, expected %t, got %t", false, nilResult)
|
||||||
}
|
}
|
||||||
|
@ -44,11 +44,13 @@ func isSameToday(t time.Time) bool {
|
|||||||
|
|
||||||
// Returns how many days ago `t` was compared to today
|
// Returns how many days ago `t` was compared to today
|
||||||
func howManyDaysAgo(t time.Time) int {
|
func howManyDaysAgo(t time.Time) int {
|
||||||
today := time.Now().In(time.UTC).Truncate(24 * time.Hour)
|
log.Msg.Debug("Calculating how many days ago", zap.Time("Input time", t))
|
||||||
input := t.In(time.UTC).Truncate(24 * time.Hour)
|
// Truncate both times to midnight in UTC timezone
|
||||||
|
today := time.Now().UTC().Truncate(24 * time.Hour)
|
||||||
|
input := t.UTC().Truncate(24 * time.Hour)
|
||||||
|
|
||||||
diff := today.Sub(input)
|
diff := today.Sub(input)
|
||||||
days := int(diff.Hours() / 24)
|
days := int(diff / (24 * time.Hour))
|
||||||
return days
|
return days
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -24,11 +24,11 @@ func TestHowManyDaysAgo(t *testing.T) {
|
|||||||
input time.Time
|
input time.Time
|
||||||
expected int
|
expected int
|
||||||
}{
|
}{
|
||||||
{time.Now(), 0}, // Today
|
{time.Now().In(time.UTC), 0}, // Today
|
||||||
{time.Now().Add(-24 * time.Hour), 1}, // Yesterday
|
{time.Now().In(time.UTC).Add(-24 * time.Hour), 1}, // Yesterday
|
||||||
{time.Now().Add(-48 * time.Hour), 2}, // Ereyesterday
|
{time.Now().In(time.UTC).Add(-48 * time.Hour), 2}, // Ereyesterday
|
||||||
{time.Now().Add(24 * time.Hour), -1}, // Tomorrow
|
{time.Now().In(time.UTC).Add(24 * time.Hour), -1}, // Tomorrow
|
||||||
{time.Now().Add(48 * time.Hour), -2}, // Overmorrow
|
{time.Now().In(time.UTC).Add(48 * time.Hour), -2}, // Overmorrow
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
|
@ -18,9 +18,11 @@ func processParsedCif(data *parsedData) error {
|
|||||||
for _, item := range data.sched {
|
for _, item := range data.sched {
|
||||||
switch item.TransactionType {
|
switch item.TransactionType {
|
||||||
case "Delete":
|
case "Delete":
|
||||||
deleteTasks = append(deleteTasks, &item)
|
deleteItem := item // Create new variable to ensure repetition of pointers
|
||||||
|
deleteTasks = append(deleteTasks, &deleteItem)
|
||||||
case "Create":
|
case "Create":
|
||||||
createTasks = append(createTasks, &item)
|
createItem := item // Create new variable to ensure repetition of pointers
|
||||||
|
createTasks = append(createTasks, &createItem)
|
||||||
default:
|
default:
|
||||||
log.Msg.Error("Unknown transaction type in CIF Schedule", zap.String("TransactionType", item.TransactionType))
|
log.Msg.Error("Unknown transaction type in CIF Schedule", zap.String("TransactionType", item.TransactionType))
|
||||||
}
|
}
|
||||||
|
@ -35,6 +35,7 @@ func TestGenerateMetadata(t *testing.T) {
|
|||||||
|
|
||||||
if result == nil {
|
if result == nil {
|
||||||
t.Errorf("generateMetadata returned nil pointer")
|
t.Errorf("generateMetadata returned nil pointer")
|
||||||
|
return // Static type checking likes this return to be here, even if it is redundant in reality.
|
||||||
}
|
}
|
||||||
|
|
||||||
if result.Doctype != expected.Doctype {
|
if result.Doctype != expected.Doctype {
|
||||||
|
@ -40,7 +40,11 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
|
|||||||
log.Msg.Error("Error processing CIF data", zap.Error(err))
|
log.Msg.Error("Error processing CIF data", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate & Write metadata
|
newMeta := generateMetadata(&parsed.header)
|
||||||
|
ok := dbAccess.PutCifMetadata(newMeta)
|
||||||
|
if !ok {
|
||||||
|
log.Msg.Warn("CIF Data updated, but metadata write failed")
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -18,10 +18,10 @@ const Doctype = "CifMetadata"
|
|||||||
// The type describing the CifMetadata 'type' in the database.
|
// The type describing the CifMetadata 'type' in the database.
|
||||||
// This type will be moved to owlboard/go-types
|
// This type will be moved to owlboard/go-types
|
||||||
type CifMetadata struct {
|
type CifMetadata struct {
|
||||||
Doctype string `json:"type"`
|
Doctype string `bson:"type"`
|
||||||
LastUpdate time.Time `json:"lastUpdate"`
|
LastUpdate time.Time `bson:"lastUpdate"`
|
||||||
LastTimestamp int64 `json:"lastTimestamp"`
|
LastTimestamp int64 `bson:"lastTimestamp"`
|
||||||
LastSequence int64 `json:"lastSequence"`
|
LastSequence int64 `bson:"lastSequence"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
|
// Fetches the CifMetadata from the database, returns nil if no metadata exists - before first initialisation for example.
|
||||||
@ -39,20 +39,24 @@ func GetCifMetadata() (*CifMetadata, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Msg.Debug("Fetched CIF Metadata from database", zap.Any("Metadata", result))
|
||||||
|
|
||||||
return &result, nil
|
return &result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Uses upsert to Insert/Update the CifMetadata in the database
|
// Uses upsert to Insert/Update the CifMetadata in the database
|
||||||
func PutCifMetadata(metadata CifMetadata) bool {
|
func PutCifMetadata(metadata *CifMetadata) bool {
|
||||||
database := MongoClient.Database(databaseName)
|
database := MongoClient.Database(databaseName)
|
||||||
collection := database.Collection(metaCollection)
|
collection := database.Collection(metaCollection)
|
||||||
options := options.Update().SetUpsert(true)
|
options := options.Update().SetUpsert(true)
|
||||||
filter := bson.M{"type": Doctype}
|
filter := bson.M{"type": Doctype}
|
||||||
update := bson.M{
|
update := bson.M{
|
||||||
|
"$set": bson.M{
|
||||||
"type": Doctype,
|
"type": Doctype,
|
||||||
"LastUpdate": metadata.LastUpdate,
|
"lastUpdate": metadata.LastUpdate,
|
||||||
"LastTimestamp": metadata.LastTimestamp,
|
"lastTimestamp": metadata.LastTimestamp,
|
||||||
"LastSequence": metadata.LastSequence,
|
"lastSequence": metadata.LastSequence,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := collection.UpdateOne(context.Background(), filter, update, options)
|
_, err := collection.UpdateOne(context.Background(), filter, update, options)
|
||||||
@ -61,14 +65,21 @@ func PutCifMetadata(metadata CifMetadata) bool {
|
|||||||
log.Msg.Error("Error updating CIF Metadata", zap.Error(err))
|
log.Msg.Error("Error updating CIF Metadata", zap.Error(err))
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Msg.Info("New CIF Metadata written", zap.Time("Update time", metadata.LastUpdate))
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handles 'Delete' tasks from CIF Schedule updates, accepts DeleteQuery types and batches deletions.
|
// Handles 'Delete' tasks from CIF Schedule updates, accepts DeleteQuery types and batches deletions.
|
||||||
func DeleteCifEntries(deletions []database.DeleteQuery) error {
|
func DeleteCifEntries(deletions []database.DeleteQuery) error {
|
||||||
collection := MongoClient.Database(databaseName).Collection(timetableCollection)
|
// Skip if deletions is empty
|
||||||
|
if len(deletions) == 0 {
|
||||||
|
log.Msg.Info("No deletions required")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Prepare deletion tasks
|
// Prepare deletion tasks
|
||||||
|
collection := MongoClient.Database(databaseName).Collection(timetableCollection)
|
||||||
bulkDeletions := make([]mongo.WriteModel, 0, len(deletions))
|
bulkDeletions := make([]mongo.WriteModel, 0, len(deletions))
|
||||||
|
|
||||||
for _, deleteQuery := range deletions {
|
for _, deleteQuery := range deletions {
|
||||||
@ -95,6 +106,12 @@ func DeleteCifEntries(deletions []database.DeleteQuery) error {
|
|||||||
|
|
||||||
// Handles 'Create' tasks for CIF Schedule updates, accepts Service structs and batches their creation.
|
// Handles 'Create' tasks for CIF Schedule updates, accepts Service structs and batches their creation.
|
||||||
func CreateCifEntries(schedules []database.Service) error {
|
func CreateCifEntries(schedules []database.Service) error {
|
||||||
|
// Skip if deletions is empty
|
||||||
|
if len(schedules) == 0 {
|
||||||
|
log.Msg.Info("No creations required")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
collection := MongoClient.Database(databaseName).Collection(timetableCollection)
|
collection := MongoClient.Database(databaseName).Collection(timetableCollection)
|
||||||
|
|
||||||
models := make([]mongo.WriteModel, 0, len(schedules))
|
models := make([]mongo.WriteModel, 0, len(schedules))
|
||||||
|
2
go.mod
2
go.mod
@ -3,7 +3,7 @@ module git.fjla.uk/owlboard/timetable-mgr
|
|||||||
go 1.21
|
go 1.21
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580
|
git.fjla.uk/owlboard/go-types v0.0.0-20240408193146-4719be9c13eb
|
||||||
github.com/go-stomp/stomp/v3 v3.0.5
|
github.com/go-stomp/stomp/v3 v3.0.5
|
||||||
go.mongodb.org/mongo-driver v1.12.0
|
go.mongodb.org/mongo-driver v1.12.0
|
||||||
go.uber.org/zap v1.24.0
|
go.uber.org/zap v1.24.0
|
||||||
|
2
go.sum
2
go.sum
@ -10,6 +10,8 @@ git.fjla.uk/owlboard/go-types v0.0.0-20240407202712-e58d7d1d9aa9 h1:aNxMYEsbBkFx
|
|||||||
git.fjla.uk/owlboard/go-types v0.0.0-20240407202712-e58d7d1d9aa9/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
|
git.fjla.uk/owlboard/go-types v0.0.0-20240407202712-e58d7d1d9aa9/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
|
||||||
git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580 h1:bEaC1JfqiSSJH65iP/NXMyBo85JMB41VBkiJdWbnHYM=
|
git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580 h1:bEaC1JfqiSSJH65iP/NXMyBo85JMB41VBkiJdWbnHYM=
|
||||||
git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
|
git.fjla.uk/owlboard/go-types v0.0.0-20240408150352-8ba2a306a580/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
|
||||||
|
git.fjla.uk/owlboard/go-types v0.0.0-20240408193146-4719be9c13eb h1:aLd0nzuU13hxycz9F4Z4PVq5dp/TxuzywPGZTJXbnq0=
|
||||||
|
git.fjla.uk/owlboard/go-types v0.0.0-20240408193146-4719be9c13eb/go.mod h1:kG+BX9UF+yJaAVnln/QSKlTdrtKRRReezMeSk1ZLMzY=
|
||||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
4
main.go
4
main.go
@ -9,7 +9,7 @@ import (
|
|||||||
_ "time/tzdata"
|
_ "time/tzdata"
|
||||||
|
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/background"
|
"git.fjla.uk/owlboard/timetable-mgr/background"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/corpus"
|
"git.fjla.uk/owlboard/timetable-mgr/cif"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||||
@ -41,7 +41,7 @@ func main() {
|
|||||||
background.InitTicker(cfg, stop)
|
background.InitTicker(cfg, stop)
|
||||||
|
|
||||||
// Test CORPUS Fetching
|
// Test CORPUS Fetching
|
||||||
go corpus.CheckCorpus(cfg)
|
go cif.CheckCif(cfg)
|
||||||
|
|
||||||
if cfg.VstpOn {
|
if cfg.VstpOn {
|
||||||
messaging.StompInit(cfg)
|
messaging.StompInit(cfg)
|
||||||
|
Loading…
Reference in New Issue
Block a user