Furnish timetable

This commit is contained in:
Fred Boniface 2023-05-30 23:03:16 +01:00
parent 3aee27d129
commit 467651c45f
3 changed files with 54 additions and 9 deletions

View File

@ -1,4 +1,4 @@
from datetime import datetime from datetime import datetime
def out(msg, level): def out(msg, level = "OTHR"):
print(datetime.now().strftime("%m/%d/%Y, %H:%M:%S") + ": " + level + ": " + msg) print(datetime.now().strftime("%m/%d/%Y, %H:%M:%S") + ": " + level + ": " + msg)

View File

@ -131,4 +131,9 @@ def putVersion(version):
collection = "meta" collection = "meta"
col = db[collection] col = db[collection]
res = col.update_one({"target": "versions"}, {"$set":{"target": "versions","dbmanager": version}}, upsert=True) res = col.update_one({"target": "versions"}, {"$set":{"target": "versions","dbmanager": version}}, upsert=True)
return return
def putTimetable(data):
collection = "timetable"
col = db[collection]
res = col.insert_many(data)

View File

@ -22,11 +22,12 @@ import zlib
import json import json
import mongo import mongo
import time import time
from datetime import datetime
# This module downloads a single TOCs Schedule data # This module downloads a single TOCs Schedule data
TOC_Code = "EF" # Business code for GWR TOC_Code = "EF" # Business code for GWR
fullDataUrl = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_{TOC_Code}_TOC_FULL_DAILY&day=toc-full" fullDataUrl = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_{TOC_Code}_TOC_FULL_DAILY&day=toc-full"
updateDataUrl = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_{TOC_Code}_TOC_UPDATE_DAILY&day=toc-update-{day}" #updateDataUrl = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_{TOC_Code}_TOC_UPDATE_DAILY&day=toc-update-{day}"
CORPUS_USER = os.getenv('OWL_LDB_CORPUSUSER') CORPUS_USER = os.getenv('OWL_LDB_CORPUSUSER')
CORPUS_PASS = os.getenv('OWL_LDB_CORPUSPASS') CORPUS_PASS = os.getenv('OWL_LDB_CORPUSPASS')
@ -44,7 +45,7 @@ def isUpdateRequired():
return False return False
def getTimetable(full = False): def getTimetable(full = False):
download_url = full_data_url if full else update_data_url downloadUrl = fullDataUrl if full else updateDataUrl
response = requests.get(downloadUrl, auth=(CORPUS_USER, CORPUS_PASS)) response = requests.get(downloadUrl, auth=(CORPUS_USER, CORPUS_PASS))
mongo.incrementCounter("schedule_api") mongo.incrementCounter("schedule_api")
return zlib.decompress(response.content, 16+zlib.MAX_WBITS) return zlib.decompress(response.content, 16+zlib.MAX_WBITS)
@ -67,7 +68,6 @@ def loopTimetable(data):
print("JsonAssociationData") print("JsonAssociationData")
# Associates trains with eachother - not planning to use yet. # Associates trains with eachother - not planning to use yet.
elif ('JsonScheduleV1' in dic): elif ('JsonScheduleV1' in dic):
print("JsonScheduleData")
document = insertSchedule(dic) document = insertSchedule(dic)
documents.append(document) documents.append(document)
return documents return documents
@ -83,28 +83,59 @@ def runUpdate():
else: else:
log.out("timetable.runUpdate: timetable update is not needed") log.out("timetable.runUpdate: timetable update is not needed")
return "done" return "done"
parsed = loopTimetable(data)
mongo.putTimetable(parsed)
## Check what happens if there is no update ## Check what happens if there is no update
def insertSchedule(sch_record): def insertSchedule(sch_record):
schedule = sch_record['JsonScheduleV1'] schedule = sch_record['JsonScheduleV1']
scheduleId = schedule['CIF_train_uid'] scheduleId = schedule['CIF_train_uid']
transactionType = schedule['transaction_type'] transactionType = schedule['transaction_type']
if ('schedule_start_date' in sch_record):
scheduleStart = _helpParseDate(sch_record['schedule_start_date'])
else:
now = datetime.now()
scheduleStart = now.replace(hour=0,minute=0,second=0,microsecond=0)
document = { document = {
'stpIndicator': schedule['CIF_stp_indicator'], 'stpIndicator': schedule['CIF_stp_indicator'],
'trainUid': scheduleId, 'trainUid': scheduleId,
'headcode': schedule['schedule_segment']['signalling_id'], 'headcode': schedule['schedule_segment']['signalling_id'],
'powerType': schedule['schedule_segment']['CIF_power_type'], 'powerType': schedule['schedule_segment']['CIF_power_type'],
'planSpeed': schedule['schedule_segment']['CIF_speed'], 'planSpeed': schedule['schedule_segment']['CIF_speed'],
'scheduleEndDate': schedule['schedule_end_date'] 'scheduleStartDate': scheduleStart,
'scheduleEndDate': _helpParseDate(schedule['schedule_end_date'], "end"),
'daysRun': _helpParseDays(schedule['schedule_days_runs'])
} }
passengerStops = []
if ('schedule_location' in schedule['schedule_segment']):
stops = _helpParseStops(schedule['schedule_segment']['schedule_location'])
document['stops'] = stops
return document return document
def _helpParseStops(schedule_segment):
return
def _helpParseDays(string):
# Incoming string contains seven numbers, each number from 0-6 representing days Mon-Sun
daysList = ["m", "t", "w", "th", "f", "s", "su"]
selectedDays = [daysList[i] for i, value in enumerate(string) if value == "1"]
return selectedDays
def _helpParseDate(string, time):
# Incoming string contains date in format %Y-%m-%d, if the time signified end of schedule,
# append 23:59:59 to the time, else append 00:00:00 to the string.
if time == "end":
string += " 235959"
else:
string += " 000000"
return datetime.strptime(string, "%Y-%m-%d %H%M%S")
# Proposed Document Schema: # Proposed Document Schema:
# { # {
# stp_indicator: "O", # stp_indicator: "O",
# train_uid: "C07284" # train_uid: "C07284"
# atoc_code: "GW" # atoc_code: "GW"
# schedule_days_runs: [0,1,2,3,4,5,6] # Sunday-Saturday # schedule_days_runs: []
# schedule_end_date: "2023-06-02" # schedule_end_date: "2023-06-02"
# headcode: "5G30" # headcode: "5G30"
# power_type: "DMU" # power_type: "DMU"
@ -120,5 +151,14 @@ def insertSchedule(sch_record):
# ] # ]
### CURRENT STATE: loopTimetable and insertSchedule builds the data into ### CURRENT STATE: loopTimetable and insertSchedule builds the data into
### a suitable format to send to Mongo, passengerStops are not yet ### a suitable format to send to Mongo, there needs to be logic around
### parsed and there needs to be logic around the transaction_type ### the transaction_type. Parsinghelper funtions implemented to keep code tidy
### Stops need parsing
# Function Usage Map =>
# runUpdate() =>
# isUpdateRequired()
# loopTimetable() =>
# insertSchedule()
# Will then need to insert into database