diff --git a/.gitignore b/.gitignore index 6d8efc6..958561c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ run.sh +env # ---> Python # Byte-compiled / optimized / DLL files diff --git a/src/logger.py b/src/logger.py index 518acd1..7a01371 100644 --- a/src/logger.py +++ b/src/logger.py @@ -1,4 +1,8 @@ from datetime import datetime -def out(msg, level = "OTHR"): - print(datetime.now().strftime("%m/%d/%Y, %H:%M:%S") + ": " + level + ": " + msg) \ No newline at end of file +def out(msg :str, level :str = "OTHR"): + logline :str = f'{datetime.now().strftime("%m/%d/%Y, %H:%M:%S")}: {level}: {msg}' + print(logline) + tmpfile = "dbman-log" + with open(tmpfile, 'a') as logfile: + logfile.write(f'{logline}\n') \ No newline at end of file diff --git a/src/mailer.py b/src/mailer.py new file mode 100644 index 0000000..3d57e6d --- /dev/null +++ b/src/mailer.py @@ -0,0 +1,57 @@ +# db-manager - Builds and manages an OwlBoard database instance - To be run on a +# cron schedule +# Copyright (C) 2023 Frederick Boniface + +# This program is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License along with this +# program. If not, see +# https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE + +import smtplib, ssl, os + +def submitLogs(): + text :str = fetchLogs() + sendMail(text) + +def fetchLogs(): + with open("dbman-log", "r") as tmpfile: + return tmpfile.read() + +def deleteLogs(): + if os.path.exists("dbman-log"): + os.remove("dbman-log") + print("Tidied log file") + else: + print("No logfile to tidy") + +def sendMail(messageBody :str): + smtpHost = os.getenv("OWL_EML_HOST") + smtpPort = os.getenv("OWL_EML_PORT") + smtpUser = os.getenv("OWL_EML_USER") + smtpPass = os.getenv("OWL_EML_PASS") + smtpFrom = os.getenv("OWL_EML_FROM") + context = ssl.create_default_context() + message = f"""Subject: OwlBoard-dbman-logs + + +{messageBody}""" + try: + server = smtplib.SMTP(smtpHost,smtpPort) + server.ehlo() + server.starttls(context=context) # Secure the connection + server.ehlo() + server.login(smtpUser, smtpPass) + server.sendmail(smtpFrom, "server-notification-receipt@fjla.uk", message) + except Exception as e: + # Print any error messages to stdout + print(e) + finally: + server.quit() + deleteLogs() \ No newline at end of file diff --git a/src/main.py b/src/main.py index 7ffe8ee..57da117 100644 --- a/src/main.py +++ b/src/main.py @@ -14,7 +14,7 @@ # program. If not, see # https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE -version = "2023.5.6" +version = "2023.5.7" print(f"main.py: Initialising db-manager v{version}") #Third Party Imports @@ -22,7 +22,7 @@ import os import time #Local Imports -import corpus, mongo, pis +import corpus, mongo, pis, mailer, timetable import logger as log log.out("main.py: db-manager Initialised", "INFO") @@ -71,6 +71,9 @@ if pisAge > 43200: # Temporarily set to 15 minutes else: log.out('main.py: Not updating PIS data until is it 1036800s old', "INFO") +## Run Timetable Update +timetable.runUpdate() + log.out('main.py: Requesting TTL Index Creation', "INFO") mongo.createTtlIndex("users", "atime", 2629800) mongo.createTtlIndex("registrations", "time", 1800) @@ -79,4 +82,6 @@ mongo.createTtlIndex("registrations", "time", 1800) mongo.putVersion(version) # END -log.out(f"main.py: db-manager v{version} Complete", "INFO") \ No newline at end of file +log.out(f"main.py: db-manager v{version} Complete", "INFO") +log.out(f"main.py: Mailing logs") +mailer.submitLogs() \ No newline at end of file diff --git a/src/mongo.py b/src/mongo.py index 9dbbd62..3b61057 100644 --- a/src/mongo.py +++ b/src/mongo.py @@ -11,7 +11,7 @@ db_user = urllib.parse.quote_plus(os.getenv('OWL_DB_USER', "owl")) db_pass = urllib.parse.quote_plus(os.getenv('OWL_DB_PASS', "twittwoo")) db_name = os.getenv('OWL_DB_NAME', "owlboard") -log.out(f"mongo.py: Connecting to database at {db_host}:{db_port}", "INFO") +log.out(f"mongo.py: Connecting to database at {db_host}:{db_port}", "DBUG") client = MongoClient(f"mongodb://{db_user}:{db_pass}@{db_host}:{db_port}") db = client[db_name] @@ -23,12 +23,12 @@ def metaCheckTime(target): if 'updated' in res: log.out(f'mongo.metaUpdateTime: {target} last updated at {res["updated"]}', "INFO") return res["updated"] - log.out(f'mongo.metaUpdatetime: {target} does not exist', "INFO") + log.out(f'mongo.metaUpdatetime: {target} does not exist', "EROR") return 0 def metaUpdateTime(target): col = db["meta"] - log.out(f'mongo.metaUpdateTime: Updating updated time for {target}', "INFO") + log.out(f'mongo.metaUpdateTime: Updating updated time for {target}', "DBUG") res = col.update_one({"target": target, "type":"collection"}, {"$set":{"updated": int(time.time()),"target":target, "type":"collection"}}, upsert=True) incrementCounter("meta") @@ -136,4 +136,13 @@ def putVersion(version): def putTimetable(data): collection = "timetable" col = db[collection] - res = col.insert_many(data) \ No newline at end of file + res = col.insert_many(data) + +def dropCollection(collection): + col = db[collection] + res = col.drop() + +def deleteTimetableData(query): + collection = "timetable" + col = db[collection] + res = col.delete_one(query) \ No newline at end of file diff --git a/src/timetable.py b/src/timetable.py index 9c5831a..bd03c01 100644 --- a/src/timetable.py +++ b/src/timetable.py @@ -22,37 +22,42 @@ import zlib import json import mongo import time -from datetime import datetime +from datetime import datetime, timedelta # This module downloads a single TOCs Schedule data +now = datetime.now() +yesterday = now - timedelta(days=1) +yesterdayDay = yesterday.strftime("%a").lower() TOC_Code = "EF" # Business code for GWR fullDataUrl = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_{TOC_Code}_TOC_FULL_DAILY&day=toc-full" -#updateDataUrl = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_{TOC_Code}_TOC_UPDATE_DAILY&day=toc-update-{day}" +updateDataUrl = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_{TOC_Code}_TOC_UPDATE_DAILY&day=toc-update-{yesterdayDay}" CORPUS_USER = os.getenv('OWL_LDB_CORPUSUSER') CORPUS_PASS = os.getenv('OWL_LDB_CORPUSPASS') # Determine state of current Timetable Database def isUpdateRequired(): timetableLength = mongo.getLength("timetable") - log.out(f"timetable.isUpdateRequired: timetable collection contains {timetableLength} documents") - timetableUpdateDate = mongo.metaUpdateTime("timetable") + log.out(f"timetable.isUpdateRequired: timetable collection contains {timetableLength} documents", "DBUG") + timetableUpdateDate = mongo.metaCheckTime("timetable") + log.out(f"timetable.isUpdateRequired: Timetable last updated at {timetableUpdateDate}", "INFO") if (not timetableLength or int(time.time()) > timetableUpdateDate + 172800): - log.out(f"timetable.isUpdateRequired: timetable collection requires rebuild") + log.out(f"timetable.isUpdateRequired: timetable collection requires rebuild", "INFO") return "full" if (int(time.time()) > (timetableUpdateDate + 86400)): - log.out(f"timetable.isUpdateRequired: timetable collection requires update") + log.out(f"timetable.isUpdateRequired: timetable collection requires update", "INFO") return "update" return False -def getTimetable(full = False): - downloadUrl = fullDataUrl if full else updateDataUrl +def getTimetable(full :bool = False): + downloadUrl :str = fullDataUrl if full else updateDataUrl response = requests.get(downloadUrl, auth=(CORPUS_USER, CORPUS_PASS)) mongo.incrementCounter("schedule_api") + log.out(f"timetable.getTimetable: Fetch (Full:{full}) response: {response.status_code}", "DBUG") return zlib.decompress(response.content, 16+zlib.MAX_WBITS) def loopTimetable(data): listify = data.splitlines() - documents = [] + documents :list = [] for item in listify: dic = json.loads(item) if ('JsonTimetableV1' in dic): @@ -75,16 +80,19 @@ def loopTimetable(data): def runUpdate(): required = isUpdateRequired() if (required == "full"): - log.out("timetable.runUpdate: Fetching full timetable data") + log.out("timetable.runUpdate: Fetching full timetable data", "INFO") data = getTimetable(full = True) elif (required == "update"): - log.out("timetable.runUpdate: Fetching todays timetable update") + log.out("timetable.runUpdate: Fetching todays timetable update", "INFO") data = getTimetable() else: - log.out("timetable.runUpdate: timetable update is not needed") + log.out("timetable.runUpdate: timetable update is not needed", "INFO") return "done" parsed = loopTimetable(data) - mongo.putTimetable(parsed) + status = _insertToDb(parsed, required) + if (status): + mongo.metaUpdateTime("timetable") + ## Check what happens if there is no update def insertSchedule(sch_record): @@ -97,6 +105,7 @@ def insertSchedule(sch_record): now = datetime.now() scheduleStart = now.replace(hour=0,minute=0,second=0,microsecond=0) document = { + 'transactionType': schedule['transaction_type'], 'stpIndicator': schedule['CIF_stp_indicator'], 'trainUid': scheduleId, 'headcode': schedule['schedule_segment']['signalling_id'], @@ -106,14 +115,51 @@ def insertSchedule(sch_record): 'scheduleEndDate': _helpParseDate(schedule['schedule_end_date'], "end"), 'daysRun': _helpParseDays(schedule['schedule_days_runs']) } - passengerStops = [] if ('schedule_location' in schedule['schedule_segment']): stops = _helpParseStops(schedule['schedule_segment']['schedule_location']) + else: + stops = [] document['stops'] = stops return document +def _insertToDb(data :list, type :str): + if type == "full": + mongo.dropCollection("timetable") + mongo.putTimetable(data) + mongo.createSingleIndex("timetable", "headcode") + elif type == "update": + for item in data: + if item['transactionType'] == "Create": + singleList = [item] + mongo.putTimetable(singleList) + elif item['transactionType'] == "Delete": + mongo.deleteTimetableData({'trainUid': item.trainUid}) + return True #If Successful else False + def _helpParseStops(schedule_segment): - return + stops = [] + for i in schedule_segment: + timing_point = {} + public_departure = i.get("public_departure") + wtt_departure = i.get("departure") + public_arrival = i.get("public_arrival") + wtt_arrival = i.get("arrival") + tiploc_code = i.get("tiploc_code") + isPublic = False + if public_departure and len(public_departure) == 4 and public_departure.isdigit(): + isPublic = True + timing_point['publicDeparture'] = public_departure + if public_arrival and len(public_arrival) == 4 and public_arrival.isdigit(): + isPublic = True + timing_point['publicArrival'] = public_arrival + if wtt_departure: + timing_point['wttDeparture'] = wtt_departure + if wtt_arrival: + timing_point['wttArrival'] = wtt_arrival + timing_point['isPublic'] = isPublic + timing_point['tiploc'] = tiploc_code + stops.append(timing_point) + return stops def _helpParseDays(string): # Incoming string contains seven numbers, each number from 0-6 representing days Mon-Sun @@ -130,35 +176,6 @@ def _helpParseDate(string, time): string += " 000000" return datetime.strptime(string, "%Y-%m-%d %H%M%S") -# Proposed Document Schema: -# { -# stp_indicator: "O", -# train_uid: "C07284" -# atoc_code: "GW" -# schedule_days_runs: [] -# schedule_end_date: "2023-06-02" -# headcode: "5G30" -# power_type: "DMU" -# speed: "090" -# catering_code: null -# service_branding: "" -# passenger_stops: [ -# { -# 'tiploc': "TIPLOC", -# 'pb_arr': "PublicArrival", -# 'pb_dep': "PublicDepartr" -# } -# ] - -### CURRENT STATE: loopTimetable and insertSchedule builds the data into -### a suitable format to send to Mongo, there needs to be logic around -### the transaction_type. Parsinghelper funtions implemented to keep code tidy -### Stops need parsing - - -# Function Usage Map => - # runUpdate() => - # isUpdateRequired() - # loopTimetable() => - # insertSchedule() - # Will then need to insert into database \ No newline at end of file +def _removeOutdatedServices(): + #Remove services with an end date before today. + return \ No newline at end of file