From 30faef09c83704cb5d998e09f95d9fdb0c9892d7 Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Sat, 3 Jun 2023 21:05:35 +0100 Subject: [PATCH] Rebase --- data/pis/gwr.yaml | 2 +- src/helpers.py | 6 ++++ src/mailer.py | 77 ++++++++++++++++++++++++++++++++++++++--------- src/main.py | 2 +- src/timetable.py | 19 ++++++++---- 5 files changed, 84 insertions(+), 22 deletions(-) create mode 100644 src/helpers.py diff --git a/data/pis/gwr.yaml b/data/pis/gwr.yaml index 9e77230..ab3a883 100644 --- a/data/pis/gwr.yaml +++ b/data/pis/gwr.yaml @@ -2314,5 +2314,5 @@ pis: ## Non Passenger Codes - code: 0015 - stops: [xxx,NOT-IN-SERVICE,xxx] + stops: [xxx,notINservice,xxx] ### All LTV \ No newline at end of file diff --git a/src/helpers.py b/src/helpers.py new file mode 100644 index 0000000..d14829b --- /dev/null +++ b/src/helpers.py @@ -0,0 +1,6 @@ +import time + +def getAgeInSeconds(updateTimeInSeconds :int): + now = int(time.time()) + ageInSeconds :int = now - updateTimeInSeconds + return ageInSeconds \ No newline at end of file diff --git a/src/mailer.py b/src/mailer.py index 7d5ec01..0e5d54a 100644 --- a/src/mailer.py +++ b/src/mailer.py @@ -15,46 +15,95 @@ # https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE import smtplib, ssl, os +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.utils import formatdate import logger as log +cif_file_path = "cif_data" + log.out("mailer.py: Mailer module loaded", "DBUG") def submitLogs(): text :str = fetchLogs() - sendMail(text) + cif_data = fetchCifData() + sendMail(text, cifData) def fetchLogs(): with open("dbman-log", "r") as tmpfile: return tmpfile.read() +def fetchCifData(): + try: + with open(cif_file_path, "r") as f: + return f.read() + except Exception as e: + log.out("mailer.fetchCifData: CIF Data is not readable") + print(e) + def deleteLogs(): if os.path.exists("dbman-log"): os.remove("dbman-log") print("Tidied log file") else: print("No logfile to tidy") + if os.path.exists(cif_file_path): + os.remove(cif_file_path) + else: + print("No cifData file to tidy") -def sendMail(messageBody :str): +#def sendMail(messageBody :str): +# smtpHost = os.getenv("OWL_EML_HOST") +# smtpPort = os.getenv("OWL_EML_PORT") +# smtpUser = os.getenv("OWL_EML_USER") +# smtpPass = os.getenv("OWL_EML_PASS") +# smtpFrom = os.getenv("OWL_EML_FROM") +# context = ssl.create_default_context() +# message = f"""Subject: OwlBoard-dbman-logs + + +#{messageBody}""" +# try: +# server = smtplib.SMTP(smtpHost,smtpPort) +# server.ehlo() +# server.starttls(context=context) # Secure the connection +# server.ehlo() +# server.login(smtpUser, smtpPass) +# server.sendmail(smtpFrom, "server-notification-receipt@fjla.uk", message) +# except Exception as e: +# # Print any error messages to stdout +# print(e) +# finally: +# server.quit() +# deleteLogs() + +def sendMail(messageBody, cif_data): smtpHost = os.getenv("OWL_EML_HOST") smtpPort = os.getenv("OWL_EML_PORT") smtpUser = os.getenv("OWL_EML_USER") smtpPass = os.getenv("OWL_EML_PASS") smtpFrom = os.getenv("OWL_EML_FROM") context = ssl.create_default_context() - message = f"""Subject: OwlBoard-dbman-logs - - -{messageBody}""" + msg = MIMEMultipart() + msg['From'] = smtpFrom + msg['To'] = "server-notification-receipt@fjla.uk" + msg['Date'] = formatdate(localtime=True) + msg['Subject'] = "OwlBoard - dbmanager Logs" + msg.attach(MIMEText(messageBody)) + if cif_data: + part = MIMEText(cif_data, 'plain') + part.add_header('Content-Disposition', 'attachment', filename="CIF-DATA") + msg.attach(part) try: - server = smtplib.SMTP(smtpHost,smtpPort) - server.ehlo() - server.starttls(context=context) # Secure the connection - server.ehlo() - server.login(smtpUser, smtpPass) - server.sendmail(smtpFrom, "server-notification-receipt@fjla.uk", message) + with smtplib.SMTP(smtpHost, smtpPort) as server: + server.ehlo() + server.starttls(context=context) # Secure the connection + server.ehlo() + server.login(smtpUser, smtpPass) + server.sendmail(smtpFrom, "server-notification-receipt@fjla.uk", msg.as_string()) except Exception as e: # Print any error messages to stdout print(e) finally: - server.quit() - deleteLogs() \ No newline at end of file + deleteLogs() + \ No newline at end of file diff --git a/src/main.py b/src/main.py index 4e7a435..45967ab 100644 --- a/src/main.py +++ b/src/main.py @@ -14,7 +14,7 @@ # program. If not, see # https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE -version = "2023.6.2" +version = "2023.6.3" print(f"main.py: Initialising db-manager v{version}") #Third Party Imports diff --git a/src/timetable.py b/src/timetable.py index 06d4d72..99a124c 100644 --- a/src/timetable.py +++ b/src/timetable.py @@ -20,6 +20,7 @@ REBUILD :bool = False ## Set to true to rebuild database import os import requests import logger as log +import helpers import zlib import json import mongo @@ -30,6 +31,11 @@ from datetime import datetime, timedelta now = datetime.now() yesterday = now - timedelta(days=1) yesterdayDay = yesterday.strftime("%a").lower() +todayDay = now.strftime("%a").lower() +oneDayinSecs = 86400 +twoDayinSecs = 86400 * 2 +isAfter0800 = (int(now.strftime("%H")) >= 8) +filePath = "cif_data" TOC_Code = "EF" # Business code for GWR fullDataUrl = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_{TOC_Code}_TOC_FULL_DAILY&day=toc-full" updateDataUrl = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_{TOC_Code}_TOC_UPDATE_DAILY&day=toc-update-{yesterdayDay}" @@ -42,12 +48,13 @@ log.out("timetable.py: Timetable module loaded", "DBUG") def isUpdateRequired(): timetableLength = mongo.getLength("timetable") log.out(f"timetable.isUpdateRequired: timetable collection contains {timetableLength} documents", "DBUG") - timetableUpdateDate = mongo.metaCheckTime("timetable") + timetableUpdateTime = mongo.metaCheckTime("timetable") log.out(f"timetable.isUpdateRequired: Timetable last updated at {timetableUpdateDate}", "INFO") - if (not timetableLength or int(time.time()) > timetableUpdateDate + 172800 or REBUILD): + timetableDataAge = helpers.getAgeInSeconds(timetableUpdateTime) + if (timetableDataAge >= twoDayinSecs and isAfter0800) or REBUILD: log.out(f"timetable.isUpdateRequired: timetable collection requires rebuild", "INFO") return "full" - if (int(time.time()) > (timetableUpdateDate + 86400)): + if (timetableDataAge >= oneDayinSecs and isAfter0800): log.out(f"timetable.isUpdateRequired: timetable collection requires update", "INFO") return "update" return False @@ -57,9 +64,9 @@ def getTimetable(full :bool = False): response = requests.get(downloadUrl, auth=(CORPUS_USER, CORPUS_PASS)) mongo.incrementCounter("schedule_api") log.out(f"timetable.getTimetable: Fetch (Full:{full}) response: {response.status_code}", "DBUG") - decompressed = zlib.decompress(response.content, 16+zlib.MAX_WBITS) - #print(decompressed) - return decompressed + with open(filePath, "wb") as f: + f.write(response.content) + return zlib.decompress(response.content, 16+zlib.MAX_WBITS) def loopTimetable(data): listify = data.splitlines()