Rebase

data/pis/gwr.yaml

@@ -2314,5 +2314,5 @@ pis:
 
   ## Non Passenger Codes
   - code: 0015
-    stops: [xxx,NOT-IN-SERVICE,xxx]
+    stops: [xxx,notINservice,xxx]
 ###          All LTV

src/helpers.py

@@ -0,0 +1,6 @@
+import time
+
+def getAgeInSeconds(updateTimeInSeconds :int):
+  now = int(time.time())
+  ageInSeconds :int = now - updateTimeInSeconds
+  return ageInSeconds

src/mailer.py

@@ -15,46 +15,95 @@
 #     https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE
 
 import smtplib, ssl, os
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.utils import formatdate
 import logger as log
 
+cif_file_path = "cif_data"
+
 log.out("mailer.py: Mailer module loaded", "DBUG")
 
 def submitLogs():
   text :str = fetchLogs()
-  sendMail(text)
+  cif_data = fetchCifData()
+  sendMail(text, cifData)
 
 def fetchLogs():
   with open("dbman-log", "r") as tmpfile:
     return tmpfile.read()
 
+def fetchCifData():
+  try:
+    with open(cif_file_path, "r") as f:
+      return f.read()
+  except Exception as e:
+    log.out("mailer.fetchCifData: CIF Data is not readable")
+    print(e)
+
 def deleteLogs():
   if os.path.exists("dbman-log"):
     os.remove("dbman-log")
     print("Tidied log file")
   else:
     print("No logfile to tidy")
+  if os.path.exists(cif_file_path):
+    os.remove(cif_file_path)
+  else:
+    print("No cifData file to tidy")
 
-def sendMail(messageBody :str):
+#def sendMail(messageBody :str):
+#  smtpHost = os.getenv("OWL_EML_HOST")
+#  smtpPort = os.getenv("OWL_EML_PORT")
+#  smtpUser = os.getenv("OWL_EML_USER")
+#  smtpPass = os.getenv("OWL_EML_PASS")
+#  smtpFrom = os.getenv("OWL_EML_FROM")
+#  context = ssl.create_default_context()
+#  message = f"""Subject: OwlBoard-dbman-logs
+
+
+#{messageBody}"""
+#  try:
+#    server = smtplib.SMTP(smtpHost,smtpPort)
+#    server.ehlo()
+#    server.starttls(context=context) # Secure the connection
+#    server.ehlo()
+#    server.login(smtpUser, smtpPass)
+#    server.sendmail(smtpFrom, "server-notification-receipt@fjla.uk", message)
+#  except Exception as e:
+#    # Print any error messages to stdout
+#    print(e)
+#  finally:
+#    server.quit()
+#    deleteLogs()
+
+def sendMail(messageBody, cif_data):
   smtpHost = os.getenv("OWL_EML_HOST")
   smtpPort = os.getenv("OWL_EML_PORT")
   smtpUser = os.getenv("OWL_EML_USER")
   smtpPass = os.getenv("OWL_EML_PASS")
   smtpFrom = os.getenv("OWL_EML_FROM")
   context = ssl.create_default_context()
-  message = f"""Subject: OwlBoard-dbman-logs
+  msg = MIMEMultipart()
-
+  msg['From'] = smtpFrom
-
+  msg['To'] = "server-notification-receipt@fjla.uk"
-{messageBody}"""
+  msg['Date'] = formatdate(localtime=True)
+  msg['Subject'] = "OwlBoard - dbmanager Logs"
+  msg.attach(MIMEText(messageBody))
+  if cif_data:
+    part = MIMEText(cif_data, 'plain')
+    part.add_header('Content-Disposition', 'attachment', filename="CIF-DATA")
+    msg.attach(part)
   try:
-    server = smtplib.SMTP(smtpHost,smtpPort)
+    with smtplib.SMTP(smtpHost, smtpPort) as server:
-    server.ehlo()
+      server.ehlo()
-    server.starttls(context=context) # Secure the connection
+      server.starttls(context=context)  # Secure the connection
-    server.ehlo()
+      server.ehlo()
-    server.login(smtpUser, smtpPass)
+      server.login(smtpUser, smtpPass)
-    server.sendmail(smtpFrom, "server-notification-receipt@fjla.uk", message)
+      server.sendmail(smtpFrom, "server-notification-receipt@fjla.uk", msg.as_string())
   except Exception as e:
     # Print any error messages to stdout
     print(e)
   finally:
-    server.quit()
+    deleteLogs()
-    deleteLogs()
+  

src/main.py

@@ -14,7 +14,7 @@
 #     program.  If not, see
 #     https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE
 
-version = "2023.6.2"
+version = "2023.6.3"
 print(f"main.py: Initialising db-manager v{version}")
 
 #Third Party Imports

src/pis.py

@@ -5,8 +5,8 @@ import mongo
 REBUILD :bool = False  # Set to True to force rebuild
 
 log.out("pis.py: PIS Module Loaded", "DBUG")
-file_location :str = "/app/data/pis/gwr.yaml" # Production & Testing
+#file_location :str = "/app/data/pis/gwr.yaml" # Production & Testing
-#file_location :str = "/home/fred.boniface/git/owlboard/db-manager/data/pis/gwr.yaml" # Local Development
+file_location :str = "/home/fred.boniface/git/owlboard/db-manager/data/pis/gwr.yaml" # Local Development
 
 def runUpdate():
   if (not requiresUpdate()):
@@ -37,7 +37,7 @@ def load():  # Programatically add a `toc` field to each entry.
     with open(file_location, "r") as data:
         try:
             pis = yaml.safe_load(data)
-            print(pis)
+            #print(pis)
             return pis["pis"]
         except yaml.YAMLError as exc:
             print(exc)

src/timetable.py

@@ -20,6 +20,7 @@ REBUILD :bool = False ## Set to true to rebuild database
 import os
 import requests
 import logger as log
+import helpers
 import zlib
 import json
 import mongo
@@ -30,6 +31,11 @@ from datetime import datetime, timedelta
 now            = datetime.now()
 yesterday      = now - timedelta(days=1)
 yesterdayDay   = yesterday.strftime("%a").lower()
+todayDay       = now.strftime("%a").lower()
+oneDayinSecs   = 86400
+twoDayinSecs   = 86400 * 2
+isAfter0800    = (int(now.strftime("%H")) >= 8)
+filePath       = "cif_data"
 TOC_Code       = "EF" # Business code for GWR
 fullDataUrl    = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_{TOC_Code}_TOC_FULL_DAILY&day=toc-full"
 updateDataUrl  = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_{TOC_Code}_TOC_UPDATE_DAILY&day=toc-update-{yesterdayDay}"
@@ -42,12 +48,13 @@ log.out("timetable.py: Timetable module loaded", "DBUG")
 def isUpdateRequired():
   timetableLength = mongo.getLength("timetable")
   log.out(f"timetable.isUpdateRequired: timetable collection contains {timetableLength} documents", "DBUG")
-  timetableUpdateDate = mongo.metaCheckTime("timetable")
+  timetableUpdateTime = mongo.metaCheckTime("timetable")
   log.out(f"timetable.isUpdateRequired: Timetable last updated at {timetableUpdateDate}", "INFO")
-  if (not timetableLength or int(time.time()) > timetableUpdateDate + 172800 or REBUILD):
+  timetableDataAge = helpers.getAgeInSeconds(timetableUpdateTime)
+  if (timetableDataAge >= twoDayinSecs and isAfter0800) or REBUILD:
     log.out(f"timetable.isUpdateRequired: timetable collection requires rebuild", "INFO")
     return "full"
-  if (int(time.time()) > (timetableUpdateDate + 86400)):
+  if (timetableDataAge >= oneDayinSecs and isAfter0800):
     log.out(f"timetable.isUpdateRequired: timetable collection requires update", "INFO")
     return "update"
   return False
@@ -57,9 +64,9 @@ def getTimetable(full :bool = False):
   response = requests.get(downloadUrl, auth=(CORPUS_USER, CORPUS_PASS))
   mongo.incrementCounter("schedule_api")
   log.out(f"timetable.getTimetable: Fetch (Full:{full}) response: {response.status_code}", "DBUG")
-  decompressed = zlib.decompress(response.content, 16+zlib.MAX_WBITS)
+  with open(filePath, "wb") as f:
-  #print(decompressed)
+    f.write(response.content)
-  return decompressed
+  return zlib.decompress(response.content, 16+zlib.MAX_WBITS)
 
 def loopTimetable(data):
   listify = data.splitlines()