From 0f7693d798f51b25aa6583a30f58e1703585dd9c Mon Sep 17 00:00:00 2001
From: Fred Boniface <fred@fjla.uk>
Date: Wed, 7 Jun 2023 21:14:49 +0100
Subject: [PATCH] Bug Fixes:  - Out of date schedules were not being removed
 from timetable  - Timetable update was at risk of missing a day  - cif-data
 file was not being removed after mailing - mainly an issue when not
 containerized  - Added additional helper variables

---
 src/corpus.py    | 1 -
 src/helpers.py   | 3 ++-
 src/mailer.py    | 5 +++++
 src/main.py      | 2 +-
 src/timetable.py | 6 +++---
 5 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/corpus.py b/src/corpus.py
index 94dfca1..fb791a2 100644
--- a/src/corpus.py
+++ b/src/corpus.py
@@ -84,5 +84,4 @@ def onlyStations(data):
             workingList.append(workingDict)
     postLength = len(workingList)
     log.out(f"corpus.onlyStations: Removed {preLength - postLength} documents", "INFO")
-    log.out(f"Yes, I am aware there are not {postLength} stations but the data includes NI, TfL, some bus, tram and closed stations too","OTHR")
     return workingList
\ No newline at end of file
diff --git a/src/helpers.py b/src/helpers.py
index a97337a..42de838 100644
--- a/src/helpers.py
+++ b/src/helpers.py
@@ -16,7 +16,8 @@ pis_file_paths = [
 corpus_data_url      = "https://publicdatafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS"
 
 ## Time Constants
-one_day_in_seconds   = 84600
+one_day_in_seconds   = (60**2)*24
+twenty_hours         = (60**2)*20
 two_day_in_seconds   = one_day_in_seconds * 2
 two_weeks_in_seconds = one_day_in_seconds * 14
 
diff --git a/src/mailer.py b/src/mailer.py
index 47c9e73..6e07cbe 100644
--- a/src/mailer.py
+++ b/src/mailer.py
@@ -47,6 +47,11 @@ def deleteLogs():
     print("Tidied log file")
   else:
     print("No logfile to tidy")
+  if os.path.exists("cif_data"):
+    os.remove("cif_data")
+    print("Removed cif_data file")
+  else:
+    print("No cif_data to tidy")
 
 
 def sendMail(msg_body :str, retry):
diff --git a/src/main.py b/src/main.py
index 654f322..ddb7c7f 100644
--- a/src/main.py
+++ b/src/main.py
@@ -14,7 +14,7 @@
 #     program.  If not, see
 #     https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE
 
-version = "2023.6.9"
+version = "2023.6.10"
 print(f"main.py: Initialising db-manager v{version}")
 
 #Third Party Imports
diff --git a/src/timetable.py b/src/timetable.py
index 1b5201b..27f3774 100644
--- a/src/timetable.py
+++ b/src/timetable.py
@@ -53,7 +53,7 @@ def isUpdateRequired():
   if (timetableDataAge >= helpers.two_day_in_seconds and isAfter0800) or REBUILD:
     log.out(f"timetable.isUpdateRequired: timetable collection requires rebuild", "INFO")
     return "full"
-  if (timetableDataAge >= helpers.one_day_in_seconds and isAfter0800):
+  if (timetableDataAge >= helpers.twenty_hours and isAfter0800):
     log.out(f"timetable.isUpdateRequired: timetable collection requires update", "INFO")
     return "update"
   return False
@@ -205,8 +205,8 @@ def _removeOutdatedServices():
   log.out("timetable._removeOutdatedServices: Removing out of date schedules", "INFO")
   pre_count = mongo.getLength("timetable")
   query = {
-    "schedule_end_date": {
-      "$lte": datetime.now()
+    "scheduleEndDate": {
+      "$lt": now
     }
   }
   mongo.deleteMany("timetable", query)