From ec6e425a0d6237fe476c576a38795ed786d666e7 Mon Sep 17 00:00:00 2001
From: Fred Boniface <fred@fjla.uk>
Date: Wed, 12 Jul 2023 21:03:41 +0100
Subject: [PATCH] Put data to DB in unordered manner

---
 src/main.py      |  2 +-
 src/mongo.py     |  4 ++--
 src/timetable.py | 30 +++++++++++++++++-------------
 3 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/src/main.py b/src/main.py
index 2becc68..28e616f 100644
--- a/src/main.py
+++ b/src/main.py
@@ -14,7 +14,7 @@
 #     program.  If not, see
 #     https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE
 
-version = "2023.7.1"
+version = "2023.7.4"
 print(f"main.py: Initialising db-manager v{version}")
 
 #Third Party Imports
diff --git a/src/mongo.py b/src/mongo.py
index ead16a4..6ebb314 100644
--- a/src/mongo.py
+++ b/src/mongo.py
@@ -55,7 +55,7 @@ def putMany(collection :str, data :list, indexed_fields :list = []):
   log.out(f"mongo.putMany: Inserting many documents to: {collection}")
   col = db[collection]
   incrementCounter(collection)
-  col.insert_many(data)
+  col.insert_many(data, ordered= False)
   metaUpdateTime(collection)
   for item in indexed_fields:
     createSingleIndex(collection, item)
@@ -92,7 +92,7 @@ def putTimetable(data):
     log.out(f"mongo.putTimetable: Adding {data_length} documents to the database")
     collection = "timetable"
     col = db[collection]
-    res = col.insert_many(data)
+    res = col.insert_many(data,ordered = False)
 
 def dropCollection(collection):
     log.out(f"mongo.dropCollection: Dropping collection '{collection}'")
diff --git a/src/timetable.py b/src/timetable.py
index c921e1e..ae870e6 100644
--- a/src/timetable.py
+++ b/src/timetable.py
@@ -34,7 +34,7 @@ todayDay       = now.strftime("%a").lower()
 isAfter0800    = (int(now.strftime("%H")) >= 8)
 filePath       = "cif_data"
 fullDataUrl    = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full"
-updateDataUrl  = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-update-{yesterdayDay}"
+updateDataUrl  = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-{yesterdayDay}"
 CORPUS_USER    = os.getenv('OWL_LDB_CORPUSUSER')
 CORPUS_PASS    = os.getenv('OWL_LDB_CORPUSPASS')
 
@@ -58,6 +58,7 @@ def isUpdateRequired():
 
 def getTimetable(full :bool = False):
   downloadUrl :str = fullDataUrl if full else updateDataUrl
+  log.out(f"timetable.getTimetable: Attempting to download timetable data from {downloadUrl}")
   response = requests.get(downloadUrl, auth=(CORPUS_USER, CORPUS_PASS))
   mongo.incrementCounter("schedule_api")
   log.out(f"timetable.getTimetable: Fetch (Full:{full}) response: {response.status_code}", "DBUG")
@@ -90,15 +91,18 @@ def loopTimetable(data):
 
 def runUpdate():
   required = isUpdateRequired()
-  if (required == "full"):
-    log.out("timetable.runUpdate: Fetching full timetable data", "INFO")
-    data = getTimetable(full = True)
-  elif (required == "update"):
-    log.out("timetable.runUpdate: Fetching todays timetable update", "INFO")
-    data = getTimetable()
-  else:
-    log.out("timetable.runUpdate: timetable update is not needed", "INFO")
-    return "done"
+  try:
+    if (required == "full"):
+      log.out("timetable.runUpdate: Fetching full timetable data", "INFO")
+      data = getTimetable(full = True)
+    elif (required == "update"):
+      log.out("timetable.runUpdate: Fetching todays timetable update", "INFO")
+      data = getTimetable()
+    else:
+      log.out("timetable.runUpdate: timetable update is not needed", "INFO")
+      return "done"
+  except Exception as e:
+    log.out(f"timetable.runUpdate: Error checking for update timetable: {e}")
   parsed = loopTimetable(data)
   status = _insertToDb(parsed, required)
   if (status):
@@ -139,21 +143,21 @@ def insertSchedule(sch_record):
 
 def _insertToDb(data :list, type :str):
   pre_count = mongo.getLength("timetable")
+  create_transactions = []
   try:
     if type == "full":
        mongo.dropCollection("timetable")
        mongo.putTimetable(data)
        mongo.createSingleIndex("timetable", "headcode")
     elif type == "update":
-      create_transactions = []
       for item in data:
         if item['transactionType'] == "Create":
-          create_transactions.append([item])
+          create_transactions.append(item)
         elif item['transactionType'] == "Delete":
           mongo.deleteTimetableData({'trainUid': item['trainUid'], 'scheduleStartDate': item['scheduleStartDate'], 'stpIndicator': item['stpIndicator']})
       mongo.putTimetable(create_transactions)
     post_count = mongo.getLength("timetable")
-    log.out(f"timetable._insertToDb: Document count difference after processing: {pre_count - post_count}", "DBUG")
+    log.out(f"timetable._insertToDb: Document count difference after processing: {post_count - pre_count}", "DBUG")
     return True #If Successfuls
   except Exception as e:
     log.out("timetable._insertToDb: Error inserting timetable data", "ERR")