Update modules to download data via HTTP

src/mailer.py

@@ -41,18 +41,19 @@ def fetchLogs():
   with open("dbman-log", "r") as tmpfile:
     return tmpfile.read()
 
-def deleteLogs():
+def tidyWorkingFiles():
   if os.path.exists("dbman-log"):
     os.remove("dbman-log")
     print("Tidied log file")
-  else:
-    print("No logfile to tidy")
   if os.path.exists("cif_data"):
     os.remove("cif_data")
     print("Removed cif_data file")
-  else:
-    print("No cif_data to tidy")
-
+  if os.path.exists("gw.yaml"):
+    os.remove("gw.yaml")
+    print("Tidied PIS Data")
+  if os.path.exists("reasoncodes.json"):
+    os.remove("reasoncodes.json")
+    print("Tidies reason code data")
 
 def sendMail(msg_body :str, retry):
   message = MIMEMultipart()
@@ -88,4 +89,4 @@ def sendMail(msg_body :str, retry):
     else:
       print("Error retrying to mail logs, giving up")
   finally:
-    deleteLogs()
+    tidyWorkingFiles()

src/pis.py

@@ -1,14 +1,20 @@
-import yaml, hashlib
+import yaml, hashlib, urllib.request
 import logger as log
 import mongo
 
 REBUILD :bool = False  # Set to True to force rebuild
 
 log.out("pis.py: PIS Module Loaded", "DBUG")
-file_location :str = "/app/data/pis/gwr.yaml" # Production & Testing
+#file_location :str = "/app/data/pis/gwr.yaml" # Production & Testing
 #file_location :str = "/home/fred.boniface/git/owlboard/db-manager/data/pis/gwr.yaml" # Local Development
+file_location :str = "gw.yaml"
+file_url :str = "https://git.fjla.uk/OwlBoard/data/raw/branch/main/pis/gw.yaml"
 
 def runUpdate():
+  state = download()
+  if (not state):
+    log.out('pis.runUpdate: Not updating PIS, unable to download data')
+    return
   if (not requiresUpdate()):
     log.out('pis.runUpdate: PIS Codes do not need updating', 'INFO')
     return
@@ -35,6 +41,15 @@ def requiresUpdate():
   log.out("pis.requiredUpdate: PIS Data is up to date", "INFO")
   return False
 
+def download():
+  log.out("pis.download: Downloading GW PIS file")
+  try:
+    urllib.request.urlretrieve(file_url, file_location)
+    return True
+  except Exception as e:
+    log.out(f"pis.download: Download error: {e}")
+    return False
+
 def load():
     with open(file_location, "r") as data:
         try:

src/reasonCodes.py

@@ -1,4 +1,4 @@
-import json, hashlib
+import json, hashlib, urllib.request
 import mongo
 import logger as log
 
@@ -6,10 +6,13 @@ REBUILD = False
 
 log.out("reasonCodes.py: reasonCodes module initialised", "DBUG")
 
-file_location :str = "/app/data/reasonCodes/reasoncodes.json" # Production & Testing
-#file_location :str = "/home/fred.boniface/git/owlboard/db-manager/data/reasonCodes/reasoncodes.json" # Local Development
+file_location :str = "reasoncodes.json"
+file_url :str = "https://git.fjla.uk/OwlBoard/data/raw/branch/main/reasonCodes/reasoncodes.json"
 
 def runUpdate():
+  state = download()
+  if (not state):
+    log.out("reasonCodes.runUpdate: Unable to update reason codes")
   if (not requiresUpdate()):
     log.out('reasonCodes.runUpdate: Reason codes do not need updating', 'INFO')
     return
@@ -19,6 +22,15 @@ def runUpdate():
   mongo.dropCollection("reasonCodes")
   mongo.putMany("reasonCodes", reason_code_data, reason_code_indexes)
 
+def download():
+  log.out("pis.download: Downloading reason codes file")
+  try:
+    urllib.request.urlretrieve(file_url, file_location)
+    return True
+  except Exception as e:
+    log.out(f"pis.download: Download error: {e}")
+    return False
+
 def requiresUpdate():
   if REBUILD:
     return True