OwlBoard/db-manager
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Releases Activity

Update modules to download data via HTTP

Browse Source
This commit is contained in:
Fred Boniface 2023-07-12 19:49:05 +01:00
parent 9491be73b9
commit bc62408313
4 changed files with 25 additions and 2326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2316
data/pis/gwr.yaml
View File

File diff suppressed because it is too large Load Diff

1
data/reasonCodes/reasoncodes.json
View File

File diff suppressed because one or more lines are too long

15
src/mailer.py
View File

@ -41,18 +41,19 @@ def fetchLogs():
with open("dbman-log", "r") as tmpfile: with open("dbman-log", "r") as tmpfile:
return tmpfile.read() return tmpfile.read()
def deleteLogs(): def tidyWorkingFiles():
if os.path.exists("dbman-log"): if os.path.exists("dbman-log"):
os.remove("dbman-log") os.remove("dbman-log")
print("Tidied log file") print("Tidied log file")
else:
print("No logfile to tidy")
if os.path.exists("cif_data"): if os.path.exists("cif_data"):
os.remove("cif_data") os.remove("cif_data")
print("Removed cif_data file") print("Removed cif_data file")
else: if os.path.exists("gw.yaml"):
print("No cif_data to tidy") os.remove("gw.yaml")
print("Tidied PIS Data")
if os.path.exists("reasoncodes.json"):
os.remove("reasoncodes.json")
print("Tidies reason code data")
def sendMail(msg_body :str, retry): def sendMail(msg_body :str, retry):
message = MIMEMultipart() message = MIMEMultipart()
@ -88,4 +89,4 @@ def sendMail(msg_body :str, retry):
else: else:
print("Error retrying to mail logs, giving up") print("Error retrying to mail logs, giving up")
finally: finally:
deleteLogs() tidyWorkingFiles()

19
src/pis.py
View File

@ -1,14 +1,20 @@
import yaml, hashlib import yaml, hashlib, urllib.request
import logger as log import logger as log
import mongo import mongo
REBUILD :bool = False # Set to True to force rebuild REBUILD :bool = False # Set to True to force rebuild
log.out("pis.py: PIS Module Loaded", "DBUG") log.out("pis.py: PIS Module Loaded", "DBUG")
file_location :str = "/app/data/pis/gwr.yaml" # Production & Testing #file_location :str = "/app/data/pis/gwr.yaml" # Production & Testing
#file_location :str = "/home/fred.boniface/git/owlboard/db-manager/data/pis/gwr.yaml" # Local Development #file_location :str = "/home/fred.boniface/git/owlboard/db-manager/data/pis/gwr.yaml" # Local Development
file_location :str = "gw.yaml"
file_url :str = "https://git.fjla.uk/OwlBoard/data/raw/branch/main/pis/gw.yaml"
def runUpdate(): def runUpdate():
state = download()
if (not state):
log.out('pis.runUpdate: Not updating PIS, unable to download data')
return
if (not requiresUpdate()): if (not requiresUpdate()):
log.out('pis.runUpdate: PIS Codes do not need updating', 'INFO') log.out('pis.runUpdate: PIS Codes do not need updating', 'INFO')
return return
@ -35,6 +41,15 @@ def requiresUpdate():
log.out("pis.requiredUpdate: PIS Data is up to date", "INFO") log.out("pis.requiredUpdate: PIS Data is up to date", "INFO")
return False return False
def download():
log.out("pis.download: Downloading GW PIS file")
try:
urllib.request.urlretrieve(file_url, file_location)
return True
except Exception as e:
log.out(f"pis.download: Download error: {e}")
return False
def load(): def load():
with open(file_location, "r") as data: with open(file_location, "r") as data:
try: try: