db-manager/src/pis.py

import yaml, hashlib
import logger as log
import mongo

REBUILD :bool = False  # Set to True to force rebuild

log.out("pis.py: PIS Module Loaded", "DBUG")
file_location :str = "/app/data/pis/gwr.yaml" # Production & Testing
#file_location :str = "/home/fred.boniface/git/owlboard/db-manager/data/pis/gwr.yaml" # Local Development

def runUpdate():
  if (not requiresUpdate()):
    log.out('pis.runUpdate: PIS Codes do not need updating', 'INFO')
    return
  log.out(f"pis.runUpdate: Update required", "INFO")
  pis_data = load()
  pis_parsed = parse(pis_data)
  mongo.dropCollection("pis")
  mongo.putMany("pis", pis_parsed)
  mongo.createSingleIndex("pis", "stops")
  mongo.createSingleIndex("pis", "tiplocs")
  
def requiresUpdate():
  if REBUILD:
    return True
  currentHash = mongo.getMetaHash("pis")
  with open(file_location, "r") as f:
    text = f.read()
    newHash = hashlib.md5(text.encode()).hexdigest()
  log.out(f"pis.requiresUpdate: Existing PIS Hash: {currentHash}","INFO")
  log.out(f"pis.requiresUpdate: New PIS hash: {newHash}", "INFO")
  if currentHash is None or newHash != currentHash:
    log.out("pis.requiredUpdate: PIS Data requires updating", "INFO")
    mongo.putMetaHash("pis", newHash)
    return True
  log.out("pis.requiredUpdate: PIS Data is up to date", "INFO")
  return False

def load():
    with open(file_location, "r") as data:
        try:
            pis = yaml.safe_load(data)
            return pis["pis"]
        except yaml.YAMLError as exc:
            print(exc)
            return exc

def parse(codeList):
    StartLen = len(codeList)
    log.out(f"pis.parse: codeList starting length: {StartLen}", "DBUG")
    log.out(f"pis.parse: Removing duplicate codes & adding TIPLOCs")
    for i in codeList:
        stops = i['stops']
        print(stops)
        code = i['code']
        for ii in codeList:
            if stops == ii['stops'] and code != ii['code']:
                log.out(f"Identical stopping pattern found: {ii['code']}","DBUG")
                codeList.remove(ii) # Instead of removing, I should add a property (duplicate: true), then I can filter this out on the backend when searching by start and end stations and just use query one for other queries, this means that when searching by code, a perfectly valid code won't show 0 results.
        tiplocs = []
        for iii in stops:
          print(iii)
          tiplocs.append(getTiploc(iii))
        i['tiplocs'] = tiplocs
    print(f"pis.parse: Removed {StartLen - len(codeList)} duplicates")
    return codeList

def getTiploc(crs :str):
  CRS = crs.upper()
  query = {
    '3ALPHA': CRS
  }
  try:
    res = mongo.query("stations", query)
    if 'TIPLOC' in res:
      return res['TIPLOC']
  except Exception as e:
    log.out(f"pis.getTiploc: Error finding tiploc: {query}", "EROR")
    log.out(f"ERROR: {e}", "EROR")