Programatically remove duplicate PIS codes

src/main.py

@@ -14,7 +14,7 @@
 #     program.  If not, see
 #     https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE
 
-version = "2.1.0"
+version = "2.1.1"
 print(f"main.py: Initialising db-manager v{version}")
 
 #Third Party Imports
@@ -60,7 +60,8 @@ log.out(f'main.py: PIS Data is {pisAge}s old', "INFO")
 if pisAge > 3600: # Temporarily set to four hours
     log.out('main.py: Updating PIS data', "INFO")
     pisData = pis.load()
-    mongo.putBulkPis(pisData)
+    pisParsed = pis.parse(pisData)
+    mongo.putBulkPis(pisParsed)
 else:
     log.out('main.py: Not updating PIS data until is it 1036800a old', "INFO")
 

src/pis.py

@@ -1,5 +1,7 @@
 import yaml
 
+print("PIS Module imported")
+
 def load():  # Programatically add a `toc` field to each entry.
     with open("/app/data/pis/gwr.yaml", "r") as data:
         try:
@@ -11,4 +13,31 @@ def load():  # Programatically add a `toc` field to each entry.
             return exc
 
 ## Do some magic here so that if any pis["pis"]["stops"][0] field contains 'reverse' then get the stops for the code stored in pis["pis"]["stops"][1]
-## reverse the stops and store that.
+## reverse the stops and store that.
+
+def parse(codeList):
+    print(f"pis.parse: codeList starting length: {len(codeList)}")
+    for i in codeList:
+        stops = i['stops']
+        code = i['code']
+        for ii in codeList:
+            if stops == ii['stops'] and code != ii['code']:
+                print("Identical stopping pattern found")
+                print(ii)
+                codeList.remove(ii)
+    print(f"pis.parse: codeList finishing length: {len(codeList)}")
+    return codeList
+
+def devLoad():  # Programatically add a `toc` field to each entry.
+    with open("/home/fred.boniface/git/owlboard/db-manager/data/pis/gwr.yaml", "r") as data:
+        try:
+            pis = yaml.safe_load(data)
+            print(pis)
+            return pis["pis"]
+        except yaml.YAMLError as exc:
+            print(exc)
+            return exc
+
+def dev():
+    data = devLoad()
+    parse(data)