Add PIS Parsing code

requirements.txt

@@ -2,6 +2,8 @@ certifi==2022.12.7
 charset-normalizer==3.0.1
 dnspython==2.3.0
 idna==3.4
+pyaml==21.10.1
 pymongo==4.3.3
+PyYAML==6.0
 requests==2.28.2
 urllib3==1.26.14

src/main.py

@@ -14,7 +14,7 @@
 #     program.  If not, see
 #     https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE
 
-version = "2.0.0-dev"
+version = "2.1.0-dev"
 print(f"main.py: Initialising db-manager v{version}")
 
 #Third Party Imports
@@ -22,7 +22,7 @@ import os
 import time
 
 #Local Imports
-import corpus, mongo
+import corpus, mongo, pis
 import logger as log
 
 log.out("main.py: db-manager Initialised", "INFO")
@@ -47,11 +47,22 @@ log.out(f'main.py: Stations is {stationsAge}s old', "INFO")
 # While the source of stations data is CORPUS, this statement is based on corpusAge, when/if changing the source, it should be changed to use stationsAge
 # if stationsAge is used now, there could be a situation where stationsAge tries to update but fails as corpusData doesn't exist.
 if corpusAge > 1036800:
-    log.out('main.py: Updating stations data until it is 1036800s old.', "INFO")
+    log.out('main.py: Updating stations data', "INFO")
     stationData = corpus.onlyStations(corpusData)
     mongo.putBulkStations(stationData)
 else:
-    log.out('main.py: Not updating stations data', "INFO")
+    log.out('main.py: Not updating stations data until it is 1036800s old.', "INFO")
+
+#Check & Update pis data:
+  # If older than 12 days then update
+pisAge = int(time.time()) - mongo.metaCheckTime("pis")
+log.out(f'main.py: PIS Data is {pisAge}s old', "INFO")
+if pisAge > 1036800:
+    log.out('main.py: Updating PIS data', "INFO")
+    pisData = pis.load()
+    mongo.putBulkPis(pisData)
+else:
+    log.out('main.py: Not updating PIS data until is it 1036800a old', "INFO")
 
 log.out('main.py: Requesting TTL Index Creation', "INFO")
 mongo.createTtlIndex("users", "atime", 2629800)

src/mongo.py

@@ -88,6 +88,23 @@ def putBulkStations(data):
     metaUpdateTime(collection)
     return
 
+def putBulkPis(data):
+    collection = "pis"
+    startCount = getLength(collection)
+    col = db[collection]
+    incrementCounter(collection)
+    if startCount > 0:
+        log.out(f'mongo.putBulkPid: Dropping {startCount} pis documents', "INFO")
+        col.drop()
+    col.insert_many(data)
+    endCount = getLength(collection)
+    log.out(f'mongo.putBulkPis: {endCount} documents inserted', "INFO")
+    log.out(f'mongo.putBulkPis: {endCount-startCount} new documents', "INFO")
+    log.out('mongo.putBulkPis: Updating meta time', "INFO")
+    metaUpdateTime(collection)
+    return
+
+
 def incrementCounter(target):
     collection = "meta"
     col = db[collection]

src/pis.py

@@ -0,0 +1,11 @@
+import yaml
+
+def load():
+    with open("../data/pis/gwr-west.yaml", "r") as data:
+        try:
+            pis = yaml.safe_load(data)
+            print(pis)
+            return pis["pis"]
+        except yaml.YAMLError as exc:
+            print(exc)
+            return exc