diff --git a/src/corpus.py b/src/corpus.py index 91f5d3c..2e63767 100644 --- a/src/corpus.py +++ b/src/corpus.py @@ -4,6 +4,7 @@ import requests import logger as log import zlib import json +import mongo CORPUS_URL = "https://publicdatafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS" @@ -15,6 +16,7 @@ CORPUS_PASS = os.getenv('OWL_LDB_CORPUSPASS') def fetch(): log.out("corpus.fetch: Fetching CORPUS Data from Network Rail", "INFO") response = requests.get(CORPUS_URL, auth=(CORPUS_USER, CORPUS_PASS)) + incrementCounter("corpus_api") log.out("corpus.fetch: Decompressing & parsing CORPUS data", "INFO") parsed = json.loads(zlib.decompress(response.content, 16+zlib.MAX_WBITS).decode()) return parsed['TIPLOCDATA'] diff --git a/src/main.py b/src/main.py index 8c1f77a..901b20b 100644 --- a/src/main.py +++ b/src/main.py @@ -14,7 +14,7 @@ # program. If not, see # https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE -version = "0.1.7" +version = "1.0.0" print(f"main.py: Initialising db-manager v{version}") #Third Party Imports diff --git a/src/mongo.py b/src/mongo.py index 2cde168..08380a2 100644 --- a/src/mongo.py +++ b/src/mongo.py @@ -18,6 +18,7 @@ db = client[db_name] def metaCheckTime(target): col = db["meta"] res = col.find_one({"target": target, "type": "collection"}) + incrementCounter("meta") if type(res) is dict: if 'updated' in res: log.out(f'mongo.metaUpdateTime: {target} last updated at {res["updated"]}', "INFO") @@ -29,22 +30,25 @@ def metaUpdateTime(target): col = db["meta"] log.out(f'mongo.metaUpdateTime: Updating updated time for {target}', "INFO") res = col.update_one({"target": target, "type":"collection"}, {"$set":{"updated": int(time.time()),"target":target, "type":"collection"}}, upsert=True) + incrementCounter("meta") def getLength(collection): col = db[collection] + incrementCounter(collection) return col.count_documents({}) def createSingleIndex(collection, field): col = db[collection] col.create_index(field) + incrementCounter(collection) log.out(f'mongo.createSingleIndex: Created index of {field} in {collection}', "INFO") return - def putBulkCorpus(data): collection = "corpus" startCount = getLength(collection) col = db[collection] + incrementCounter(collection) if startCount > 0: log.out(f'mongo.putBulkCorpus: Dropping {startCount} CORPUS documents', "INFO") col.drop() @@ -63,6 +67,7 @@ def putBulkStations(data): collection = "stations" startCount = getLength(collection) col = db[collection] + incrementCounter("stations") if startCount > 0: log.out(f'mongo.putBulkStations: Dropping {startCount} station documents', "INFO") col.drop() @@ -78,9 +83,16 @@ def putBulkStations(data): metaUpdateTime(collection) return +def incrementCounter(target): + collection = "meta" + col = db[collection] + col.update_one({"target": "counters"}, {"$inc":target}) + return + def metaCounters(): collection = "meta" col = db[collection] + incrementCounter(collection) res = col.find_one({"target": "counters","type": "count"}) log.out(f'mongo.metaCounters: Query returned `{res}`', "DEBG") if type(res) is dict: