Add creation of counters document

This commit is contained in:
Fred Boniface 2023-02-12 21:36:41 +00:00
parent 171291408c
commit 0afcbb4633
3 changed files with 26 additions and 5 deletions

View File

@ -40,13 +40,19 @@ def removeEmpty(data):
def onlyStations(data): def onlyStations(data):
# DATA: List of Dictionaries # DATA: List of Dictionaries
# Iterates through the list and checks each item (which is a dictionary) for the STANOX key # Iterates through the list and checks each item (which is a dictionary) for the STANOX key
# If the 3ALPHA exists the dictionary will be appended to the `workingList` which is returned once iteration is complete # If the 3ALPHA exists the relevant fields will be appended to the `workingDict` which is appended to the workingList
# the workingList is finally returned.
preLength = len(data) preLength = len(data)
log.out(f"corpus.onlyStations: Removing items without `3ALPHA` from {preLength} documents", "INFO") log.out(f"corpus.onlyStations: Removing items without `3ALPHA` from {preLength} documents", "INFO")
workingList = [] workingList = []
for item in data: for item in data:
workingDict = {}
if '3ALPHA' in item: if '3ALPHA' in item:
workingList.append(item) workingDict.update(item)
workingDict.pop("NLC", False)
workingDict.pop("NLCDESC16", False)
workingDict.pop("UIC", False)
workingList.append(workingDict)
postLength = len(workingList) postLength = len(workingList)
log.out(f"corpus.onlyStations: Removed {preLength - postLength} documents", "INFO") log.out(f"corpus.onlyStations: Removed {preLength - postLength} documents", "INFO")
log.out(f"Yes, I am aware there are not {postLength} stations but the data includes NI, TfL, some bus, tram and closed stations too","OTHR") log.out(f"Yes, I am aware there are not {postLength} stations but the data includes NI, TfL, some bus, tram and closed stations too","OTHR")

View File

@ -50,7 +50,7 @@ else:
log.out('main.py: Not updating stations data', "INFO") log.out('main.py: Not updating stations data', "INFO")
#Ensure count document exists in meta: #Ensure count document exists in meta:
mongo.metaCounters()
# END # END
log.out(f"main.py: db-manager v{version} Complete", "INFO") log.out(f"main.py: db-manager v{version} Complete", "INFO")

View File

@ -18,8 +18,12 @@ db = client[db_name]
def metaCheckTime(target): def metaCheckTime(target):
col = db["meta"] col = db["meta"]
res = col.find_one({"target": target, "type": "collection"}) res = col.find_one({"target": target, "type": "collection"})
log.out(f'mongo.metaUpdateTime: {target} last updated at {res["updated"]}', "INFO") if 'updated' in res:
return res["updated"] log.out(f'mongo.metaUpdateTime: {target} last updated at {res["updated"]}', "INFO")
return res["updated"]
else:
log.out(f'mongo.metaUpdatetime: {target} does not exist', "INFO")
return 0
def metaUpdateTime(target): def metaUpdateTime(target):
col = db["meta"] col = db["meta"]
@ -72,4 +76,15 @@ def putBulkStations(data):
createSingleIndex(collection, "TIPLOC") createSingleIndex(collection, "TIPLOC")
log.out('mongo.putBulkStations: Updating meta time',"INFO") log.out('mongo.putBulkStations: Updating meta time',"INFO")
metaUpdateTime(collection) metaUpdateTime(collection)
return
def metaCounters():
collection = "meta"
col = db[collection]
res = col.find_one({"target": "counters","type": "count"})
if 'since' not in res:
log.out('mongo.metaCounters: counters does not exist, creating', "INFO")
col.update_one({"target": "counters","type": "count"}, {"target": "counters","type": "count","since": int(time.time())})
else:
log.out('mongo.metaCounters: counters already exists, skipping', "INFO")
return return