pis-entry #1

Merged
fred.boniface merged 23 commits from pis-entry into dev 2023-05-06 21:51:54 +01:00
4 changed files with 45 additions and 4 deletions
Showing only changes of commit 0bc82d03f9 - Show all commits

View File

@ -2,6 +2,8 @@ certifi==2022.12.7
charset-normalizer==3.0.1
dnspython==2.3.0
idna==3.4
pyaml==21.10.1
pymongo==4.3.3
PyYAML==6.0
requests==2.28.2
urllib3==1.26.14

View File

@ -14,7 +14,7 @@
# program. If not, see
# https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE
version = "2.0.0-dev"
version = "2.1.0-dev"
print(f"main.py: Initialising db-manager v{version}")
#Third Party Imports
@ -22,7 +22,7 @@ import os
import time
#Local Imports
import corpus, mongo
import corpus, mongo, pis
import logger as log
log.out("main.py: db-manager Initialised", "INFO")
@ -47,11 +47,22 @@ log.out(f'main.py: Stations is {stationsAge}s old', "INFO")
# While the source of stations data is CORPUS, this statement is based on corpusAge, when/if changing the source, it should be changed to use stationsAge
# if stationsAge is used now, there could be a situation where stationsAge tries to update but fails as corpusData doesn't exist.
if corpusAge > 1036800:
log.out('main.py: Updating stations data until it is 1036800s old.', "INFO")
log.out('main.py: Updating stations data', "INFO")
stationData = corpus.onlyStations(corpusData)
mongo.putBulkStations(stationData)
else:
log.out('main.py: Not updating stations data', "INFO")
log.out('main.py: Not updating stations data until it is 1036800s old.', "INFO")
#Check & Update pis data:
# If older than 12 days then update
pisAge = int(time.time()) - mongo.metaCheckTime("pis")
log.out(f'main.py: PIS Data is {pisAge}s old', "INFO")
if pisAge > 1036800:
log.out('main.py: Updating PIS data', "INFO")
pisData = pis.load()
mongo.putBulkPis(pisData)
else:
log.out('main.py: Not updating PIS data until is it 1036800a old', "INFO")
log.out('main.py: Requesting TTL Index Creation', "INFO")
mongo.createTtlIndex("users", "atime", 2629800)

View File

@ -88,6 +88,23 @@ def putBulkStations(data):
metaUpdateTime(collection)
return
def putBulkPis(data):
collection = "pis"
startCount = getLength(collection)
col = db[collection]
incrementCounter(collection)
if startCount > 0:
log.out(f'mongo.putBulkPid: Dropping {startCount} pis documents', "INFO")
col.drop()
col.insert_many(data)
endCount = getLength(collection)
log.out(f'mongo.putBulkPis: {endCount} documents inserted', "INFO")
log.out(f'mongo.putBulkPis: {endCount-startCount} new documents', "INFO")
log.out('mongo.putBulkPis: Updating meta time', "INFO")
metaUpdateTime(collection)
return
def incrementCounter(target):
collection = "meta"
col = db[collection]

11
src/pis.py Normal file
View File

@ -0,0 +1,11 @@
import yaml
def load():
with open("../data/pis/gwr-west.yaml", "r") as data:
try:
pis = yaml.safe_load(data)
print(pis)
return pis["pis"]
except yaml.YAMLError as exc:
print(exc)
return exc