pis-entry #1
@ -2,6 +2,8 @@ certifi==2022.12.7
|
||||
charset-normalizer==3.0.1
|
||||
dnspython==2.3.0
|
||||
idna==3.4
|
||||
pyaml==21.10.1
|
||||
pymongo==4.3.3
|
||||
PyYAML==6.0
|
||||
requests==2.28.2
|
||||
urllib3==1.26.14
|
||||
|
19
src/main.py
19
src/main.py
@ -14,7 +14,7 @@
|
||||
# program. If not, see
|
||||
# https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE
|
||||
|
||||
version = "2.0.0-dev"
|
||||
version = "2.1.0-dev"
|
||||
print(f"main.py: Initialising db-manager v{version}")
|
||||
|
||||
#Third Party Imports
|
||||
@ -22,7 +22,7 @@ import os
|
||||
import time
|
||||
|
||||
#Local Imports
|
||||
import corpus, mongo
|
||||
import corpus, mongo, pis
|
||||
import logger as log
|
||||
|
||||
log.out("main.py: db-manager Initialised", "INFO")
|
||||
@ -47,11 +47,22 @@ log.out(f'main.py: Stations is {stationsAge}s old', "INFO")
|
||||
# While the source of stations data is CORPUS, this statement is based on corpusAge, when/if changing the source, it should be changed to use stationsAge
|
||||
# if stationsAge is used now, there could be a situation where stationsAge tries to update but fails as corpusData doesn't exist.
|
||||
if corpusAge > 1036800:
|
||||
log.out('main.py: Updating stations data until it is 1036800s old.', "INFO")
|
||||
log.out('main.py: Updating stations data', "INFO")
|
||||
stationData = corpus.onlyStations(corpusData)
|
||||
mongo.putBulkStations(stationData)
|
||||
else:
|
||||
log.out('main.py: Not updating stations data', "INFO")
|
||||
log.out('main.py: Not updating stations data until it is 1036800s old.', "INFO")
|
||||
|
||||
#Check & Update pis data:
|
||||
# If older than 12 days then update
|
||||
pisAge = int(time.time()) - mongo.metaCheckTime("pis")
|
||||
log.out(f'main.py: PIS Data is {pisAge}s old', "INFO")
|
||||
if pisAge > 1036800:
|
||||
log.out('main.py: Updating PIS data', "INFO")
|
||||
pisData = pis.load()
|
||||
mongo.putBulkPis(pisData)
|
||||
else:
|
||||
log.out('main.py: Not updating PIS data until is it 1036800a old', "INFO")
|
||||
|
||||
log.out('main.py: Requesting TTL Index Creation', "INFO")
|
||||
mongo.createTtlIndex("users", "atime", 2629800)
|
||||
|
17
src/mongo.py
17
src/mongo.py
@ -88,6 +88,23 @@ def putBulkStations(data):
|
||||
metaUpdateTime(collection)
|
||||
return
|
||||
|
||||
def putBulkPis(data):
|
||||
collection = "pis"
|
||||
startCount = getLength(collection)
|
||||
col = db[collection]
|
||||
incrementCounter(collection)
|
||||
if startCount > 0:
|
||||
log.out(f'mongo.putBulkPid: Dropping {startCount} pis documents', "INFO")
|
||||
col.drop()
|
||||
col.insert_many(data)
|
||||
endCount = getLength(collection)
|
||||
log.out(f'mongo.putBulkPis: {endCount} documents inserted', "INFO")
|
||||
log.out(f'mongo.putBulkPis: {endCount-startCount} new documents', "INFO")
|
||||
log.out('mongo.putBulkPis: Updating meta time', "INFO")
|
||||
metaUpdateTime(collection)
|
||||
return
|
||||
|
||||
|
||||
def incrementCounter(target):
|
||||
collection = "meta"
|
||||
col = db[collection]
|
||||
|
11
src/pis.py
Normal file
11
src/pis.py
Normal file
@ -0,0 +1,11 @@
|
||||
import yaml
|
||||
|
||||
def load():
|
||||
with open("../data/pis/gwr-west.yaml", "r") as data:
|
||||
try:
|
||||
pis = yaml.safe_load(data)
|
||||
print(pis)
|
||||
return pis["pis"]
|
||||
except yaml.YAMLError as exc:
|
||||
print(exc)
|
||||
return exc
|
Reference in New Issue
Block a user