import os from pymongo import MongoClient import time import urllib.parse import logger as log log.out("mongo.py: Fetching configuration", "INFO") db_host = os.getenv('OWL_DB_HOST', 'localhost') db_port = os.getenv('OWL_DB_PORT', 27017) db_user = urllib.parse.quote_plus(os.getenv('OWL_DB_USER', "owl")) db_pass = urllib.parse.quote_plus(os.getenv('OWL_DB_PASS', "twittwoo")) db_name = os.getenv('OWL_DB_NAME', "owlboard") log.out(f"mongo.py: Connecting to database at {db_host}:{db_port}", "INFOm") client = MongoClient(f"mongodb://{db_user}:{db_pass}@{db_host}:{db_port}") db = client[db_name] def metaCheckTime(target): col = db["meta"] res = col.find_one({"target": target, "type": "collection"}) log.out(f'mongo.metaUpdateTime: {target} last updated at {res["updated"]}', "INFO") return res["updated"] def metaUpdateTime(target): col = db["meta"] log.out(f'mongo.metaUpdateTime: Updating updated time for {target}', "INFO") res = col.update_one({"target": target, "type":"collection"}, {"$set":{"updated": int(time.time()),"target":target, "type":"collection"}}, upsert=True) def getLength(collection): col = db[collection] return col.count_documents({}) def putBulkCorpus(data): startCount = getLength("corpus") col = db["corpus"] if startCount > 0: log.out(f'mongo.putBulkCorpus: Dropping {startCount} CORPUS documents', "INFO") col.drop() col.insert_many(data) endCount = getLength("corpus") log.out(f'mongo.putBulkCorpus: {endCount} documents inserted', "INFO") log.out(f'mongo.putBulkCorpus: {endCount - startCount} new documents', "INFO") #log.out('mongo.putBulkCorpus: Building collection indexes',"INFO") # ADD INDEXES HERE log.out('mongo.putBulkCorpus: Updating meta time',"INFO") metaUpdateTime("corpus") return def putBulkStations(data): startCount = getLength("stations") col = db["stations"] if startCount > 0: log.out(f'mongo.putBulkStations: Dropping {startCount} station documents', "INFO") col.drop() col.insert_many(data) endCount = getLength("stations") log.out(f'mongo.putBulkStations: {endCount} documents inserted', "INFO") log.out(f'mongo.putBulkStations: {endCount - startCount} new documents', "INFO") #log.out('mongo.putBulkStations: Building collection indexes',"INFO") # ADD INDEXES HERE log.out('mongo.putBulkStations: Updating meta time',"INFO") metaUpdateTime("stations") return