db-manager/src/mongo.py

import os
from pymongo import MongoClient
import time
import urllib.parse
import logger as log

log.out("mongo.py: Fetching configuration", "INFO")
db_host = os.getenv('OWL_DB_HOST', 'localhost')
db_port = os.getenv('OWL_DB_PORT', 27017)
db_user = urllib.parse.quote_plus(os.getenv('OWL_DB_USER', "owl"))
db_pass = urllib.parse.quote_plus(os.getenv('OWL_DB_PASS', "twittwoo"))
db_name = os.getenv('OWL_DB_NAME', "owlboard")

log.out(f"mongo.py: Connecting to database at {db_host}:{db_port}", "INFOm")
client = MongoClient(f"mongodb://{db_user}:{db_pass}@{db_host}:{db_port}")
db = client[db_name]

def metaCheckTime(target):
    col = db["meta"]
    res = col.find_one({"target": target, "type": "collection"})
    log.out(f'mongo.metaUpdateTime: {target} last updated at {res["updated"]}', "INFO")
    return res["updated"]

def metaUpdateTime(target):
    col = db["meta"]
    log.out(f'mongo.metaUpdateTime: Updating updated time for {target}', "INFO")
    res = col.update_one({"target": target, "type":"collection"}, {"$set":{"updated": int(time.time()),"target":target, "type":"collection"}}, upsert=True)

def getLength(collection):
    col = db[collection]
    return col.count_documents({})

def putBulkCorpus(data):
    startCount = getLength("corpus")
    col = db["corpus"]
    if startCount > 0:
        log.out(f'mongo.putBulkCorpus: Dropping {startCount} CORPUS documents', "INFO")
        col.drop()
    col.insert_many(data)
    endCount = getLength("corpus")
    log.out(f'mongo.putBulkCorpus: {endCount} documents inserted', "INFO")
    log.out(f'mongo.putBulkCorpus: {endCount - startCount} new documents', "INFO")
    #log.out('mongo.putBulkCorpus: Building collection indexes',"INFO")
    # ADD INDEXES HERE
    log.out('mongo.putBulkCorpus: Updating meta time',"INFO")
    metaUpdateTime("corpus")
    return

def putBulkStations(data):
    startCount = getLength("stations")
    col = db["stations"]
    if startCount > 0:
        log.out(f'mongo.putBulkStations: Dropping {startCount} station documents', "INFO")
        col.drop()
    col.insert_many(data)
    endCount = getLength("stations")
    log.out(f'mongo.putBulkStations: {endCount} documents inserted', "INFO")
    log.out(f'mongo.putBulkStations: {endCount - startCount} new documents', "INFO")
    #log.out('mongo.putBulkStations: Building collection indexes',"INFO")
    # ADD INDEXES HERE
    log.out('mongo.putBulkStations: Updating meta time',"INFO")
    metaUpdateTime("stations")
    return