db-manager/src/main.py

#     db-manager - Builds and manages an OwlBoard database instance - To be run on a
#     cron schedule
#     Copyright (C) 2023  Frederick Boniface

#     This program is free software: you can redistribute it and/or modify it under the
#     terms of the GNU General Public License as published by the Free Software
#     Foundation, either version 3 of the License, or (at your option) any later version.

#     This program is distributed in the hope that it will be useful, but WITHOUT ANY
#     WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
#     PARTICULAR PURPOSE.  See the GNU General Public License for more details.

#     You should have received a copy of the GNU General Public License along with this
#     program.  If not, see
#     https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE

version = "2023.5.7"
print(f"main.py: Initialising db-manager v{version}")

#Third Party Imports
import os
import time

#Local Imports
import corpus, mongo, pis, mailer, timetable
import logger as log

log.out("main.py: db-manager Initialised", "INFO")


#Ensure count document exists in meta, wrap in while look to prevent crashing if the DB is not ready:
dbReady = False
while dbReady is False:
    try:
        mongo.metaCounters()
        dbReady = True
    except:
        dbReady = False

#Check & Update corpus/stations:
  # If older than 12 days then update
corpusAge = int(time.time()) - mongo.metaCheckTime("corpus")
log.out(f'main.py: Corpus is {corpusAge}s old', "INFO")
if corpusAge > 1036800:
    log.out('main.py: Updating CORPUS data', "INFO")
    corpusData = corpus.removeEmpty(corpus.fetch())
    mongo.putBulkCorpus(corpusData)
else:
    log.out('main.py: Not updating CORPUS data until it is 1036800s old.', "INFO")

stationsAge = int(time.time()) - mongo.metaCheckTime("stations")
log.out(f'main.py: Stations is {stationsAge}s old', "INFO")
# While the source of stations data is CORPUS, this statement is based on corpusAge, when/if changing the source, it should be changed to use stationsAge
# if stationsAge is used now, there could be a situation where stationsAge tries to update but fails as corpusData doesn't exist.
if corpusAge > 1036800:
    log.out('main.py: Updating stations data', "INFO")
    stationData = corpus.onlyStations(corpusData)
    mongo.putBulkStations(stationData)
else:
    log.out('main.py: Not updating stations data until it is 1036800s old.', "INFO")

#Check & Update pis data:
  # If older than 2 days then update
pisAge = int(time.time()) - mongo.metaCheckTime("pis")
log.out(f'main.py: PIS Data is {pisAge}s old', "INFO")
if pisAge > 43200: # Temporarily set to 15 minutes
    log.out('main.py: Updating PIS data', "INFO")
    pisData = pis.load()
    pisParsed = pis.parse(pisData)
    mongo.putBulkPis(pisParsed)
else:
    log.out('main.py: Not updating PIS data until is it 1036800s old', "INFO")

## Run Timetable Update
timetable.runUpdate()

log.out('main.py: Requesting TTL Index Creation', "INFO")
mongo.createTtlIndex("users", "atime", 2629800)
mongo.createTtlIndex("registrations", "time", 1800)

# Push version number to database for reporting
mongo.putVersion(version)

# END
log.out(f"main.py: db-manager v{version} Complete", "INFO")
log.out(f"main.py: Mailing logs")
mailer.submitLogs()