Put data to DB in unordered manner
This commit is contained in:
parent
591cbb2e46
commit
ec6e425a0d
@ -14,7 +14,7 @@
|
||||
# program. If not, see
|
||||
# https://git.fjla.uk/OwlBoard/db-manager/src/branch/main/LICENSE
|
||||
|
||||
version = "2023.7.1"
|
||||
version = "2023.7.4"
|
||||
print(f"main.py: Initialising db-manager v{version}")
|
||||
|
||||
#Third Party Imports
|
||||
|
@ -55,7 +55,7 @@ def putMany(collection :str, data :list, indexed_fields :list = []):
|
||||
log.out(f"mongo.putMany: Inserting many documents to: {collection}")
|
||||
col = db[collection]
|
||||
incrementCounter(collection)
|
||||
col.insert_many(data)
|
||||
col.insert_many(data, ordered= False)
|
||||
metaUpdateTime(collection)
|
||||
for item in indexed_fields:
|
||||
createSingleIndex(collection, item)
|
||||
@ -92,7 +92,7 @@ def putTimetable(data):
|
||||
log.out(f"mongo.putTimetable: Adding {data_length} documents to the database")
|
||||
collection = "timetable"
|
||||
col = db[collection]
|
||||
res = col.insert_many(data)
|
||||
res = col.insert_many(data,ordered = False)
|
||||
|
||||
def dropCollection(collection):
|
||||
log.out(f"mongo.dropCollection: Dropping collection '{collection}'")
|
||||
|
@ -34,7 +34,7 @@ todayDay = now.strftime("%a").lower()
|
||||
isAfter0800 = (int(now.strftime("%H")) >= 8)
|
||||
filePath = "cif_data"
|
||||
fullDataUrl = "https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-full"
|
||||
updateDataUrl = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_FULL_DAILY&day=toc-update-{yesterdayDay}"
|
||||
updateDataUrl = f"https://publicdatafeeds.networkrail.co.uk/ntrod/CifFileAuthenticate?type=CIF_ALL_UPDATE_DAILY&day=toc-update-{yesterdayDay}"
|
||||
CORPUS_USER = os.getenv('OWL_LDB_CORPUSUSER')
|
||||
CORPUS_PASS = os.getenv('OWL_LDB_CORPUSPASS')
|
||||
|
||||
@ -58,6 +58,7 @@ def isUpdateRequired():
|
||||
|
||||
def getTimetable(full :bool = False):
|
||||
downloadUrl :str = fullDataUrl if full else updateDataUrl
|
||||
log.out(f"timetable.getTimetable: Attempting to download timetable data from {downloadUrl}")
|
||||
response = requests.get(downloadUrl, auth=(CORPUS_USER, CORPUS_PASS))
|
||||
mongo.incrementCounter("schedule_api")
|
||||
log.out(f"timetable.getTimetable: Fetch (Full:{full}) response: {response.status_code}", "DBUG")
|
||||
@ -90,15 +91,18 @@ def loopTimetable(data):
|
||||
|
||||
def runUpdate():
|
||||
required = isUpdateRequired()
|
||||
if (required == "full"):
|
||||
log.out("timetable.runUpdate: Fetching full timetable data", "INFO")
|
||||
data = getTimetable(full = True)
|
||||
elif (required == "update"):
|
||||
log.out("timetable.runUpdate: Fetching todays timetable update", "INFO")
|
||||
data = getTimetable()
|
||||
else:
|
||||
log.out("timetable.runUpdate: timetable update is not needed", "INFO")
|
||||
return "done"
|
||||
try:
|
||||
if (required == "full"):
|
||||
log.out("timetable.runUpdate: Fetching full timetable data", "INFO")
|
||||
data = getTimetable(full = True)
|
||||
elif (required == "update"):
|
||||
log.out("timetable.runUpdate: Fetching todays timetable update", "INFO")
|
||||
data = getTimetable()
|
||||
else:
|
||||
log.out("timetable.runUpdate: timetable update is not needed", "INFO")
|
||||
return "done"
|
||||
except Exception as e:
|
||||
log.out(f"timetable.runUpdate: Error checking for update timetable: {e}")
|
||||
parsed = loopTimetable(data)
|
||||
status = _insertToDb(parsed, required)
|
||||
if (status):
|
||||
@ -139,21 +143,21 @@ def insertSchedule(sch_record):
|
||||
|
||||
def _insertToDb(data :list, type :str):
|
||||
pre_count = mongo.getLength("timetable")
|
||||
create_transactions = []
|
||||
try:
|
||||
if type == "full":
|
||||
mongo.dropCollection("timetable")
|
||||
mongo.putTimetable(data)
|
||||
mongo.createSingleIndex("timetable", "headcode")
|
||||
elif type == "update":
|
||||
create_transactions = []
|
||||
for item in data:
|
||||
if item['transactionType'] == "Create":
|
||||
create_transactions.append([item])
|
||||
create_transactions.append(item)
|
||||
elif item['transactionType'] == "Delete":
|
||||
mongo.deleteTimetableData({'trainUid': item['trainUid'], 'scheduleStartDate': item['scheduleStartDate'], 'stpIndicator': item['stpIndicator']})
|
||||
mongo.putTimetable(create_transactions)
|
||||
post_count = mongo.getLength("timetable")
|
||||
log.out(f"timetable._insertToDb: Document count difference after processing: {pre_count - post_count}", "DBUG")
|
||||
log.out(f"timetable._insertToDb: Document count difference after processing: {post_count - pre_count}", "DBUG")
|
||||
return True #If Successfuls
|
||||
except Exception as e:
|
||||
log.out("timetable._insertToDb: Error inserting timetable data", "ERR")
|
||||
|
Reference in New Issue
Block a user