db-manager/src/corpus.py

#Imports
import os
import requests
import logger as log
import zlib

CORPUS_URL = "https://publicdatafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS"

#Fetch Configuration
log.out("corpus: Fetching CORPUS Configuration", "INFO")
CORPUS_USER = os.getenv('OWL_LDB_CORPUSUSER')
CORPUS_PASS = os.getenv('OWL_LDB_CORPUSPASS')

def fetch():
    log.out("corpus.fetch: Fetching CORPUS Data from Network Rail", "INFO")
    response = requests.get(CORPUS_URL, auth=(CORPUS_USER, CORPUS_PASS))
    log.out("corpus.fetch: Decompressing CORPUS data", "INFO")
    return zlib.decompress(response.content, 16+zlib.MAX_WBITS)

def removeEmpty(list):
    for dict in list:
      print(dict)


ORIGINAL METHOD FROM FILE:
#! /usr/bin/python3

import json

print("Opening CORPUSExtract.json")
with open("./CORPUSExtract.json", "r") as input_file:
    dict = json.load(input_file)
    list = dict['TIPLOCDATA']

cleanList = []

print("Processing data")
for item in list:
    if item["3ALPHA"] != ' ' and item["STANOX"] != ' ':
        item.pop("UIC")
        item.pop("NLCDESC16")
        item.pop("NLC")
        cleanList.append(item)

print("Saving data")

cleanDict = {"data":cleanList}
with open("CorpusClean.json", "w") as output_file:
    output_file.write(json.dumps(cleanDict))

print(cleanList)
print("Processed.")