Adjust CORPUS Handling

This commit is contained in:
Fred Boniface 2022-12-21 20:09:24 +00:00
parent 8a113354e4
commit 6002d486ec
2 changed files with 2 additions and 11 deletions

2
.gitignore vendored
View File

@ -1,5 +1,3 @@
.test-tools
# ---> Node # ---> Node
# Logs # Logs
logs logs

View File

@ -7,8 +7,7 @@
// initAll() : Exported: Uses the internal functions to return a full CORPUS object. // initAll() : Exported: Uses the internal functions to return a full CORPUS object.
// get() : Get the CORPUS data from Network Rail as a gzip file. // get() : Get the CORPUS data from Network Rail as a gzip file.
// extract() : Extract the CORPUS JSON file from the GZIP file. // extract() : Extract the CORPUS JSON file from the GZIP file.
// clean() : Cleans the CORPUS data, removing unneccesary keys and objects from the data. // clean() : Cleans the CORPUS data, removing unneccesary non-stations from the data.
// removeBlanks(): Removes keys with the value " ".
const axios = require('axios') const axios = require('axios')
const gz = require('node-gzip') const gz = require('node-gzip')
@ -25,8 +24,7 @@ async function initSubset() {
async function initAll(){ async function initAll(){
var gzipData = await get() var gzipData = await get()
var allCorpus = await extract(gzipData) var allCorpus = await extract(gzipData)
var tidied = await removeBlanks(allCorpus) return allCorpus
return tidied
} }
async function get() { async function get() {
@ -39,7 +37,6 @@ async function get() {
responseType: 'arraybuffer' responseType: 'arraybuffer'
} }
var { data } = await axios.get(url, options) var { data } = await axios.get(url, options)
console.log(data)
return data return data
} }
@ -63,10 +60,6 @@ async function clean(input) {
return clean; return clean;
} }
async function removeBlanks(input){
return input;
}
module.exports = { module.exports = {
initAll, initAll,
initSubset initSubset