Adjust CORPUS Handling
This commit is contained in:
parent
8a113354e4
commit
6002d486ec
2
.gitignore
vendored
2
.gitignore
vendored
@ -1,5 +1,3 @@
|
|||||||
.test-tools
|
|
||||||
|
|
||||||
# ---> Node
|
# ---> Node
|
||||||
# Logs
|
# Logs
|
||||||
logs
|
logs
|
||||||
|
@ -7,8 +7,7 @@
|
|||||||
// initAll() : Exported: Uses the internal functions to return a full CORPUS object.
|
// initAll() : Exported: Uses the internal functions to return a full CORPUS object.
|
||||||
// get() : Get the CORPUS data from Network Rail as a gzip file.
|
// get() : Get the CORPUS data from Network Rail as a gzip file.
|
||||||
// extract() : Extract the CORPUS JSON file from the GZIP file.
|
// extract() : Extract the CORPUS JSON file from the GZIP file.
|
||||||
// clean() : Cleans the CORPUS data, removing unneccesary keys and objects from the data.
|
// clean() : Cleans the CORPUS data, removing unneccesary non-stations from the data.
|
||||||
// removeBlanks(): Removes keys with the value " ".
|
|
||||||
|
|
||||||
const axios = require('axios')
|
const axios = require('axios')
|
||||||
const gz = require('node-gzip')
|
const gz = require('node-gzip')
|
||||||
@ -25,8 +24,7 @@ async function initSubset() {
|
|||||||
async function initAll(){
|
async function initAll(){
|
||||||
var gzipData = await get()
|
var gzipData = await get()
|
||||||
var allCorpus = await extract(gzipData)
|
var allCorpus = await extract(gzipData)
|
||||||
var tidied = await removeBlanks(allCorpus)
|
return allCorpus
|
||||||
return tidied
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function get() {
|
async function get() {
|
||||||
@ -39,7 +37,6 @@ async function get() {
|
|||||||
responseType: 'arraybuffer'
|
responseType: 'arraybuffer'
|
||||||
}
|
}
|
||||||
var { data } = await axios.get(url, options)
|
var { data } = await axios.get(url, options)
|
||||||
console.log(data)
|
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -63,10 +60,6 @@ async function clean(input) {
|
|||||||
return clean;
|
return clean;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function removeBlanks(input){
|
|
||||||
return input;
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
initAll,
|
initAll,
|
||||||
initSubset
|
initSubset
|
||||||
|
Reference in New Issue
Block a user