corpus.utils now working

package-lock.json

@@ -13,6 +13,7 @@
         "express": "^4.18.2",
         "ldbs-json": "^1.2.1",
         "needle": "^3.2.0",
+        "node-gzip": "^1.1.2",
         "sqlite3": "^5.1.2"
       }
     },
@@ -1279,6 +1280,11 @@
         "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
       }
     },
+    "node_modules/node-gzip": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/node-gzip/-/node-gzip-1.1.2.tgz",
+      "integrity": "sha512-ZB6zWpfZHGtxZnPMrJSKHVPrRjURoUzaDbLFj3VO70mpLTW5np96vXyHwft4Id0o+PYIzgDkBUjIzaNHhQ8srw=="
+    },
     "node_modules/nopt": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz",
@@ -2862,6 +2868,11 @@
         }
       }
     },
+    "node-gzip": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/node-gzip/-/node-gzip-1.1.2.tgz",
+      "integrity": "sha512-ZB6zWpfZHGtxZnPMrJSKHVPrRjURoUzaDbLFj3VO70mpLTW5np96vXyHwft4Id0o+PYIzgDkBUjIzaNHhQ8srw=="
+    },
     "nopt": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz",

package.json

@@ -4,6 +4,7 @@
     "express": "^4.18.2",
     "ldbs-json": "^1.2.1",
     "needle": "^3.2.0",
+    "node-gzip": "^1.1.2",
     "sqlite3": "^5.1.2"
   },
   "name": "owlboard",

src/utils/corpus.utils.js

@@ -2,50 +2,57 @@
 
 // Network Rail Datafeed user and pass must be stored in `/srv/keys/owlboard/keys.config.js`
 
-// FUNCTIONS
+// FUNCTIONS/
 //  init()   : Exported: Uses the internal functions to return a clean CORPUS file.
 //  get()    : Get the CORPUS data from Network Rail as a gzip file.
 //  extract(): Extract the CORPUS JSON file from the GZIP file.
-//  clean()  : Cleans the CORPUS data, removing unneccesary keys from the data.
+//  clean()  : Cleans the CORPUS data, removing unneccesary keys and objects from the data.
 
-const keys    = require('/srv/keys/owlboard/keys.configs');
+const keys   = require('/srv/keys/owlboard/keys.configs')
-const axios   = require('axios');
+const axios  = require('axios')
-const zlib    = require('zlib');
+const gz     = require('node-gzip')
 
 async function init() {
-    var gzipData = await get();
+    var gzipData = await get()
-    console.log(gzipData);
+    var corpus = await extract(gzipData)
-    var rawData = await extract(gzipData);
+    console.log(corpus)
-    console.log(rawData);
+    var cleanCorpus = await clean(corpus)
-    var cleanData = await clean(rawData);
+    console.log(cleanCorpus)
-    console.log(cleanData)
+    return cleanCorpus
-    return cleanData;
 }
 
 async function get() {
-    authHead = Buffer.from(`${keys.nr_user}:${keys.nr_pass}`).toString('base64');
+    authHead = Buffer.from(`${keys.nr_user}:${keys.nr_pass}`).toString('base64')
     const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
     const options = {
         method: 'get',
         timeout: 20000,
         headers: {'Authorization': `Basic ${authHead}`},
         responseType: 'arraybuffer'
-    };
+    }
-    var { output } = await axios.get(url, options)
+    var { data } = await axios.get(url, options)
-    console.log(output)
+    console.log(data)
-    return output;
+    return data
 }
 
 async function extract(input) {
-    var { data } = zlib.Gunzip(input);
+    var raw = await gz.ungzip(input)
-    let string = data.toString()
+    var obj = await JSON.parse(raw)
-    let obj = JSON.parse(string)
+    var output = obj.TIPLOCDATA
-    //console.log(obj);
+    return output
-    return obj; // Find out how to get data out of callback -> See pinned tab on laptop2.ws
 }
 
-async function clean() {
+async function clean(input) {
-    return "Empty Function: clean()"
+    let clean = [];
+    for (const element of input) {
+        if (element.STANOX != ' ' && element['3ALPHA'] != ' '){
+            delete(element.UIC);
+            delete(element.NLCDESC16);
+            delete(element.NLC);
+            clean.push(element);
+        }
+      }
+    return clean;
 }
 
 module.exports = {