From b0a6fcd52208f8ad9dd0b2b5959fd833507f1407 Mon Sep 17 00:00:00 2001
From: Fred Boniface <fred@fjla.uk>
Date: Wed, 21 Dec 2022 00:42:32 +0000
Subject: [PATCH] Updated list options

---
 README.md                           | 16 ++++++++++++++-
 src/controllers/list.controllers.js | 12 ++++++++++-
 src/routes/list.routes.js           |  1 +
 src/services/list.services.js       | 10 +++++++--
 src/utils/corpus.utils.js           | 32 ++++++++++++++++++++---------
 5 files changed, 57 insertions(+), 14 deletions(-)

diff --git a/README.md b/README.md
index 77aa119..d4dd476 100644
--- a/README.md
+++ b/README.md
@@ -80,4 +80,18 @@ The app is designed to be run within Kubernetes or within a Docker container, as
 
 In the case that OWL_LDB_SVKEY is not available, staff versions of departure board, etc. will not be available.
 
-In the case that OWL_GIT_ISSUEBOT is not available, the 'Report Issue' page will not be able to POST data.
\ No newline at end of file
+In the case that OWL_GIT_ISSUEBOT is not available, the 'Report Issue' page will not be able to POST data.
+
+## Database Layout
+
+The OwlBoard application will build the database if required at startup.  All it needs is authentication details for a MongoDB server.
+
+### Collections
+
+|Collection|Contents|Purpose|
+|:--------:|:------:|:-----:|
+|corpus|Raw CORPUS data with blank keys removed|Code lookups|
+|stations|Cleaned CORPUS Data, any objects with blank 3ALPHA & STANOX fields are removed|Validation before fetching Arr/Dep boards|
+|meta|Lists the update time of corpus and station data|Will be used to update after a predetermined time period|
+
+Note that even after removing all objects from the CORPUS with a blank 3ALPHA & STANOX, many items remain which are not stations and will not have a board available.  Going forwards methods to remove non-stations from this data will be introduced.
\ No newline at end of file
diff --git a/src/controllers/list.controllers.js b/src/controllers/list.controllers.js
index fdf9023..1ccd2f1 100644
--- a/src/controllers/list.controllers.js
+++ b/src/controllers/list.controllers.js
@@ -9,6 +9,16 @@ async function get(req, res, next){
     }
 }
 
+async function getAll(req, res, next){
+    try {
+        res.json(await list.getAll(req.body))
+    } catch (err) {
+        console.erroe(`Unknown Error`, err.message);
+        next(err);
+    }
+}
+
 module.exports = {
-    get
+    get,
+    getAll
 }
\ No newline at end of file
diff --git a/src/routes/list.routes.js b/src/routes/list.routes.js
index a0e1f84..131c098 100644
--- a/src/routes/list.routes.js
+++ b/src/routes/list.routes.js
@@ -15,5 +15,6 @@ const listController = require('../controllers/list.controllers');
 //router.delete('/:id', programmingLanguagesController.remove);
 
 router.get('/', listController.get);
+router.get('/all', listController.getAll);
 
 module.exports = router;
\ No newline at end of file
diff --git a/src/services/list.services.js b/src/services/list.services.js
index daebb67..23bbdba 100644
--- a/src/services/list.services.js
+++ b/src/services/list.services.js
@@ -3,9 +3,15 @@ const corpus = require('../utils/corpus.utils');
 
 async function get(){
     // Databse lookup, get list of all stations as per the Python script output
-    return corpus.init();
+    return corpus.initSubset();
+}
+
+async function getAll(){
+    // Raw corpus data
+    return corpus.initAll();
 }
 
 module.exports = {
-    get
+    get,
+    getAll
 }
\ No newline at end of file
diff --git a/src/utils/corpus.utils.js b/src/utils/corpus.utils.js
index ea23016..9bbfdfa 100644
--- a/src/utils/corpus.utils.js
+++ b/src/utils/corpus.utils.js
@@ -3,10 +3,12 @@
 // Network Rail Datafeed user and pass must be stored in `/srv/keys/owlboard/keys.config.js`
 
 // FUNCTIONS/
-//  init()   : Exported: Uses the internal functions to return a clean CORPUS file.
-//  get()    : Get the CORPUS data from Network Rail as a gzip file.
-//  extract(): Extract the CORPUS JSON file from the GZIP file.
-//  clean()  : Cleans the CORPUS data, removing unneccesary keys and objects from the data.
+//  initSubset()  : Exported: Uses the internal functions to return a clean CORPUS object.
+//  initAll()     : Exported: Uses the internal functions to return a full CORPUS object.
+//  get()         : Get the CORPUS data from Network Rail as a gzip file.
+//  extract()     : Extract the CORPUS JSON file from the GZIP file.
+//  clean()       : Cleans the CORPUS data, removing unneccesary keys and objects from the data.
+//  removeBlanks(): Removes keys with the value " ".
 
 const axios  = require('axios')
 const gz     = require('node-gzip')
@@ -14,13 +16,19 @@ const gz     = require('node-gzip')
 const corpusUser = process.env.OWL_LDB_CORPUSUSER
 const corpusPass = process.env.OWL_LDB_CORPUSPASS
 
-async function init() {
-    var gzipData = await get()
-    var corpus = await extract(gzipData)
-    var cleanCorpus = await clean(corpus)
+async function initSubset() {
+    var allCorpus = await initAll()
+    var cleanCorpus = await clean(allCorpus)
     return cleanCorpus
 }
 
+async function initAll(){
+    var gzipData = await get()
+    var allCorpus = await extract(gzipData)
+    var tidied = await removeBlanks(allCorpus)
+    return tidied
+}
+
 async function get() {
     authHead = Buffer.from(`${corpusUser}:${corpusPass}`).toString('base64')
     const url = 'https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS'
@@ -55,7 +63,11 @@ async function clean(input) {
     return clean;
 }
 
+async function removeBlanks(input){
+    return input;
+}
+
 module.exports = {
-    init,
-    outputToFile
+    initAll,
+    initSubset
 }
\ No newline at end of file