Fix JSONL inclusion?

Merge new PIS Updates

Co-authored-by: Fred Boniface <fred@fjla.uk>
Co-authored-by: Robotic Owl <owlbot@owlboard.info>
Reviewed-on: #1

.gitea/workflows/auto_pr.yaml

@@ -9,7 +9,7 @@ on:
             - pis-*
 
 jobs:
-    validate_and_release:
+    validate_and_PR:
         runs-on: ubuntu-latest
         permissions:
             contents: write
@@ -27,6 +27,7 @@ jobs:
               run: node ./scripts/merge-yaml.js
 
             - name: Commit Changes
+              id: commit
               uses: https://github.com/stefanzweifel/git-auto-commit-action@v5
               with:
                   commit_message: Sort & Merge PIS Files

.gitea/workflows/release.yaml

@@ -20,6 +20,21 @@ jobs:
               id: timestamp
               run: echo "::set-output name=date::$(date +'%Y%m%d_%H%M%S')"
 
+            - name: Install Dependencies
+              run: npm i
+
+            - name: Create JSON Lines
+              run: node ./scripts/ndjson-out.js ./pis/codes.yaml
+
+            - name: Verify JSONL Output
+              run: |
+                if [ ! -s pis-objects.jsonl ]; then
+                  echo "Error: pis-objects.jsonl is empty or missong"
+                  exit 1
+                fi
+                echo "File verified"
+                pwd
+
             - name: Create Release
               if: success()
               uses: akkuman/gitea-release-action@v1
@@ -30,3 +45,5 @@ jobs:
                 tag_name: ${{ steps.timestamp.outputs.date }}
                 name: ${{ steps.timestamp.outputs.date }}
                 token: ${{ secrets.OWLBOT_KEY }}
+                files: |-
+                  pis-objects.jsonl

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+package-lock.json

pis/codes.yaml

@@ -1835,12 +1835,6 @@ pis:
   - code: "5351"
     stops: [exc,exd,ntc,cdi,yeo,cop,mrd,egg,umb,bnp]
     toc: "gw"
-  - code: "5351"
-    stops: [exc,exd,ntc,cdi,yeo,cop,mrd,egg,umb,bnp]
-    toc: "gw"
-  - code: "5352"
-    stops: [exc,exd,ntc,cdi,yeo,cop,mrd,lap,egg,kgn,pma,umb,cpn,bnp]
-    toc: "gw"
   - code: "5352"
     stops: [exc,exd,ntc,cdi,yeo,cop,mrd,lap,egg,kgn,pma,umb,cpn,bnp]
     toc: "gw"
@@ -1931,9 +1925,6 @@ pis:
   - code: "5381"
     stops: [exm,lym,lyc,exn,top,nco,dig,pol,sjp,exc,exd,ext,scs,dww,dwl,tgm,nta,trr,tqy,pgn]
     toc: "gw"
-  - code: "5381"
-    stops: [exm,lym,lyc,exn,top,nco,dig,pol,sjp,exc,exd,ext,scs,dww,dwl,tgm,nta,trr,tqy,pgn]
-    toc: "gw"
   - code: "5382"
     stops: [exm,lym,top,nco,dig,exc,exd,dwl,tgm,nta,trr,tqy,pgn]
     toc: "gw"
@@ -3857,9 +3848,6 @@ pis:
   - code: "6051"
     stops: [exc,exd,ntc,cdi,oke]
     toc: "gw"
-  - code: "6051"
-    stops: [exc,exd,ntc,cdi,oke]
-    toc: "gw"
   - code: "6052"
     stops: [exd,cdi,oke]
     toc: "gw"
@@ -4202,9 +4190,6 @@ pis:
   - code: "6182"
     stops: [exm,lym,top,nco,dig,exc,exd,ext,mbt,scs,dww,dwl,tgm,nta,trr,tqy,pgn]
     toc: "gw"
-  - code: "6182"
-    stops: [exm,lym,top,nco,dig,exc,exd,ext,mbt,scs,dww,dwl,tgm,nta,trr,tqy,pgn]
-    toc: "gw"
   - code: "6183"
     stops: [pgn,tqy,trr,nta,tgm,dwl,dww,scs,mbt,ext,exd,exc,dig,nco,top,lym,exm]
     toc: "gw"
@@ -4415,9 +4400,6 @@ pis:
   - code: "6293"
     stops: [exm,lym,top,nco,dig,exc,exd,ext,mbt,scs,dwl,tgm,nta,trr,tqy,pgn]
     toc: "gw"
-  - code: "6293"
-    stops: [exm,lym,top,nco,dig,exc,exd,ext,mbt,scs,dwl,tgm,nta,trr,tqy,pgn]
-    toc: "gw"
   - code: "6294"
     stops: [exm,lym,lyc,exn,top,nco,dig,pol,sjp,exc,exd,dwl,tgm,nta,trr,tqy,pgn]
     toc: "gw"
@@ -4622,6 +4604,9 @@ pis:
   - code: "6384"
     stops: [bri,lwh,srd,asd,fit]
     toc: "gw"
+  - code: "6404"
+    stops: [cdf,nwp,fit,bri,bth,boa,tro,wsb,wmn,sal]
+    toc: "gw"
   - code: "6406"
     stops: [pmh,pms,ftn,csa,frm,sou]
     toc: "gw"
@@ -4637,6 +4622,9 @@ pis:
   - code: "6413"
     stops: [exm,lym,lyc,exn,top,dig,pol,exc,exd,ext,mbt,scs,dww,dwl,tgm,nta,trr,tqy,pgn]
     toc: "gw"
+  - code: "6417"
+    stops: [sal,wmn,wsb,tro,boa,brh,bri,fit,stj,nwp,cdf]
+    toc: "gw"
   - code: "6418"
     stops: [cdf,nwp,fit,bri,bth,boa,tro,wsb,wmn]
     toc: "gw"

scripts/merge-yaml.js

@@ -41,7 +41,6 @@ function sortAndMergeYAMLFiles() {
                 seenCodes.add(item.code);
                 outputString += `  - code: "${item.code}"\n`;
                 outputString += `    stops: [${item.stops.map(stop => `${stop}`).join(',')}]\n`;
-                outputString += `    toc: "${item.toc || "gw"}"\n`;
             }
         });
 
@@ -52,9 +51,9 @@ function sortAndMergeYAMLFiles() {
             console.error("Error writing codes.yaml", err)
         }
 
-        // Remove any files created by dgp2
+        // Remove any new files
         files.forEach(file => {
-            if (file.startsWith('auto-')) {
+            if (file !== 'codes.yaml') {
                 fs.unlinkSync(directoryPath + file);
                 console.log(`Deleted file: ${file}`);
             }

scripts/ndjson-out.js

@@ -0,0 +1,27 @@
+const fs = require('fs');
+const yaml = require('yaml');
+
+const inputFile = process.argv[2];
+const outputFile = 'pis-objects.jsonl';
+
+if (!inputFile) {
+    console.error('Please provide an input YAML file');
+    process.exit(1);
+}
+
+try {
+    const fileContent = fs.readFileSync(inputFile, 'utf8');
+    const data = yaml.parse(fileContent);
+
+    if (!data || !data.pis || !Array.isArray(data.pis)) {
+        throw new Error('Invalid structure: Expected object with "pis" array.');
+    }
+
+    const jsonl = data.pis.map(item => JSON.stringify(item));
+
+    fs.writeFileSync(outputFile, jsonl.join('\n') + '\n');
+    console.log(`Successfully processed ${data.pis.length} items.`);
+} catch (e) {
+    console.error("Conversion failed: ", e.message);
+    process.exit(1);
+}