Feature Complete:

- MongoDB Check before Initialising
- Output to file or S3
- Output to correct format for JSON Schema

src/index.ts

@@ -1,12 +1,18 @@
 import { log } from './logger.js'
 import { Readable } from 'node:stream'
-import { createWriteStream } from 'node:fs'
-import { finished } from 'node:stream/promises'
+import { processAndStore } from './sss.js'
 
 import { getLatestPackageName, getRequestStream } from './sources/gitea.js'
 import { processPisStream } from './process.js'
+import { isPackageProcessed } from './database.js'
 
 async function main() {
+    const SERVICE_NAME = process.env.SERVICE_NAME;
+    if (!SERVICE_NAME) {
+        log('ERROR', "SERVICE_NAME env variable must be set");
+        process.exit(1);
+    }
+
     try {
         const packageInfo = await getLatestPackageName();
         log('INFO', `Latest PIS Package: ${packageInfo.name}`);
@@ -16,12 +22,19 @@ async function main() {
             process.exit(9);
         }
 
+        if (await isPackageProcessed(SERVICE_NAME, packageInfo.name)) {
+            log('INFO', `Database matches latest release.  Exiting`);
+            process.exit(0);
+        }
+
         const inputStream: Readable = await getRequestStream(packageInfo.assets[0].browser_download_url);
+        const objectGenerator = processPisStream(inputStream);
 
-        const outputPath = './review_output.ndjson';
-        log('DEBUG', `Processing stream to: ${outputPath}`);
+        const filename = `${packageInfo.name.replace(/\s+/g, '_')}_pis_data_ndjson`;
 
-        await writeToTemporaryFile(processPisStream(inputStream), outputPath);
+        log('DEBUG', `Processing stream to: ${filename}`);
+
+        await processAndStore(objectGenerator, filename);
         log('DEBUG', 'Done');
     } catch (err) {
         log('ERROR', 'Fatal error in pipeline: ', err);
@@ -29,18 +42,4 @@ async function main() {
     }
 }
 
-async function writeToTemporaryFile(generator: AsyncGenerator<any>, path: string) {
-    const writer = createWriteStream(path);
-    for await (const record of generator) {
-        const line = JSON.stringify(record) + '\n';
-
-        if (!writer.write(line)) {
-            await new Promise((resolve) => writer.once('drain', resolve));
-        }
-    }
-
-    writer.end();
-    return finished(writer);
-}
-
 main();