Feature Complete:

- MongoDB Check before Initialising
- Output to file or S3
- Output to correct format for JSON Schema

src/sss.ts

@@ -0,0 +1,58 @@
+import { S3Client } from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { createWriteStream } from "node:fs";
+import { Readable } from "node:stream";
+import { DataIngressPisData } from "@owlboard/backend-data-contracts";
+import { log } from "./logger.js";
+
+export async function processAndStore(
+    generator: AsyncGenerator<DataIngressPisData.PisObjects>,
+    filename: string
+) {
+    const ndjsonStream = Readable.from((async function* () {
+        for await (const record of generator) {
+            yield JSON.stringify(record) + "\n";
+        }
+    })());
+
+    const useS3 = process.env.S3_ENDPOINT && process.env.S3_BUCKET;
+
+    if (useS3) {
+        if (!process.env.S3_ENDPOINT || process.env.S3_BUCKET || !process.env.S3_REGION || !process.env.S3_ACCESS_KEY || !process.env.S3_SECRET_KEY) {
+            log("DEBUG", "Missing required variables for S3 Support - EXIT CODE 24");
+            process.exit(24);
+        }
+        log('INFO', `Streaming to S3: ${process.env.S3_BUCKET}/${filename}`);
+
+        const client = new S3Client({
+            endpoint: process.env.S3_ENDPOINT,
+            region: process.env.S3_REGION,
+            credentials: {
+                accessKeyId: process.env.S3_ACCESS_KEY!,
+                secretAccessKey: process.env.S3_SECRET_KEY,
+            },
+            forcePathStyle: true,
+        });
+
+        const upload = new Upload({
+            client,
+            params: {
+                Bucket: process.env.S3_BUCKET,
+                Key: filename,
+                Body: ndjsonStream,
+                ContentType: "application/x-ndjson",
+            },
+        });
+
+        await upload.done();
+    } else {
+        log('INFO', `Streaming to local filesystem at: ${filename}`);
+        const fileWriter = createWriteStream(filename);
+
+        return new Promise((resolve, reject) => {
+            ndjsonStream.pipe(fileWriter)
+              .on('finish', resolve)
+              .on('error', reject);
+        })
+    }
+}