pis-data-ingress/src/sss.ts

import { S3Client } from "@aws-sdk/client-s3";
import { Upload } from "@aws-sdk/lib-storage";
import { createWriteStream } from "node:fs";
import { Readable } from "node:stream";
import { DataIngressPisData } from "@owlboard/backend-data-contracts";
import { log } from "./logger.js";

export async function processAndStore(
    generator: AsyncGenerator<DataIngressPisData.PisObjects>,
    filename: string
) {
    const ndjsonStream = Readable.from((async function* () {
        for await (const record of generator) {
            yield JSON.stringify(record) + "\n";
        }
    })());

    const useS3 = process.env.S3_ENDPOINT && process.env.S3_BUCKET;

    if (useS3) {
        if (!process.env.S3_ENDPOINT || process.env.S3_BUCKET || !process.env.S3_REGION || !process.env.S3_ACCESS_KEY || !process.env.S3_SECRET_KEY) {
            log("DEBUG", "Missing required variables for S3 Support - EXIT CODE 24");
            process.exit(24);
        }
        log('INFO', `Streaming to S3: ${process.env.S3_BUCKET}/${filename}`);

        const client = new S3Client({
            endpoint: process.env.S3_ENDPOINT,
            region: process.env.S3_REGION,
            credentials: {
                accessKeyId: process.env.S3_ACCESS_KEY!,
                secretAccessKey: process.env.S3_SECRET_KEY,
            },
            forcePathStyle: true,
        });

        const upload = new Upload({
            client,
            params: {
                Bucket: process.env.S3_BUCKET,
                Key: filename,
                Body: ndjsonStream,
                ContentType: "application/x-ndjson",
            },
        });

        await upload.done();
    } else {
        log('INFO', `Streaming to local filesystem at: ${filename}`);
        const fileWriter = createWriteStream(filename);

        return new Promise((resolve, reject) => {
            ndjsonStream.pipe(fileWriter)
              .on('finish', resolve)
              .on('error', reject);
        })
    }
}