OwlBoard/pis-data-ingress
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch to central config loader using mounted secret files.

Browse Source
This commit is contained in:
Fred Boniface
2026-01-13 19:53:08 +00:00
parent cb37774a3a
commit c90163cdce
6 changed files with 127 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/sss.ts
View File

@@ -4,8 +4,10 @@ import { createWriteStream } from "node:fs";
import { Readable } from "node:stream";
import { DataIngressPisData } from "@owlboard/backend-data-contracts";
import { log } from "./logger.js";
import type { S3 } from "./config.js";
export async function processAndStore(
cfg: S3,
generator: AsyncGenerator<DataIngressPisData.PisObjects>,
filename: string
) {
@@ -15,21 +17,21 @@ export async function processAndStore(
}
})());
const useS3 = process.env.S3_ENDPOINT && process.env.S3_BUCKET;
const useS3 = cfg.S3_ENDPOINT && cfg.S3_BUCKET;
if (useS3) {
if (!process.env.S3_ENDPOINT || process.env.S3_BUCKET || !process.env.S3_REGION || !process.env.S3_ACCESS_KEY || !process.env.S3_SECRET_KEY) {
if (!cfg.S3_ENDPOINT || !cfg.S3_BUCKET || !cfg.S3_REGION || !cfg.S3_ACCESS_KEY || !cfg.S3_SECRET_KEY) {
log("DEBUG", "Missing required variables for S3 Support - EXIT CODE 24");
process.exit(24);
}
log('INFO', `Streaming to S3: ${process.env.S3_BUCKET}/${filename}`);
log('INFO', `Streaming to S3: ${cfg.S3_BUCKET}/${filename}`);
const client = new S3Client({
endpoint: process.env.S3_ENDPOINT,
region: process.env.S3_REGION,
endpoint: cfg.S3_ENDPOINT,
region: cfg.S3_REGION,
credentials: {
accessKeyId: process.env.S3_ACCESS_KEY!,
secretAccessKey: process.env.S3_SECRET_KEY,
accessKeyId: cfg.S3_ACCESS_KEY!,
secretAccessKey: cfg.S3_SECRET_KEY,
},
forcePathStyle: true,
});
@@ -37,7 +39,7 @@ export async function processAndStore(
const upload = new Upload({
client,
params: {
Bucket: process.env.S3_BUCKET,
Bucket: cfg.S3_BUCKET,
Key: filename,
Body: ndjsonStream,
ContentType: "application/x-ndjson",