Ensure S3 configured to create bucket if not exists

src/index.ts

@@ -26,6 +26,7 @@ async function main() {
         }
 
         // Check if package already processed HERE...
+        log('WARN', 'Version check disabled, downloading data from source');
         // exit(0) if done, else continue
 
         const inputStream: Readable = await getRequestStream(packageInfo.assets[0].browser_download_url);

src/process.ts

@@ -36,7 +36,7 @@ export async function* processPisStream(cfg: GeneralConfig, inputStream: Readabl
         if (!line.trim()) continue;
 
         const record = JSON.parse(line) as InputRecord;
-        log('DEBUG', `JSON Line Parsed: ${JSON.stringify(record)}`)
+        // log('DEBUG', `JSON Line Parsed: ${JSON.stringify(record)}`)
 
         const crsHash = XXH.h64(record.stops.join('|'), SEED);
         const tiplocStops = await mapStopsToTiploc(record.stops);

src/sss.ts

@@ -1,4 +1,4 @@
-import { S3Client } from "@aws-sdk/client-s3";
+import { S3Client, CreateBucketCommand, HeadBucketCommand, S3ServiceException } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { createWriteStream } from "node:fs";
 import { Readable } from "node:stream";
@@ -24,7 +24,8 @@ export async function processAndStore(
             log("DEBUG", "Missing required variables for S3 Support - EXIT CODE 24");
             process.exit(24);
         }
-        log('INFO', `Streaming to S3: ${cfg.S3_BUCKET}/${filename}`);
+
+        log("DEBUG", "Opening connection to S3 Server");
 
         const client = new S3Client({
             endpoint: cfg.S3_ENDPOINT,
@@ -36,6 +37,21 @@ export async function processAndStore(
             forcePathStyle: true,
         });
 
+        // Check bucket exists, and create if needed
+        try {
+            await client.send(new HeadBucketCommand({ Bucket: cfg.S3_BUCKET }));
+        } catch (err) {
+            if (err instanceof S3ServiceException && err.$metadata.httpStatusCode === 404) {
+                log('INFO', `Bucket ${cfg.S3_BUCKET} does not exist, creating...`);
+                await client.send(new CreateBucketCommand({ Bucket: cfg.S3_BUCKET }));
+            } else {
+                log(`ERROR`, `Failed to verify bucket: ${cfg.S3_BUCKET}`);
+                process.exit(21);
+            }
+        }
+
+        log('INFO', `Streaming to S3: ${cfg.S3_BUCKET}/${filename}`);
+
         const upload = new Upload({
             client,
             params: {