Adjust file write functions to reduce memory load

cif/helpers.go

@@ -1,12 +1,10 @@
 package cif
 
 import (
-	"encoding/json"
 	"errors"
 	"os"
 	"time"
 
-	"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
 	"git.fjla.uk/owlboard/timetable-mgr/helpers"
 	"git.fjla.uk/owlboard/timetable-mgr/log"
 	"go.uber.org/zap"
@@ -108,34 +106,6 @@ func parseDaysRun(daysBinary *string) []string {
 	return result
 }
 
-func debugWriteFile(header *upstreamApi.JsonTimetableV1, schedule *[]upstreamApi.JsonScheduleV1) {
-	if helpers.Runtime == "debug" {
-		log.Msg.Debug("Writing CIF Header and Schedule elements to file")
-		filepath := "./cif_debug_data/"
-		filename := time.Now().In(londonTimezone).Format("2006-01-02_15:04:05_ParsedCIF")
-
-		data, err := json.MarshalIndent(map[string]interface{}{
-			"header":   header,
-			"schedule": schedule,
-		}, "", "  ")
-		if err != nil {
-			log.Msg.Error("Error marshalling data", zap.Error(err))
-			return
-		}
-
-		err = os.MkdirAll(filepath, 0777)
-		if err != nil {
-			log.Msg.Error("Error creating directory", zap.Error(err))
-			return
-		}
-
-		writeErr := os.WriteFile(filepath+filename+".json", data, 0777)
-		if writeErr != nil {
-			log.Msg.Error("Error writing debug file to disk", zap.Error(writeErr))
-		}
-	}
-}
-
 func debugWriteDownload(input *[]byte) {
 	if helpers.Runtime == "debug" {
 		log.Msg.Debug("Writing CIF Download to file")
@@ -148,9 +118,25 @@ func debugWriteDownload(input *[]byte) {
 			return
 		}
 
-		err = os.WriteFile(filepath+filename+".jsonl", *input, 0777)
+		file, err := os.Create(filepath + filename + ".jsonl")
 		if err != nil {
-			log.Msg.Error("Error writing debug file to disk", zap.Error(err))
+			log.Msg.Error("Error creating file", zap.Error(err))
+			return
+		}
+		defer file.Close()
+
+		// Write data to file in smaller chunks
+		bufferSize := 4096 // Adjust the buffer size as needed
+		for i := 0; i < len(*input); i += bufferSize {
+			end := i + bufferSize
+			if end > len(*input) {
+				end = len(*input)
+			}
+			_, err := file.Write((*input)[i:end])
+			if err != nil {
+				log.Msg.Error("Error writing data to file", zap.Error(err))
+				return
+			}
 		}
 	}
 }