From a2c52f7b8bf253f1efd1fb75c2186f38b6a89288 Mon Sep 17 00:00:00 2001 From: Fred Boniface Date: Tue, 9 Apr 2024 21:07:21 +0100 Subject: [PATCH] Increase use of pointers to improve memory usage. Currently uses around 16GB RAM. --- cif/parse.go | 10 ++++++++-- cif/update.go | 12 +++++++++--- corpus/update.go | 2 +- nrod/download.go | 8 ++++---- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/cif/parse.go b/cif/parse.go index 46f0fe7..9b5579c 100644 --- a/cif/parse.go +++ b/cif/parse.go @@ -3,6 +3,7 @@ package cif import ( "bytes" "encoding/json" + "errors" "git.fjla.uk/owlboard/go-types/pkg/upstreamApi" "git.fjla.uk/owlboard/timetable-mgr/log" @@ -10,9 +11,14 @@ import ( ) // Unmarshalls data into the correct types for processing -func parseCifData(data []byte) (*parsedData, error) { +// This function suffers from extremely high memory usage +func parseCifData(data *[]byte) (*parsedData, error) { + if data == nil { + err := errors.New("unable to parse nil pointer") + return nil, err + } // Split the data into lines - lines := bytes.Split(data, []byte("\n")) + lines := bytes.Split(*data, []byte("\n")) // Initialise variable for the parsed data var parsed parsedData diff --git a/cif/update.go b/cif/update.go index 5be9bc7..bbdac84 100644 --- a/cif/update.go +++ b/cif/update.go @@ -27,7 +27,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error { // If debug mode is on, call debugWriteDownload if helpers.Runtime == "debug" { - debugWriteDownload(&data) + debugWriteDownload(data) } // Parse CIF file @@ -37,6 +37,9 @@ func runCifFullDownload(cfg *helpers.Configuration) error { return err } + // Make `data` a nil pointer as it is no longer required + data = nil + // Drop timetable collection dbAccess.DropCollection(dbAccess.TimetableCollection) // I should edit this to prevent removal of VSTP entries in the database. @@ -77,7 +80,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta // If debug mode is on, call debugWriteDownload if helpers.Runtime == "debug" { - debugWriteDownload(&data) + debugWriteDownload(data) } // Parse CIF file @@ -87,6 +90,9 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta return err } + // Make `data` a nil pointer as it is no longer required + data = nil + // If debug mode is on, call debugWriteFile if helpers.Runtime == "debug" { debugWriteFile(&parsed.header, &parsed.sched) @@ -123,7 +129,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta } // Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day -func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) { +func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) { url, err := getUpdateUrl("daily") if err != nil { return nil, err diff --git a/corpus/update.go b/corpus/update.go index a9053d3..fcd907a 100644 --- a/corpus/update.go +++ b/corpus/update.go @@ -16,7 +16,7 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error { return err } - unsortedCorpusData, err := parseCorpusData(&resp) + unsortedCorpusData, err := parseCorpusData(resp) if err != nil { log.Msg.Error("Error parsing Corpus data", zap.Error(err)) return err diff --git a/nrod/download.go b/nrod/download.go index bdecc73..240af66 100644 --- a/nrod/download.go +++ b/nrod/download.go @@ -13,7 +13,7 @@ import ( ) // Downloads NROD Data over HTTP from the given URL, extracted data is returned -func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) { +func NrodDownload(url string, cfg *helpers.Configuration) (*[]byte, error) { log.Msg.Debug("Fetching NROD data", zap.String("Request URL", url)) client := http.Client{ Timeout: time.Second * 10, @@ -49,7 +49,7 @@ func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) { } // Extracts GZIP Data from an HTTP Response and returns the decompresses data as a byte array -func nrodExtract(resp http.Response) ([]byte, error) { +func nrodExtract(resp http.Response) (*[]byte, error) { log.Msg.Debug("Extracting HTTP Response Data") gzReader, err := gzip.NewReader(resp.Body) if err != nil { @@ -59,7 +59,7 @@ func nrodExtract(resp http.Response) ([]byte, error) { log.Msg.Error("Unable to read response body") return nil, err } - return data, nil + return &data, nil } defer gzReader.Close() @@ -70,5 +70,5 @@ func nrodExtract(resp http.Response) ([]byte, error) { log.Msg.Error("Failed to read GZIPped data", zap.Error(err)) } - return extractedData, nil + return &extractedData, nil }