Increase use of pointers to improve memory usage. Currently uses around 16GB RAM.

cif/parse.go

@@ -3,6 +3,7 @@ package cif
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 
 	"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
 	"git.fjla.uk/owlboard/timetable-mgr/log"
@@ -10,9 +11,14 @@ import (
 )
 
 // Unmarshalls data into the correct types for processing
-func parseCifData(data []byte) (*parsedData, error) {
+// This function suffers from extremely high memory usage
+func parseCifData(data *[]byte) (*parsedData, error) {
+	if data == nil {
+		err := errors.New("unable to parse nil pointer")
+		return nil, err
+	}
 	// Split the data into lines
-	lines := bytes.Split(data, []byte("\n"))
+	lines := bytes.Split(*data, []byte("\n"))
 
 	// Initialise variable for the parsed data
 	var parsed parsedData

cif/update.go

@@ -27,7 +27,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
 
 	// If debug mode is on, call debugWriteDownload
 	if helpers.Runtime == "debug" {
-		debugWriteDownload(&data)
+		debugWriteDownload(data)
 	}
 
 	// Parse CIF file
@@ -37,6 +37,9 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
 		return err
 	}
 
+	// Make `data` a nil pointer as it is no longer required
+	data = nil
+
 	// Drop timetable collection
 	dbAccess.DropCollection(dbAccess.TimetableCollection) // I should edit this to prevent removal of VSTP entries in the database.
 
@@ -77,7 +80,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 
 		// If debug mode is on, call debugWriteDownload
 		if helpers.Runtime == "debug" {
-			debugWriteDownload(&data)
+			debugWriteDownload(data)
 		}
 
 		// Parse CIF file
@@ -87,6 +90,9 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 			return err
 		}
 
+		// Make `data` a nil pointer as it is no longer required
+		data = nil
+
 		// If debug mode is on, call debugWriteFile
 		if helpers.Runtime == "debug" {
 			debugWriteFile(&parsed.header, &parsed.sched)
@@ -123,7 +129,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 }
 
 // Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
-func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) {
+func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
 	url, err := getUpdateUrl("daily")
 	if err != nil {
 		return nil, err

corpus/update.go

@@ -16,7 +16,7 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error {
 		return err
 	}
 
-	unsortedCorpusData, err := parseCorpusData(&resp)
+	unsortedCorpusData, err := parseCorpusData(resp)
 	if err != nil {
 		log.Msg.Error("Error parsing Corpus data", zap.Error(err))
 		return err

nrod/download.go

@@ -13,7 +13,7 @@ import (
 )
 
 // Downloads NROD Data over HTTP from the given URL, extracted data is returned
-func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) {
+func NrodDownload(url string, cfg *helpers.Configuration) (*[]byte, error) {
 	log.Msg.Debug("Fetching NROD data", zap.String("Request URL", url))
 	client := http.Client{
 		Timeout: time.Second * 10,
@@ -49,7 +49,7 @@ func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) {
 }
 
 // Extracts GZIP Data from an HTTP Response and returns the decompresses data as a byte array
-func nrodExtract(resp http.Response) ([]byte, error) {
+func nrodExtract(resp http.Response) (*[]byte, error) {
 	log.Msg.Debug("Extracting HTTP Response Data")
 	gzReader, err := gzip.NewReader(resp.Body)
 	if err != nil {
@@ -59,7 +59,7 @@ func nrodExtract(resp http.Response) ([]byte, error) {
 			log.Msg.Error("Unable to read response body")
 			return nil, err
 		}
-		return data, nil
+		return &data, nil
 	}
 
 	defer gzReader.Close()
@@ -70,5 +70,5 @@ func nrodExtract(resp http.Response) ([]byte, error) {
 		log.Msg.Error("Failed to read GZIPped data", zap.Error(err))
 	}
 
-	return extractedData, nil
+	return &extractedData, nil
 }