Increase use of pointers to improve memory usage. Currently uses around 16GB RAM.

cif/parse.go

@@ -3,6 +3,7 @@ package cif
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 
 	"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
 	"git.fjla.uk/owlboard/timetable-mgr/log"
@@ -10,9 +11,14 @@ import (
 )
 
 // Unmarshalls data into the correct types for processing
-func parseCifData(data []byte) (*parsedData, error) {
+// This function suffers from extremely high memory usage
+func parseCifData(data *[]byte) (*parsedData, error) {
+	if data == nil {
+		err := errors.New("unable to parse nil pointer")
+		return nil, err
+	}
 	// Split the data into lines
-	lines := bytes.Split(data, []byte("\n"))
+	lines := bytes.Split(*data, []byte("\n"))
 
 	// Initialise variable for the parsed data
 	var parsed parsedData

cif/update.go

@@ -27,7 +27,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
 
 	// If debug mode is on, call debugWriteDownload
 	if helpers.Runtime == "debug" {
-		debugWriteDownload(&data)
+		debugWriteDownload(data)
 	}
 
 	// Parse CIF file
@@ -37,6 +37,9 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
 		return err
 	}
 
+	// Make `data` a nil pointer as it is no longer required
+	data = nil
+
 	// Drop timetable collection
 	dbAccess.DropCollection(dbAccess.TimetableCollection) // I should edit this to prevent removal of VSTP entries in the database.
 
@@ -77,7 +80,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 
 		// If debug mode is on, call debugWriteDownload
 		if helpers.Runtime == "debug" {
-			debugWriteDownload(&data)
+			debugWriteDownload(data)
 		}
 
 		// Parse CIF file
@@ -87,6 +90,9 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 			return err
 		}
 
+		// Make `data` a nil pointer as it is no longer required
+		data = nil
+
 		// If debug mode is on, call debugWriteFile
 		if helpers.Runtime == "debug" {
 			debugWriteFile(&parsed.header, &parsed.sched)
@@ -123,7 +129,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
 }
 
 // Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
-func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) {
+func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
 	url, err := getUpdateUrl("daily")
 	if err != nil {
 		return nil, err