Increase use of pointers to improve memory usage. Currently uses around 16GB RAM.
This commit is contained in:
parent
259f514b3d
commit
a2c52f7b8b
10
cif/parse.go
10
cif/parse.go
@ -3,6 +3,7 @@ package cif
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
|
||||||
"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
|
"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||||
@ -10,9 +11,14 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Unmarshalls data into the correct types for processing
|
// Unmarshalls data into the correct types for processing
|
||||||
func parseCifData(data []byte) (*parsedData, error) {
|
// This function suffers from extremely high memory usage
|
||||||
|
func parseCifData(data *[]byte) (*parsedData, error) {
|
||||||
|
if data == nil {
|
||||||
|
err := errors.New("unable to parse nil pointer")
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
// Split the data into lines
|
// Split the data into lines
|
||||||
lines := bytes.Split(data, []byte("\n"))
|
lines := bytes.Split(*data, []byte("\n"))
|
||||||
|
|
||||||
// Initialise variable for the parsed data
|
// Initialise variable for the parsed data
|
||||||
var parsed parsedData
|
var parsed parsedData
|
||||||
|
@ -27,7 +27,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
|
|||||||
|
|
||||||
// If debug mode is on, call debugWriteDownload
|
// If debug mode is on, call debugWriteDownload
|
||||||
if helpers.Runtime == "debug" {
|
if helpers.Runtime == "debug" {
|
||||||
debugWriteDownload(&data)
|
debugWriteDownload(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse CIF file
|
// Parse CIF file
|
||||||
@ -37,6 +37,9 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Make `data` a nil pointer as it is no longer required
|
||||||
|
data = nil
|
||||||
|
|
||||||
// Drop timetable collection
|
// Drop timetable collection
|
||||||
dbAccess.DropCollection(dbAccess.TimetableCollection) // I should edit this to prevent removal of VSTP entries in the database.
|
dbAccess.DropCollection(dbAccess.TimetableCollection) // I should edit this to prevent removal of VSTP entries in the database.
|
||||||
|
|
||||||
@ -77,7 +80,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
|||||||
|
|
||||||
// If debug mode is on, call debugWriteDownload
|
// If debug mode is on, call debugWriteDownload
|
||||||
if helpers.Runtime == "debug" {
|
if helpers.Runtime == "debug" {
|
||||||
debugWriteDownload(&data)
|
debugWriteDownload(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse CIF file
|
// Parse CIF file
|
||||||
@ -87,6 +90,9 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Make `data` a nil pointer as it is no longer required
|
||||||
|
data = nil
|
||||||
|
|
||||||
// If debug mode is on, call debugWriteFile
|
// If debug mode is on, call debugWriteFile
|
||||||
if helpers.Runtime == "debug" {
|
if helpers.Runtime == "debug" {
|
||||||
debugWriteFile(&parsed.header, &parsed.sched)
|
debugWriteFile(&parsed.header, &parsed.sched)
|
||||||
@ -123,7 +129,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
|
// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
|
||||||
func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) {
|
func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
|
||||||
url, err := getUpdateUrl("daily")
|
url, err := getUpdateUrl("daily")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -16,7 +16,7 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
unsortedCorpusData, err := parseCorpusData(&resp)
|
unsortedCorpusData, err := parseCorpusData(resp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Msg.Error("Error parsing Corpus data", zap.Error(err))
|
log.Msg.Error("Error parsing Corpus data", zap.Error(err))
|
||||||
return err
|
return err
|
||||||
|
@ -13,7 +13,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Downloads NROD Data over HTTP from the given URL, extracted data is returned
|
// Downloads NROD Data over HTTP from the given URL, extracted data is returned
|
||||||
func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) {
|
func NrodDownload(url string, cfg *helpers.Configuration) (*[]byte, error) {
|
||||||
log.Msg.Debug("Fetching NROD data", zap.String("Request URL", url))
|
log.Msg.Debug("Fetching NROD data", zap.String("Request URL", url))
|
||||||
client := http.Client{
|
client := http.Client{
|
||||||
Timeout: time.Second * 10,
|
Timeout: time.Second * 10,
|
||||||
@ -49,7 +49,7 @@ func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Extracts GZIP Data from an HTTP Response and returns the decompresses data as a byte array
|
// Extracts GZIP Data from an HTTP Response and returns the decompresses data as a byte array
|
||||||
func nrodExtract(resp http.Response) ([]byte, error) {
|
func nrodExtract(resp http.Response) (*[]byte, error) {
|
||||||
log.Msg.Debug("Extracting HTTP Response Data")
|
log.Msg.Debug("Extracting HTTP Response Data")
|
||||||
gzReader, err := gzip.NewReader(resp.Body)
|
gzReader, err := gzip.NewReader(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -59,7 +59,7 @@ func nrodExtract(resp http.Response) ([]byte, error) {
|
|||||||
log.Msg.Error("Unable to read response body")
|
log.Msg.Error("Unable to read response body")
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return data, nil
|
return &data, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
defer gzReader.Close()
|
defer gzReader.Close()
|
||||||
@ -70,5 +70,5 @@ func nrodExtract(resp http.Response) ([]byte, error) {
|
|||||||
log.Msg.Error("Failed to read GZIPped data", zap.Error(err))
|
log.Msg.Error("Failed to read GZIPped data", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
return extractedData, nil
|
return &extractedData, nil
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user