timetable-extension #1
10
cif/parse.go
10
cif/parse.go
@ -3,6 +3,7 @@ package cif
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
|
||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||
@ -10,9 +11,14 @@ import (
|
||||
)
|
||||
|
||||
// Unmarshalls data into the correct types for processing
|
||||
func parseCifData(data []byte) (*parsedData, error) {
|
||||
// This function suffers from extremely high memory usage
|
||||
func parseCifData(data *[]byte) (*parsedData, error) {
|
||||
if data == nil {
|
||||
err := errors.New("unable to parse nil pointer")
|
||||
return nil, err
|
||||
}
|
||||
// Split the data into lines
|
||||
lines := bytes.Split(data, []byte("\n"))
|
||||
lines := bytes.Split(*data, []byte("\n"))
|
||||
|
||||
// Initialise variable for the parsed data
|
||||
var parsed parsedData
|
||||
|
@ -27,7 +27,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
|
||||
|
||||
// If debug mode is on, call debugWriteDownload
|
||||
if helpers.Runtime == "debug" {
|
||||
debugWriteDownload(&data)
|
||||
debugWriteDownload(data)
|
||||
}
|
||||
|
||||
// Parse CIF file
|
||||
@ -37,6 +37,9 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Make `data` a nil pointer as it is no longer required
|
||||
data = nil
|
||||
|
||||
// Drop timetable collection
|
||||
dbAccess.DropCollection(dbAccess.TimetableCollection) // I should edit this to prevent removal of VSTP entries in the database.
|
||||
|
||||
@ -77,7 +80,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
||||
|
||||
// If debug mode is on, call debugWriteDownload
|
||||
if helpers.Runtime == "debug" {
|
||||
debugWriteDownload(&data)
|
||||
debugWriteDownload(data)
|
||||
}
|
||||
|
||||
// Parse CIF file
|
||||
@ -87,6 +90,9 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
||||
return err
|
||||
}
|
||||
|
||||
// Make `data` a nil pointer as it is no longer required
|
||||
data = nil
|
||||
|
||||
// If debug mode is on, call debugWriteFile
|
||||
if helpers.Runtime == "debug" {
|
||||
debugWriteFile(&parsed.header, &parsed.sched)
|
||||
@ -123,7 +129,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
|
||||
}
|
||||
|
||||
// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
|
||||
func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) {
|
||||
func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
|
||||
url, err := getUpdateUrl("daily")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -16,7 +16,7 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error {
|
||||
return err
|
||||
}
|
||||
|
||||
unsortedCorpusData, err := parseCorpusData(&resp)
|
||||
unsortedCorpusData, err := parseCorpusData(resp)
|
||||
if err != nil {
|
||||
log.Msg.Error("Error parsing Corpus data", zap.Error(err))
|
||||
return err
|
||||
|
@ -13,7 +13,7 @@ import (
|
||||
)
|
||||
|
||||
// Downloads NROD Data over HTTP from the given URL, extracted data is returned
|
||||
func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) {
|
||||
func NrodDownload(url string, cfg *helpers.Configuration) (*[]byte, error) {
|
||||
log.Msg.Debug("Fetching NROD data", zap.String("Request URL", url))
|
||||
client := http.Client{
|
||||
Timeout: time.Second * 10,
|
||||
@ -49,7 +49,7 @@ func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) {
|
||||
}
|
||||
|
||||
// Extracts GZIP Data from an HTTP Response and returns the decompresses data as a byte array
|
||||
func nrodExtract(resp http.Response) ([]byte, error) {
|
||||
func nrodExtract(resp http.Response) (*[]byte, error) {
|
||||
log.Msg.Debug("Extracting HTTP Response Data")
|
||||
gzReader, err := gzip.NewReader(resp.Body)
|
||||
if err != nil {
|
||||
@ -59,7 +59,7 @@ func nrodExtract(resp http.Response) ([]byte, error) {
|
||||
log.Msg.Error("Unable to read response body")
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
return &data, nil
|
||||
}
|
||||
|
||||
defer gzReader.Close()
|
||||
@ -70,5 +70,5 @@ func nrodExtract(resp http.Response) ([]byte, error) {
|
||||
log.Msg.Error("Failed to read GZIPped data", zap.Error(err))
|
||||
}
|
||||
|
||||
return extractedData, nil
|
||||
return &extractedData, nil
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user