timetable-extension #1

Open
fred.boniface wants to merge 154 commits from timetable-extension into main
4 changed files with 22 additions and 10 deletions
Showing only changes of commit a2c52f7b8b - Show all commits

View File

@ -3,6 +3,7 @@ package cif
import (
"bytes"
"encoding/json"
"errors"
"git.fjla.uk/owlboard/go-types/pkg/upstreamApi"
"git.fjla.uk/owlboard/timetable-mgr/log"
@ -10,9 +11,14 @@ import (
)
// Unmarshalls data into the correct types for processing
func parseCifData(data []byte) (*parsedData, error) {
// This function suffers from extremely high memory usage
func parseCifData(data *[]byte) (*parsedData, error) {
if data == nil {
err := errors.New("unable to parse nil pointer")
return nil, err
}
// Split the data into lines
lines := bytes.Split(data, []byte("\n"))
lines := bytes.Split(*data, []byte("\n"))
// Initialise variable for the parsed data
var parsed parsedData

View File

@ -27,7 +27,7 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
// If debug mode is on, call debugWriteDownload
if helpers.Runtime == "debug" {
debugWriteDownload(&data)
debugWriteDownload(data)
}
// Parse CIF file
@ -37,6 +37,9 @@ func runCifFullDownload(cfg *helpers.Configuration) error {
return err
}
// Make `data` a nil pointer as it is no longer required
data = nil
// Drop timetable collection
dbAccess.DropCollection(dbAccess.TimetableCollection) // I should edit this to prevent removal of VSTP entries in the database.
@ -77,7 +80,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
// If debug mode is on, call debugWriteDownload
if helpers.Runtime == "debug" {
debugWriteDownload(&data)
debugWriteDownload(data)
}
// Parse CIF file
@ -87,6 +90,9 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
return err
}
// Make `data` a nil pointer as it is no longer required
data = nil
// If debug mode is on, call debugWriteFile
if helpers.Runtime == "debug" {
debugWriteFile(&parsed.header, &parsed.sched)
@ -123,7 +129,7 @@ func runCifUpdateDownload(cfg *helpers.Configuration, metadata *dbAccess.CifMeta
}
// Wraps nrod.NrodDownload() into a function which can handle downloading data for a given day
func fetchUpdate(t time.Time, cfg *helpers.Configuration) ([]byte, error) {
func fetchUpdate(t time.Time, cfg *helpers.Configuration) (*[]byte, error) {
url, err := getUpdateUrl("daily")
if err != nil {
return nil, err

View File

@ -16,7 +16,7 @@ func RunCorpusUpdate(cfg *helpers.Configuration) error {
return err
}
unsortedCorpusData, err := parseCorpusData(&resp)
unsortedCorpusData, err := parseCorpusData(resp)
if err != nil {
log.Msg.Error("Error parsing Corpus data", zap.Error(err))
return err

View File

@ -13,7 +13,7 @@ import (
)
// Downloads NROD Data over HTTP from the given URL, extracted data is returned
func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) {
func NrodDownload(url string, cfg *helpers.Configuration) (*[]byte, error) {
log.Msg.Debug("Fetching NROD data", zap.String("Request URL", url))
client := http.Client{
Timeout: time.Second * 10,
@ -49,7 +49,7 @@ func NrodDownload(url string, cfg *helpers.Configuration) ([]byte, error) {
}
// Extracts GZIP Data from an HTTP Response and returns the decompresses data as a byte array
func nrodExtract(resp http.Response) ([]byte, error) {
func nrodExtract(resp http.Response) (*[]byte, error) {
log.Msg.Debug("Extracting HTTP Response Data")
gzReader, err := gzip.NewReader(resp.Body)
if err != nil {
@ -59,7 +59,7 @@ func nrodExtract(resp http.Response) ([]byte, error) {
log.Msg.Error("Unable to read response body")
return nil, err
}
return data, nil
return &data, nil
}
defer gzReader.Close()
@ -70,5 +70,5 @@ func nrodExtract(resp http.Response) ([]byte, error) {
log.Msg.Error("Failed to read GZIPped data", zap.Error(err))
}
return extractedData, nil
return &extractedData, nil
}