timetable-mgr/src/corpus/fetch.go

package corpus

import (
	"encoding/json"
	"errors"

	"git.fjla.uk/owlboard/go-types/pkg/database"
	"git.fjla.uk/owlboard/timetable-mgr/helpers"
	"git.fjla.uk/owlboard/timetable-mgr/log"
	"git.fjla.uk/owlboard/timetable-mgr/nrod"
	"go.uber.org/zap"
)

const url string = "https://publicdatafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS"

func fetchCorpus(cfg *helpers.Configuration) (*[]byte, error) {
	log.Msg.Info("Fetching CORPUS Data")
	data, err := nrod.NrodDownload(url, cfg)
	if err != nil {
		log.Msg.Error("Corpus update failed")
		return nil, err
	}
	return &data, nil
}

func parseCorpusData(jsonData *[]byte) ([]database.CorpusEntry, error) {
	log.Msg.Info("Unmarshalling CORPUS Data")

	var dataMap map[string]interface{}
	err := json.Unmarshal(*jsonData, &dataMap)
	if err != nil {
		log.Msg.Error("Unable to unmarshal CORPUS data", zap.Error(err))
	}

	corpusDataArrayInterface, ok := dataMap["TIPLOCDATA"]
	if !ok {
		err := errors.New("corpus Data not in expected format")
		log.Msg.Error("Error parsing CORPUS Data", zap.Error(err))
		return nil, err
	}

	corpusDataArray, ok := corpusDataArrayInterface.([]interface{})
	if !ok {
		err := errors.New("corpus data missing the data array")
		log.Msg.Error("Error parsing CORPUS Data", zap.Error(err))
		return nil, err
	}

	var corpusEntries []database.CorpusEntry
	for _, item := range corpusDataArray {
		jsonItem, err := json.Marshal(item)
		if err != nil {
			log.Msg.Error("Error parsing CORPUS Data", zap.Error(err))
			return nil, err
		}

		var corpusEntry database.CorpusEntry
		err = json.Unmarshal(jsonItem, &corpusEntry)
		if err != nil {
			log.Msg.Error("Error parsing CORPUS Data", zap.Error(err))
			return nil, err
		}

		corpusEntries = append(corpusEntries, corpusEntry)
	}

	return corpusEntries, nil
}

func pruneCorpusEntries(corpusEntries []database.CorpusEntry) []database.CorpusEntry {
	for i := range corpusEntries {
		if corpusEntries[i].CRS == " " {
			corpusEntries[i].CRS = ""
		}
		if corpusEntries[i].TIPLOC == " " {
			corpusEntries[i].TIPLOC = ""
		}
		if corpusEntries[i].NLCDESC16 == " " {
			corpusEntries[i].NLCDESC16 = ""
		}
		if corpusEntries[i].STANOX == " " {
			corpusEntries[i].STANOX = ""
		}
		if corpusEntries[i].UIC == " " {
			corpusEntries[i].UIC = ""
		}
	}

	return corpusEntries
}