88 lines
2.3 KiB
Go
88 lines
2.3 KiB
Go
package corpus
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
|
|
"git.fjla.uk/owlboard/go-types/pkg/database"
|
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
// Accepts CORPUS data as a byte array and formats it ready for database insertion
|
|
func parseCorpusData(stream io.ReadCloser) (*[]database.CorpusEntry, error) {
|
|
defer stream.Close()
|
|
|
|
log.Debug("Starting CORPUS Data parsing")
|
|
|
|
var corpusEntries []database.CorpusEntry
|
|
decoder := json.NewDecoder(stream)
|
|
|
|
// Expect an object at the root of the JSON stream
|
|
if _, err := decoder.Token(); err != nil {
|
|
log.Error("Error parsing CORPUS Data", zap.Error(err))
|
|
return nil, err
|
|
}
|
|
|
|
// Search for the "TIPLOCDATA" key
|
|
for decoder.More() {
|
|
// Decode the next JSON token
|
|
if tok, err := decoder.Token(); err != nil {
|
|
log.Error("Error parsing CORPUS Data", zap.Error(err))
|
|
return nil, err
|
|
} else if tok == "TIPLOCDATA" {
|
|
// Found the "TIPLOCDATA" key, expect the associated array
|
|
if !decoder.More() {
|
|
err := errors.New("missing array after TIPLOCDATA key")
|
|
log.Error("Error parsing CORPUS Data", zap.Error(err))
|
|
return nil, err
|
|
}
|
|
|
|
// Start reading the array associated with the "TIPLOCDATA" key
|
|
if _, err := decoder.Token(); err != nil {
|
|
log.Error("Error parsing CORPUS Data", zap.Error(err))
|
|
return nil, err
|
|
}
|
|
|
|
// Iterate over the JSON array
|
|
for decoder.More() {
|
|
var corpusEntry database.CorpusEntry
|
|
if err := decoder.Decode(&corpusEntry); err != nil {
|
|
log.Error("Error parsing CORPUS Data", zap.Error(err))
|
|
return nil, err
|
|
}
|
|
corpusEntries = append(corpusEntries, corpusEntry)
|
|
}
|
|
break // Exit loop after processing "TIPLOCDATA" array
|
|
}
|
|
}
|
|
|
|
log.Debug("CORPUS parsing complete")
|
|
|
|
return &corpusEntries, nil
|
|
}
|
|
|
|
// Removes empty fields from CORPUS entries
|
|
func pruneCorpusEntries(corpusEntries []database.CorpusEntry) *[]database.CorpusEntry {
|
|
for i := range corpusEntries {
|
|
if corpusEntries[i].CRS == " " {
|
|
corpusEntries[i].CRS = ""
|
|
}
|
|
if corpusEntries[i].TIPLOC == " " {
|
|
corpusEntries[i].TIPLOC = ""
|
|
}
|
|
if corpusEntries[i].NLCDESC16 == " " {
|
|
corpusEntries[i].NLCDESC16 = ""
|
|
}
|
|
if corpusEntries[i].STANOX == " " {
|
|
corpusEntries[i].STANOX = ""
|
|
}
|
|
if corpusEntries[i].UIC == " " {
|
|
corpusEntries[i].UIC = ""
|
|
}
|
|
}
|
|
|
|
return &corpusEntries
|
|
}
|