56 lines
1.6 KiB
Go
56 lines
1.6 KiB
Go
package corpus
|
|
|
|
import (
|
|
"git.fjla.uk/owlboard/timetable-mgr/dbAccess"
|
|
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
|
"git.fjla.uk/owlboard/timetable-mgr/nrod"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
// Runs all stages of the CORPUS Update process
|
|
func RunCorpusUpdate(cfg *helpers.Configuration) error {
|
|
resp, err := nrod.NrodStream(url, cfg)
|
|
if err != nil {
|
|
log.Error("Failed to fetch CORPUS data", zap.Error(err))
|
|
return err
|
|
}
|
|
|
|
unsortedCorpusData, err := parseCorpusData(resp)
|
|
if err != nil {
|
|
log.Error("Error parsing Corpus data", zap.Error(err))
|
|
return err
|
|
}
|
|
|
|
corpusData := pruneCorpusEntries(*unsortedCorpusData)
|
|
//stationData := createStationEntries(corpusData)
|
|
|
|
if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil {
|
|
log.Warn("CORPUS data may be incomplete")
|
|
log.Error("Error dropping CORPUS Data", zap.Error(err))
|
|
return err
|
|
}
|
|
//if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil {
|
|
// log.Warn("Stations data may be incomplete")
|
|
// log.Error("Error dropping stations Data", zap.Error(err))
|
|
// return err
|
|
//}
|
|
|
|
if err := dbAccess.PutManyCorpus(corpusData); err != nil {
|
|
log.Warn("CORPUS data may be incomplete")
|
|
log.Error("Error inserting CORPUS Data", zap.Error(err))
|
|
return err
|
|
}
|
|
//if err := dbAccess.PutManyStations(stationData); err != nil {
|
|
// log.Warn("Stations data may be incomplete")
|
|
// log.Error("Error inserting stations data", zap.Error(err))
|
|
// return err
|
|
//}
|
|
|
|
if err := dbAccess.CreateCorpusIndexes(); err != nil {
|
|
log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err))
|
|
}
|
|
|
|
return nil
|
|
}
|