package corpus import ( "git.fjla.uk/owlboard/timetable-mgr/dbAccess" "git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/log" "git.fjla.uk/owlboard/timetable-mgr/nrod" "go.uber.org/zap" ) // Runs all stages of the CORPUS Update process func RunCorpusUpdate(cfg *helpers.Configuration) error { resp, err := nrod.NrodStream(url, cfg) if err != nil { log.Error("Failed to fetch CORPUS data", zap.Error(err)) return err } unsortedCorpusData, err := parseCorpusData(resp) if err != nil { log.Error("Error parsing Corpus data", zap.Error(err)) return err } corpusData := pruneCorpusEntries(*unsortedCorpusData) stationData := createStationEntries(corpusData) if err := dbAccess.DropCollection(dbAccess.CorpusCollection); err != nil { log.Warn("CORPUS data may be incomplete") log.Error("Error dropping CORPUS Data", zap.Error(err)) return err } if err := dbAccess.DropCollection(dbAccess.StationsCollection); err != nil { log.Warn("Stations data may be incomplete") log.Error("Error dropping stations Data", zap.Error(err)) return err } if err := dbAccess.PutManyCorpus(corpusData); err != nil { log.Warn("CORPUS data may be incomplete") log.Error("Error inserting CORPUS Data", zap.Error(err)) return err } if err := dbAccess.PutManyStations(stationData); err != nil { log.Warn("Stations data may be incomplete") log.Error("Error inserting stations data", zap.Error(err)) return err } if err := dbAccess.CreateCorpusIndexes(); err != nil { log.Error("Corpus Indexes creation failed, application speed will be reduced", zap.Error(err)) } return nil }