Set unordered for bulk CIF deletions from DB & reorganise dbAccess module.
This commit is contained in:
parent
76ae25fab6
commit
251a190558
@ -2,79 +2,16 @@ package dbAccess
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.fjla.uk/owlboard/go-types/pkg/database"
|
"git.fjla.uk/owlboard/go-types/pkg/database"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
"git.fjla.uk/owlboard/timetable-mgr/helpers"
|
||||||
"git.fjla.uk/owlboard/timetable-mgr/log"
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||||
"go.mongodb.org/mongo-driver/bson"
|
"go.mongodb.org/mongo-driver/bson"
|
||||||
"go.mongodb.org/mongo-driver/mongo/options"
|
"go.mongodb.org/mongo-driver/mongo/options"
|
||||||
"go.uber.org/zap"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const timetableCollection string = "timetable"
|
const timetableCollection string = "timetable"
|
||||||
|
|
||||||
// CAUTION: Drops the collection named in collectionName
|
|
||||||
func DropCollection(collectionName string) error {
|
|
||||||
log.Msg.Info("Dropping collection", zap.String("Collection Name", collectionName))
|
|
||||||
database := MongoClient.Database(databaseName)
|
|
||||||
collection := database.Collection(collectionName)
|
|
||||||
|
|
||||||
err := collection.Drop(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
log.Msg.Error("Error dropping collection", zap.String("Collection Name", collectionName), zap.Error(err))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checks the update time (unix timestamp) of the collection named in collectionName, uses 'type: collection' entries in the meta collection
|
|
||||||
func CheckUpdateTime(collectionName string) (int64, error) {
|
|
||||||
database := MongoClient.Database(databaseName)
|
|
||||||
collection := database.Collection(metaCollection)
|
|
||||||
filter := bson.D{
|
|
||||||
{Key: "target", Value: collectionName},
|
|
||||||
{Key: "type", Value: "collection"},
|
|
||||||
}
|
|
||||||
var result struct {
|
|
||||||
Updated int64 `bson:"updated"`
|
|
||||||
}
|
|
||||||
err := collection.FindOne(context.Background(), filter).Decode(&result)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
return result.Updated, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sets a new update time (unix timestamp) of the collection named in collectionName. The update time is calculated within the function.
|
|
||||||
func SetUpdateTime(collectionName string) error {
|
|
||||||
log.Msg.Info("Setting update time", zap.String("collection", collectionName))
|
|
||||||
database := MongoClient.Database(databaseName)
|
|
||||||
collection := database.Collection("meta")
|
|
||||||
options := options.Update().SetUpsert(true)
|
|
||||||
updateTime := time.Now().Unix()
|
|
||||||
filter := bson.M{
|
|
||||||
"target": collectionName,
|
|
||||||
"type": "collection",
|
|
||||||
}
|
|
||||||
update := bson.M{
|
|
||||||
"$set": bson.M{
|
|
||||||
"updated": updateTime,
|
|
||||||
"updated_time": time.Now().In(time.UTC),
|
|
||||||
"target": collectionName,
|
|
||||||
"type": "collection",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
_, err := collection.UpdateOne(context.Background(), filter, update, options)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Msg.Error("Error setting update time", zap.String("collection", collectionName), zap.Error(err))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pushes the current version of this application to the data base 'versions' collection.
|
// Pushes the current version of this application to the data base 'versions' collection.
|
||||||
// Currently uses the old name of mq-client
|
// Currently uses the old name of mq-client
|
||||||
func PushVersionToDb() {
|
func PushVersionToDb() {
|
||||||
|
@ -93,7 +93,9 @@ func DeleteCifEntries(deletions []database.DeleteQuery) error {
|
|||||||
bulkDeletions = append(bulkDeletions, mongo.NewDeleteManyModel().SetFilter(filter))
|
bulkDeletions = append(bulkDeletions, mongo.NewDeleteManyModel().SetFilter(filter))
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := collection.BulkWrite(context.Background(), bulkDeletions)
|
bulkWriteOptions := options.BulkWrite().SetOrdered(false)
|
||||||
|
|
||||||
|
_, err := collection.BulkWrite(context.Background(), bulkDeletions, bulkWriteOptions)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Msg.Error("Error deleting documents", zap.Error(err))
|
log.Msg.Error("Error deleting documents", zap.Error(err))
|
||||||
return err
|
return err
|
||||||
|
@ -1 +1,72 @@
|
|||||||
package dbAccess
|
package dbAccess
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.fjla.uk/owlboard/timetable-mgr/log"
|
||||||
|
"go.mongodb.org/mongo-driver/bson"
|
||||||
|
"go.mongodb.org/mongo-driver/mongo/options"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CAUTION: Drops the collection named in collectionName
|
||||||
|
func DropCollection(collectionName string) error {
|
||||||
|
log.Msg.Info("Dropping collection", zap.String("Collection Name", collectionName))
|
||||||
|
database := MongoClient.Database(databaseName)
|
||||||
|
collection := database.Collection(collectionName)
|
||||||
|
|
||||||
|
err := collection.Drop(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
log.Msg.Error("Error dropping collection", zap.String("Collection Name", collectionName), zap.Error(err))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Checks the update time (unix timestamp) of the collection named in collectionName, uses 'type: collection' entries in the meta collection
|
||||||
|
func CheckUpdateTime(collectionName string) (int64, error) {
|
||||||
|
database := MongoClient.Database(databaseName)
|
||||||
|
collection := database.Collection(metaCollection)
|
||||||
|
filter := bson.D{
|
||||||
|
{Key: "target", Value: collectionName},
|
||||||
|
{Key: "type", Value: "collection"},
|
||||||
|
}
|
||||||
|
var result struct {
|
||||||
|
Updated int64 `bson:"updated"`
|
||||||
|
}
|
||||||
|
err := collection.FindOne(context.Background(), filter).Decode(&result)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return result.Updated, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sets a new update time (unix timestamp) of the collection named in collectionName. The update time is calculated within the function.
|
||||||
|
func SetUpdateTime(collectionName string) error {
|
||||||
|
log.Msg.Info("Setting update time", zap.String("collection", collectionName))
|
||||||
|
database := MongoClient.Database(databaseName)
|
||||||
|
collection := database.Collection("meta")
|
||||||
|
options := options.Update().SetUpsert(true)
|
||||||
|
updateTime := time.Now().Unix()
|
||||||
|
filter := bson.M{
|
||||||
|
"target": collectionName,
|
||||||
|
"type": "collection",
|
||||||
|
}
|
||||||
|
update := bson.M{
|
||||||
|
"$set": bson.M{
|
||||||
|
"updated": updateTime,
|
||||||
|
"updated_time": time.Now().In(time.UTC),
|
||||||
|
"target": collectionName,
|
||||||
|
"type": "collection",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_, err := collection.UpdateOne(context.Background(), filter, update, options)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Msg.Error("Error setting update time", zap.String("collection", collectionName), zap.Error(err))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
@ -5,4 +5,3 @@ const CorpusCollection string = "corpus"
|
|||||||
const StationsCollection string = "stations"
|
const StationsCollection string = "stations"
|
||||||
const metaCollection string = "meta"
|
const metaCollection string = "meta"
|
||||||
const TimetableCollection string = "timetable"
|
const TimetableCollection string = "timetable"
|
||||||
const batchsize int = 100
|
|
||||||
|
6
dbAccess/types.go
Normal file
6
dbAccess/types.go
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
package dbAccess
|
||||||
|
|
||||||
|
// This file should define types used within dbAccess.
|
||||||
|
// Any types representing database or upstream API resources should
|
||||||
|
// instead be defined in git.fjla.uk/owlboard/go-types and also be
|
||||||
|
// reflected in git.fjla.uk/owlboard/ts-types
|
Loading…
Reference in New Issue
Block a user