Set unordered for bulk CIF deletions from DB & reorganise dbAccess module.

This commit is contained in:
Fred Boniface 2024-04-11 21:06:10 +01:00
parent 76ae25fab6
commit 251a190558
5 changed files with 80 additions and 65 deletions

View File

@ -2,79 +2,16 @@ package dbAccess
import ( import (
"context" "context"
"time"
"git.fjla.uk/owlboard/go-types/pkg/database" "git.fjla.uk/owlboard/go-types/pkg/database"
"git.fjla.uk/owlboard/timetable-mgr/helpers" "git.fjla.uk/owlboard/timetable-mgr/helpers"
"git.fjla.uk/owlboard/timetable-mgr/log" "git.fjla.uk/owlboard/timetable-mgr/log"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"go.uber.org/zap"
) )
const timetableCollection string = "timetable" const timetableCollection string = "timetable"
// CAUTION: Drops the collection named in collectionName
func DropCollection(collectionName string) error {
log.Msg.Info("Dropping collection", zap.String("Collection Name", collectionName))
database := MongoClient.Database(databaseName)
collection := database.Collection(collectionName)
err := collection.Drop(context.Background())
if err != nil {
log.Msg.Error("Error dropping collection", zap.String("Collection Name", collectionName), zap.Error(err))
return err
}
return nil
}
// Checks the update time (unix timestamp) of the collection named in collectionName, uses 'type: collection' entries in the meta collection
func CheckUpdateTime(collectionName string) (int64, error) {
database := MongoClient.Database(databaseName)
collection := database.Collection(metaCollection)
filter := bson.D{
{Key: "target", Value: collectionName},
{Key: "type", Value: "collection"},
}
var result struct {
Updated int64 `bson:"updated"`
}
err := collection.FindOne(context.Background(), filter).Decode(&result)
if err != nil {
return 0, err
}
return result.Updated, nil
}
// Sets a new update time (unix timestamp) of the collection named in collectionName. The update time is calculated within the function.
func SetUpdateTime(collectionName string) error {
log.Msg.Info("Setting update time", zap.String("collection", collectionName))
database := MongoClient.Database(databaseName)
collection := database.Collection("meta")
options := options.Update().SetUpsert(true)
updateTime := time.Now().Unix()
filter := bson.M{
"target": collectionName,
"type": "collection",
}
update := bson.M{
"$set": bson.M{
"updated": updateTime,
"updated_time": time.Now().In(time.UTC),
"target": collectionName,
"type": "collection",
},
}
_, err := collection.UpdateOne(context.Background(), filter, update, options)
if err != nil {
log.Msg.Error("Error setting update time", zap.String("collection", collectionName), zap.Error(err))
return err
}
return nil
}
// Pushes the current version of this application to the data base 'versions' collection. // Pushes the current version of this application to the data base 'versions' collection.
// Currently uses the old name of mq-client // Currently uses the old name of mq-client
func PushVersionToDb() { func PushVersionToDb() {

View File

@ -93,7 +93,9 @@ func DeleteCifEntries(deletions []database.DeleteQuery) error {
bulkDeletions = append(bulkDeletions, mongo.NewDeleteManyModel().SetFilter(filter)) bulkDeletions = append(bulkDeletions, mongo.NewDeleteManyModel().SetFilter(filter))
} }
_, err := collection.BulkWrite(context.Background(), bulkDeletions) bulkWriteOptions := options.BulkWrite().SetOrdered(false)
_, err := collection.BulkWrite(context.Background(), bulkDeletions, bulkWriteOptions)
if err != nil { if err != nil {
log.Msg.Error("Error deleting documents", zap.Error(err)) log.Msg.Error("Error deleting documents", zap.Error(err))
return err return err

View File

@ -1 +1,72 @@
package dbAccess package dbAccess
import (
"context"
"time"
"git.fjla.uk/owlboard/timetable-mgr/log"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
"go.uber.org/zap"
)
// CAUTION: Drops the collection named in collectionName
func DropCollection(collectionName string) error {
log.Msg.Info("Dropping collection", zap.String("Collection Name", collectionName))
database := MongoClient.Database(databaseName)
collection := database.Collection(collectionName)
err := collection.Drop(context.Background())
if err != nil {
log.Msg.Error("Error dropping collection", zap.String("Collection Name", collectionName), zap.Error(err))
return err
}
return nil
}
// Checks the update time (unix timestamp) of the collection named in collectionName, uses 'type: collection' entries in the meta collection
func CheckUpdateTime(collectionName string) (int64, error) {
database := MongoClient.Database(databaseName)
collection := database.Collection(metaCollection)
filter := bson.D{
{Key: "target", Value: collectionName},
{Key: "type", Value: "collection"},
}
var result struct {
Updated int64 `bson:"updated"`
}
err := collection.FindOne(context.Background(), filter).Decode(&result)
if err != nil {
return 0, err
}
return result.Updated, nil
}
// Sets a new update time (unix timestamp) of the collection named in collectionName. The update time is calculated within the function.
func SetUpdateTime(collectionName string) error {
log.Msg.Info("Setting update time", zap.String("collection", collectionName))
database := MongoClient.Database(databaseName)
collection := database.Collection("meta")
options := options.Update().SetUpsert(true)
updateTime := time.Now().Unix()
filter := bson.M{
"target": collectionName,
"type": "collection",
}
update := bson.M{
"$set": bson.M{
"updated": updateTime,
"updated_time": time.Now().In(time.UTC),
"target": collectionName,
"type": "collection",
},
}
_, err := collection.UpdateOne(context.Background(), filter, update, options)
if err != nil {
log.Msg.Error("Error setting update time", zap.String("collection", collectionName), zap.Error(err))
return err
}
return nil
}

View File

@ -5,4 +5,3 @@ const CorpusCollection string = "corpus"
const StationsCollection string = "stations" const StationsCollection string = "stations"
const metaCollection string = "meta" const metaCollection string = "meta"
const TimetableCollection string = "timetable" const TimetableCollection string = "timetable"
const batchsize int = 100

6
dbAccess/types.go Normal file
View File

@ -0,0 +1,6 @@
package dbAccess
// This file should define types used within dbAccess.
// Any types representing database or upstream API resources should
// instead be defined in git.fjla.uk/owlboard/go-types and also be
// reflected in git.fjla.uk/owlboard/ts-types