public static WriteModel <TEntity> ToMongoDbWriteModel <TEntity>([NotNull] this IUpdateEntry updateEntry)
{
Check.NotNull(updateEntry, nameof(updateEntry));
if (!typeof(TEntity).GetTypeInfo().IsAssignableFrom(updateEntry.EntityType.ClrType))
{
throw new InvalidOperationException($"Entity must derive from {nameof(TEntity)}.");
}
if (updateEntry.EntityState != EntityState.Added &&
updateEntry.EntityState != EntityState.Modified &&
updateEntry.EntityState != EntityState.Deleted)
{
throw new InvalidOperationException($"Entity state must be Added, Modified, or Deleted.");
}
WriteModel <TEntity> writeModel;
switch (updateEntry.EntityState)
{
case EntityState.Added:
writeModel = ToInsertModel <TEntity>(updateEntry as InternalEntityEntry);
break;
case EntityState.Modified:
writeModel = new ReplaceOneModel <TEntity>(
GetIdFilter <TEntity>(updateEntry),
(TEntity)updateEntry.ToEntityEntry().Entity);
break;
default:
writeModel = new DeleteOneModel <TEntity>(GetIdFilter <TEntity>(updateEntry));
break;
}
return(writeModel);
}
public void BulkWrite_with_DeleteOne(
[Values(false, true)] bool async)
{
var subject = CreateSubject();
var model = new DeleteOneModel <B>(_providedFilter);
var options = new BulkWriteOptions();
if (async)
{
subject.BulkWriteAsync(new[] { model }, options, CancellationToken.None);
_mockDerivedCollection.Verify(
c => c.BulkWriteAsync(
It.Is <IEnumerable <WriteModel <B> > >(v => v.OfType <DeleteOneModel <B> >()
.Where(m => RenderFilter(m.Filter).Equals(_expectedFilter)).Count() == 1),
options,
CancellationToken.None),
Times.Once);
}
else
{
subject.BulkWrite(new[] { model }, options, CancellationToken.None);
_mockDerivedCollection.Verify(
c => c.BulkWrite(
It.Is <IEnumerable <WriteModel <B> > >(v => v.OfType <DeleteOneModel <B> >()
.Where(m => RenderFilter(m.Filter).Equals(_expectedFilter)).Count() == 1),
options,
CancellationToken.None),
Times.Once);
}
}
private void OnDeleted(SaveDocumentsChangesContext <TDocument> context)
{
var filter = PrimaryKeyFilter;
var writeModel = new DeleteOneModel <TDocument>(filter);
context.AddWriteModel(writeModel);
}
public void UpdateClients(List <BannedClientRecord> newBannedClientRecords)
{
var filter = Builders <BannedClientRecord> .Filter;
var update = Builders <BannedClientRecord> .Update;
var updatesLength = newBannedClientRecords.Count;
var updates = new WriteModel <BannedClientRecord> [updatesLength];
for (int i = 0; i < updatesLength; i++)
{
var banRecord = newBannedClientRecords[i];
var source = banRecord.Source;
var currentAccFilter = filter.Eq(a => a.Source, source);
if (banRecord.BanCounts == 0)
{
updates[i] = new DeleteOneModel <BannedClientRecord>(currentAccFilter);
}
else
{
updates[i] = new ReplaceOneModel <BannedClientRecord>(currentAccFilter, banRecord)
{
IsUpsert = true
}
};
}
bannedClientsCollection.BulkWrite(updates);
}
}
private List <WriteModel <AccountModel> > GetAccountUpdates(List <DiffObject.Account> accounts)
{
if (accounts == null || accounts.Count < 1)
{
return(null);
}
var filter = Builders <AccountModel> .Filter;
var update = Builders <AccountModel> .Update;
var accLength = accounts.Count;
var updates = new List <WriteModel <AccountModel> >(accLength);
updates.AddRange(Enumerable.Repeat(default(WriteModel <AccountModel>), accLength));
Parallel.For(0, accLength, (i) =>
{
var acc = accounts[i];
var currentAccFilter = filter.Eq(a => a.Id, acc.Id);
if (acc.IsInserted)
{
updates[i] = new InsertOneModel <AccountModel>(new AccountModel
{
Id = acc.Id,
Nonce = acc.Nonce,
PubKey = acc.PubKey,
RequestRateLimits = acc.RequestRateLimits,
Withdrawal = (acc.Withdrawal ?? 0)
});
}
else if (acc.IsDeleted)
{
updates[i] = new DeleteOneModel <AccountModel>(currentAccFilter);
}
else
{
var updateDefs = new List <UpdateDefinition <AccountModel> >();
if (acc.Nonce != 0)
{
updateDefs.Add(update.Set(a => a.Nonce, acc.Nonce));
}
if (acc.RequestRateLimits != null)
{
updateDefs.Add(update.Set(a => a.RequestRateLimits, acc.RequestRateLimits));
}
if (acc.Withdrawal.HasValue)
{
updateDefs.Add(update.Set(a => a.Withdrawal, acc.Withdrawal.Value));
}
updates[i] = new UpdateOneModel <AccountModel>(currentAccFilter, update.Combine(updateDefs));
}
});
return(updates);
}
private WriteModel <AccountModel>[] GetAccountUpdates(List <DiffObject.Account> accounts)
{
if (accounts == null || accounts.Count < 1)
{
return(null);
}
var filter = Builders <AccountModel> .Filter;
var update = Builders <AccountModel> .Update;
var accLength = accounts.Count;
var updates = new WriteModel <AccountModel> [accLength];
for (int i = 0; i < accLength; i++)
{
var acc = accounts[i];
var currentAccFilter = filter.Eq(a => a.Id, acc.Id);
if (acc.IsInserted)
{
updates[i] = new InsertOneModel <AccountModel>(new AccountModel
{
Id = acc.Id,
Nonce = acc.Nonce,
PubKey = acc.PubKey,
RequestRateLimits = acc.RequestRateLimits,
Withdrawal = (acc.Withdrawal.HasValue ? acc.Withdrawal.Value : 0)
});
}
else if (acc.IsDeleted)
{
updates[i] = new DeleteOneModel <AccountModel>(currentAccFilter);
}
else
{
var updateDefs = new List <UpdateDefinition <AccountModel> >();
if (acc.Nonce != 0)
{
updateDefs.Add(update.Set(a => a.Nonce, acc.Nonce));
}
if (acc.RequestRateLimits != null)
{
updateDefs.Add(update.Set(a => a.RequestRateLimits, acc.RequestRateLimits));
}
if (acc.Withdrawal.HasValue)
{
updateDefs.Add(update.Set(a => a.Withdrawal, acc.Withdrawal.Value));
}
updates[i] = new UpdateOneModel <AccountModel>(currentAccFilter, update.Combine(updateDefs));
}
}
return(updates);
}
public override void RemoveHash(string key)
{
if (key == null)
{
throw new ArgumentNullException(nameof(key));
}
var filter = new BsonDocument(nameof(HashDto.Key), key);
var writeModel = new DeleteOneModel <BsonDocument>(filter);
_writeModels.Add(writeModel);
}
public override void RemoveFromSet(string key, string value)
{
if (key == null)
{
throw new ArgumentNullException(nameof(key));
}
var filter = CreateSetFilter(key, value);
var writeModel = new DeleteOneModel <BsonDocument>(filter);
_writeModels.Add(writeModel);
}
public void BulkWriteAsync_with_DeleteOne()
{
var subject = CreateSubject();
var model = new DeleteOneModel <B>(_providedFilter);
var options = new BulkWriteOptions();
subject.BulkWriteAsync(new[] { model }, options, CancellationToken.None);
_derivedCollection.Received().BulkWriteAsync(
Arg.Is <IEnumerable <WriteModel <B> > >(v => v.OfType <DeleteOneModel <B> >()
.Where(m => RenderFilter(m.Filter).Equals(_expectedFilter)).Count() == 1),
options,
CancellationToken.None);
}
private List <WriteModel <OrderModel> > GetOrderUpdates(List <DiffObject.Order> orders)
{
if (orders == null || orders.Count < 1)
{
return(null);
}
unchecked
{
var filter = Builders <OrderModel> .Filter;
var update = Builders <OrderModel> .Update;
var ordersLength = orders.Count;
var updates = new List <WriteModel <OrderModel> >(ordersLength);
updates.AddRange(Enumerable.Repeat(default(WriteModel <OrderModel>), ordersLength));
Parallel.For(0, ordersLength, (i) =>
{
var order = orders[i];
var currentOrderFilter = filter.And(filter.Eq(s => s.Id, (long)order.OrderId));
if (order.IsInserted)
{
updates[i] = new InsertOneModel <OrderModel>(new OrderModel
{
Id = (long)order.OrderId,
Amount = order.AmountDiff,
QuoteAmount = order.QuoteAmountDiff,
Price = order.Price,
Account = order.Account
});
}
else if (order.IsDeleted)
{
updates[i] = new DeleteOneModel <OrderModel>(currentOrderFilter);
}
else
{
updates[i] = new UpdateOneModel <OrderModel>(
currentOrderFilter,
update.Inc(b => b.Amount, order.AmountDiff).Inc(b => b.QuoteAmount, order.QuoteAmountDiff)
);
}
});
return(updates);
}
}
private WriteModel <OrderModel>[] GetOrderUpdates(List <DiffObject.Order> orders)
{
if (orders == null || orders.Count < 1)
{
return(null);
}
unchecked
{
var filter = Builders <OrderModel> .Filter;
var update = Builders <OrderModel> .Update;
var ordersLength = orders.Count;
var updates = new WriteModel <OrderModel> [ordersLength];
for (int i = 0; i < ordersLength; i++)
{
var order = orders[i];
var currentOrderFilter = filter.And(filter.Eq(s => s.Id, (long)order.OrderId));
if (order.IsInserted)
{
updates[i] = new InsertOneModel <OrderModel>(new OrderModel
{
Id = (long)order.OrderId,
Amount = order.AmountDiff,
QuoteAmount = order.QuoteAmountDiff,
Price = order.Price,
Account = order.Account
});
}
else if (order.IsDeleted)
{
updates[i] = new DeleteOneModel <OrderModel>(currentOrderFilter);
}
else
{
updates[i] = new UpdateOneModel <OrderModel>(
currentOrderFilter,
update.Inc(b => b.Amount, order.AmountDiff).Inc(b => b.QuoteAmount, order.QuoteAmountDiff)
);
}
}
return(updates);
}
}
public override void RemoveFromSet(string key, string value)
{
if (key == null)
{
throw new ArgumentNullException(nameof(key));
}
var filter = new BsonDocument("$and", new BsonArray
{
new BsonDocument(nameof(KeyJobDto.Key), key),
new BsonDocument(nameof(SetDto.Value), value),
new BsonDocument("_t", nameof(SetDto))
});
var writeModel = new DeleteOneModel <BsonDocument>(filter);
_writeModels.Add(writeModel);
}
private List <WriteModel <BalanceModel> > GetBalanceUpdates(List <DiffObject.Balance> balances)
{
if (balances == null || balances.Count < 1)
{
return(null);
}
var filter = Builders <BalanceModel> .Filter;
var update = Builders <BalanceModel> .Update;
var balancesLength = balances.Count;
var updates = new List <WriteModel <BalanceModel> >(balancesLength);
updates.AddRange(Enumerable.Repeat(default(WriteModel <BalanceModel>), balancesLength));
Parallel.For(0, balancesLength, (i) =>
{
var balance = balances[i];
var currentBalanceFilter = filter.Eq(s => s.Id, balance.Id);
if (balance.IsInserted)
{
updates[i] = new InsertOneModel <BalanceModel>(new BalanceModel
{
Id = balance.Id,
Amount = balance.AmountDiff,
Liabilities = balance.LiabilitiesDiff
});
}
else if (balance.IsDeleted)
{
updates[i] = new DeleteOneModel <BalanceModel>(currentBalanceFilter);
}
else
{
updates[i] = new UpdateOneModel <BalanceModel>(
currentBalanceFilter,
update
.Inc(b => b.Amount, balance.AmountDiff)
.Inc(b => b.Liabilities, balance.LiabilitiesDiff)
);
}
});
return(updates);
}
private WriteModel <BalanceModel>[] GetBalanceUpdates(List <DiffObject.Balance> balances)
{
if (balances == null || balances.Count < 1)
{
return(null);
}
var filter = Builders <BalanceModel> .Filter;
var update = Builders <BalanceModel> .Update;
var balancesLength = balances.Count;
var updates = new WriteModel <BalanceModel> [balancesLength];
for (int i = 0; i < balancesLength; i++)
{
var balance = balances[i];
var currentBalanceFilter = filter.Eq(s => s.Id, balance.Id);
if (balance.IsInserted)
{
updates[i] = new InsertOneModel <BalanceModel>(new BalanceModel
{
Id = balance.Id,
Amount = balance.AmountDiff,
Liabilities = balance.LiabilitiesDiff
});
}
else if (balance.IsDeleted)
{
updates[i] = new DeleteOneModel <BalanceModel>(currentBalanceFilter);
}
else
{
updates[i] = new UpdateOneModel <BalanceModel>(
currentBalanceFilter,
update
.Inc(b => b.Amount, balance.AmountDiff)
.Inc(b => b.Liabilities, balance.LiabilitiesDiff)
);
}
}
return(updates);
}
private void RunDeleteCommand(GridFSBucket bucket, BsonDocument command)
{
var collectionName = command["delete"].AsString;
var collection = bucket.Database.GetCollection <BsonDocument>(collectionName);
var requests = new List <WriteModel <BsonDocument> >();
foreach (BsonDocument deleteStatement in command["deletes"].AsBsonArray)
{
var filter = deleteStatement["q"].AsBsonDocument;
var limit = deleteStatement["limit"].ToInt32();
WriteModel <BsonDocument> request;
if (limit == 1)
{
request = new DeleteOneModel <BsonDocument>(filter);
}
else
{
request = new DeleteManyModel <BsonDocument>(filter);
}
requests.Add(request);
}
collection.BulkWrite(requests);
}
static void SafeMain()
{
Console.WriteLine("MongoTransfer - Copyright (C) 2018-" + DateTime.Now.Year + " Simon Mourier. All rights reserved.");
Console.WriteLine();
if (CommandLine.HelpRequested)
{
Help();
return;
}
var inCnx = CommandLine.GetNullifiedArgument("incnx", _defaultCnx);
var outCnx = CommandLine.GetNullifiedArgument("outcnx");
var inCollName = CommandLine.GetNullifiedArgument("inc");
var outCollName = CommandLine.GetNullifiedArgument("outc");
var inDbName = CommandLine.GetNullifiedArgument("ind");
var outDbName = CommandLine.GetNullifiedArgument("outd", inDbName);
if (outCnx == null || inCollName == null || inDbName == null)
{
Help();
return;
}
if (outCnx == "*")
{
if (outCollName == null)
{
Console.WriteLine("ERROR: same connection strings and output collection unspecified.");
Help();
return;
}
outCnx = inCnx;
}
if (inCnx.EqualsIgnoreCase(outCnx) && inCollName.EqualsIgnoreCase(outCollName))
{
Console.WriteLine("ERROR: same connection strings and same collection names.");
Help();
return;
}
var mm = CommandLine.GetArgument("mirror", MirrorMode.None);
var batch = CommandLine.GetArgument("batch", 50000);
var inSettings = MongoClientSettings.FromConnectionString(inCnx);
var outSettings = MongoClientSettings.FromConnectionString(outCnx);
Console.WriteLine("Input server : " + inSettings.Server);
Console.WriteLine("Input database : " + inDbName);
Console.WriteLine("Input collection : " + inCollName);
Console.WriteLine();
Console.WriteLine("Output server : " + string.Join(", ", outSettings.Servers));
Console.WriteLine("Output database : " + outDbName);
Console.WriteLine("Output collection : " + outCollName);
Console.WriteLine();
Console.WriteLine("Mirror mode : " + mm);
Console.WriteLine("Batch size : " + batch);
var sw = new Stopwatch();
sw.Start();
var inClient = new MongoClient(inSettings);
var outClient = new MongoClient(outSettings);
var inDb = inClient.GetDatabase(inDbName);
var outDb = outClient.GetDatabase(outDbName);
var inColl = inDb.GetCollection <BsonDocument>(inCollName);
var outColl = outDb.GetCollection <BsonDocument>(outCollName);
var outIds = new HashSet <BsonValue>();
if (mm != MirrorMode.None)
{
Console.WriteLine();
var projection = Builders <BsonDocument> .Projection.Include(_idName);
foreach (var doc in outColl.Find(d => true).Project(projection).ToEnumerable())
{
outIds.Add(doc[_idName]);
}
if (outIds.Count == 0)
{
Console.WriteLine("No document exist in the output collection.");
}
else
{
var path = Path.GetFullPath("mirror." + Guid.NewGuid().ToString("N") + ".json");
using (var writer = new StreamWriter(path, false, Encoding.UTF8))
{
using (var jw = new JsonWriter(writer))
{
BsonSerializer.Serialize(jw, outIds);
}
}
Console.WriteLine(outIds.Count + " document(s) exist in the output collection. Ids have been output to '" + path + "'.");
}
}
Console.WriteLine();
Copy(inColl, outColl, batch, outIds).Wait();
if (mm != MirrorMode.None)
{
Console.WriteLine();
if (outIds.Count == 0)
{
Console.WriteLine("No document existed in the output collection and not in the input collection. Mirror is implicit.");
}
else
{
var path = Path.GetFullPath("mirror." + Guid.NewGuid().ToString("N") + ".out.json");
using (var writer = new StreamWriter(path, false, Encoding.UTF8))
{
using (var jw = new JsonWriter(writer))
{
BsonSerializer.Serialize(jw, outIds);
}
}
Console.WriteLine(outIds.Count + " document(s) exist in the output collection and not in the input collection. Ids have been output to '" + path + "'.");
if (mm == MirrorMode.Delete)
{
var models = new List <DeleteOneModel <BsonDocument> >();
foreach (var id in outIds)
{
var idFilter = Builders <BsonDocument> .Filter.Eq(_idName, id);
var model = new DeleteOneModel <BsonDocument>(idFilter);
models.Add(model);
}
outColl.BulkWriteAsync(models).Wait();
Console.WriteLine("Delete mode. " + outIds.Count + " document(s) have been deleted from the output collection.");
}
else
{
Console.WriteLine("Test mode. Nothing was deleted.");
}
}
}
Console.WriteLine();
Console.WriteLine("Elapsed: " + sw.Elapsed);
}
private Task RunDeleteCommandAsync(GridFSBucket bucket, BsonDocument command)
{
var collectionName = command["delete"].AsString;
var collection = bucket.Database.GetCollection<BsonDocument>(collectionName);
var requests = new List<WriteModel<BsonDocument>>();
foreach (BsonDocument deleteStatement in command["deletes"].AsBsonArray)
{
var filter = deleteStatement["q"].AsBsonDocument;
var limit = deleteStatement["limit"].ToInt32();
WriteModel<BsonDocument> request;
if (limit == 1)
{
request = new DeleteOneModel<BsonDocument>(filter);
}
else
{
request = new DeleteManyModel<BsonDocument>(filter);
}
requests.Add(request);
}
return collection.BulkWriteAsync(requests);
}