public BulkWriteResult <Vehicle> BulkWrite(IClientSessionHandle session, IEnumerable <WriteModel <Vehicle> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
public BulkWriteResult <T> BulkWrite(string collectionName, IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null)
{
return(infrastructure.Exec(database =>
{
return database.GetCollection <T>(collectionName).BulkWrite(requests, options);
}));
}
/// <summary>
/// Replace a collection of documents from the mongodb collection.
/// <remarks>
/// The transaction filter is created using the property Id from <see cref="IDocumentRoot"/>. This behaviour is by design.
/// </remarks>
/// </summary>
/// <param name="context">Mongodb collection context</param>
/// <param name="documents">Collection of documents</param>
/// <param name="handle"><see cref="IClientSessionHandle"/></param>
/// <param name="options"><see cref="BulkWriteOptions"/></param>
/// <typeparam name="TDocument">Document type</typeparam>
/// <returns><see cref="BulkWriteResult{T}"/></returns>
public static async ValueTask <BulkWriteResult <TDocument> > BulkReplace <TDocument>(this ICallistoCollectionContext <TDocument> context, IEnumerable <TDocument> documents,
IClientSessionHandle handle,
BulkWriteOptions options = null)
where TDocument : class, IDocumentRoot, IReplaceableDocument
{
IEnumerable <TDocument> updatableDocuments = documents.ToList();
ValidateOperationParameters(updatableDocuments, context, "bulk-replace");
return(await WriteWithHandle(context, documents.Select(a => new ReplaceOneModel <TDocument>(a.FilterById(), a)).ToList(), handle, options));
}
public override BulkWriteResult <TDocument> BulkWrite(IEnumerable <WriteModel <TDocument> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default(CancellationToken))
{
throw new NotImplementedException();
}
public BulkWriteResult <TModel> BulkWrite(IEnumerable <WriteModel <TModel> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default(CancellationToken))
{
return(_collection.BulkWrite(requests, options, cancellationToken));
}
public Task <BulkWriteResult <T> > BulkWriteAsync(IClientSessionHandle session, IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default) => BulkWriteAsync(requests, options, cancellationToken);
public BulkWriteResult <AnimatedImage> BulkWrite(IEnumerable <WriteModel <AnimatedImage> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
/// <inheritdoc/>
public BulkWriteResult <T> BulkWrite(IClientSessionHandle session, IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default)
{
return(_actualCollection.BulkWrite(session, requests, options, cancellationToken));
}
/// <inheritdoc/>
public Task <BulkWriteResult <T> > BulkWriteAsync(IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default)
{
return(_actualCollection.BulkWriteAsync(requests, options, cancellationToken));
}
public static void ParallelBulkWrite <TDocument>(this IMongoCollection <TDocument> @this,
IEnumerable <TDocument> docs,
int batchSize, int parallelBatches,
Func <TDocument, WriteModel <TDocument> > createWriteModel,
CancellationToken cancellToken = default(CancellationToken))
{
if (@this == null)
{
throw new ArgumentNullException(nameof(@this));
}
if (docs == null)
{
throw new ArgumentNullException(nameof(docs));
}
if (batchSize < 1)
{
throw new ArgumentException("'batchSize' must be at least 1.", nameof(batchSize));
}
if (parallelBatches < 1)
{
throw new ArgumentException("'parallelBatches' must be at least 1.", nameof(parallelBatches));
}
if (createWriteModel == null)
{
throw new ArgumentNullException(nameof(createWriteModel));
}
using (var cancellTokenSrc = cancellToken == default(CancellationToken)
? new CancellationTokenSource()
: CancellationTokenSource.CreateLinkedTokenSource(cancellToken))
{
try
{
var bulkWriteOpts = new BulkWriteOptions {
IsOrdered = false
};
var bulkWritesEnumr = docs
.Select(createWriteModel)
.BatchesOf(batchSize)
.Select(x => @this.BulkWriteAsync(x, bulkWriteOpts, cancellToken))
.GetEnumerator();
Func <int, Task <BulkWriteResult <TDocument> >[]> startBulkWrites = upToN =>
{
var started = new List <Task <BulkWriteResult <TDocument> > >();
while (started.Count < upToN && bulkWritesEnumr.MoveNext())
{
started.Add(bulkWritesEnumr.Current);
}
return(started.ToArray());
};
var runningBulkWrites = startBulkWrites(parallelBatches);
while (runningBulkWrites.Length > 0)
{
var complBulkWriteInx = Task.WaitAny(runningBulkWrites, cancellToken);
var complBulkWrite = runningBulkWrites[complBulkWriteInx];
if (complBulkWrite.Status != TaskStatus.RanToCompletion) // TODO: Consider parameterized error tolerance.
{
throw new Exception("Error occurred during a bulkWrite. ", complBulkWrite.Exception);
}
runningBulkWrites = runningBulkWrites
.Where(x => !ReferenceEquals(x, complBulkWrite))
.Union(startBulkWrites(1))
.ToArray();
}
}
catch (Exception)
{
try
{
if (!cancellTokenSrc.IsCancellationRequested)
{
cancellTokenSrc.Cancel();
}
}
catch (Exception)
{
}
throw;
}
}
}
public Task <BulkWriteResult <T> > BulkWriteAsync(IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default(CancellationToken))
{
return(_Repository.BulkWriteAsync(requests, options, cancellationToken));
}
public abstract BulkWriteResult <TDocument> BulkWrite(IEnumerable <WriteModel <TDocument> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default(CancellationToken));
public InsertStats InsertTransactions(SyncBlockTransactionsOperation item)
{
var stats = new InsertStats {
Items = new List <MapTransactionAddress>()
};
if (item.BlockInfo != null)
{
// remove all transactions from the memory pool
item.Transactions.ForEach(t =>
{
data.MemoryTransactions.TryRemove(t.GetHash().ToString(), out Transaction outer);
});
// break the work in to batches of transactions
var queue = new Queue <NBitcoin.Transaction>(item.Transactions);
do
{
var items = GetBatch(configuration.MongoBatchSize, queue).ToList();
try
{
if (item.BlockInfo != null)
{
var inserts = items.Select(s => new MapTransactionBlock {
BlockIndex = item.BlockInfo.Height, TransactionId = s.GetHash().ToString()
}).ToList();
stats.Transactions += inserts.Count;
data.MapTransactionBlock.InsertMany(inserts, new InsertManyOptions {
IsOrdered = false
});
}
}
catch (MongoBulkWriteException mbwex)
{
if (mbwex.WriteErrors.Any(e => e.Category != ServerErrorCategory.DuplicateKey))//.Message.Contains("E11000 duplicate key error collection"))
{
throw;
}
}
// insert inputs and add to the list for later to use on the notification task.
var inputs = CreateInputs(item.BlockInfo.Height, items).ToList();
var outputs = CreateOutputs(items, item.BlockInfo.Height).ToList();
inputs.AddRange(outputs);
var queueInner = new Queue <MapTransactionAddress>(inputs);
do
{
try
{
var itemsInner = GetBatch(configuration.MongoBatchSize, queueInner).ToList();
var ops = new Dictionary <string, WriteModel <MapTransactionAddress> >();
var writeOptions = new BulkWriteOptions()
{
IsOrdered = false
};
foreach (MapTransactionAddress mapTransactionAddress in itemsInner)
{
if (mapTransactionAddress.SpendingTransactionId == null)
{
ops.Add(mapTransactionAddress.Id, new InsertOneModel <MapTransactionAddress>(mapTransactionAddress));
}
else
{
if (ops.TryGetValue(mapTransactionAddress.Id, out WriteModel <MapTransactionAddress> mta))
{
// in case a utxo is spent in the same block
// we just modify the inserted item directly
var imta = mta as InsertOneModel <MapTransactionAddress>;
imta.Document.SpendingTransactionId = mapTransactionAddress.SpendingTransactionId;
imta.Document.SpendingBlockIndex = mapTransactionAddress.SpendingBlockIndex;
}
else
{
FilterDefinition <MapTransactionAddress> filter = Builders <MapTransactionAddress> .Filter.Eq(addr => addr.Id, mapTransactionAddress.Id);
UpdateDefinition <MapTransactionAddress> update = Builders <MapTransactionAddress> .Update
.Set(blockInfo => blockInfo.SpendingTransactionId, mapTransactionAddress.SpendingTransactionId)
.Set(blockInfo => blockInfo.SpendingBlockIndex, mapTransactionAddress.SpendingBlockIndex);
ops.Add(mapTransactionAddress.Id, new UpdateOneModel <MapTransactionAddress>(filter, update));
}
}
}
if (itemsInner.Any())
{
stats.Items.AddRange(itemsInner);
stats.InputsOutputs += ops.Count;
data.MapTransactionAddress.BulkWrite(ops.Values, writeOptions);
}
}
catch (MongoBulkWriteException mbwex)
{
if (mbwex.WriteErrors.Any(e => e.Category != ServerErrorCategory.DuplicateKey))//.Message.Contains("E11000 duplicate key error collection"))
{
throw;
}
}
}while (queueInner.Any());
// If insert trx supported then push trx in batches.
if (configuration.StoreRawTransactions)
{
try
{
var inserts = items.Select(t => new MapTransaction {
TransactionId = t.GetHash().ToString(), RawTransaction = t.ToBytes(syncConnection.Network.Consensus.ConsensusFactory)
}).ToList();
stats.RawTransactions = inserts.Count;
data.MapTransaction.InsertMany(inserts, new InsertManyOptions {
IsOrdered = false
});
}
catch (MongoBulkWriteException mbwex)
{
if (mbwex.WriteErrors.Any(e => e.Category != ServerErrorCategory.DuplicateKey))//.Message.Contains("E11000 duplicate key error collection"))
{
throw;
}
}
}
}while (queue.Any());
// mark the block as synced.
CompleteBlock(item.BlockInfo);
}
else
{
// memory transaction push in to the pool.
item.Transactions.ForEach(t =>
{
data.MemoryTransactions.TryAdd(t.GetHash().ToString(), t);
});
stats.Transactions = data.MemoryTransactions.Count();
// todo: for accuracy - remove transactions from the mongo memory pool that are not anymore in the syncing pool
// remove all transactions from the memory pool
// this can be done using the SyncingBlocks objects - see method SyncOperations.FindPoolInternal()
// add to the list for later to use on the notification task.
var inputs = CreateInputs(-1, item.Transactions).ToList();
stats.Items.AddRange(inputs);
}
return(stats);
}
public Task <BulkWriteResult <T> > BulkWriteAsync(string collectionName, IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default)
{
return(infrastructure.Exec(database =>
{
return database.GetCollection <T>(collectionName).BulkWriteAsync(requests, options, cancellationToken);
}));
}
public Task <BulkWriteResult <Vehicle> > BulkWriteAsync(IEnumerable <WriteModel <Vehicle> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
/// <inheritdoc/>
public BulkWriteResult <T> BulkWrite(IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default(CancellationToken))
{
throw new System.NotImplementedException();
}
public Task <BulkWriteResult <T> > BulkWriteAsync(IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default) => Task.FromResult(BulkWrite(requests, options, cancellationToken));
/// <summary>
/// Update a collection of documents from the mongodb collection.
/// </summary>
/// <param name="context">Mongodb collection context</param>
/// <param name="documents">Collection of documents</param>
/// <param name="options"><see cref="BulkWriteOptions"/></param>
/// <typeparam name="TDocument">Document type</typeparam>
/// <returns><see cref="BulkWriteResult{T}"/></returns>
public static async ValueTask <BulkWriteResult <TDocument> > BulkUpdate <TDocument>(this ICallistoCollectionContext <TDocument> context,
IEnumerable <TDocument> documents, BulkWriteOptions options = null)
where TDocument : class, IDocumentRoot, IUpdatableDocument <TDocument>
{
List <TDocument> updatableDocuments = documents.ToList();
ValidateOperationParameters(updatableDocuments, context, "bulk-update");
BulkWriteResult <TDocument> result = await Write(context, updatableDocuments.SelectMany(a => a.PendingUpdates, (root, model) => model), options);
updatableDocuments.ForEach(a => a.PendingUpdates.Clear());
return(result);
}
public BulkWriteResult <T> BulkWrite(IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default)
{
var bypassDocumentValidation = options?.BypassDocumentValidation;
long deleted = 0;
long inserted = 0;
long matched = 0;
long modified = 0;
int count = 0;
var upserts = new List <BulkWriteUpsert>();
foreach (var req in requests)
{
switch (req)
{
case InsertOneModel <T> insert:
InsertOne(insert.Document);
inserted++;
break;
case ReplaceOneModel <T> replace:
var rOneResult = ReplaceOne(replace.Filter, replace.Replacement, new UpdateOptions {
IsUpsert = replace.IsUpsert, BypassDocumentValidation = bypassDocumentValidation
});
matched += rOneResult.MatchedCount;
if (rOneResult.MatchedCount == 0)
{
upserts.Add(INMBulkWriteUpsert.Create(count, rOneResult.UpsertedId));
inserted++;
}
break;
case UpdateOneModel <T> update:
var uOneResult = UpdateOne(update.Filter, update.Update, new UpdateOptions {
ArrayFilters = update.ArrayFilters, IsUpsert = update.IsUpsert, BypassDocumentValidation = bypassDocumentValidation
});
matched += uOneResult.MatchedCount;
modified += uOneResult.ModifiedCount;
if (uOneResult.ModifiedCount != uOneResult.MatchedCount)
{
upserts.Add(INMBulkWriteUpsert.Create(count, uOneResult.UpsertedId));
}
break;
case UpdateManyModel <T> updateMany:
var uManuResult = UpdateMany(updateMany.Filter, updateMany.Update, new UpdateOptions {
ArrayFilters = updateMany.ArrayFilters, IsUpsert = updateMany.IsUpsert, BypassDocumentValidation = bypassDocumentValidation
});
matched += uManuResult.MatchedCount;
modified += uManuResult.ModifiedCount;
if (uManuResult.ModifiedCount != uManuResult.MatchedCount)
{
upserts.Add(INMBulkWriteUpsert.Create(count, uManuResult.UpsertedId));
}
break;
case DeleteOneModel <T> delete:
var dOneResult = DeleteOne(delete.Filter);
deleted += dOneResult.DeletedCount;
break;
case DeleteManyModel <T> deleteMany:
var dManyResult = DeleteMany(deleteMany.Filter);
deleted += dManyResult.DeletedCount;
break;
default:
throw new InMemoryDatabaseException($"Unknown WriteModel type '{req.GetType().Name}'");
}
count++;
}
return(new INMBulkWriteResult <T>(count, requests)
{
INMDeletedCount = deleted,
IMNUpserts = upserts,
INMInsertedCount = inserted,
INMMatchedCount = matched,
INMModifiedCount = modified
});
}
/// <summary>
/// Insert a collection of documents into the mongodb collection.
/// </summary>
/// <param name="context">Mongodb collection context</param>
/// <param name="documents">Collection of documents to be inserted</param>
/// <param name="options"><see cref="BulkWriteOptions"/></param>
/// <typeparam name="TDocument">Collection type</typeparam>
/// <returns><see cref="BulkWriteResult{T}"/></returns>
public static async ValueTask <BulkWriteResult <TDocument> > BulkInsert <TDocument>(this ICallistoCollectionContext <TDocument> context, IEnumerable <TDocument> documents,
BulkWriteOptions options = null)
where TDocument : class, IDocumentRoot, IInsertableDocument
{
IEnumerable <TDocument> updatableDocuments = documents.ToList();
ValidateOperationParameters(updatableDocuments, context, "bulk-insert");
return(await Write(context, documents.Select(a => new InsertOneModel <TDocument>(a)).ToList(), options));
}
public Task <BulkWriteResult <AnimatedImage> > BulkWriteAsync(IClientSessionHandle session, IEnumerable <WriteModel <AnimatedImage> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
}
/// <summary>
/// Update one document from the MongoDb collection.
/// </summary>
/// <param name="context">Mongodb collection context</param>
/// <param name="document">The document to be updated</param>
/// <param name="handle"><see cref="IClientSessionHandle"/></param>
/// <param name="options"><see cref="UpdateOptions"/></param>
/// <typeparam name="TDocument">Type of the document</typeparam>
/// <returns><see cref="BulkWriteResult{TDocument}"/></returns>
public static async ValueTask <BulkWriteResult <TDocument> > Update <TDocument>(this ICallistoCollectionContext <TDocument> context, TDocument document,
IClientSessionHandle handle, BulkWriteOptions options = null)
where TDocument : class, IDocumentRoot, IUpdatableDocument <TDocument>
{
ValidateOperationParameters(document, context, "update-one");
BulkWriteResult <TDocument> result = await WriteWithHandle(context, CheckPendingUpdatesListFilter(document), handle, options);
document.PendingUpdates.Clear();
return(result);
}
public async Task UpdateAsync()
{
const int SizeOfBatch = 1000;
const int SizeOfQueue = 20;
var collectionOld = database.GetCollection<BsonDocument>("Events");
var collectionNew = database.GetCollection<BsonDocument>("Events2");
var batchBlock = new BatchBlock<BsonDocument>(SizeOfBatch, new GroupingDataflowBlockOptions
{
BoundedCapacity = SizeOfQueue * SizeOfBatch
});
var writeOptions = new BulkWriteOptions
{
IsOrdered = false
};
var actionBlock = new ActionBlock<BsonDocument[]>(async batch =>
{
var updates = new List<WriteModel<BsonDocument>>();
foreach (var document in batch)
{
var eventStream = document["EventStream"].AsString;
if (TryGetAppId(document, out var appId))
{
var indexOfType = eventStream.IndexOf('-');
var indexOfId = indexOfType + 1;
var indexOfOldId = eventStream.LastIndexOf("--", StringComparison.OrdinalIgnoreCase);
if (indexOfOldId > 0)
{
indexOfId = indexOfOldId + 2;
}
var domainType = eventStream.Substring(0, indexOfType);
var domainId = eventStream[indexOfId..];
if (!eventStream.StartsWith("app-", StringComparison.OrdinalIgnoreCase))
{
var newDomainId = DomainId.Combine(DomainId.Create(appId), DomainId.Create(domainId)).ToString();
var newStreamName = $"{domainType}-{newDomainId}";
document["EventStream"] = newStreamName;
foreach (var @event in document["Events"].AsBsonArray)
{
var metadata = @event["Metadata"].AsBsonDocument;
metadata["AggregateId"] = newDomainId;
}
}
foreach (var @event in document["Events"].AsBsonArray)
{
var metadata = @event["Metadata"].AsBsonDocument;
metadata.Remove("AppId");
}
}
private static async ValueTask <BulkWriteResult <T> > WriteWithHandle <T>(ICallistoCollectionContext <T> context,
IEnumerable <WriteModel <T> > models,
IClientSessionHandle handle, BulkWriteOptions options)
where T : class, IDocumentRoot
{
return(await context.Collection.BulkWriteAsync(handle, models, options));
}
public Task <BulkWriteResult <TModel> > BulkWriteAsync(IClientSessionHandle session, IEnumerable <WriteModel <TModel> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default(CancellationToken))
{
return(_collection.BulkWriteAsync(session, requests, options, cancellationToken));
}
/// <summary>
/// 批量写操作
/// </summary>
/// <param name="requests"></param>
/// <param name="options"></param>
/// <returns></returns>
public BulkWriteResult <T> BulkWrite(IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null)
{
return(BulkWrite(collectionTypeName, requests, options));
}
