public BulkWriteResult <Vehicle> BulkWrite(IClientSessionHandle session, IEnumerable <WriteModel <Vehicle> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); }
public BulkWriteResult <T> BulkWrite(string collectionName, IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null) { return(infrastructure.Exec(database => { return database.GetCollection <T>(collectionName).BulkWrite(requests, options); })); }
/// <summary> /// Replace a collection of documents from the mongodb collection. /// <remarks> /// The transaction filter is created using the property Id from <see cref="IDocumentRoot"/>. This behaviour is by design. /// </remarks> /// </summary> /// <param name="context">Mongodb collection context</param> /// <param name="documents">Collection of documents</param> /// <param name="handle"><see cref="IClientSessionHandle"/></param> /// <param name="options"><see cref="BulkWriteOptions"/></param> /// <typeparam name="TDocument">Document type</typeparam> /// <returns><see cref="BulkWriteResult{T}"/></returns> public static async ValueTask <BulkWriteResult <TDocument> > BulkReplace <TDocument>(this ICallistoCollectionContext <TDocument> context, IEnumerable <TDocument> documents, IClientSessionHandle handle, BulkWriteOptions options = null) where TDocument : class, IDocumentRoot, IReplaceableDocument { IEnumerable <TDocument> updatableDocuments = documents.ToList(); ValidateOperationParameters(updatableDocuments, context, "bulk-replace"); return(await WriteWithHandle(context, documents.Select(a => new ReplaceOneModel <TDocument>(a.FilterById(), a)).ToList(), handle, options)); }
public override BulkWriteResult <TDocument> BulkWrite(IEnumerable <WriteModel <TDocument> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default(CancellationToken)) { throw new NotImplementedException(); }
public BulkWriteResult <TModel> BulkWrite(IEnumerable <WriteModel <TModel> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default(CancellationToken)) { return(_collection.BulkWrite(requests, options, cancellationToken)); }
public Task <BulkWriteResult <T> > BulkWriteAsync(IClientSessionHandle session, IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default) => BulkWriteAsync(requests, options, cancellationToken);
public BulkWriteResult <AnimatedImage> BulkWrite(IEnumerable <WriteModel <AnimatedImage> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); }
/// <inheritdoc/> public BulkWriteResult <T> BulkWrite(IClientSessionHandle session, IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default) { return(_actualCollection.BulkWrite(session, requests, options, cancellationToken)); }
/// <inheritdoc/> public Task <BulkWriteResult <T> > BulkWriteAsync(IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default) { return(_actualCollection.BulkWriteAsync(requests, options, cancellationToken)); }
public static void ParallelBulkWrite <TDocument>(this IMongoCollection <TDocument> @this, IEnumerable <TDocument> docs, int batchSize, int parallelBatches, Func <TDocument, WriteModel <TDocument> > createWriteModel, CancellationToken cancellToken = default(CancellationToken)) { if (@this == null) { throw new ArgumentNullException(nameof(@this)); } if (docs == null) { throw new ArgumentNullException(nameof(docs)); } if (batchSize < 1) { throw new ArgumentException("'batchSize' must be at least 1.", nameof(batchSize)); } if (parallelBatches < 1) { throw new ArgumentException("'parallelBatches' must be at least 1.", nameof(parallelBatches)); } if (createWriteModel == null) { throw new ArgumentNullException(nameof(createWriteModel)); } using (var cancellTokenSrc = cancellToken == default(CancellationToken) ? new CancellationTokenSource() : CancellationTokenSource.CreateLinkedTokenSource(cancellToken)) { try { var bulkWriteOpts = new BulkWriteOptions { IsOrdered = false }; var bulkWritesEnumr = docs .Select(createWriteModel) .BatchesOf(batchSize) .Select(x => @this.BulkWriteAsync(x, bulkWriteOpts, cancellToken)) .GetEnumerator(); Func <int, Task <BulkWriteResult <TDocument> >[]> startBulkWrites = upToN => { var started = new List <Task <BulkWriteResult <TDocument> > >(); while (started.Count < upToN && bulkWritesEnumr.MoveNext()) { started.Add(bulkWritesEnumr.Current); } return(started.ToArray()); }; var runningBulkWrites = startBulkWrites(parallelBatches); while (runningBulkWrites.Length > 0) { var complBulkWriteInx = Task.WaitAny(runningBulkWrites, cancellToken); var complBulkWrite = runningBulkWrites[complBulkWriteInx]; if (complBulkWrite.Status != TaskStatus.RanToCompletion) // TODO: Consider parameterized error tolerance. { throw new Exception("Error occurred during a bulkWrite. ", complBulkWrite.Exception); } runningBulkWrites = runningBulkWrites .Where(x => !ReferenceEquals(x, complBulkWrite)) .Union(startBulkWrites(1)) .ToArray(); } } catch (Exception) { try { if (!cancellTokenSrc.IsCancellationRequested) { cancellTokenSrc.Cancel(); } } catch (Exception) { } throw; } } }
public Task <BulkWriteResult <T> > BulkWriteAsync(IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default(CancellationToken)) { return(_Repository.BulkWriteAsync(requests, options, cancellationToken)); }
public abstract BulkWriteResult <TDocument> BulkWrite(IEnumerable <WriteModel <TDocument> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default(CancellationToken));
public InsertStats InsertTransactions(SyncBlockTransactionsOperation item) { var stats = new InsertStats { Items = new List <MapTransactionAddress>() }; if (item.BlockInfo != null) { // remove all transactions from the memory pool item.Transactions.ForEach(t => { data.MemoryTransactions.TryRemove(t.GetHash().ToString(), out Transaction outer); }); // break the work in to batches of transactions var queue = new Queue <NBitcoin.Transaction>(item.Transactions); do { var items = GetBatch(configuration.MongoBatchSize, queue).ToList(); try { if (item.BlockInfo != null) { var inserts = items.Select(s => new MapTransactionBlock { BlockIndex = item.BlockInfo.Height, TransactionId = s.GetHash().ToString() }).ToList(); stats.Transactions += inserts.Count; data.MapTransactionBlock.InsertMany(inserts, new InsertManyOptions { IsOrdered = false }); } } catch (MongoBulkWriteException mbwex) { if (mbwex.WriteErrors.Any(e => e.Category != ServerErrorCategory.DuplicateKey))//.Message.Contains("E11000 duplicate key error collection")) { throw; } } // insert inputs and add to the list for later to use on the notification task. var inputs = CreateInputs(item.BlockInfo.Height, items).ToList(); var outputs = CreateOutputs(items, item.BlockInfo.Height).ToList(); inputs.AddRange(outputs); var queueInner = new Queue <MapTransactionAddress>(inputs); do { try { var itemsInner = GetBatch(configuration.MongoBatchSize, queueInner).ToList(); var ops = new Dictionary <string, WriteModel <MapTransactionAddress> >(); var writeOptions = new BulkWriteOptions() { IsOrdered = false }; foreach (MapTransactionAddress mapTransactionAddress in itemsInner) { if (mapTransactionAddress.SpendingTransactionId == null) { ops.Add(mapTransactionAddress.Id, new InsertOneModel <MapTransactionAddress>(mapTransactionAddress)); } else { if (ops.TryGetValue(mapTransactionAddress.Id, out WriteModel <MapTransactionAddress> mta)) { // in case a utxo is spent in the same block // we just modify the inserted item directly var imta = mta as InsertOneModel <MapTransactionAddress>; imta.Document.SpendingTransactionId = mapTransactionAddress.SpendingTransactionId; imta.Document.SpendingBlockIndex = mapTransactionAddress.SpendingBlockIndex; } else { FilterDefinition <MapTransactionAddress> filter = Builders <MapTransactionAddress> .Filter.Eq(addr => addr.Id, mapTransactionAddress.Id); UpdateDefinition <MapTransactionAddress> update = Builders <MapTransactionAddress> .Update .Set(blockInfo => blockInfo.SpendingTransactionId, mapTransactionAddress.SpendingTransactionId) .Set(blockInfo => blockInfo.SpendingBlockIndex, mapTransactionAddress.SpendingBlockIndex); ops.Add(mapTransactionAddress.Id, new UpdateOneModel <MapTransactionAddress>(filter, update)); } } } if (itemsInner.Any()) { stats.Items.AddRange(itemsInner); stats.InputsOutputs += ops.Count; data.MapTransactionAddress.BulkWrite(ops.Values, writeOptions); } } catch (MongoBulkWriteException mbwex) { if (mbwex.WriteErrors.Any(e => e.Category != ServerErrorCategory.DuplicateKey))//.Message.Contains("E11000 duplicate key error collection")) { throw; } } }while (queueInner.Any()); // If insert trx supported then push trx in batches. if (configuration.StoreRawTransactions) { try { var inserts = items.Select(t => new MapTransaction { TransactionId = t.GetHash().ToString(), RawTransaction = t.ToBytes(syncConnection.Network.Consensus.ConsensusFactory) }).ToList(); stats.RawTransactions = inserts.Count; data.MapTransaction.InsertMany(inserts, new InsertManyOptions { IsOrdered = false }); } catch (MongoBulkWriteException mbwex) { if (mbwex.WriteErrors.Any(e => e.Category != ServerErrorCategory.DuplicateKey))//.Message.Contains("E11000 duplicate key error collection")) { throw; } } } }while (queue.Any()); // mark the block as synced. CompleteBlock(item.BlockInfo); } else { // memory transaction push in to the pool. item.Transactions.ForEach(t => { data.MemoryTransactions.TryAdd(t.GetHash().ToString(), t); }); stats.Transactions = data.MemoryTransactions.Count(); // todo: for accuracy - remove transactions from the mongo memory pool that are not anymore in the syncing pool // remove all transactions from the memory pool // this can be done using the SyncingBlocks objects - see method SyncOperations.FindPoolInternal() // add to the list for later to use on the notification task. var inputs = CreateInputs(-1, item.Transactions).ToList(); stats.Items.AddRange(inputs); } return(stats); }
public Task <BulkWriteResult <T> > BulkWriteAsync(string collectionName, IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default) { return(infrastructure.Exec(database => { return database.GetCollection <T>(collectionName).BulkWriteAsync(requests, options, cancellationToken); })); }
public Task <BulkWriteResult <Vehicle> > BulkWriteAsync(IEnumerable <WriteModel <Vehicle> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); }
/// <inheritdoc/> public BulkWriteResult <T> BulkWrite(IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default(CancellationToken)) { throw new System.NotImplementedException(); }
public Task <BulkWriteResult <T> > BulkWriteAsync(IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default) => Task.FromResult(BulkWrite(requests, options, cancellationToken));
/// <summary> /// Update a collection of documents from the mongodb collection. /// </summary> /// <param name="context">Mongodb collection context</param> /// <param name="documents">Collection of documents</param> /// <param name="options"><see cref="BulkWriteOptions"/></param> /// <typeparam name="TDocument">Document type</typeparam> /// <returns><see cref="BulkWriteResult{T}"/></returns> public static async ValueTask <BulkWriteResult <TDocument> > BulkUpdate <TDocument>(this ICallistoCollectionContext <TDocument> context, IEnumerable <TDocument> documents, BulkWriteOptions options = null) where TDocument : class, IDocumentRoot, IUpdatableDocument <TDocument> { List <TDocument> updatableDocuments = documents.ToList(); ValidateOperationParameters(updatableDocuments, context, "bulk-update"); BulkWriteResult <TDocument> result = await Write(context, updatableDocuments.SelectMany(a => a.PendingUpdates, (root, model) => model), options); updatableDocuments.ForEach(a => a.PendingUpdates.Clear()); return(result); }
public BulkWriteResult <T> BulkWrite(IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default) { var bypassDocumentValidation = options?.BypassDocumentValidation; long deleted = 0; long inserted = 0; long matched = 0; long modified = 0; int count = 0; var upserts = new List <BulkWriteUpsert>(); foreach (var req in requests) { switch (req) { case InsertOneModel <T> insert: InsertOne(insert.Document); inserted++; break; case ReplaceOneModel <T> replace: var rOneResult = ReplaceOne(replace.Filter, replace.Replacement, new UpdateOptions { IsUpsert = replace.IsUpsert, BypassDocumentValidation = bypassDocumentValidation }); matched += rOneResult.MatchedCount; if (rOneResult.MatchedCount == 0) { upserts.Add(INMBulkWriteUpsert.Create(count, rOneResult.UpsertedId)); inserted++; } break; case UpdateOneModel <T> update: var uOneResult = UpdateOne(update.Filter, update.Update, new UpdateOptions { ArrayFilters = update.ArrayFilters, IsUpsert = update.IsUpsert, BypassDocumentValidation = bypassDocumentValidation }); matched += uOneResult.MatchedCount; modified += uOneResult.ModifiedCount; if (uOneResult.ModifiedCount != uOneResult.MatchedCount) { upserts.Add(INMBulkWriteUpsert.Create(count, uOneResult.UpsertedId)); } break; case UpdateManyModel <T> updateMany: var uManuResult = UpdateMany(updateMany.Filter, updateMany.Update, new UpdateOptions { ArrayFilters = updateMany.ArrayFilters, IsUpsert = updateMany.IsUpsert, BypassDocumentValidation = bypassDocumentValidation }); matched += uManuResult.MatchedCount; modified += uManuResult.ModifiedCount; if (uManuResult.ModifiedCount != uManuResult.MatchedCount) { upserts.Add(INMBulkWriteUpsert.Create(count, uManuResult.UpsertedId)); } break; case DeleteOneModel <T> delete: var dOneResult = DeleteOne(delete.Filter); deleted += dOneResult.DeletedCount; break; case DeleteManyModel <T> deleteMany: var dManyResult = DeleteMany(deleteMany.Filter); deleted += dManyResult.DeletedCount; break; default: throw new InMemoryDatabaseException($"Unknown WriteModel type '{req.GetType().Name}'"); } count++; } return(new INMBulkWriteResult <T>(count, requests) { INMDeletedCount = deleted, IMNUpserts = upserts, INMInsertedCount = inserted, INMMatchedCount = matched, INMModifiedCount = modified }); }
/// <summary> /// Insert a collection of documents into the mongodb collection. /// </summary> /// <param name="context">Mongodb collection context</param> /// <param name="documents">Collection of documents to be inserted</param> /// <param name="options"><see cref="BulkWriteOptions"/></param> /// <typeparam name="TDocument">Collection type</typeparam> /// <returns><see cref="BulkWriteResult{T}"/></returns> public static async ValueTask <BulkWriteResult <TDocument> > BulkInsert <TDocument>(this ICallistoCollectionContext <TDocument> context, IEnumerable <TDocument> documents, BulkWriteOptions options = null) where TDocument : class, IDocumentRoot, IInsertableDocument { IEnumerable <TDocument> updatableDocuments = documents.ToList(); ValidateOperationParameters(updatableDocuments, context, "bulk-insert"); return(await Write(context, documents.Select(a => new InsertOneModel <TDocument>(a)).ToList(), options)); }
public Task <BulkWriteResult <AnimatedImage> > BulkWriteAsync(IClientSessionHandle session, IEnumerable <WriteModel <AnimatedImage> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); }
/// <summary> /// Update one document from the MongoDb collection. /// </summary> /// <param name="context">Mongodb collection context</param> /// <param name="document">The document to be updated</param> /// <param name="handle"><see cref="IClientSessionHandle"/></param> /// <param name="options"><see cref="UpdateOptions"/></param> /// <typeparam name="TDocument">Type of the document</typeparam> /// <returns><see cref="BulkWriteResult{TDocument}"/></returns> public static async ValueTask <BulkWriteResult <TDocument> > Update <TDocument>(this ICallistoCollectionContext <TDocument> context, TDocument document, IClientSessionHandle handle, BulkWriteOptions options = null) where TDocument : class, IDocumentRoot, IUpdatableDocument <TDocument> { ValidateOperationParameters(document, context, "update-one"); BulkWriteResult <TDocument> result = await WriteWithHandle(context, CheckPendingUpdatesListFilter(document), handle, options); document.PendingUpdates.Clear(); return(result); }
public async Task UpdateAsync() { const int SizeOfBatch = 1000; const int SizeOfQueue = 20; var collectionOld = database.GetCollection<BsonDocument>("Events"); var collectionNew = database.GetCollection<BsonDocument>("Events2"); var batchBlock = new BatchBlock<BsonDocument>(SizeOfBatch, new GroupingDataflowBlockOptions { BoundedCapacity = SizeOfQueue * SizeOfBatch }); var writeOptions = new BulkWriteOptions { IsOrdered = false }; var actionBlock = new ActionBlock<BsonDocument[]>(async batch => { var updates = new List<WriteModel<BsonDocument>>(); foreach (var document in batch) { var eventStream = document["EventStream"].AsString; if (TryGetAppId(document, out var appId)) { var indexOfType = eventStream.IndexOf('-'); var indexOfId = indexOfType + 1; var indexOfOldId = eventStream.LastIndexOf("--", StringComparison.OrdinalIgnoreCase); if (indexOfOldId > 0) { indexOfId = indexOfOldId + 2; } var domainType = eventStream.Substring(0, indexOfType); var domainId = eventStream[indexOfId..]; if (!eventStream.StartsWith("app-", StringComparison.OrdinalIgnoreCase)) { var newDomainId = DomainId.Combine(DomainId.Create(appId), DomainId.Create(domainId)).ToString(); var newStreamName = $"{domainType}-{newDomainId}"; document["EventStream"] = newStreamName; foreach (var @event in document["Events"].AsBsonArray) { var metadata = @event["Metadata"].AsBsonDocument; metadata["AggregateId"] = newDomainId; } } foreach (var @event in document["Events"].AsBsonArray) { var metadata = @event["Metadata"].AsBsonDocument; metadata.Remove("AppId"); } }
private static async ValueTask <BulkWriteResult <T> > WriteWithHandle <T>(ICallistoCollectionContext <T> context, IEnumerable <WriteModel <T> > models, IClientSessionHandle handle, BulkWriteOptions options) where T : class, IDocumentRoot { return(await context.Collection.BulkWriteAsync(handle, models, options)); }
public Task <BulkWriteResult <TModel> > BulkWriteAsync(IClientSessionHandle session, IEnumerable <WriteModel <TModel> > requests, BulkWriteOptions options = null, CancellationToken cancellationToken = default(CancellationToken)) { return(_collection.BulkWriteAsync(session, requests, options, cancellationToken)); }
/// <summary> /// 批量写操作 /// </summary> /// <param name="requests"></param> /// <param name="options"></param> /// <returns></returns> public BulkWriteResult <T> BulkWrite(IEnumerable <WriteModel <T> > requests, BulkWriteOptions options = null) { return(BulkWrite(collectionTypeName, requests, options)); }