public MongoUpdateBatch(IMongoCollection <TRecord> collection, uint batchSize = 10000, CancellationToken?cancellationToken = null) { _block = BatchedBlockingBlock <FindAndModifyArgs <TRecord> > .CreateBlock(batchSize); _block.LinkTo(new ActionBlock <FindAndModifyArgs <TRecord>[]>(UpdateAll), new DataflowLinkOptions { PropagateCompletion = true }); _collection = collection; _cancellationToken = cancellationToken == null ? CancellationToken.None : cancellationToken.Value; }
public MongoInsertBatch(IMongoCollection <TRecord> collection, uint batchSize = 1000, CancellationToken?cancellationToken = null) { _batchesSent = 0; _batchBlock = BatchedBlockingBlock <TRecord> .CreateBlock(batchSize); _cancellationToken = cancellationToken == null ? CancellationToken.None : cancellationToken.Value; Func <TRecord[], Task> targetAction = InsertAll; _actionBlock = new ActionBlock <TRecord[]>(targetAction, new ExecutionDataflowBlockOptions { BoundedCapacity = (int)batchSize, CancellationToken = _cancellationToken }); _batchBlock.LinkTo(_actionBlock, new DataflowLinkOptions { PropagateCompletion = true }); _collection = collection; }
/// <summary> /// /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task <DataImportResult> Import(CancellationToken?cancellationToken = null, bool truncateDestination = false) { if (Options.TotalEntryLimit > 0) { _harvester.LimitEntries(Options.TotalEntryLimit); } if (Options.ShardLimit > 0) { _harvester.LimitShards(Options.ShardLimit); } var database = MongoHelper.GetDatabase(); if (truncateDestination) { database.Truncate(OutputDestinationCollection.OutputCollection); //Debug.WriteLine($"Created temp collections: {dstCollection.GetCollectionName()} & {OutputDestinationCollection.ReducedOutputCollection}"); } var dstCollection = database.GetCollection <BsonDocument>(OutputDestinationCollection.OutputCollection); var batchesInserted = 0; var batchSize = _options.ReadBlockSize; var executionOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = 1, }; var toBsonDocBlock = new TransformBlock <ExpandoObject, BsonDocument>((o) => { var doc = o.ToBsonDocument(); //Apply encoding here if (EncodeOnImport) { EncodeImportDocument(doc); } return(doc); });//BsonConverter.CreateBlock(new ExecutionDataflowBlockOptions { BoundedCapacity = 1 }); var readBatcher = BatchedBlockingBlock <BsonDocument> .CreateBlock(batchSize); //readBatcher.LinkTo(toBsonDocBlock, new DataflowLinkOptions { PropagateCompletion = true }); var inserterBlock = new ActionBlock <IEnumerable <BsonDocument> >(x => { Debug.WriteLine($"Inserting batch {batchesInserted + 1} [{x.Count()}]"); dstCollection.InsertMany(x); Interlocked.Increment(ref batchesInserted); Debug.WriteLine($"Inserted batch {batchesInserted}"); }, executionOptions); toBsonDocBlock.LinkTo(readBatcher, new DataflowLinkOptions { PropagateCompletion = true }); readBatcher.LinkTo(inserterBlock, new DataflowLinkOptions { PropagateCompletion = true }); var result = await _harvester.ReadAll(toBsonDocBlock, cancellationToken); await Task.WhenAll(inserterBlock.Completion, toBsonDocBlock.Completion); foreach (var index in _options.IndexesToCreate) { dstCollection.EnsureIndex(index); } var output = new DataImportResult(result, dstCollection, _integration); return(output); }