public MongoUpdateBatch(IMongoCollection <TRecord> collection, uint batchSize = 10000, CancellationToken?cancellationToken = null)
{
_block = BatchedBlockingBlock <FindAndModifyArgs <TRecord> > .CreateBlock(batchSize);
_block.LinkTo(new ActionBlock <FindAndModifyArgs <TRecord>[]>(UpdateAll), new DataflowLinkOptions {
PropagateCompletion = true
});
_collection = collection;
_cancellationToken = cancellationToken == null ? CancellationToken.None : cancellationToken.Value;
}
public MongoInsertBatch(IMongoCollection <TRecord> collection, uint batchSize = 1000, CancellationToken?cancellationToken = null)
{
_batchesSent = 0;
_batchBlock = BatchedBlockingBlock <TRecord> .CreateBlock(batchSize);
_cancellationToken = cancellationToken == null ? CancellationToken.None : cancellationToken.Value;
Func <TRecord[], Task> targetAction = InsertAll;
_actionBlock = new ActionBlock <TRecord[]>(targetAction, new ExecutionDataflowBlockOptions
{
BoundedCapacity = (int)batchSize,
CancellationToken = _cancellationToken
});
_batchBlock.LinkTo(_actionBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
_collection = collection;
}
/// <summary>
///
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <DataImportResult> Import(CancellationToken?cancellationToken = null, bool truncateDestination = false)
{
if (Options.TotalEntryLimit > 0)
{
_harvester.LimitEntries(Options.TotalEntryLimit);
}
if (Options.ShardLimit > 0)
{
_harvester.LimitShards(Options.ShardLimit);
}
var database = MongoHelper.GetDatabase();
if (truncateDestination)
{
database.Truncate(OutputDestinationCollection.OutputCollection);
//Debug.WriteLine($"Created temp collections: {dstCollection.GetCollectionName()} & {OutputDestinationCollection.ReducedOutputCollection}");
}
var dstCollection = database.GetCollection <BsonDocument>(OutputDestinationCollection.OutputCollection);
var batchesInserted = 0;
var batchSize = _options.ReadBlockSize;
var executionOptions = new ExecutionDataflowBlockOptions {
BoundedCapacity = 1,
};
var toBsonDocBlock = new TransformBlock <ExpandoObject, BsonDocument>((o) =>
{
var doc = o.ToBsonDocument();
//Apply encoding here
if (EncodeOnImport)
{
EncodeImportDocument(doc);
}
return(doc);
});//BsonConverter.CreateBlock(new ExecutionDataflowBlockOptions { BoundedCapacity = 1 });
var readBatcher = BatchedBlockingBlock <BsonDocument> .CreateBlock(batchSize);
//readBatcher.LinkTo(toBsonDocBlock, new DataflowLinkOptions { PropagateCompletion = true });
var inserterBlock = new ActionBlock <IEnumerable <BsonDocument> >(x =>
{
Debug.WriteLine($"Inserting batch {batchesInserted + 1} [{x.Count()}]");
dstCollection.InsertMany(x);
Interlocked.Increment(ref batchesInserted);
Debug.WriteLine($"Inserted batch {batchesInserted}");
}, executionOptions);
toBsonDocBlock.LinkTo(readBatcher, new DataflowLinkOptions {
PropagateCompletion = true
});
readBatcher.LinkTo(inserterBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
var result = await _harvester.ReadAll(toBsonDocBlock, cancellationToken);
await Task.WhenAll(inserterBlock.Completion, toBsonDocBlock.Completion);
foreach (var index in _options.IndexesToCreate)
{
dstCollection.EnsureIndex(index);
}
var output = new DataImportResult(result, dstCollection, _integration);
return(output);
}
