public async Task ReadDocumentsAsync(SourceFilter filter,
ChannelWriter <List <ReplaceOneModel <BsonDocument> > > batchWriter,
int batchSize,
string[] keyFields,
bool upsert,
IDestinationDocumentFinder?documentFinder,
CancellationToken token)
{
_logger.Debug("Creating a cursor to the source with batch size {Batch}", batchSize);
using var cursor = await _collection.FindAsync(
filter.ToBsonDocument(), new FindOptions <BsonDocument>
{
BatchSize = batchSize
}, token);
_logger.Debug("Started reading documents from source");
try
{
while (await cursor.MoveNextAsync(token))
{
if (cursor.Current == null || !cursor.Current.Any())
{
continue;
}
var currentBatch = cursor.Current.ToList();
var destinationDocuments = new Dictionary <BsonValue, BsonDocument>();
if (documentFinder != null)
{
var dest = await documentFinder.GetFieldsAsync(currentBatch, keyFields, token);
foreach (var doc in dest)
{
destinationDocuments[doc["_id"]] = doc;
}
}
var replaceModels = currentBatch.Select(srcDoc => CreateReplaceModel(
destinationDocuments.TryGetValue(srcDoc["_id"], out var destDoc)
? destDoc
: srcDoc, srcDoc, keyFields, upsert))
.ToList();
await batchWriter.WriteAsync(replaceModels, token);
}
}
finally
{
batchWriter.Complete();
}
}
public async Task ReadDocumentsAsync_ShouldWriteAllDocumentsIntoChannel_FilterByDate()
{
// Arrange
var channel = Channel.CreateUnbounded <List <ReplaceOneModel <BsonDocument> > >();
var currentDate = Fixture.CreateMongoDbDate();
await _sourceCollection.InsertManyAsync(Enumerable.Range(0, 100).Select(idx => new BsonDocument
{
["Modified"] = new BsonDateTime(currentDate.AddMinutes(idx))
}));
var filerByDate = new SourceFilter("Modified", currentDate.AddMinutes(50));
// Act
await _sut.ReadDocumentsAsync(filerByDate, channel, 10, Array.Empty <string>(),
false, null, CancellationToken.None);
// Assert
var actual = await ReadDocumentsFromChannelAsync(channel);
var existing = _sourceCollection.Find(filerByDate.ToBsonDocument()).ToList();
actual.Should().BeEquivalentTo(existing);
}
public async Task <long> CountLagAsync(SourceFilter filter, CancellationToken token)
{
return(await _collection.CountDocumentsAsync(filter.ToBsonDocument(), cancellationToken : token));
}