public async Task ReadDocumentsAsync(SourceFilter filter, ChannelWriter <List <ReplaceOneModel <BsonDocument> > > batchWriter, int batchSize, string[] keyFields, bool upsert, IDestinationDocumentFinder?documentFinder, CancellationToken token) { _logger.Debug("Creating a cursor to the source with batch size {Batch}", batchSize); using var cursor = await _collection.FindAsync( filter.ToBsonDocument(), new FindOptions <BsonDocument> { BatchSize = batchSize }, token); _logger.Debug("Started reading documents from source"); try { while (await cursor.MoveNextAsync(token)) { if (cursor.Current == null || !cursor.Current.Any()) { continue; } var currentBatch = cursor.Current.ToList(); var destinationDocuments = new Dictionary <BsonValue, BsonDocument>(); if (documentFinder != null) { var dest = await documentFinder.GetFieldsAsync(currentBatch, keyFields, token); foreach (var doc in dest) { destinationDocuments[doc["_id"]] = doc; } } var replaceModels = currentBatch.Select(srcDoc => CreateReplaceModel( destinationDocuments.TryGetValue(srcDoc["_id"], out var destDoc) ? destDoc : srcDoc, srcDoc, keyFields, upsert)) .ToList(); await batchWriter.WriteAsync(replaceModels, token); } } finally { batchWriter.Complete(); } }
public async Task ReadDocumentsAsync_ShouldWriteAllDocumentsIntoChannel_FilterByDate() { // Arrange var channel = Channel.CreateUnbounded <List <ReplaceOneModel <BsonDocument> > >(); var currentDate = Fixture.CreateMongoDbDate(); await _sourceCollection.InsertManyAsync(Enumerable.Range(0, 100).Select(idx => new BsonDocument { ["Modified"] = new BsonDateTime(currentDate.AddMinutes(idx)) })); var filerByDate = new SourceFilter("Modified", currentDate.AddMinutes(50)); // Act await _sut.ReadDocumentsAsync(filerByDate, channel, 10, Array.Empty <string>(), false, null, CancellationToken.None); // Assert var actual = await ReadDocumentsFromChannelAsync(channel); var existing = _sourceCollection.Find(filerByDate.ToBsonDocument()).ToList(); actual.Should().BeEquivalentTo(existing); }
public async Task <long> CountLagAsync(SourceFilter filter, CancellationToken token) { return(await _collection.CountDocumentsAsync(filter.ToBsonDocument(), cancellationToken : token)); }