public async Task WhenLeasesHaveContinuationTokenNullReturn0()
{
ChangeFeedProcessor processor = this.Container
.CreateChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) =>
{
return(Task.CompletedTask);
})
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await processor.StopAsync();
long?receivedEstimation = null;
ChangeFeedProcessor estimator = this.Container
.CreateChangeFeedEstimatorBuilder("test", (long estimation, CancellationToken token) =>
{
receivedEstimation = estimation;
return(Task.CompletedTask);
}, TimeSpan.FromSeconds(1))
.WithLeaseContainer(this.LeaseContainer).Build();
await estimator.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await estimator.StopAsync();
Assert.IsTrue(receivedEstimation.HasValue);
Assert.AreEqual(0, receivedEstimation);
}
public static async Task Main(string[] args)
{
using CosmosClient client = new CosmosClient(endpoint, apikey);
Database db = client.GetDatabase("StoreDB");
Container container = db.GetContainer("CartContainer");
Container leasecontainer = await db.CreateContainerIfNotExistsAsync("Lease", "/id");
ChangeFeedProcessorBuilder builder = container.GetChangeFeedProcessorBuilder(
"sampleProcessor",
async(IReadOnlyCollection <Item> changes, CancellationToken cancellationToken) =>
{
await Console.Out.WriteLineAsync("***Changes occured***");
foreach (Item change in changes)
{
await Console.Out.WriteLineAsync(change.id + " - " + change.location + " - " + change.name);
}
});
ChangeFeedProcessor processor = builder
.WithInstanceName("firstinstance")
.WithLeaseContainer(leasecontainer)
.Build();
await processor.StartAsync();
Console.ReadKey();
await processor.StopAsync();
}
public async Task WhenLeasesHaveContinuationTokenNullReturn0()
{
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask)
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await processor.StopAsync();
ManualResetEvent manualResetEvent = new ManualResetEvent(false);
long receivedEstimation = 0;
ChangeFeedProcessor estimator = this.Container
.GetChangeFeedEstimatorBuilder("test", (long estimation, CancellationToken token) =>
{
receivedEstimation = estimation;
manualResetEvent.Set();
return(Task.CompletedTask);
}, TimeSpan.FromSeconds(1))
.WithLeaseContainer(this.LeaseContainer).Build();
await estimator.StartAsync();
Assert.IsTrue(manualResetEvent.WaitOne(BaseChangeFeedClientHelper.ChangeFeedCleanupTime), "Not received estimation in the expected time");
await estimator.StopAsync();
Assert.AreEqual(0, receivedEstimation);
}
public async Task CloseAsync()
{
ChangeFeedProcessor processorSnapshot = this.changeFeedProcessor;
if (processorSnapshot != null)
{
await processorSnapshot.StopAsync().ConfigureAwait(false);
}
CosmosClient clientSnapshot = this.destinationCollectionClient;
if (clientSnapshot != null)
{
clientSnapshot.Dispose();
}
clientSnapshot = this.sourceCollectionClient;
if (clientSnapshot != null)
{
clientSnapshot.Dispose();
}
clientSnapshot = this.destinationCollectionClient;
if (clientSnapshot != null)
{
clientSnapshot.Dispose();
}
this.containerToStoreDocuments = null;
this.changeFeedProcessor = null;
}
/// <summary>
/// Reading the Change Feed since the beginning of time.
/// </summary>
/// <remarks>
/// StartTime only works if the leaseContainer is empty or contains no leases for the particular processor name.
/// </remarks>
public static async Task RunStartFromBeginningChangeFeed(
string databaseId,
CosmosClient client)
{
await Program.InitializeAsync(databaseId, client);
Console.WriteLine("Generating 10 items that will be picked up by the delegate...");
await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer));
// <StartFromBeginningInitialization>
Container leaseContainer = client.GetContainer(databaseId, Program.leasesContainer);
Container monitoredContainer = client.GetContainer(databaseId, Program.monitoredContainer);
ChangeFeedProcessor changeFeedProcessor = monitoredContainer
.GetChangeFeedProcessorBuilder <ToDoItem>("changeFeedBeginning", Program.HandleChangesAsync)
.WithInstanceName("consoleHost")
.WithLeaseContainer(leaseContainer)
.WithStartTime(DateTime.MinValue.ToUniversalTime())
.Build();
// </StartFromBeginningInitialization>
Console.WriteLine($"Starting Change Feed Processor with changes since the beginning...");
await changeFeedProcessor.StartAsync();
Console.WriteLine("Change Feed Processor started.");
// Wait random time for the delegate to output all messages after initialization is done
await Task.Delay(5000);
Console.WriteLine("Press any key to continue with the next demo...");
Console.ReadKey();
await changeFeedProcessor.StopAsync();
}
public async Task StartProcessor <T>(string feedName, Container.ChangesHandler <T> handler)
{
_changeFeedProcessor = _container
.GetChangeFeedProcessorBuilder(feedName, handler).WithLeaseContainer(_leaseContainer)
.WithInstanceName(feedName).WithPollInterval(TimeSpan.FromMilliseconds(50)).Build();
await _changeFeedProcessor.StartAsync();
}
public async Task WhenLeasesHaveContinuationTokenNullReturn0_Pull()
{
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask)
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await processor.StopAsync();
long receivedEstimation = 0;
ChangeFeedEstimator estimator = ((ContainerInternal)this.Container)
.GetChangeFeedEstimator(
processorName: "test",
this.LeaseContainer);
using FeedIterator <ChangeFeedProcessorState> feedIterator = estimator.GetCurrentStateIterator();
while (feedIterator.HasMoreResults)
{
FeedResponse <ChangeFeedProcessorState> response = await feedIterator.ReadNextAsync();
receivedEstimation += response.Sum(r => r.EstimatedLag);
}
Assert.AreEqual(0, receivedEstimation);
}
public async Task Schema_DefaultsToNoPartitionId()
{
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) => Task.CompletedTask)
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
// Letting processor initialize
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
// Verify that no leases have PartitionId (V2 contract)
using FeedIterator <dynamic> iterator = this.LeaseContainer.GetItemQueryIterator <dynamic>();
while (iterator.HasMoreResults)
{
FeedResponse <dynamic> page = await iterator.ReadNextAsync();
foreach (dynamic lease in page)
{
string leaseId = lease.id;
if (leaseId.Contains(".info") || leaseId.Contains(".lock"))
{
// These are the store initialization marks
continue;
}
Assert.IsNotNull(lease.LeaseToken);
Assert.IsNull(lease.PartitionId);
}
}
await processor.StopAsync();
}
public async Task TestReducePageSizeScenario()
{
int partitionKey = 0;
// Create some docs to make sure that one separate response is returned for 1st execute of query before retries.
// These are to make sure continuation token is passed along during retries.
string sprocId = "createTwoDocs";
string sprocBody = @"function(startIndex) { for (var i = 0; i < 2; ++i) __.createDocument(
__.getSelfLink(),
{ id: 'doc' + (i + startIndex).toString(), value: 'y'.repeat(1500000), pk:0 },
err => { if (err) throw err;}
);}";
Scripts scripts = this.Container.Scripts;
StoredProcedureResponse storedProcedureResponse =
await scripts.CreateStoredProcedureAsync(new StoredProcedureProperties(sprocId, sprocBody));
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) =>
{
processedDocCount += docs.Count();
foreach (var doc in docs)
{
accumulator += doc.id.ToString() + ".";
}
if (processedDocCount == 5)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithStartFromBeginning()
.WithInstanceName("random")
.WithMaxItems(6)
.WithLeaseContainer(this.LeaseContainer).Build();
// Generate the payload
await scripts.ExecuteStoredProcedureAsync <int, object>(sprocId, 0, new PartitionKey(partitionKey));
// Create 3 docs each 1.5MB. All 3 do not fit into MAX_RESPONSE_SIZE (4 MB). 2nd and 3rd are in same transaction.
var content = string.Format("{{\"id\": \"doc2\", \"value\": \"{0}\", \"pk\": 0}}", new string('x', 1500000));
await this.Container.CreateItemAsync(JsonConvert.DeserializeObject <dynamic>(content), new PartitionKey(partitionKey));
await scripts.ExecuteStoredProcedureAsync <int, object>(sprocId, 3, new PartitionKey(partitionKey));
await processor.StartAsync();
// Letting processor initialize and pickup changes
var isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("doc0.doc1.doc2.doc3.doc4.", accumulator);
}
public async Task TestWithFixedLeaseContainer()
{
await NonPartitionedContainerHelper.CreateNonPartitionedContainer(
this.database,
"fixedLeases");
Container fixedLeasesContainer = this.cosmosClient.GetContainer(this.database.Id, "fixedLeases");
try
{
int partitionKey = 0;
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, Stream stream, CancellationToken token) =>
{
this.ValidateContext(context);
IEnumerable <JObject> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream);
processedDocCount += asEnumerable.Count();
foreach (JObject doc in asEnumerable)
{
accumulator += doc["id"].ToString() + ".";
}
if (processedDocCount == 10)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithInstanceName("random")
.WithLeaseContainer(fixedLeasesContainer).Build();
// Start the processor, insert 1 document to generate a checkpoint
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
foreach (int id in Enumerable.Range(0, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString(), pk = partitionKey });
}
bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("0.1.2.3.4.5.6.7.8.9.", accumulator);
}
finally
{
await fixedLeasesContainer.DeleteContainerAsync();
}
}
public CosmosSubscriptionProcessor(
ISubscriptionTelemetry telemetry,
ChangeFeedProcessor processor,
ConsumerGroup consumerGroup)
{
this.telemetry = telemetry;
this.processor = processor;
this.consumerGroup = consumerGroup;
}
public async Task TestWithRunningProcessor_WithManualCheckpoint()
{
int partitionKey = 0;
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilderWithManualCheckpoint("test", async(ChangeFeedProcessorContext context, Stream stream, Func <Task> checkpointAsync, CancellationToken token) =>
{
this.ValidateContext(context);
IEnumerable <JObject> docs = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream);
processedDocCount += docs.Count();
foreach (dynamic doc in docs)
{
accumulator += doc.id.ToString() + ".";
}
if (processedDocCount == 3)
{
// Throwing on the 3rd document, since we checkpointed only on the 1st, we would repeat 2nd and 3rd
throw new Exception("Stop here");
}
if (processedDocCount == 1)
{
// Checkpointing on the first document to be able to have a point to rollback to
await checkpointAsync();
}
if (processedDocCount == 12)
{
allDocsProcessed.Set();
}
})
.WithInstanceName("random")
.WithMaxItems(1)
.WithLeaseContainer(this.LeaseContainer).Build();
// Start the processor, insert 1 document to generate a checkpoint
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
foreach (int id in Enumerable.Range(0, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString(), pk = partitionKey });
}
bool isStartOk = allDocsProcessed.WaitOne(30 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("0.1.2.1.2.3.4.5.6.7.8.9.", accumulator);
}
public async Task CountPendingDocuments()
{
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder(
processorName: "test",
onChangesDelegate: (IReadOnlyCollection <dynamic> docs, CancellationToken token) =>
{
return(Task.CompletedTask);
})
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer)
.Build();
await processor.StartAsync();
// Letting processor initialize
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
// Inserting documents
foreach (int id in Enumerable.Range(0, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() });
}
// Waiting on all notifications to finish
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await processor.StopAsync();
long?receivedEstimation = null;
ChangeFeedProcessor estimator = this.Container
.GetChangeFeedEstimatorBuilder(
processorName: "test",
estimationDelegate: (long estimation, CancellationToken token) =>
{
receivedEstimation = estimation;
return(Task.CompletedTask);
}, TimeSpan.FromSeconds(1))
.WithLeaseContainer(this.LeaseContainer)
.Build();
// Inserting more documents
foreach (int id in Enumerable.Range(11, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() });
}
await estimator.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await estimator.StopAsync();
Assert.IsTrue(receivedEstimation.HasValue);
Assert.AreEqual(10, receivedEstimation);
}
public ChangeFeedProcessorTests()
{
_changeFeedProcessor = new ChangeFeedProcessor(
_changeFeedRetrieveService,
_fhirTransactionPipeline,
_syncStateService,
NullLogger <ChangeFeedProcessor> .Instance);
SetupSyncState();
}
public async Task StartAsync(CancellationToken cancellationToken)
{
this.client = clientFactory();
logger.LogInformation($"Starting {nameof(ChangeFeedHostedService)} - {client.Endpoint}");
this.processor = await this.processorProvider(this.client);
await this.processor.StartAsync();
logger.LogInformation($"Started {nameof(ChangeFeedHostedService)}");
}
public async Task TestWithStartTime_CustomTime()
{
int partitionKey = 0;
foreach (int id in Enumerable.Range(0, 5))
{
await this.Container.CreateItemAsync <dynamic>(new { id = $"doc{id}", pk = partitionKey });
}
await Task.Delay(1000);
DateTime now = DateTime.UtcNow;
await Task.Delay(1000);
foreach (int id in Enumerable.Range(5, 5))
{
await this.Container.CreateItemAsync <dynamic>(new { id = $"doc{id}", pk = partitionKey });
}
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, IReadOnlyCollection <dynamic> docs, CancellationToken token) =>
{
this.ValidateContext(context);
Assert.IsTrue(docs.Count > 0);
processedDocCount += docs.Count;
foreach (dynamic doc in docs)
{
accumulator += doc.id.ToString() + ".";
}
if (processedDocCount == 5)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithStartTime(now)
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
// Letting processor initialize and pickup changes
bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("doc5.doc6.doc7.doc8.doc9.", accumulator);
}
public async Task CountPendingDocuments_Pull()
{
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder(
processorName: "test",
onChangesDelegate: (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask)
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer)
.Build();
await processor.StartAsync();
// Letting processor initialize
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
// Inserting documents
foreach (int id in Enumerable.Range(0, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() });
}
// Waiting on all notifications to finish
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await processor.StopAsync();
ManualResetEvent manualResetEvent = new ManualResetEvent(false);
long receivedEstimation = 0;
ChangeFeedEstimator estimator = ((ContainerInternal)this.Container)
.GetChangeFeedEstimator(
processorName: "test",
this.LeaseContainer);
// Inserting more documents
foreach (int id in Enumerable.Range(11, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() });
}
using FeedIterator <ChangeFeedProcessorState> feedIterator = estimator.GetCurrentStateIterator();
while (feedIterator.HasMoreResults)
{
FeedResponse <ChangeFeedProcessorState> response = await feedIterator.ReadNextAsync();
receivedEstimation += response.Sum(r => r.EstimatedLag);
Assert.IsTrue(response.Headers.RequestCharge > 0);
Assert.IsNotNull(response.Diagnostics);
string asString = response.Diagnostics.ToString();
Assert.IsTrue(asString.Length > 0);
Assert.IsTrue(asString.Contains("cosmos-netstandard-sdk"));
}
Assert.AreEqual(10, receivedEstimation);
}
public async Task StartProcessorAsync()
{
_changeFeedProcessor = _container
.GetChangeFeedProcessorBuilder <DailyDeviceReading>("DeviceSimChangeFeedProc", HandleChangesAsync)
.WithInstanceName("consoleHost1")
.WithLeaseContainer(_leaseContainer)
//.WithStartTime(particularPointInTime)
.Build();
await _changeFeedProcessor.StartAsync();
}
static async Task Main(string[] args)
{
IConfiguration configuration = BuildConfiguration();
CosmosClient cosmosClient = BuildCosmosClient(configuration);
await InitializeContainersAsync(cosmosClient, configuration);
ChangeFeedProcessor processor = await StartChangeFeedProcessorAsync(cosmosClient, configuration);
await GenerateItemsAsync(cosmosClient, processor, configuration);
}
private async void CreateDatabaseAsync()
{
Database database = await this.CreateDatabaseIfNotExistsAsync(cosmosDbSetting.DatabaseName);
this.GetCustomerDataContainer = await database.CreateContainerIfNotExistsAsync(new ContainerProperties(cosmosDbSetting.CustomerDataContainerName, "/id"));
this.GetCustomerLeaseContainer = await database.CreateContainerIfNotExistsAsync(new ContainerProperties(cosmosDbSetting.CustomerLeaseContainerName, "/id"));
this.GetCustomerSearchContainer = await database.CreateContainerIfNotExistsAsync(new ContainerProperties(cosmosDbSetting.CustomerSearchContainerName, "/postCode"));
this.processor = await StartChangeFeedProcessorAsync();
}
public async Task NotExistentLeaseContainer()
{
Container notFoundContainer = this.cosmosClient.GetContainer(this.database.Id, "NonExistent");
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) => Task.CompletedTask)
.WithInstanceName("random")
.WithLeaseContainer(notFoundContainer).Build();
CosmosException exception = await Assert.ThrowsExceptionAsync <CosmosException>(() => processor.StartAsync());
Assert.AreEqual(HttpStatusCode.NotFound, exception.StatusCode);
}
private static async Task <ChangeFeedProcessor> StartChangeFeedProcessorAsync()
{
// Delete the source container if it exists
try
{
await cosmosClient.GetContainer(databaseName, sourceContainerName)
.DeleteContainerAsync();
}
catch (Exception e)
{
Console.WriteLine($"No container to delete");
}
// Create the source container and add an item
await cosmosClient.GetDatabase(databaseName).CreateContainerAsync(sourceContainerName, "/id");
Container sourceContainer = cosmosClient.GetContainer(databaseName, sourceContainerName);
ToDo firstToDo = new ToDo()
{
Name = "First"
};
await sourceContainer.CreateItemAsync(firstToDo, new PartitionKey(firstToDo.id));
// Delete the lease container if it exists
try
{
await cosmosClient.GetContainer(databaseName, leaseContainerName)
.DeleteContainerAsync();
}
catch (Exception e)
{
Console.WriteLine($"No container to delete");
}
// Create a new lease container
await cosmosClient.GetDatabase(databaseName).CreateContainerAsync(leaseContainerName, "/id");
Container leaseContainer = cosmosClient.GetContainer(databaseName, leaseContainerName);
ChangeFeedProcessor changeFeedProcessor = sourceContainer
.GetChangeFeedProcessorBuilder <ToDo>("changeFeedBeginning", Program.HandleChangesAsync)
.WithInstanceName("consoleHost")
.WithLeaseContainer(leaseContainer)
.WithStartTime(DateTime.MinValue.ToUniversalTime())
.WithPollInterval(TimeSpan.FromSeconds(1))
.Build();
Console.WriteLine("Starting Change Feed Processor...");
await changeFeedProcessor.StartAsync();
Console.WriteLine("Change Feed Processor started.");
return(changeFeedProcessor);
}
public async Task ExceptionsRetryBatch()
{
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
bool thrown = false;
IEnumerable <int> expectedIds = Enumerable.Range(0, 3);
List <int> receivedIds = new List <int>();
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) =>
{
if (receivedIds.Count == 1 &&
!thrown)
{
thrown = true;
throw new Exception("Retry batch");
}
foreach (TestClass doc in docs)
{
receivedIds.Add(int.Parse(doc.id));
}
if (receivedIds.Count == 3)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithInstanceName("random")
.WithMaxItems(1)
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
// Letting processor initialize
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
// Inserting documents
foreach (int id in expectedIds)
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() });
}
// Waiting on all notifications to finish
bool isStartOk = allDocsProcessed.WaitOne(30 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
// Verify that we maintain order
CollectionAssert.AreEqual(expectedIds.ToList(), receivedIds);
}
public async Task StartAsync_ShouldThrowIfContainerDoesNotExist()
{
ChangeFeedProcessor estimator = this.cosmosClient.GetContainer(this.database.Id, "DoesNotExist")
.GetChangeFeedEstimatorBuilder("test", (long estimation, CancellationToken token) =>
{
return(Task.CompletedTask);
}, TimeSpan.FromSeconds(1))
.WithLeaseContainer(this.LeaseContainer).Build();
CosmosException exception = await Assert.ThrowsExceptionAsync <CosmosException>(() => estimator.StartAsync());
Assert.AreEqual(HttpStatusCode.NotFound, exception.StatusCode);
}
public async IAsyncEnumerable <T> FetchFeed([EnumeratorCancellation] CancellationToken cancellationToken = default)
{
IReadOnlyCollection <T> changesToProcess = null;
SemaphoreSlim changesToProcessSignal = new SemaphoreSlim(0, 1);
SemaphoreSlim changesProcessedSignal = new SemaphoreSlim(0, 1);
ChangeFeedProcessor changeFeedProcessor = _container.GetChangeFeedProcessorBuilder <T>($"{typeof(T).Name}_ChangeFeedProcessor", async(changes, cancellationToken) =>
{
if ((changes != null) && changes.Count > 0)
{
changesToProcess = changes;
changesToProcessSignal.Release();
try
{
await changesProcessedSignal.WaitAsync(cancellationToken);
}
catch (OperationCanceledException)
{ }
}
})
.WithInstanceName("Demo.AspNetCore.Changefeed")
.WithStartTime(_startTime)
.WithPollInterval(_pollInterval)
.WithLeaseContainer(_leaseContainer)
.Build();
await changeFeedProcessor.StartAsync();
try
{
while (!cancellationToken.IsCancellationRequested)
{
await changesToProcessSignal.WaitAsync(cancellationToken);
foreach (T item in changesToProcess)
{
yield return(item);
}
changesToProcess = null;
changesProcessedSignal.Release();
}
}
finally
{
await changeFeedProcessor.StopAsync();
}
}
/// <summary>
/// Exposing progress with the Estimator.
/// </summary>
/// <remarks>
/// The Estimator uses the same processorName and the same lease configuration as the existing processor to measure progress.
/// </remarks>
public static async Task RunEstimatorChangeFeed(
string databaseId,
CosmosClient client)
{
await Program.InitializeAsync(databaseId, client);
// <StartProcessorEstimator>
Container leaseContainer = client.GetContainer(databaseId, Program.leasesContainer);
Container monitoredContainer = client.GetContainer(databaseId, Program.monitoredContainer);
ChangeFeedProcessor changeFeedProcessor = monitoredContainer
.GetChangeFeedProcessorBuilder <ToDoItem>("changeFeedEstimator", Program.HandleChangesAsync)
.WithInstanceName("consoleHost")
.WithLeaseContainer(leaseContainer)
.Build();
// </StartProcessorEstimator>
Console.WriteLine($"Starting Change Feed Processor...");
await changeFeedProcessor.StartAsync();
Console.WriteLine("Change Feed Processor started.");
Console.WriteLine("Generating 10 items that will be picked up by the delegate...");
await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer));
// Wait random time for the delegate to output all messages after initialization is done
await Task.Delay(5000);
// <StartEstimator>
ChangeFeedProcessor changeFeedEstimator = monitoredContainer
.GetChangeFeedEstimatorBuilder("changeFeedEstimator", Program.HandleEstimationAsync, TimeSpan.FromMilliseconds(1000))
.WithLeaseContainer(leaseContainer)
.Build();
// </StartEstimator>
Console.WriteLine($"Starting Change Feed Estimator...");
await changeFeedEstimator.StartAsync();
Console.WriteLine("Change Feed Estimator started.");
Console.WriteLine("Generating 10 items that will be picked up by the delegate and reported by the Estimator...");
await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer));
Console.WriteLine("Press any key to continue with the next demo...");
Console.ReadKey();
await changeFeedProcessor.StopAsync();
await changeFeedEstimator.StopAsync();
}
static async Task Main(string[] args)
{
CosmosClient cosmosClient = new CosmosClientBuilder(endpointUri, primaryKey).Build();
Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(databaseName);
await database.CreateContainerIfNotExistsAsync(new ContainerProperties(sourceContainerName, "/id"));
await database.CreateContainerIfNotExistsAsync(new ContainerProperties(leaseContainerName, "/id"));
ChangeFeedProcessor processor = await StartChangeFeedProcessorAsync(cosmosClient);
await GenerateItemsAsync(cosmosClient);
}
public async Task RunPollTabulationProcess()
{
await Initialize();
Container leaseContainer = _cosmosClient.GetContainer("PollDb", "PollLeases");
Container monitoredContainer = _cosmosClient.GetContainer("PollDb", "PollData");
ChangeFeedProcessor changeFeedProcessor = monitoredContainer
.GetChangeFeedProcessorBuilder <JObject>("changeFeedBasic", HandleChangesAsync)
.WithInstanceName("consoleHost")
.WithLeaseContainer(leaseContainer)
.WithStartTime(DateTime.MinValue.ToUniversalTime())
.Build();
await changeFeedProcessor.StartAsync();
}
public async Task CloseAsync()
{
if (GetDestinationCollectionClient() != null)
{
GetDestinationCollectionClient().Dispose();
}
if (changeFeedProcessor != null)
{
await changeFeedProcessor.StopAsync();
}
destinationCollectionClient = null;
changeFeedProcessor = null;
}
static async Task Main(string[] args)
{
ChangeFeedProcessor changeFeedProcessor = await StartChangeFeedProcessorAsync();
Thread.Sleep(TimeSpan.FromSeconds(5));
Container container = cosmosClient.GetContainer(databaseName, sourceContainerName);
ToDo secondToDo = new ToDo()
{
Name = "Second"
};
await container.CreateItemAsync(secondToDo, new PartitionKey(secondToDo.id));
Console.ReadLine();
await changeFeedProcessor.StopAsync();
}
