public async Task WhenLeasesHaveContinuationTokenNullReturn0()
{
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask)
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await processor.StopAsync();
ManualResetEvent manualResetEvent = new ManualResetEvent(false);
long receivedEstimation = 0;
ChangeFeedProcessor estimator = this.Container
.GetChangeFeedEstimatorBuilder("test", (long estimation, CancellationToken token) =>
{
receivedEstimation = estimation;
manualResetEvent.Set();
return(Task.CompletedTask);
}, TimeSpan.FromSeconds(1))
.WithLeaseContainer(this.LeaseContainer).Build();
await estimator.StartAsync();
Assert.IsTrue(manualResetEvent.WaitOne(BaseChangeFeedClientHelper.ChangeFeedCleanupTime), "Not received estimation in the expected time");
await estimator.StopAsync();
Assert.AreEqual(0, receivedEstimation);
}
public async Task Start()
{
// Warmup of the connection
await _container.GetItemLinqQueryable <Item>().ToFeedIterator().ReadNextAsync();
await _changeFeedProcessor.StartAsync();
}
public static async Task Main(string[] args)
{
using CosmosClient client = new CosmosClient(endpoint, apikey);
Database db = client.GetDatabase("StoreDB");
Container container = db.GetContainer("CartContainer");
Container leasecontainer = await db.CreateContainerIfNotExistsAsync("Lease", "/id");
ChangeFeedProcessorBuilder builder = container.GetChangeFeedProcessorBuilder(
"sampleProcessor",
async(IReadOnlyCollection <Item> changes, CancellationToken cancellationToken) =>
{
await Console.Out.WriteLineAsync("***Changes occured***");
foreach (Item change in changes)
{
await Console.Out.WriteLineAsync(change.id + " - " + change.location + " - " + change.name);
}
});
ChangeFeedProcessor processor = builder
.WithInstanceName("firstinstance")
.WithLeaseContainer(leasecontainer)
.Build();
await processor.StartAsync();
Console.ReadKey();
await processor.StopAsync();
}
public async Task WhenLeasesHaveContinuationTokenNullReturn0()
{
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) =>
{
return(Task.CompletedTask);
})
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await processor.StopAsync();
long?receivedEstimation = null;
ChangeFeedProcessor estimator = this.Container
.GetChangeFeedEstimatorBuilder("test", (long estimation, CancellationToken token) =>
{
receivedEstimation = estimation;
return(Task.CompletedTask);
}, TimeSpan.FromSeconds(1))
.WithLeaseContainer(this.LeaseContainer).Build();
await estimator.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await estimator.StopAsync();
Assert.IsTrue(receivedEstimation.HasValue);
Assert.AreEqual(0, receivedEstimation);
}
public async Task TestReducePageSizeScenario()
{
int partitionKey = 0;
// Create some docs to make sure that one separate response is returned for 1st execute of query before retries.
// These are to make sure continuation token is passed along during retries.
string sprocId = "createTwoDocs";
string sprocBody = @"function(startIndex) { for (var i = 0; i < 2; ++i) __.createDocument(
__.getSelfLink(),
{ id: 'doc' + (i + startIndex).toString(), value: 'y'.repeat(1500000), pk:0 },
err => { if (err) throw err;}
);}";
CosmosScripts scripts = this.Container.GetScripts();
StoredProcedureResponse storedProcedureResponse =
await scripts.CreateStoredProcedureAsync(new CosmosStoredProcedureSettings(sprocId, sprocBody));
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.CreateChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) =>
{
processedDocCount += docs.Count();
foreach (var doc in docs)
{
accumulator += doc.id.ToString() + ".";
}
if (processedDocCount == 5)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithStartFromBeginning()
.WithInstanceName("random")
.WithMaxItems(6)
.WithLeaseContainer(this.LeaseContainer).Build();
// Generate the payload
await scripts.ExecuteStoredProcedureAsync <int, object>(new PartitionKey(partitionKey), sprocId, 0);
// Create 3 docs each 1.5MB. All 3 do not fit into MAX_RESPONSE_SIZE (4 MB). 2nd and 3rd are in same transaction.
var content = string.Format("{{\"id\": \"doc2\", \"value\": \"{0}\", \"pk\": 0}}", new string('x', 1500000));
await this.Container.CreateItemAsync(JsonConvert.DeserializeObject <dynamic>(content), new PartitionKey(partitionKey));
await scripts.ExecuteStoredProcedureAsync <int, object>(new PartitionKey(partitionKey), sprocId, 3);
await processor.StartAsync();
// Letting processor initialize and pickup changes
var isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("doc0.doc1.doc2.doc3.doc4.", accumulator);
}
/// <summary>
/// Reading the Change Feed since the beginning of time.
/// </summary>
/// <remarks>
/// StartTime only works if the leaseContainer is empty or contains no leases for the particular processor name.
/// </remarks>
public static async Task RunStartFromBeginningChangeFeed(
string databaseId,
CosmosClient client)
{
await Program.InitializeAsync(databaseId, client);
Console.WriteLine("Generating 10 items that will be picked up by the delegate...");
await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer));
// <StartFromBeginningInitialization>
Container leaseContainer = client.GetContainer(databaseId, Program.leasesContainer);
Container monitoredContainer = client.GetContainer(databaseId, Program.monitoredContainer);
ChangeFeedProcessor changeFeedProcessor = monitoredContainer
.GetChangeFeedProcessorBuilder <ToDoItem>("changeFeedBeginning", Program.HandleChangesAsync)
.WithInstanceName("consoleHost")
.WithLeaseContainer(leaseContainer)
.WithStartTime(DateTime.MinValue.ToUniversalTime())
.Build();
// </StartFromBeginningInitialization>
Console.WriteLine($"Starting Change Feed Processor with changes since the beginning...");
await changeFeedProcessor.StartAsync();
Console.WriteLine("Change Feed Processor started.");
// Wait random time for the delegate to output all messages after initialization is done
await Task.Delay(5000);
Console.WriteLine("Press any key to continue with the next demo...");
Console.ReadKey();
await changeFeedProcessor.StopAsync();
}
public async Task Schema_DefaultsToNoPartitionId()
{
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) => Task.CompletedTask)
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
// Letting processor initialize
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
// Verify that no leases have PartitionId (V2 contract)
using FeedIterator <dynamic> iterator = this.LeaseContainer.GetItemQueryIterator <dynamic>();
while (iterator.HasMoreResults)
{
FeedResponse <dynamic> page = await iterator.ReadNextAsync();
foreach (dynamic lease in page)
{
string leaseId = lease.id;
Assert.IsNull(lease.partitionKey);
if (leaseId.Contains(".info") || leaseId.Contains(".lock"))
{
// These are the store initialization marks
continue;
}
Assert.IsNotNull(lease.LeaseToken);
Assert.IsNull(lease.PartitionId);
}
}
await processor.StopAsync();
}
public async Task StartProcessor <T>(string feedName, Container.ChangesHandler <T> handler)
{
_changeFeedProcessor = _container
.GetChangeFeedProcessorBuilder(feedName, handler).WithLeaseContainer(_leaseContainer)
.WithInstanceName(feedName).WithPollInterval(TimeSpan.FromMilliseconds(50)).Build();
await _changeFeedProcessor.StartAsync();
}
public async Task WhenLeasesHaveContinuationTokenNullReturn0_Pull()
{
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask)
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await processor.StopAsync();
long receivedEstimation = 0;
ChangeFeedEstimator estimator = ((ContainerInternal)this.Container)
.GetChangeFeedEstimator(
processorName: "test",
this.LeaseContainer);
using FeedIterator <ChangeFeedProcessorState> feedIterator = estimator.GetCurrentStateIterator();
while (feedIterator.HasMoreResults)
{
FeedResponse <ChangeFeedProcessorState> response = await feedIterator.ReadNextAsync();
receivedEstimation += response.Sum(r => r.EstimatedLag);
}
Assert.AreEqual(0, receivedEstimation);
}
public async Task StartAsync()
{
await processor
.StartAsync()
.ConfigureAwait(false);
activity = telemetry.SubscriptionStarted(consumerGroup);
}
public async Task TestWithFixedLeaseContainer()
{
await NonPartitionedContainerHelper.CreateNonPartitionedContainer(
this.database,
"fixedLeases");
Container fixedLeasesContainer = this.cosmosClient.GetContainer(this.database.Id, "fixedLeases");
try
{
int partitionKey = 0;
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, Stream stream, CancellationToken token) =>
{
this.ValidateContext(context);
IEnumerable <JObject> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream);
processedDocCount += asEnumerable.Count();
foreach (JObject doc in asEnumerable)
{
accumulator += doc["id"].ToString() + ".";
}
if (processedDocCount == 10)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithInstanceName("random")
.WithLeaseContainer(fixedLeasesContainer).Build();
// Start the processor, insert 1 document to generate a checkpoint
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
foreach (int id in Enumerable.Range(0, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString(), pk = partitionKey });
}
bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("0.1.2.3.4.5.6.7.8.9.", accumulator);
}
finally
{
await fixedLeasesContainer.DeleteContainerAsync();
}
}
public IChangeToken Watch()
{
Task.Run(async() =>
{
await _processor.StartAsync();
});
return(_reloadToken);
}
public async Task CountPendingDocuments()
{
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder(
processorName: "test",
onChangesDelegate: (IReadOnlyCollection <dynamic> docs, CancellationToken token) =>
{
return(Task.CompletedTask);
})
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer)
.Build();
await processor.StartAsync();
// Letting processor initialize
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
// Inserting documents
foreach (int id in Enumerable.Range(0, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() });
}
// Waiting on all notifications to finish
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await processor.StopAsync();
long?receivedEstimation = null;
ChangeFeedProcessor estimator = this.Container
.GetChangeFeedEstimatorBuilder(
processorName: "test",
estimationDelegate: (long estimation, CancellationToken token) =>
{
receivedEstimation = estimation;
return(Task.CompletedTask);
}, TimeSpan.FromSeconds(1))
.WithLeaseContainer(this.LeaseContainer)
.Build();
// Inserting more documents
foreach (int id in Enumerable.Range(11, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() });
}
await estimator.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await estimator.StopAsync();
Assert.IsTrue(receivedEstimation.HasValue);
Assert.AreEqual(10, receivedEstimation);
}
public async Task TestWithRunningProcessor_WithManualCheckpoint()
{
int partitionKey = 0;
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilderWithManualCheckpoint("test", async(ChangeFeedProcessorContext context, Stream stream, Func <Task> checkpointAsync, CancellationToken token) =>
{
this.ValidateContext(context);
IEnumerable <JObject> docs = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream);
processedDocCount += docs.Count();
foreach (dynamic doc in docs)
{
accumulator += doc.id.ToString() + ".";
}
if (processedDocCount == 3)
{
// Throwing on the 3rd document, since we checkpointed only on the 1st, we would repeat 2nd and 3rd
throw new Exception("Stop here");
}
if (processedDocCount == 1)
{
// Checkpointing on the first document to be able to have a point to rollback to
await checkpointAsync();
}
if (processedDocCount == 12)
{
allDocsProcessed.Set();
}
})
.WithInstanceName("random")
.WithMaxItems(1)
.WithLeaseContainer(this.LeaseContainer).Build();
// Start the processor, insert 1 document to generate a checkpoint
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
foreach (int id in Enumerable.Range(0, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString(), pk = partitionKey });
}
bool isStartOk = allDocsProcessed.WaitOne(30 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("0.1.2.1.2.3.4.5.6.7.8.9.", accumulator);
}
public async Task TestWithStartTime_CustomTime()
{
int partitionKey = 0;
foreach (int id in Enumerable.Range(0, 5))
{
await this.Container.CreateItemAsync <dynamic>(new { id = $"doc{id}", pk = partitionKey });
}
await Task.Delay(1000);
DateTime now = DateTime.UtcNow;
await Task.Delay(1000);
foreach (int id in Enumerable.Range(5, 5))
{
await this.Container.CreateItemAsync <dynamic>(new { id = $"doc{id}", pk = partitionKey });
}
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, IReadOnlyCollection <dynamic> docs, CancellationToken token) =>
{
this.ValidateContext(context);
Assert.IsTrue(docs.Count > 0);
processedDocCount += docs.Count;
foreach (dynamic doc in docs)
{
accumulator += doc.id.ToString() + ".";
}
if (processedDocCount == 5)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithStartTime(now)
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
// Letting processor initialize and pickup changes
bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("doc5.doc6.doc7.doc8.doc9.", accumulator);
}
public async Task StartProcessorAsync()
{
_changeFeedProcessor = _container
.GetChangeFeedProcessorBuilder <DailyDeviceReading>("DeviceSimChangeFeedProc", HandleChangesAsync)
.WithInstanceName("consoleHost1")
.WithLeaseContainer(_leaseContainer)
//.WithStartTime(particularPointInTime)
.Build();
await _changeFeedProcessor.StartAsync();
}
public async Task CountPendingDocuments_Pull()
{
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder(
processorName: "test",
onChangesDelegate: (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask)
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer)
.Build();
await processor.StartAsync();
// Letting processor initialize
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
// Inserting documents
foreach (int id in Enumerable.Range(0, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() });
}
// Waiting on all notifications to finish
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime);
await processor.StopAsync();
ManualResetEvent manualResetEvent = new ManualResetEvent(false);
long receivedEstimation = 0;
ChangeFeedEstimator estimator = ((ContainerInternal)this.Container)
.GetChangeFeedEstimator(
processorName: "test",
this.LeaseContainer);
// Inserting more documents
foreach (int id in Enumerable.Range(11, 10))
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() });
}
using FeedIterator <ChangeFeedProcessorState> feedIterator = estimator.GetCurrentStateIterator();
while (feedIterator.HasMoreResults)
{
FeedResponse <ChangeFeedProcessorState> response = await feedIterator.ReadNextAsync();
receivedEstimation += response.Sum(r => r.EstimatedLag);
Assert.IsTrue(response.Headers.RequestCharge > 0);
Assert.IsNotNull(response.Diagnostics);
string asString = response.Diagnostics.ToString();
Assert.IsTrue(asString.Length > 0);
Assert.IsTrue(asString.Contains("cosmos-netstandard-sdk"));
}
Assert.AreEqual(10, receivedEstimation);
}
private static async Task <ChangeFeedProcessor> StartChangeFeedProcessorAsync()
{
// Delete the source container if it exists
try
{
await cosmosClient.GetContainer(databaseName, sourceContainerName)
.DeleteContainerAsync();
}
catch (Exception e)
{
Console.WriteLine($"No container to delete");
}
// Create the source container and add an item
await cosmosClient.GetDatabase(databaseName).CreateContainerAsync(sourceContainerName, "/id");
Container sourceContainer = cosmosClient.GetContainer(databaseName, sourceContainerName);
ToDo firstToDo = new ToDo()
{
Name = "First"
};
await sourceContainer.CreateItemAsync(firstToDo, new PartitionKey(firstToDo.id));
// Delete the lease container if it exists
try
{
await cosmosClient.GetContainer(databaseName, leaseContainerName)
.DeleteContainerAsync();
}
catch (Exception e)
{
Console.WriteLine($"No container to delete");
}
// Create a new lease container
await cosmosClient.GetDatabase(databaseName).CreateContainerAsync(leaseContainerName, "/id");
Container leaseContainer = cosmosClient.GetContainer(databaseName, leaseContainerName);
ChangeFeedProcessor changeFeedProcessor = sourceContainer
.GetChangeFeedProcessorBuilder <ToDo>("changeFeedBeginning", Program.HandleChangesAsync)
.WithInstanceName("consoleHost")
.WithLeaseContainer(leaseContainer)
.WithStartTime(DateTime.MinValue.ToUniversalTime())
.WithPollInterval(TimeSpan.FromSeconds(1))
.Build();
Console.WriteLine("Starting Change Feed Processor...");
await changeFeedProcessor.StartAsync();
Console.WriteLine("Change Feed Processor started.");
return(changeFeedProcessor);
}
public async Task ExceptionsRetryBatch()
{
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
bool thrown = false;
IEnumerable <int> expectedIds = Enumerable.Range(0, 3);
List <int> receivedIds = new List <int>();
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) =>
{
if (receivedIds.Count == 1 &&
!thrown)
{
thrown = true;
throw new Exception("Retry batch");
}
foreach (TestClass doc in docs)
{
receivedIds.Add(int.Parse(doc.id));
}
if (receivedIds.Count == 3)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithInstanceName("random")
.WithMaxItems(1)
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
// Letting processor initialize
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
// Inserting documents
foreach (int id in expectedIds)
{
await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() });
}
// Waiting on all notifications to finish
bool isStartOk = allDocsProcessed.WaitOne(30 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
// Verify that we maintain order
CollectionAssert.AreEqual(expectedIds.ToList(), receivedIds);
}
/// <summary>
/// Exposing progress with the Estimator.
/// </summary>
/// <remarks>
/// The Estimator uses the same processorName and the same lease configuration as the existing processor to measure progress.
/// </remarks>
public static async Task RunEstimatorChangeFeed(
string databaseId,
CosmosClient client)
{
await Program.InitializeAsync(databaseId, client);
// <StartProcessorEstimator>
Container leaseContainer = client.GetContainer(databaseId, Program.leasesContainer);
Container monitoredContainer = client.GetContainer(databaseId, Program.monitoredContainer);
ChangeFeedProcessor changeFeedProcessor = monitoredContainer
.GetChangeFeedProcessorBuilder <ToDoItem>("changeFeedEstimator", Program.HandleChangesAsync)
.WithInstanceName("consoleHost")
.WithLeaseContainer(leaseContainer)
.Build();
// </StartProcessorEstimator>
Console.WriteLine($"Starting Change Feed Processor...");
await changeFeedProcessor.StartAsync();
Console.WriteLine("Change Feed Processor started.");
Console.WriteLine("Generating 10 items that will be picked up by the delegate...");
await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer));
// Wait random time for the delegate to output all messages after initialization is done
await Task.Delay(5000);
// <StartEstimator>
ChangeFeedProcessor changeFeedEstimator = monitoredContainer
.GetChangeFeedEstimatorBuilder("changeFeedEstimator", Program.HandleEstimationAsync, TimeSpan.FromMilliseconds(1000))
.WithLeaseContainer(leaseContainer)
.Build();
// </StartEstimator>
Console.WriteLine($"Starting Change Feed Estimator...");
await changeFeedEstimator.StartAsync();
Console.WriteLine("Change Feed Estimator started.");
Console.WriteLine("Generating 10 items that will be picked up by the delegate and reported by the Estimator...");
await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer));
Console.WriteLine("Press any key to continue with the next demo...");
Console.ReadKey();
await changeFeedProcessor.StopAsync();
await changeFeedEstimator.StopAsync();
}
public async IAsyncEnumerable <T> FetchFeed([EnumeratorCancellation] CancellationToken cancellationToken = default)
{
IReadOnlyCollection <T> changesToProcess = null;
SemaphoreSlim changesToProcessSignal = new SemaphoreSlim(0, 1);
SemaphoreSlim changesProcessedSignal = new SemaphoreSlim(0, 1);
ChangeFeedProcessor changeFeedProcessor = _container.GetChangeFeedProcessorBuilder <T>($"{typeof(T).Name}_ChangeFeedProcessor", async(changes, cancellationToken) =>
{
if ((changes != null) && changes.Count > 0)
{
changesToProcess = changes;
changesToProcessSignal.Release();
try
{
await changesProcessedSignal.WaitAsync(cancellationToken);
}
catch (OperationCanceledException)
{ }
}
})
.WithInstanceName("Demo.AspNetCore.Changefeed")
.WithStartTime(_startTime)
.WithPollInterval(_pollInterval)
.WithLeaseContainer(_leaseContainer)
.Build();
await changeFeedProcessor.StartAsync();
try
{
while (!cancellationToken.IsCancellationRequested)
{
await changesToProcessSignal.WaitAsync(cancellationToken);
foreach (T item in changesToProcess)
{
yield return(item);
}
changesToProcess = null;
changesProcessedSignal.Release();
}
}
finally
{
await changeFeedProcessor.StopAsync();
}
}
public async Task RunPollTabulationProcess()
{
await Initialize();
Container leaseContainer = _cosmosClient.GetContainer("PollDb", "PollLeases");
Container monitoredContainer = _cosmosClient.GetContainer("PollDb", "PollData");
ChangeFeedProcessor changeFeedProcessor = monitoredContainer
.GetChangeFeedProcessorBuilder <JObject>("changeFeedBasic", HandleChangesAsync)
.WithInstanceName("consoleHost")
.WithLeaseContainer(leaseContainer)
.WithStartTime(DateTime.MinValue.ToUniversalTime())
.Build();
await changeFeedProcessor.StartAsync();
}
private static async Task <ChangeFeedProcessor> StartChangeFeedProcessorAsync(
CosmosClient cosmosClient)
{
Container leaseContainer = cosmosClient.GetContainer(_productDatabase.Id, _productLeaseContainer.Id);
ChangeFeedProcessor changeFeedProcessor = cosmosClient.GetContainer(_productDatabase.Id, _productContainer.Id)
.GetChangeFeedProcessorBuilder <Product>("cosmosBackupSample", HandleChangesAsync)
.WithInstanceName("consoleHost")
.WithLeaseContainer(leaseContainer)
.Build();
Console.WriteLine("Starting Change Feed Processor for Product Backups...");
await changeFeedProcessor.StartAsync();
Console.WriteLine("Product Backup started");
return(changeFeedProcessor);
}
private static async Task <ChangeFeedProcessor> StartChangeFeedProcessorAsync(
CosmosClient cosmosClient)
{
Container leaseContainer = cosmosClient.GetContainer(databaseName, leaseContainerName);
ChangeFeedProcessor changeFeedProcessor = cosmosClient.GetContainer(databaseName, sourceContainerName)
.GetChangeFeedProcessorBuilder <Person>(processorName: "changeFeedSample", HandleChangesAsync)
.WithInstanceName("consoleHost")
.WithLeaseContainer(leaseContainer)
.Build();
Console.WriteLine("Starting Change Feed Processor...");
await changeFeedProcessor.StartAsync();
Console.WriteLine("Change Feed Processor started.");
return(changeFeedProcessor);
}
public Task StartAsync(string instanceName)
{
CosmosClient client = new CosmosClient(_endpointUrl, _authorizationKey);
Container eventContainer = client.GetContainer(_databaseId, _eventContainerId);
Container leaseContainer = client.GetContainer(_databaseId, _leaseContainerId);
_changeFeedProcessor = eventContainer
.GetChangeFeedProcessorBuilder <Change>("MSSQLProjections", HandleChangesAsync)
.WithInstanceName(instanceName)
.WithLeaseContainer(leaseContainer)
.WithStartTime(new DateTime(2020, 5, 1, 0, 0, 0, DateTimeKind.Utc))
.Build();
return(_changeFeedProcessor.StartAsync());
}
public Task Run(CancellationToken token)
{
_logger.LogInformation($"{nameof(Run)}: Starting monitoring");
CancellationTokenSource tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
Console.CancelKeyPress += (sender, args) =>
{
_logger.LogInformation($"{nameof(Run)}: Canceling monitoring");
args.Cancel = true;
tokenSource.Cancel();
};
var cts = new TaskCompletionSource <bool>();
_ = Task.Run(async() =>
{
CosmosClient cosmosClient = new CosmosClient(_watcherOption.GetResolvedConnectionString());
Database database = cosmosClient.GetDatabase(_watcherOption.DatabaseName);
Container traceContainer = await database.CreateContainerIfNotExistsAsync("Trace", Constants.DefaultPartitionKey);
Container leaseContainer = await database.CreateContainerIfNotExistsAsync("_feedLease", Constants.DefaultPartitionKey);
ChangeFeedProcessor changeFeedProcessor = traceContainer.GetChangeFeedProcessorBuilder <TraceRecord>("traceChangeFeed", HandleChangesAsync)
.WithInstanceName(nameof(WatcherCmd))
.WithLeaseContainer(leaseContainer)
.Build();
await changeFeedProcessor.StartAsync();
_logger.LogInformation("Feed process has started");
try
{
await Task.Delay(-1, tokenSource.Token);
}
catch { }
finally
{
await changeFeedProcessor.StopAsync();
_logger.LogInformation("Feed process has stopped");
}
cts.SetResult(true);
});
return(cts.Task);
}
public async Task TestWithStartTime_Beginning()
{
int partitionKey = 0;
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = this.Container
.GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, Stream stream, CancellationToken token) =>
{
this.ValidateContext(context);
IEnumerable <JObject> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream);
processedDocCount += asEnumerable.Count();
foreach (JObject doc in asEnumerable)
{
accumulator += doc["id"].ToString() + ".";
}
if (processedDocCount == 5)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithStartTime(DateTime.MinValue.ToUniversalTime())
.WithInstanceName("random")
.WithLeaseContainer(this.LeaseContainer).Build();
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
foreach (int id in Enumerable.Range(0, 5))
{
await this.Container.CreateItemAsync <dynamic>(new { id = $"doc{id}", pk = partitionKey });
}
// Letting processor initialize and pickup changes
bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("doc0.doc1.doc2.doc3.doc4.", accumulator);
}
private static async Task StartEstimatorAsync(CosmosClient client)
{
Container container = client.GetContainer("OnDotNet", "changefeed-items");
Container leasesContainer = client.GetContainer("OnDotNet", "changefeed-leases");
ChangeFeedProcessor processor = container.GetChangeFeedEstimatorBuilder("OnDotNet", Program.ShowEstimation, TimeSpan.FromSeconds(1))
.WithLeaseContainer(leasesContainer)
.Build();
await processor.StartAsync();
Console.WriteLine($"Estimator started.");
Console.WriteLine("Press any key to stop.");
Console.ReadKey();
await processor.StopAsync();
Console.WriteLine($"Estimator stopped.");
}
public async Task DoesNotUseUserSerializer()
{
CosmosClient cosmosClient = TestCommon.CreateCosmosClient(builder => builder.WithCustomSerializer(new FailedUserSerializer()));
ManualResetEvent allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
ChangeFeedProcessor processor = cosmosClient.GetContainer(this.database.Id, this.Container.Id)
.GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) =>
{
processedDocCount += docs.Count();
foreach (TestClass doc in docs)
{
accumulator += doc.id.ToString() + ".";
}
if (processedDocCount == 10)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
})
.WithInstanceName("random")
.WithLeaseContainer(cosmosClient.GetContainer(this.database.Id, this.LeaseContainer.Id)).Build();
// Start the processor, insert 1 document to generate a checkpoint
await processor.StartAsync();
await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime);
foreach (int id in Enumerable.Range(0, 10))
{
await this.Container.CreateItemAsync <TestClass>(new TestClass { id = id.ToString() });
}
bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime);
await processor.StopAsync();
Assert.IsTrue(isStartOk, "Timed out waiting for docs to process");
Assert.AreEqual("0.1.2.3.4.5.6.7.8.9.", accumulator);
}
private static async Task StartProcessorAsync(CosmosClient client, string instanceName)
{
Container container = client.GetContainer("OnDotNet", "changefeed-items");
Container leasesContainer = client.GetContainer("OnDotNet", "changefeed-leases");
ChangeFeedProcessor processor = container.GetChangeFeedProcessorBuilder <Model>("OnDotNet", Program.ProcessChangesAsync)
.WithInstanceName(instanceName)
.WithLeaseContainer(leasesContainer)
.Build();
await processor.StartAsync();
Console.WriteLine($"Instance {instanceName} started.");
Console.WriteLine("Press any key to stop.");
Console.ReadKey();
await processor.StopAsync();
Console.WriteLine($"Instance {instanceName} stopped.");
}
