public async Task WhenLeasesHaveContinuationTokenNullReturn0_Pull() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await processor.StopAsync(); long receivedEstimation = 0; ChangeFeedEstimator estimator = ((ContainerInternal)this.Container) .GetChangeFeedEstimator( processorName: "test", this.LeaseContainer); using FeedIterator <ChangeFeedProcessorState> feedIterator = estimator.GetCurrentStateIterator(); while (feedIterator.HasMoreResults) { FeedResponse <ChangeFeedProcessorState> response = await feedIterator.ReadNextAsync(); receivedEstimation += response.Sum(r => r.EstimatedLag); } Assert.AreEqual(0, receivedEstimation); }
/// <summary> /// Used for tests /// </summary> internal ChangeFeedEstimatorRunner( ChangesEstimationHandler initialEstimateDelegate, TimeSpan?estimatorPeriod, ChangeFeedEstimator remainingWorkEstimator) : this(initialEstimateDelegate, estimatorPeriod) { this.remainingWorkEstimator = remainingWorkEstimator; }
public FeedEstimatorRunner( ChangesEstimationHandler dispatchEstimation, ChangeFeedEstimator remainingWorkEstimator, TimeSpan?estimationPeriod = null) { this.dispatchEstimation = dispatchEstimation; this.remainingWorkEstimator = remainingWorkEstimator; this.monitoringDelay = estimationPeriod ?? FeedEstimatorRunner.defaultMonitoringDelay; }
public async Task CountPendingDocuments_Pull() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder( processorName: "test", onChangesDelegate: (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer) .Build(); await processor.StartAsync(); // Letting processor initialize await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); // Inserting documents foreach (int id in Enumerable.Range(0, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() }); } // Waiting on all notifications to finish await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await processor.StopAsync(); ManualResetEvent manualResetEvent = new ManualResetEvent(false); long receivedEstimation = 0; ChangeFeedEstimator estimator = ((ContainerInternal)this.Container) .GetChangeFeedEstimator( processorName: "test", this.LeaseContainer); // Inserting more documents foreach (int id in Enumerable.Range(11, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() }); } using FeedIterator <ChangeFeedProcessorState> feedIterator = estimator.GetCurrentStateIterator(); while (feedIterator.HasMoreResults) { FeedResponse <ChangeFeedProcessorState> response = await feedIterator.ReadNextAsync(); receivedEstimation += response.Sum(r => r.EstimatedLag); Assert.IsTrue(response.Headers.RequestCharge > 0); Assert.IsNotNull(response.Diagnostics); string asString = response.Diagnostics.ToString(); Assert.IsTrue(asString.Length > 0); Assert.IsTrue(asString.Contains("cosmos-netstandard-sdk")); } Assert.AreEqual(10, receivedEstimation); }
private FeedEstimatorRunner BuildFeedEstimatorRunner() { if (this.remainingWorkEstimator == null) { this.remainingWorkEstimator = new ChangeFeedEstimatorCore( this.changeFeedLeaseOptions.LeasePrefix, this.monitoredContainer, this.leaseContainer); } return(new FeedEstimatorRunner(this.initialEstimateDelegate, this.remainingWorkEstimator, this.estimatorPeriod)); }
public async Task WhenNoLeasesExist_Pull() { ChangeFeedEstimator estimator = ((ContainerInternal)this.Container) .GetChangeFeedEstimator( processorName: "test", this.LeaseContainer); long receivedEstimation = 0; using FeedIterator <ChangeFeedProcessorState> feedIterator = estimator.GetCurrentStateIterator(); while (feedIterator.HasMoreResults) { FeedResponse <ChangeFeedProcessorState> response = await feedIterator.ReadNextAsync(); receivedEstimation += response.Sum(r => r.EstimatedLag); } Assert.AreEqual(0, receivedEstimation); }
/// <summary> /// Exposing progress with the Estimator with the detailed iterator. /// </summary> /// <remarks> /// The Estimator uses the same processorName and the same lease configuration as the existing processor to measure progress. /// The iterator exposes detailed, per-lease, information on estimation and ownership. /// </remarks> public static async Task RunEstimatorPullChangeFeed( string databaseId, CosmosClient client) { await Program.InitializeAsync(databaseId, client); // <StartProcessorEstimatorDetailed> Container leaseContainer = client.GetContainer(databaseId, Program.leasesContainer); Container monitoredContainer = client.GetContainer(databaseId, Program.monitoredContainer); ChangeFeedProcessor changeFeedProcessor = monitoredContainer .GetChangeFeedProcessorBuilder <ToDoItem>("changeFeedEstimator", Program.HandleChangesAsync) .WithInstanceName("consoleHost") .WithLeaseContainer(leaseContainer) .Build(); // </StartProcessorEstimatorDetailed> Console.WriteLine($"Starting Change Feed Processor..."); await changeFeedProcessor.StartAsync(); Console.WriteLine("Change Feed Processor started."); // Wait some seconds for instances to acquire leases await Task.Delay(5000); Console.WriteLine("Generating 10 items that will be picked up by the delegate..."); await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer)); // Wait random time for the delegate to output all messages after initialization is done await Task.Delay(5000); // <StartEstimatorDetailed> ChangeFeedEstimator changeFeedEstimator = monitoredContainer .GetChangeFeedEstimator("changeFeedEstimator", leaseContainer); // </StartEstimatorDetailed> // <GetIteratorEstimatorDetailed> Console.WriteLine("Checking estimation..."); using FeedIterator <ChangeFeedProcessorState> estimatorIterator = changeFeedEstimator.GetCurrentStateIterator(); while (estimatorIterator.HasMoreResults) { FeedResponse <ChangeFeedProcessorState> states = await estimatorIterator.ReadNextAsync(); foreach (ChangeFeedProcessorState leaseState in states) { string host = leaseState.InstanceName == null ? $"not owned by any host currently" : $"owned by host {leaseState.InstanceName}"; Console.WriteLine($"Lease [{leaseState.LeaseToken}] {host} reports {leaseState.EstimatedLag} as estimated lag."); } } // </GetIteratorEstimatorDetailed> Console.WriteLine("Stopping processor to show how the lag increases if no processing is happening."); await changeFeedProcessor.StopAsync(); // Wait for processor to shutdown completely so the next items generate lag await Task.Delay(5000); Console.WriteLine("Generating 10 items that will be seen by the Estimator..."); await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer)); Console.WriteLine("Checking estimation..."); using FeedIterator <ChangeFeedProcessorState> estimatorIteratorAfter = changeFeedEstimator.GetCurrentStateIterator(); while (estimatorIteratorAfter.HasMoreResults) { FeedResponse <ChangeFeedProcessorState> states = await estimatorIteratorAfter.ReadNextAsync(); foreach (ChangeFeedProcessorState leaseState in states) { // Host ownership should be empty as we have already stopped the estimator string host = leaseState.InstanceName == null ? $"not owned by any host currently" : $"owned by host {leaseState.InstanceName}"; Console.WriteLine($"Lease [{leaseState.LeaseToken}] {host} reports {leaseState.EstimatedLag} as estimated lag."); } } Console.WriteLine("Press any key to continue with the next demo..."); Console.ReadKey(); }