public async Task WhenLeasesHaveContinuationTokenNullReturn0() { ChangeFeedProcessor processor = this.Container .CreateChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => { return(Task.CompletedTask); }) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await processor.StopAsync(); long?receivedEstimation = null; ChangeFeedProcessor estimator = this.Container .CreateChangeFeedEstimatorBuilder("test", (long estimation, CancellationToken token) => { receivedEstimation = estimation; return(Task.CompletedTask); }, TimeSpan.FromSeconds(1)) .WithLeaseContainer(this.LeaseContainer).Build(); await estimator.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await estimator.StopAsync(); Assert.IsTrue(receivedEstimation.HasValue); Assert.AreEqual(0, receivedEstimation); }
public static async Task Main(string[] args) { using CosmosClient client = new CosmosClient(endpoint, apikey); Database db = client.GetDatabase("StoreDB"); Container container = db.GetContainer("CartContainer"); Container leasecontainer = await db.CreateContainerIfNotExistsAsync("Lease", "/id"); ChangeFeedProcessorBuilder builder = container.GetChangeFeedProcessorBuilder( "sampleProcessor", async(IReadOnlyCollection <Item> changes, CancellationToken cancellationToken) => { await Console.Out.WriteLineAsync("***Changes occured***"); foreach (Item change in changes) { await Console.Out.WriteLineAsync(change.id + " - " + change.location + " - " + change.name); } }); ChangeFeedProcessor processor = builder .WithInstanceName("firstinstance") .WithLeaseContainer(leasecontainer) .Build(); await processor.StartAsync(); Console.ReadKey(); await processor.StopAsync(); }
public async Task WhenLeasesHaveContinuationTokenNullReturn0() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await processor.StopAsync(); ManualResetEvent manualResetEvent = new ManualResetEvent(false); long receivedEstimation = 0; ChangeFeedProcessor estimator = this.Container .GetChangeFeedEstimatorBuilder("test", (long estimation, CancellationToken token) => { receivedEstimation = estimation; manualResetEvent.Set(); return(Task.CompletedTask); }, TimeSpan.FromSeconds(1)) .WithLeaseContainer(this.LeaseContainer).Build(); await estimator.StartAsync(); Assert.IsTrue(manualResetEvent.WaitOne(BaseChangeFeedClientHelper.ChangeFeedCleanupTime), "Not received estimation in the expected time"); await estimator.StopAsync(); Assert.AreEqual(0, receivedEstimation); }
public async Task CloseAsync() { ChangeFeedProcessor processorSnapshot = this.changeFeedProcessor; if (processorSnapshot != null) { await processorSnapshot.StopAsync().ConfigureAwait(false); } CosmosClient clientSnapshot = this.destinationCollectionClient; if (clientSnapshot != null) { clientSnapshot.Dispose(); } clientSnapshot = this.sourceCollectionClient; if (clientSnapshot != null) { clientSnapshot.Dispose(); } clientSnapshot = this.destinationCollectionClient; if (clientSnapshot != null) { clientSnapshot.Dispose(); } this.containerToStoreDocuments = null; this.changeFeedProcessor = null; }
/// <summary> /// Reading the Change Feed since the beginning of time. /// </summary> /// <remarks> /// StartTime only works if the leaseContainer is empty or contains no leases for the particular processor name. /// </remarks> public static async Task RunStartFromBeginningChangeFeed( string databaseId, CosmosClient client) { await Program.InitializeAsync(databaseId, client); Console.WriteLine("Generating 10 items that will be picked up by the delegate..."); await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer)); // <StartFromBeginningInitialization> Container leaseContainer = client.GetContainer(databaseId, Program.leasesContainer); Container monitoredContainer = client.GetContainer(databaseId, Program.monitoredContainer); ChangeFeedProcessor changeFeedProcessor = monitoredContainer .GetChangeFeedProcessorBuilder <ToDoItem>("changeFeedBeginning", Program.HandleChangesAsync) .WithInstanceName("consoleHost") .WithLeaseContainer(leaseContainer) .WithStartTime(DateTime.MinValue.ToUniversalTime()) .Build(); // </StartFromBeginningInitialization> Console.WriteLine($"Starting Change Feed Processor with changes since the beginning..."); await changeFeedProcessor.StartAsync(); Console.WriteLine("Change Feed Processor started."); // Wait random time for the delegate to output all messages after initialization is done await Task.Delay(5000); Console.WriteLine("Press any key to continue with the next demo..."); Console.ReadKey(); await changeFeedProcessor.StopAsync(); }
public async Task StartProcessor <T>(string feedName, Container.ChangesHandler <T> handler) { _changeFeedProcessor = _container .GetChangeFeedProcessorBuilder(feedName, handler).WithLeaseContainer(_leaseContainer) .WithInstanceName(feedName).WithPollInterval(TimeSpan.FromMilliseconds(50)).Build(); await _changeFeedProcessor.StartAsync(); }
public async Task WhenLeasesHaveContinuationTokenNullReturn0_Pull() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await processor.StopAsync(); long receivedEstimation = 0; ChangeFeedEstimator estimator = ((ContainerInternal)this.Container) .GetChangeFeedEstimator( processorName: "test", this.LeaseContainer); using FeedIterator <ChangeFeedProcessorState> feedIterator = estimator.GetCurrentStateIterator(); while (feedIterator.HasMoreResults) { FeedResponse <ChangeFeedProcessorState> response = await feedIterator.ReadNextAsync(); receivedEstimation += response.Sum(r => r.EstimatedLag); } Assert.AreEqual(0, receivedEstimation); }
public async Task Schema_DefaultsToNoPartitionId() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) => Task.CompletedTask) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); // Letting processor initialize await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); // Verify that no leases have PartitionId (V2 contract) using FeedIterator <dynamic> iterator = this.LeaseContainer.GetItemQueryIterator <dynamic>(); while (iterator.HasMoreResults) { FeedResponse <dynamic> page = await iterator.ReadNextAsync(); foreach (dynamic lease in page) { string leaseId = lease.id; if (leaseId.Contains(".info") || leaseId.Contains(".lock")) { // These are the store initialization marks continue; } Assert.IsNotNull(lease.LeaseToken); Assert.IsNull(lease.PartitionId); } } await processor.StopAsync(); }
public async Task TestReducePageSizeScenario() { int partitionKey = 0; // Create some docs to make sure that one separate response is returned for 1st execute of query before retries. // These are to make sure continuation token is passed along during retries. string sprocId = "createTwoDocs"; string sprocBody = @"function(startIndex) { for (var i = 0; i < 2; ++i) __.createDocument( __.getSelfLink(), { id: 'doc' + (i + startIndex).toString(), value: 'y'.repeat(1500000), pk:0 }, err => { if (err) throw err;} );}"; Scripts scripts = this.Container.Scripts; StoredProcedureResponse storedProcedureResponse = await scripts.CreateStoredProcedureAsync(new StoredProcedureProperties(sprocId, sprocBody)); ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => { processedDocCount += docs.Count(); foreach (var doc in docs) { accumulator += doc.id.ToString() + "."; } if (processedDocCount == 5) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithStartFromBeginning() .WithInstanceName("random") .WithMaxItems(6) .WithLeaseContainer(this.LeaseContainer).Build(); // Generate the payload await scripts.ExecuteStoredProcedureAsync <int, object>(sprocId, 0, new PartitionKey(partitionKey)); // Create 3 docs each 1.5MB. All 3 do not fit into MAX_RESPONSE_SIZE (4 MB). 2nd and 3rd are in same transaction. var content = string.Format("{{\"id\": \"doc2\", \"value\": \"{0}\", \"pk\": 0}}", new string('x', 1500000)); await this.Container.CreateItemAsync(JsonConvert.DeserializeObject <dynamic>(content), new PartitionKey(partitionKey)); await scripts.ExecuteStoredProcedureAsync <int, object>(sprocId, 3, new PartitionKey(partitionKey)); await processor.StartAsync(); // Letting processor initialize and pickup changes var isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("doc0.doc1.doc2.doc3.doc4.", accumulator); }
public async Task TestWithFixedLeaseContainer() { await NonPartitionedContainerHelper.CreateNonPartitionedContainer( this.database, "fixedLeases"); Container fixedLeasesContainer = this.cosmosClient.GetContainer(this.database.Id, "fixedLeases"); try { int partitionKey = 0; ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, Stream stream, CancellationToken token) => { this.ValidateContext(context); IEnumerable <JObject> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream); processedDocCount += asEnumerable.Count(); foreach (JObject doc in asEnumerable) { accumulator += doc["id"].ToString() + "."; } if (processedDocCount == 10) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithInstanceName("random") .WithLeaseContainer(fixedLeasesContainer).Build(); // Start the processor, insert 1 document to generate a checkpoint await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); foreach (int id in Enumerable.Range(0, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString(), pk = partitionKey }); } bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("0.1.2.3.4.5.6.7.8.9.", accumulator); } finally { await fixedLeasesContainer.DeleteContainerAsync(); } }
public CosmosSubscriptionProcessor( ISubscriptionTelemetry telemetry, ChangeFeedProcessor processor, ConsumerGroup consumerGroup) { this.telemetry = telemetry; this.processor = processor; this.consumerGroup = consumerGroup; }
public async Task TestWithRunningProcessor_WithManualCheckpoint() { int partitionKey = 0; ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilderWithManualCheckpoint("test", async(ChangeFeedProcessorContext context, Stream stream, Func <Task> checkpointAsync, CancellationToken token) => { this.ValidateContext(context); IEnumerable <JObject> docs = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream); processedDocCount += docs.Count(); foreach (dynamic doc in docs) { accumulator += doc.id.ToString() + "."; } if (processedDocCount == 3) { // Throwing on the 3rd document, since we checkpointed only on the 1st, we would repeat 2nd and 3rd throw new Exception("Stop here"); } if (processedDocCount == 1) { // Checkpointing on the first document to be able to have a point to rollback to await checkpointAsync(); } if (processedDocCount == 12) { allDocsProcessed.Set(); } }) .WithInstanceName("random") .WithMaxItems(1) .WithLeaseContainer(this.LeaseContainer).Build(); // Start the processor, insert 1 document to generate a checkpoint await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); foreach (int id in Enumerable.Range(0, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString(), pk = partitionKey }); } bool isStartOk = allDocsProcessed.WaitOne(30 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("0.1.2.1.2.3.4.5.6.7.8.9.", accumulator); }
public async Task CountPendingDocuments() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder( processorName: "test", onChangesDelegate: (IReadOnlyCollection <dynamic> docs, CancellationToken token) => { return(Task.CompletedTask); }) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer) .Build(); await processor.StartAsync(); // Letting processor initialize await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); // Inserting documents foreach (int id in Enumerable.Range(0, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() }); } // Waiting on all notifications to finish await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await processor.StopAsync(); long?receivedEstimation = null; ChangeFeedProcessor estimator = this.Container .GetChangeFeedEstimatorBuilder( processorName: "test", estimationDelegate: (long estimation, CancellationToken token) => { receivedEstimation = estimation; return(Task.CompletedTask); }, TimeSpan.FromSeconds(1)) .WithLeaseContainer(this.LeaseContainer) .Build(); // Inserting more documents foreach (int id in Enumerable.Range(11, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() }); } await estimator.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await estimator.StopAsync(); Assert.IsTrue(receivedEstimation.HasValue); Assert.AreEqual(10, receivedEstimation); }
public ChangeFeedProcessorTests() { _changeFeedProcessor = new ChangeFeedProcessor( _changeFeedRetrieveService, _fhirTransactionPipeline, _syncStateService, NullLogger <ChangeFeedProcessor> .Instance); SetupSyncState(); }
public async Task StartAsync(CancellationToken cancellationToken) { this.client = clientFactory(); logger.LogInformation($"Starting {nameof(ChangeFeedHostedService)} - {client.Endpoint}"); this.processor = await this.processorProvider(this.client); await this.processor.StartAsync(); logger.LogInformation($"Started {nameof(ChangeFeedHostedService)}"); }
public async Task TestWithStartTime_CustomTime() { int partitionKey = 0; foreach (int id in Enumerable.Range(0, 5)) { await this.Container.CreateItemAsync <dynamic>(new { id = $"doc{id}", pk = partitionKey }); } await Task.Delay(1000); DateTime now = DateTime.UtcNow; await Task.Delay(1000); foreach (int id in Enumerable.Range(5, 5)) { await this.Container.CreateItemAsync <dynamic>(new { id = $"doc{id}", pk = partitionKey }); } ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, IReadOnlyCollection <dynamic> docs, CancellationToken token) => { this.ValidateContext(context); Assert.IsTrue(docs.Count > 0); processedDocCount += docs.Count; foreach (dynamic doc in docs) { accumulator += doc.id.ToString() + "."; } if (processedDocCount == 5) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithStartTime(now) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); // Letting processor initialize and pickup changes bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("doc5.doc6.doc7.doc8.doc9.", accumulator); }
public async Task CountPendingDocuments_Pull() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder( processorName: "test", onChangesDelegate: (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer) .Build(); await processor.StartAsync(); // Letting processor initialize await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); // Inserting documents foreach (int id in Enumerable.Range(0, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() }); } // Waiting on all notifications to finish await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await processor.StopAsync(); ManualResetEvent manualResetEvent = new ManualResetEvent(false); long receivedEstimation = 0; ChangeFeedEstimator estimator = ((ContainerInternal)this.Container) .GetChangeFeedEstimator( processorName: "test", this.LeaseContainer); // Inserting more documents foreach (int id in Enumerable.Range(11, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() }); } using FeedIterator <ChangeFeedProcessorState> feedIterator = estimator.GetCurrentStateIterator(); while (feedIterator.HasMoreResults) { FeedResponse <ChangeFeedProcessorState> response = await feedIterator.ReadNextAsync(); receivedEstimation += response.Sum(r => r.EstimatedLag); Assert.IsTrue(response.Headers.RequestCharge > 0); Assert.IsNotNull(response.Diagnostics); string asString = response.Diagnostics.ToString(); Assert.IsTrue(asString.Length > 0); Assert.IsTrue(asString.Contains("cosmos-netstandard-sdk")); } Assert.AreEqual(10, receivedEstimation); }
public async Task StartProcessorAsync() { _changeFeedProcessor = _container .GetChangeFeedProcessorBuilder <DailyDeviceReading>("DeviceSimChangeFeedProc", HandleChangesAsync) .WithInstanceName("consoleHost1") .WithLeaseContainer(_leaseContainer) //.WithStartTime(particularPointInTime) .Build(); await _changeFeedProcessor.StartAsync(); }
static async Task Main(string[] args) { IConfiguration configuration = BuildConfiguration(); CosmosClient cosmosClient = BuildCosmosClient(configuration); await InitializeContainersAsync(cosmosClient, configuration); ChangeFeedProcessor processor = await StartChangeFeedProcessorAsync(cosmosClient, configuration); await GenerateItemsAsync(cosmosClient, processor, configuration); }
private async void CreateDatabaseAsync() { Database database = await this.CreateDatabaseIfNotExistsAsync(cosmosDbSetting.DatabaseName); this.GetCustomerDataContainer = await database.CreateContainerIfNotExistsAsync(new ContainerProperties(cosmosDbSetting.CustomerDataContainerName, "/id")); this.GetCustomerLeaseContainer = await database.CreateContainerIfNotExistsAsync(new ContainerProperties(cosmosDbSetting.CustomerLeaseContainerName, "/id")); this.GetCustomerSearchContainer = await database.CreateContainerIfNotExistsAsync(new ContainerProperties(cosmosDbSetting.CustomerSearchContainerName, "/postCode")); this.processor = await StartChangeFeedProcessorAsync(); }
public async Task NotExistentLeaseContainer() { Container notFoundContainer = this.cosmosClient.GetContainer(this.database.Id, "NonExistent"); ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) => Task.CompletedTask) .WithInstanceName("random") .WithLeaseContainer(notFoundContainer).Build(); CosmosException exception = await Assert.ThrowsExceptionAsync <CosmosException>(() => processor.StartAsync()); Assert.AreEqual(HttpStatusCode.NotFound, exception.StatusCode); }
private static async Task <ChangeFeedProcessor> StartChangeFeedProcessorAsync() { // Delete the source container if it exists try { await cosmosClient.GetContainer(databaseName, sourceContainerName) .DeleteContainerAsync(); } catch (Exception e) { Console.WriteLine($"No container to delete"); } // Create the source container and add an item await cosmosClient.GetDatabase(databaseName).CreateContainerAsync(sourceContainerName, "/id"); Container sourceContainer = cosmosClient.GetContainer(databaseName, sourceContainerName); ToDo firstToDo = new ToDo() { Name = "First" }; await sourceContainer.CreateItemAsync(firstToDo, new PartitionKey(firstToDo.id)); // Delete the lease container if it exists try { await cosmosClient.GetContainer(databaseName, leaseContainerName) .DeleteContainerAsync(); } catch (Exception e) { Console.WriteLine($"No container to delete"); } // Create a new lease container await cosmosClient.GetDatabase(databaseName).CreateContainerAsync(leaseContainerName, "/id"); Container leaseContainer = cosmosClient.GetContainer(databaseName, leaseContainerName); ChangeFeedProcessor changeFeedProcessor = sourceContainer .GetChangeFeedProcessorBuilder <ToDo>("changeFeedBeginning", Program.HandleChangesAsync) .WithInstanceName("consoleHost") .WithLeaseContainer(leaseContainer) .WithStartTime(DateTime.MinValue.ToUniversalTime()) .WithPollInterval(TimeSpan.FromSeconds(1)) .Build(); Console.WriteLine("Starting Change Feed Processor..."); await changeFeedProcessor.StartAsync(); Console.WriteLine("Change Feed Processor started."); return(changeFeedProcessor); }
public async Task ExceptionsRetryBatch() { ManualResetEvent allDocsProcessed = new ManualResetEvent(false); bool thrown = false; IEnumerable <int> expectedIds = Enumerable.Range(0, 3); List <int> receivedIds = new List <int>(); ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) => { if (receivedIds.Count == 1 && !thrown) { thrown = true; throw new Exception("Retry batch"); } foreach (TestClass doc in docs) { receivedIds.Add(int.Parse(doc.id)); } if (receivedIds.Count == 3) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithInstanceName("random") .WithMaxItems(1) .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); // Letting processor initialize await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); // Inserting documents foreach (int id in expectedIds) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() }); } // Waiting on all notifications to finish bool isStartOk = allDocsProcessed.WaitOne(30 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); // Verify that we maintain order CollectionAssert.AreEqual(expectedIds.ToList(), receivedIds); }
public async Task StartAsync_ShouldThrowIfContainerDoesNotExist() { ChangeFeedProcessor estimator = this.cosmosClient.GetContainer(this.database.Id, "DoesNotExist") .GetChangeFeedEstimatorBuilder("test", (long estimation, CancellationToken token) => { return(Task.CompletedTask); }, TimeSpan.FromSeconds(1)) .WithLeaseContainer(this.LeaseContainer).Build(); CosmosException exception = await Assert.ThrowsExceptionAsync <CosmosException>(() => estimator.StartAsync()); Assert.AreEqual(HttpStatusCode.NotFound, exception.StatusCode); }
public async IAsyncEnumerable <T> FetchFeed([EnumeratorCancellation] CancellationToken cancellationToken = default) { IReadOnlyCollection <T> changesToProcess = null; SemaphoreSlim changesToProcessSignal = new SemaphoreSlim(0, 1); SemaphoreSlim changesProcessedSignal = new SemaphoreSlim(0, 1); ChangeFeedProcessor changeFeedProcessor = _container.GetChangeFeedProcessorBuilder <T>($"{typeof(T).Name}_ChangeFeedProcessor", async(changes, cancellationToken) => { if ((changes != null) && changes.Count > 0) { changesToProcess = changes; changesToProcessSignal.Release(); try { await changesProcessedSignal.WaitAsync(cancellationToken); } catch (OperationCanceledException) { } } }) .WithInstanceName("Demo.AspNetCore.Changefeed") .WithStartTime(_startTime) .WithPollInterval(_pollInterval) .WithLeaseContainer(_leaseContainer) .Build(); await changeFeedProcessor.StartAsync(); try { while (!cancellationToken.IsCancellationRequested) { await changesToProcessSignal.WaitAsync(cancellationToken); foreach (T item in changesToProcess) { yield return(item); } changesToProcess = null; changesProcessedSignal.Release(); } } finally { await changeFeedProcessor.StopAsync(); } }
/// <summary> /// Exposing progress with the Estimator. /// </summary> /// <remarks> /// The Estimator uses the same processorName and the same lease configuration as the existing processor to measure progress. /// </remarks> public static async Task RunEstimatorChangeFeed( string databaseId, CosmosClient client) { await Program.InitializeAsync(databaseId, client); // <StartProcessorEstimator> Container leaseContainer = client.GetContainer(databaseId, Program.leasesContainer); Container monitoredContainer = client.GetContainer(databaseId, Program.monitoredContainer); ChangeFeedProcessor changeFeedProcessor = monitoredContainer .GetChangeFeedProcessorBuilder <ToDoItem>("changeFeedEstimator", Program.HandleChangesAsync) .WithInstanceName("consoleHost") .WithLeaseContainer(leaseContainer) .Build(); // </StartProcessorEstimator> Console.WriteLine($"Starting Change Feed Processor..."); await changeFeedProcessor.StartAsync(); Console.WriteLine("Change Feed Processor started."); Console.WriteLine("Generating 10 items that will be picked up by the delegate..."); await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer)); // Wait random time for the delegate to output all messages after initialization is done await Task.Delay(5000); // <StartEstimator> ChangeFeedProcessor changeFeedEstimator = monitoredContainer .GetChangeFeedEstimatorBuilder("changeFeedEstimator", Program.HandleEstimationAsync, TimeSpan.FromMilliseconds(1000)) .WithLeaseContainer(leaseContainer) .Build(); // </StartEstimator> Console.WriteLine($"Starting Change Feed Estimator..."); await changeFeedEstimator.StartAsync(); Console.WriteLine("Change Feed Estimator started."); Console.WriteLine("Generating 10 items that will be picked up by the delegate and reported by the Estimator..."); await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer)); Console.WriteLine("Press any key to continue with the next demo..."); Console.ReadKey(); await changeFeedProcessor.StopAsync(); await changeFeedEstimator.StopAsync(); }
static async Task Main(string[] args) { CosmosClient cosmosClient = new CosmosClientBuilder(endpointUri, primaryKey).Build(); Database database = await cosmosClient.CreateDatabaseIfNotExistsAsync(databaseName); await database.CreateContainerIfNotExistsAsync(new ContainerProperties(sourceContainerName, "/id")); await database.CreateContainerIfNotExistsAsync(new ContainerProperties(leaseContainerName, "/id")); ChangeFeedProcessor processor = await StartChangeFeedProcessorAsync(cosmosClient); await GenerateItemsAsync(cosmosClient); }
public async Task RunPollTabulationProcess() { await Initialize(); Container leaseContainer = _cosmosClient.GetContainer("PollDb", "PollLeases"); Container monitoredContainer = _cosmosClient.GetContainer("PollDb", "PollData"); ChangeFeedProcessor changeFeedProcessor = monitoredContainer .GetChangeFeedProcessorBuilder <JObject>("changeFeedBasic", HandleChangesAsync) .WithInstanceName("consoleHost") .WithLeaseContainer(leaseContainer) .WithStartTime(DateTime.MinValue.ToUniversalTime()) .Build(); await changeFeedProcessor.StartAsync(); }
public async Task CloseAsync() { if (GetDestinationCollectionClient() != null) { GetDestinationCollectionClient().Dispose(); } if (changeFeedProcessor != null) { await changeFeedProcessor.StopAsync(); } destinationCollectionClient = null; changeFeedProcessor = null; }
static async Task Main(string[] args) { ChangeFeedProcessor changeFeedProcessor = await StartChangeFeedProcessorAsync(); Thread.Sleep(TimeSpan.FromSeconds(5)); Container container = cosmosClient.GetContainer(databaseName, sourceContainerName); ToDo secondToDo = new ToDo() { Name = "Second" }; await container.CreateItemAsync(secondToDo, new PartitionKey(secondToDo.id)); Console.ReadLine(); await changeFeedProcessor.StopAsync(); }