public async Task WhenLeasesHaveContinuationTokenNullReturn0() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await processor.StopAsync(); ManualResetEvent manualResetEvent = new ManualResetEvent(false); long receivedEstimation = 0; ChangeFeedProcessor estimator = this.Container .GetChangeFeedEstimatorBuilder("test", (long estimation, CancellationToken token) => { receivedEstimation = estimation; manualResetEvent.Set(); return(Task.CompletedTask); }, TimeSpan.FromSeconds(1)) .WithLeaseContainer(this.LeaseContainer).Build(); await estimator.StartAsync(); Assert.IsTrue(manualResetEvent.WaitOne(BaseChangeFeedClientHelper.ChangeFeedCleanupTime), "Not received estimation in the expected time"); await estimator.StopAsync(); Assert.AreEqual(0, receivedEstimation); }
public async Task Start() { // Warmup of the connection await _container.GetItemLinqQueryable <Item>().ToFeedIterator().ReadNextAsync(); await _changeFeedProcessor.StartAsync(); }
public static async Task Main(string[] args) { using CosmosClient client = new CosmosClient(endpoint, apikey); Database db = client.GetDatabase("StoreDB"); Container container = db.GetContainer("CartContainer"); Container leasecontainer = await db.CreateContainerIfNotExistsAsync("Lease", "/id"); ChangeFeedProcessorBuilder builder = container.GetChangeFeedProcessorBuilder( "sampleProcessor", async(IReadOnlyCollection <Item> changes, CancellationToken cancellationToken) => { await Console.Out.WriteLineAsync("***Changes occured***"); foreach (Item change in changes) { await Console.Out.WriteLineAsync(change.id + " - " + change.location + " - " + change.name); } }); ChangeFeedProcessor processor = builder .WithInstanceName("firstinstance") .WithLeaseContainer(leasecontainer) .Build(); await processor.StartAsync(); Console.ReadKey(); await processor.StopAsync(); }
public async Task WhenLeasesHaveContinuationTokenNullReturn0() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => { return(Task.CompletedTask); }) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await processor.StopAsync(); long?receivedEstimation = null; ChangeFeedProcessor estimator = this.Container .GetChangeFeedEstimatorBuilder("test", (long estimation, CancellationToken token) => { receivedEstimation = estimation; return(Task.CompletedTask); }, TimeSpan.FromSeconds(1)) .WithLeaseContainer(this.LeaseContainer).Build(); await estimator.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await estimator.StopAsync(); Assert.IsTrue(receivedEstimation.HasValue); Assert.AreEqual(0, receivedEstimation); }
public async Task TestReducePageSizeScenario() { int partitionKey = 0; // Create some docs to make sure that one separate response is returned for 1st execute of query before retries. // These are to make sure continuation token is passed along during retries. string sprocId = "createTwoDocs"; string sprocBody = @"function(startIndex) { for (var i = 0; i < 2; ++i) __.createDocument( __.getSelfLink(), { id: 'doc' + (i + startIndex).toString(), value: 'y'.repeat(1500000), pk:0 }, err => { if (err) throw err;} );}"; CosmosScripts scripts = this.Container.GetScripts(); StoredProcedureResponse storedProcedureResponse = await scripts.CreateStoredProcedureAsync(new CosmosStoredProcedureSettings(sprocId, sprocBody)); ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .CreateChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => { processedDocCount += docs.Count(); foreach (var doc in docs) { accumulator += doc.id.ToString() + "."; } if (processedDocCount == 5) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithStartFromBeginning() .WithInstanceName("random") .WithMaxItems(6) .WithLeaseContainer(this.LeaseContainer).Build(); // Generate the payload await scripts.ExecuteStoredProcedureAsync <int, object>(new PartitionKey(partitionKey), sprocId, 0); // Create 3 docs each 1.5MB. All 3 do not fit into MAX_RESPONSE_SIZE (4 MB). 2nd and 3rd are in same transaction. var content = string.Format("{{\"id\": \"doc2\", \"value\": \"{0}\", \"pk\": 0}}", new string('x', 1500000)); await this.Container.CreateItemAsync(JsonConvert.DeserializeObject <dynamic>(content), new PartitionKey(partitionKey)); await scripts.ExecuteStoredProcedureAsync <int, object>(new PartitionKey(partitionKey), sprocId, 3); await processor.StartAsync(); // Letting processor initialize and pickup changes var isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("doc0.doc1.doc2.doc3.doc4.", accumulator); }
/// <summary> /// Reading the Change Feed since the beginning of time. /// </summary> /// <remarks> /// StartTime only works if the leaseContainer is empty or contains no leases for the particular processor name. /// </remarks> public static async Task RunStartFromBeginningChangeFeed( string databaseId, CosmosClient client) { await Program.InitializeAsync(databaseId, client); Console.WriteLine("Generating 10 items that will be picked up by the delegate..."); await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer)); // <StartFromBeginningInitialization> Container leaseContainer = client.GetContainer(databaseId, Program.leasesContainer); Container monitoredContainer = client.GetContainer(databaseId, Program.monitoredContainer); ChangeFeedProcessor changeFeedProcessor = monitoredContainer .GetChangeFeedProcessorBuilder <ToDoItem>("changeFeedBeginning", Program.HandleChangesAsync) .WithInstanceName("consoleHost") .WithLeaseContainer(leaseContainer) .WithStartTime(DateTime.MinValue.ToUniversalTime()) .Build(); // </StartFromBeginningInitialization> Console.WriteLine($"Starting Change Feed Processor with changes since the beginning..."); await changeFeedProcessor.StartAsync(); Console.WriteLine("Change Feed Processor started."); // Wait random time for the delegate to output all messages after initialization is done await Task.Delay(5000); Console.WriteLine("Press any key to continue with the next demo..."); Console.ReadKey(); await changeFeedProcessor.StopAsync(); }
public async Task Schema_DefaultsToNoPartitionId() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) => Task.CompletedTask) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); // Letting processor initialize await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); // Verify that no leases have PartitionId (V2 contract) using FeedIterator <dynamic> iterator = this.LeaseContainer.GetItemQueryIterator <dynamic>(); while (iterator.HasMoreResults) { FeedResponse <dynamic> page = await iterator.ReadNextAsync(); foreach (dynamic lease in page) { string leaseId = lease.id; Assert.IsNull(lease.partitionKey); if (leaseId.Contains(".info") || leaseId.Contains(".lock")) { // These are the store initialization marks continue; } Assert.IsNotNull(lease.LeaseToken); Assert.IsNull(lease.PartitionId); } } await processor.StopAsync(); }
public async Task StartProcessor <T>(string feedName, Container.ChangesHandler <T> handler) { _changeFeedProcessor = _container .GetChangeFeedProcessorBuilder(feedName, handler).WithLeaseContainer(_leaseContainer) .WithInstanceName(feedName).WithPollInterval(TimeSpan.FromMilliseconds(50)).Build(); await _changeFeedProcessor.StartAsync(); }
public async Task WhenLeasesHaveContinuationTokenNullReturn0_Pull() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await processor.StopAsync(); long receivedEstimation = 0; ChangeFeedEstimator estimator = ((ContainerInternal)this.Container) .GetChangeFeedEstimator( processorName: "test", this.LeaseContainer); using FeedIterator <ChangeFeedProcessorState> feedIterator = estimator.GetCurrentStateIterator(); while (feedIterator.HasMoreResults) { FeedResponse <ChangeFeedProcessorState> response = await feedIterator.ReadNextAsync(); receivedEstimation += response.Sum(r => r.EstimatedLag); } Assert.AreEqual(0, receivedEstimation); }
public async Task StartAsync() { await processor .StartAsync() .ConfigureAwait(false); activity = telemetry.SubscriptionStarted(consumerGroup); }
public async Task TestWithFixedLeaseContainer() { await NonPartitionedContainerHelper.CreateNonPartitionedContainer( this.database, "fixedLeases"); Container fixedLeasesContainer = this.cosmosClient.GetContainer(this.database.Id, "fixedLeases"); try { int partitionKey = 0; ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, Stream stream, CancellationToken token) => { this.ValidateContext(context); IEnumerable <JObject> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream); processedDocCount += asEnumerable.Count(); foreach (JObject doc in asEnumerable) { accumulator += doc["id"].ToString() + "."; } if (processedDocCount == 10) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithInstanceName("random") .WithLeaseContainer(fixedLeasesContainer).Build(); // Start the processor, insert 1 document to generate a checkpoint await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); foreach (int id in Enumerable.Range(0, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString(), pk = partitionKey }); } bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("0.1.2.3.4.5.6.7.8.9.", accumulator); } finally { await fixedLeasesContainer.DeleteContainerAsync(); } }
public IChangeToken Watch() { Task.Run(async() => { await _processor.StartAsync(); }); return(_reloadToken); }
public async Task CountPendingDocuments() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder( processorName: "test", onChangesDelegate: (IReadOnlyCollection <dynamic> docs, CancellationToken token) => { return(Task.CompletedTask); }) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer) .Build(); await processor.StartAsync(); // Letting processor initialize await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); // Inserting documents foreach (int id in Enumerable.Range(0, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() }); } // Waiting on all notifications to finish await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await processor.StopAsync(); long?receivedEstimation = null; ChangeFeedProcessor estimator = this.Container .GetChangeFeedEstimatorBuilder( processorName: "test", estimationDelegate: (long estimation, CancellationToken token) => { receivedEstimation = estimation; return(Task.CompletedTask); }, TimeSpan.FromSeconds(1)) .WithLeaseContainer(this.LeaseContainer) .Build(); // Inserting more documents foreach (int id in Enumerable.Range(11, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() }); } await estimator.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await estimator.StopAsync(); Assert.IsTrue(receivedEstimation.HasValue); Assert.AreEqual(10, receivedEstimation); }
public async Task TestWithRunningProcessor_WithManualCheckpoint() { int partitionKey = 0; ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilderWithManualCheckpoint("test", async(ChangeFeedProcessorContext context, Stream stream, Func <Task> checkpointAsync, CancellationToken token) => { this.ValidateContext(context); IEnumerable <JObject> docs = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream); processedDocCount += docs.Count(); foreach (dynamic doc in docs) { accumulator += doc.id.ToString() + "."; } if (processedDocCount == 3) { // Throwing on the 3rd document, since we checkpointed only on the 1st, we would repeat 2nd and 3rd throw new Exception("Stop here"); } if (processedDocCount == 1) { // Checkpointing on the first document to be able to have a point to rollback to await checkpointAsync(); } if (processedDocCount == 12) { allDocsProcessed.Set(); } }) .WithInstanceName("random") .WithMaxItems(1) .WithLeaseContainer(this.LeaseContainer).Build(); // Start the processor, insert 1 document to generate a checkpoint await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); foreach (int id in Enumerable.Range(0, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString(), pk = partitionKey }); } bool isStartOk = allDocsProcessed.WaitOne(30 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("0.1.2.1.2.3.4.5.6.7.8.9.", accumulator); }
public async Task TestWithStartTime_CustomTime() { int partitionKey = 0; foreach (int id in Enumerable.Range(0, 5)) { await this.Container.CreateItemAsync <dynamic>(new { id = $"doc{id}", pk = partitionKey }); } await Task.Delay(1000); DateTime now = DateTime.UtcNow; await Task.Delay(1000); foreach (int id in Enumerable.Range(5, 5)) { await this.Container.CreateItemAsync <dynamic>(new { id = $"doc{id}", pk = partitionKey }); } ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, IReadOnlyCollection <dynamic> docs, CancellationToken token) => { this.ValidateContext(context); Assert.IsTrue(docs.Count > 0); processedDocCount += docs.Count; foreach (dynamic doc in docs) { accumulator += doc.id.ToString() + "."; } if (processedDocCount == 5) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithStartTime(now) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); // Letting processor initialize and pickup changes bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("doc5.doc6.doc7.doc8.doc9.", accumulator); }
public async Task StartProcessorAsync() { _changeFeedProcessor = _container .GetChangeFeedProcessorBuilder <DailyDeviceReading>("DeviceSimChangeFeedProc", HandleChangesAsync) .WithInstanceName("consoleHost1") .WithLeaseContainer(_leaseContainer) //.WithStartTime(particularPointInTime) .Build(); await _changeFeedProcessor.StartAsync(); }
public async Task CountPendingDocuments_Pull() { ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder( processorName: "test", onChangesDelegate: (IReadOnlyCollection <dynamic> docs, CancellationToken token) => Task.CompletedTask) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer) .Build(); await processor.StartAsync(); // Letting processor initialize await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); // Inserting documents foreach (int id in Enumerable.Range(0, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() }); } // Waiting on all notifications to finish await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedCleanupTime); await processor.StopAsync(); ManualResetEvent manualResetEvent = new ManualResetEvent(false); long receivedEstimation = 0; ChangeFeedEstimator estimator = ((ContainerInternal)this.Container) .GetChangeFeedEstimator( processorName: "test", this.LeaseContainer); // Inserting more documents foreach (int id in Enumerable.Range(11, 10)) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() }); } using FeedIterator <ChangeFeedProcessorState> feedIterator = estimator.GetCurrentStateIterator(); while (feedIterator.HasMoreResults) { FeedResponse <ChangeFeedProcessorState> response = await feedIterator.ReadNextAsync(); receivedEstimation += response.Sum(r => r.EstimatedLag); Assert.IsTrue(response.Headers.RequestCharge > 0); Assert.IsNotNull(response.Diagnostics); string asString = response.Diagnostics.ToString(); Assert.IsTrue(asString.Length > 0); Assert.IsTrue(asString.Contains("cosmos-netstandard-sdk")); } Assert.AreEqual(10, receivedEstimation); }
private static async Task <ChangeFeedProcessor> StartChangeFeedProcessorAsync() { // Delete the source container if it exists try { await cosmosClient.GetContainer(databaseName, sourceContainerName) .DeleteContainerAsync(); } catch (Exception e) { Console.WriteLine($"No container to delete"); } // Create the source container and add an item await cosmosClient.GetDatabase(databaseName).CreateContainerAsync(sourceContainerName, "/id"); Container sourceContainer = cosmosClient.GetContainer(databaseName, sourceContainerName); ToDo firstToDo = new ToDo() { Name = "First" }; await sourceContainer.CreateItemAsync(firstToDo, new PartitionKey(firstToDo.id)); // Delete the lease container if it exists try { await cosmosClient.GetContainer(databaseName, leaseContainerName) .DeleteContainerAsync(); } catch (Exception e) { Console.WriteLine($"No container to delete"); } // Create a new lease container await cosmosClient.GetDatabase(databaseName).CreateContainerAsync(leaseContainerName, "/id"); Container leaseContainer = cosmosClient.GetContainer(databaseName, leaseContainerName); ChangeFeedProcessor changeFeedProcessor = sourceContainer .GetChangeFeedProcessorBuilder <ToDo>("changeFeedBeginning", Program.HandleChangesAsync) .WithInstanceName("consoleHost") .WithLeaseContainer(leaseContainer) .WithStartTime(DateTime.MinValue.ToUniversalTime()) .WithPollInterval(TimeSpan.FromSeconds(1)) .Build(); Console.WriteLine("Starting Change Feed Processor..."); await changeFeedProcessor.StartAsync(); Console.WriteLine("Change Feed Processor started."); return(changeFeedProcessor); }
public async Task ExceptionsRetryBatch() { ManualResetEvent allDocsProcessed = new ManualResetEvent(false); bool thrown = false; IEnumerable <int> expectedIds = Enumerable.Range(0, 3); List <int> receivedIds = new List <int>(); ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) => { if (receivedIds.Count == 1 && !thrown) { thrown = true; throw new Exception("Retry batch"); } foreach (TestClass doc in docs) { receivedIds.Add(int.Parse(doc.id)); } if (receivedIds.Count == 3) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithInstanceName("random") .WithMaxItems(1) .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); // Letting processor initialize await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); // Inserting documents foreach (int id in expectedIds) { await this.Container.CreateItemAsync <dynamic>(new { id = id.ToString() }); } // Waiting on all notifications to finish bool isStartOk = allDocsProcessed.WaitOne(30 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); // Verify that we maintain order CollectionAssert.AreEqual(expectedIds.ToList(), receivedIds); }
/// <summary> /// Exposing progress with the Estimator. /// </summary> /// <remarks> /// The Estimator uses the same processorName and the same lease configuration as the existing processor to measure progress. /// </remarks> public static async Task RunEstimatorChangeFeed( string databaseId, CosmosClient client) { await Program.InitializeAsync(databaseId, client); // <StartProcessorEstimator> Container leaseContainer = client.GetContainer(databaseId, Program.leasesContainer); Container monitoredContainer = client.GetContainer(databaseId, Program.monitoredContainer); ChangeFeedProcessor changeFeedProcessor = monitoredContainer .GetChangeFeedProcessorBuilder <ToDoItem>("changeFeedEstimator", Program.HandleChangesAsync) .WithInstanceName("consoleHost") .WithLeaseContainer(leaseContainer) .Build(); // </StartProcessorEstimator> Console.WriteLine($"Starting Change Feed Processor..."); await changeFeedProcessor.StartAsync(); Console.WriteLine("Change Feed Processor started."); Console.WriteLine("Generating 10 items that will be picked up by the delegate..."); await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer)); // Wait random time for the delegate to output all messages after initialization is done await Task.Delay(5000); // <StartEstimator> ChangeFeedProcessor changeFeedEstimator = monitoredContainer .GetChangeFeedEstimatorBuilder("changeFeedEstimator", Program.HandleEstimationAsync, TimeSpan.FromMilliseconds(1000)) .WithLeaseContainer(leaseContainer) .Build(); // </StartEstimator> Console.WriteLine($"Starting Change Feed Estimator..."); await changeFeedEstimator.StartAsync(); Console.WriteLine("Change Feed Estimator started."); Console.WriteLine("Generating 10 items that will be picked up by the delegate and reported by the Estimator..."); await Program.GenerateItems(10, client.GetContainer(databaseId, Program.monitoredContainer)); Console.WriteLine("Press any key to continue with the next demo..."); Console.ReadKey(); await changeFeedProcessor.StopAsync(); await changeFeedEstimator.StopAsync(); }
public async IAsyncEnumerable <T> FetchFeed([EnumeratorCancellation] CancellationToken cancellationToken = default) { IReadOnlyCollection <T> changesToProcess = null; SemaphoreSlim changesToProcessSignal = new SemaphoreSlim(0, 1); SemaphoreSlim changesProcessedSignal = new SemaphoreSlim(0, 1); ChangeFeedProcessor changeFeedProcessor = _container.GetChangeFeedProcessorBuilder <T>($"{typeof(T).Name}_ChangeFeedProcessor", async(changes, cancellationToken) => { if ((changes != null) && changes.Count > 0) { changesToProcess = changes; changesToProcessSignal.Release(); try { await changesProcessedSignal.WaitAsync(cancellationToken); } catch (OperationCanceledException) { } } }) .WithInstanceName("Demo.AspNetCore.Changefeed") .WithStartTime(_startTime) .WithPollInterval(_pollInterval) .WithLeaseContainer(_leaseContainer) .Build(); await changeFeedProcessor.StartAsync(); try { while (!cancellationToken.IsCancellationRequested) { await changesToProcessSignal.WaitAsync(cancellationToken); foreach (T item in changesToProcess) { yield return(item); } changesToProcess = null; changesProcessedSignal.Release(); } } finally { await changeFeedProcessor.StopAsync(); } }
public async Task RunPollTabulationProcess() { await Initialize(); Container leaseContainer = _cosmosClient.GetContainer("PollDb", "PollLeases"); Container monitoredContainer = _cosmosClient.GetContainer("PollDb", "PollData"); ChangeFeedProcessor changeFeedProcessor = monitoredContainer .GetChangeFeedProcessorBuilder <JObject>("changeFeedBasic", HandleChangesAsync) .WithInstanceName("consoleHost") .WithLeaseContainer(leaseContainer) .WithStartTime(DateTime.MinValue.ToUniversalTime()) .Build(); await changeFeedProcessor.StartAsync(); }
private static async Task <ChangeFeedProcessor> StartChangeFeedProcessorAsync( CosmosClient cosmosClient) { Container leaseContainer = cosmosClient.GetContainer(_productDatabase.Id, _productLeaseContainer.Id); ChangeFeedProcessor changeFeedProcessor = cosmosClient.GetContainer(_productDatabase.Id, _productContainer.Id) .GetChangeFeedProcessorBuilder <Product>("cosmosBackupSample", HandleChangesAsync) .WithInstanceName("consoleHost") .WithLeaseContainer(leaseContainer) .Build(); Console.WriteLine("Starting Change Feed Processor for Product Backups..."); await changeFeedProcessor.StartAsync(); Console.WriteLine("Product Backup started"); return(changeFeedProcessor); }
private static async Task <ChangeFeedProcessor> StartChangeFeedProcessorAsync( CosmosClient cosmosClient) { Container leaseContainer = cosmosClient.GetContainer(databaseName, leaseContainerName); ChangeFeedProcessor changeFeedProcessor = cosmosClient.GetContainer(databaseName, sourceContainerName) .GetChangeFeedProcessorBuilder <Person>(processorName: "changeFeedSample", HandleChangesAsync) .WithInstanceName("consoleHost") .WithLeaseContainer(leaseContainer) .Build(); Console.WriteLine("Starting Change Feed Processor..."); await changeFeedProcessor.StartAsync(); Console.WriteLine("Change Feed Processor started."); return(changeFeedProcessor); }
public Task StartAsync(string instanceName) { CosmosClient client = new CosmosClient(_endpointUrl, _authorizationKey); Container eventContainer = client.GetContainer(_databaseId, _eventContainerId); Container leaseContainer = client.GetContainer(_databaseId, _leaseContainerId); _changeFeedProcessor = eventContainer .GetChangeFeedProcessorBuilder <Change>("MSSQLProjections", HandleChangesAsync) .WithInstanceName(instanceName) .WithLeaseContainer(leaseContainer) .WithStartTime(new DateTime(2020, 5, 1, 0, 0, 0, DateTimeKind.Utc)) .Build(); return(_changeFeedProcessor.StartAsync()); }
public Task Run(CancellationToken token) { _logger.LogInformation($"{nameof(Run)}: Starting monitoring"); CancellationTokenSource tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); Console.CancelKeyPress += (sender, args) => { _logger.LogInformation($"{nameof(Run)}: Canceling monitoring"); args.Cancel = true; tokenSource.Cancel(); }; var cts = new TaskCompletionSource <bool>(); _ = Task.Run(async() => { CosmosClient cosmosClient = new CosmosClient(_watcherOption.GetResolvedConnectionString()); Database database = cosmosClient.GetDatabase(_watcherOption.DatabaseName); Container traceContainer = await database.CreateContainerIfNotExistsAsync("Trace", Constants.DefaultPartitionKey); Container leaseContainer = await database.CreateContainerIfNotExistsAsync("_feedLease", Constants.DefaultPartitionKey); ChangeFeedProcessor changeFeedProcessor = traceContainer.GetChangeFeedProcessorBuilder <TraceRecord>("traceChangeFeed", HandleChangesAsync) .WithInstanceName(nameof(WatcherCmd)) .WithLeaseContainer(leaseContainer) .Build(); await changeFeedProcessor.StartAsync(); _logger.LogInformation("Feed process has started"); try { await Task.Delay(-1, tokenSource.Token); } catch { } finally { await changeFeedProcessor.StopAsync(); _logger.LogInformation("Feed process has stopped"); } cts.SetResult(true); }); return(cts.Task); }
public async Task TestWithStartTime_Beginning() { int partitionKey = 0; ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = this.Container .GetChangeFeedProcessorBuilder("test", (ChangeFeedProcessorContext context, Stream stream, CancellationToken token) => { this.ValidateContext(context); IEnumerable <JObject> asEnumerable = CosmosFeedResponseSerializer.FromFeedResponseStream <JObject>(this.serializerCore, stream); processedDocCount += asEnumerable.Count(); foreach (JObject doc in asEnumerable) { accumulator += doc["id"].ToString() + "."; } if (processedDocCount == 5) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithStartTime(DateTime.MinValue.ToUniversalTime()) .WithInstanceName("random") .WithLeaseContainer(this.LeaseContainer).Build(); await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); foreach (int id in Enumerable.Range(0, 5)) { await this.Container.CreateItemAsync <dynamic>(new { id = $"doc{id}", pk = partitionKey }); } // Letting processor initialize and pickup changes bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("doc0.doc1.doc2.doc3.doc4.", accumulator); }
private static async Task StartEstimatorAsync(CosmosClient client) { Container container = client.GetContainer("OnDotNet", "changefeed-items"); Container leasesContainer = client.GetContainer("OnDotNet", "changefeed-leases"); ChangeFeedProcessor processor = container.GetChangeFeedEstimatorBuilder("OnDotNet", Program.ShowEstimation, TimeSpan.FromSeconds(1)) .WithLeaseContainer(leasesContainer) .Build(); await processor.StartAsync(); Console.WriteLine($"Estimator started."); Console.WriteLine("Press any key to stop."); Console.ReadKey(); await processor.StopAsync(); Console.WriteLine($"Estimator stopped."); }
public async Task DoesNotUseUserSerializer() { CosmosClient cosmosClient = TestCommon.CreateCosmosClient(builder => builder.WithCustomSerializer(new FailedUserSerializer())); ManualResetEvent allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; ChangeFeedProcessor processor = cosmosClient.GetContainer(this.database.Id, this.Container.Id) .GetChangeFeedProcessorBuilder("test", (IReadOnlyCollection <TestClass> docs, CancellationToken token) => { processedDocCount += docs.Count(); foreach (TestClass doc in docs) { accumulator += doc.id.ToString() + "."; } if (processedDocCount == 10) { allDocsProcessed.Set(); } return(Task.CompletedTask); }) .WithInstanceName("random") .WithLeaseContainer(cosmosClient.GetContainer(this.database.Id, this.LeaseContainer.Id)).Build(); // Start the processor, insert 1 document to generate a checkpoint await processor.StartAsync(); await Task.Delay(BaseChangeFeedClientHelper.ChangeFeedSetupTime); foreach (int id in Enumerable.Range(0, 10)) { await this.Container.CreateItemAsync <TestClass>(new TestClass { id = id.ToString() }); } bool isStartOk = allDocsProcessed.WaitOne(10 * BaseChangeFeedClientHelper.ChangeFeedSetupTime); await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("0.1.2.3.4.5.6.7.8.9.", accumulator); }
private static async Task StartProcessorAsync(CosmosClient client, string instanceName) { Container container = client.GetContainer("OnDotNet", "changefeed-items"); Container leasesContainer = client.GetContainer("OnDotNet", "changefeed-leases"); ChangeFeedProcessor processor = container.GetChangeFeedProcessorBuilder <Model>("OnDotNet", Program.ProcessChangesAsync) .WithInstanceName(instanceName) .WithLeaseContainer(leasesContainer) .Build(); await processor.StartAsync(); Console.WriteLine($"Instance {instanceName} started."); Console.WriteLine("Press any key to stop."); Console.ReadKey(); await processor.StopAsync(); Console.WriteLine($"Instance {instanceName} stopped."); }