static async Task Main(string[] args) { log4net.Config.XmlConfigurator.Configure(); // for using System.Diagnostics.TraceSource logging uncomment the line bellow // LogProvider.SetCurrentLogProvider(new TraceLogProvider()); var dbUri = "https://localhost:8081/"; var key = "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="; var collectionName = "Input"; await SetupEnvironmentAsync(dbUri, key, collectionName); CancellationTokenSource cts = new CancellationTokenSource(); Task feedingTask = StartFeedingDataAsync(dbUri, key, collectionName, cts.Token); IChangeFeedProcessor processor = await RunChangeFeedProcessorAsync(dbUri, key, collectionName); Console.WriteLine("Running...[Press ENTER to stop]"); Console.ReadLine(); Console.WriteLine("Stopping..."); cts.Cancel(); await feedingTask.ConfigureAwait(false); await processor.StopAsync().ConfigureAwait(false); Console.WriteLine("Stopped"); Console.ReadLine(); }
static async Task Main(string[] args) { var dbUri = "https://localhost:8081/"; var key = "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="; var collectionName = "Input"; await SetupEnvironmentAsync(dbUri, key, collectionName); CancellationTokenSource cts = new CancellationTokenSource(); Task feedingTask = StartFeedingDataAsync(dbUri, key, collectionName, cts.Token); Task monitoringTask = StartMonitoringAsync(dbUri, key, collectionName, cts.Token); IChangeFeedProcessor processor = await RunChangeFeedProcessorAsync(dbUri, key, collectionName); Console.WriteLine("Running...[Press ENTER to stop]"); Console.ReadLine(); Console.WriteLine("Stopping..."); cts.Cancel(); await feedingTask.ConfigureAwait(false); await processor.StopAsync().ConfigureAwait(false); await monitoringTask.ConfigureAwait(false); Console.WriteLine("Stopped"); Console.ReadLine(); }
private async void OnStopping() // private void OnStopping() { _logger.LogInformation("OnStopping has been called."); // Perform on-stopping activities here await iChangeFeedProcessor.StopAsync(); }
public bool UnRegisterDependency(string key) { lock (this) { if (DependencyChangeCorrelator.Instance.UnRegisterDependency(key)) { _changeFeedProcessor.StopAsync(); _changeFeedProcessor = null; } } return(true); }
public async Task Schema_DefaultsToNoLeaseToken() { TestObserverFactory observerFactory = new TestObserverFactory( openProcessor: null, (FeedProcessing.IChangeFeedObserverContext context, IReadOnlyList <Document> docs) => { return(Task.CompletedTask); }); IChangeFeedProcessor changeFeedProcessorBuilder = await new ChangeFeedProcessorBuilder() .WithObserverFactory(observerFactory) .WithHostName("smoke_test") .WithFeedCollection(this.MonitoredCollectionInfo) .WithLeaseCollection(this.LeaseCollectionInfo) .BuildAsync(); await changeFeedProcessorBuilder.StartAsync(); await this.WaitUntilLeaseStoreIsInitializedAsync(new CancellationTokenSource(5000).Token); await changeFeedProcessorBuilder.StopAsync(); // Verify that no leases have LeaseToken (V3 contract) int leasesProcessed = 0; using (DocumentClient client = new DocumentClient(this.LeaseCollectionInfo.Uri, this.LeaseCollectionInfo.MasterKey, this.LeaseCollectionInfo.ConnectionPolicy)) { Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.LeaseCollectionInfo.DatabaseName, this.LeaseCollectionInfo.CollectionName); IDocumentQuery <JObject> query = client.CreateDocumentQuery <JObject>(collectionUri, "SELECT * FROM c").AsDocumentQuery(); while (query.HasMoreResults) { foreach (JObject lease in await query.ExecuteNextAsync()) { string leaseId = lease.Value <string>("id"); if (leaseId.Contains(".info")) { // These are the store initialization marks continue; } Assert.NotNull(lease.Value <string>("PartitionId")); Assert.Null(lease.Value <string>("LeaseToken")); leasesProcessed++; } } } Assert.True(leasesProcessed > 0); }
public async Task CloseAsync() { if (GetDestinationCollectionClient() != null) { GetDestinationCollectionClient().Dispose(); } if (changeFeedProcessor != null) { await changeFeedProcessor.StopAsync(); } destinationCollectionClient = null; changeFeedProcessor = null; }
public async Task StopAsync(CancellationToken cancellationToken) { try { if (_host != null) { await _host.StopAsync(); _listenerStatus = ListenerNotRegistered; } } catch (Exception ex) { _logger.LogWarning($"Stopping the observer failed, potentially it was never started. Exception: {ex.Message}."); } }
static async Task MainAsync(string[] args) { try { //setup our DI var serviceProvider = new ServiceCollection() .AddLogging((builder) => { builder .AddConsole() .SetMinimumLevel(LogLevel.Information); }) .AddSingleton <CosmosDbSink>() .BuildServiceProvider(); var loggerFactory = serviceProvider.GetService <ILoggerFactory>(); var logger = loggerFactory.CreateLogger <Program>(); logger.LogDebug("Starting application"); AppDomain.CurrentDomain.FirstChanceException += new FirstChanceExceptionLogger(loggerFactory).FirstChanceHandler; // Set the maximum number of concurrent connections ServicePointManager.DefaultConnectionLimit = 1200; var changeFeed = new ChangeFeed(new CosmosDbSink(loggerFactory), loggerFactory); IChangeFeedProcessor processor = await changeFeed.StartAsync(); logger.LogInformation("Changefeed started successfully"); Console.WriteLine("Press any key to stop listening for changes..."); Console.ReadKey(); await processor.StopAsync(); } catch (Exception error) { Console.WriteLine("UNHANDLED EXCEPTION: {0}", error); // TODO fabianm REMOVE Console.ReadKey(); throw; } }
public override async Task StopAsync(CancellationToken cancellationToken) { await _processor?.StopAsync(); await base.StopAsync(cancellationToken); }
public Task StopAsync() { return(_changeFeedProcessor.StopAsync()); }
public async Task Schema_OnV2MigrationMaintainLeaseToken() { const int batchSize = 10; int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo); int openedCount = 0; ManualResetEvent allObserversStarted = new ManualResetEvent(false); List <int> expectedIds = Enumerable.Range(0, batchSize * 2).ToList(); ManualResetEvent firstSetOfResultsProcessed = new ManualResetEvent(false); ManualResetEvent secondSetOfResultsProcessed = new ManualResetEvent(false); List <int> receivedIds = new List <int>(); TestObserverFactory observerFactory = new TestObserverFactory( context => { int newCount = Interlocked.Increment(ref openedCount); if (newCount == partitionCount) { allObserversStarted.Set(); } return(Task.CompletedTask); }, (FeedProcessing.IChangeFeedObserverContext context, IReadOnlyList <Document> docs) => { foreach (Document doc in docs) { receivedIds.Add(int.Parse(doc.Id)); } if (receivedIds.Count == batchSize) { firstSetOfResultsProcessed.Set(); } if (receivedIds.Count == batchSize * 2) { secondSetOfResultsProcessed.Set(); } return(Task.CompletedTask); }); IChangeFeedProcessor changeFeedProcessorBuilder = await new ChangeFeedProcessorBuilder() .WithObserverFactory(observerFactory) .WithHostName("smoke_test") .WithFeedCollection(this.MonitoredCollectionInfo) .WithLeaseCollection(this.LeaseCollectionInfo) .BuildAsync(); await changeFeedProcessorBuilder.StartAsync(); await this.WaitUntilLeaseStoreIsInitializedAsync(new CancellationTokenSource(5000).Token); // Inserting some documents using (DocumentClient client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy)) { Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName); foreach (int id in expectedIds.Take(10)) { await client.CreateDocumentAsync(collectionUri, new { id = id.ToString() }); } } Assert.True(firstSetOfResultsProcessed.WaitOne(IntegrationTest.changeWaitTimeout), "Timed out waiting for first set of items to be received."); await changeFeedProcessorBuilder.StopAsync(); // At this point we have leases for V2, so we will simulate V3 by manually adding LeaseToken and removing PartitionId using (DocumentClient client = new DocumentClient(this.LeaseCollectionInfo.Uri, this.LeaseCollectionInfo.MasterKey, this.LeaseCollectionInfo.ConnectionPolicy)) { Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.LeaseCollectionInfo.DatabaseName, this.LeaseCollectionInfo.CollectionName); IDocumentQuery <JObject> query = client.CreateDocumentQuery <JObject>(collectionUri, "SELECT * FROM c").AsDocumentQuery(); while (query.HasMoreResults) { foreach (JObject lease in await query.ExecuteNextAsync()) { string leaseId = lease.Value <string>("id"); if (leaseId.Contains(".info")) { // These are the store initialization marks continue; } // create the LeaseToken property lease.Add("LeaseToken", lease.Value <string>("PartitionId")); lease.Remove("PartitionId"); await client.UpsertDocumentAsync(collectionUri, lease); } } } // Now all leases are V3 leases, start another processor that should migrate to V2 schema and maintain LeaseToken for compatibility openedCount = 0; allObserversStarted.Reset(); changeFeedProcessorBuilder = await new ChangeFeedProcessorBuilder() .WithObserverFactory(observerFactory) .WithHostName("smoke_test") .WithFeedCollection(this.MonitoredCollectionInfo) .WithLeaseCollection(this.LeaseCollectionInfo) .BuildAsync(); await changeFeedProcessorBuilder.StartAsync(); Assert.True(allObserversStarted.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout), "Timed out waiting for observres to start"); // Create the rest of the documents using (DocumentClient client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy)) { Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName); foreach (int id in expectedIds.TakeLast(10)) { await client.CreateDocumentAsync(collectionUri, new { id = id.ToString() }); } } Assert.True(secondSetOfResultsProcessed.WaitOne(IntegrationTest.changeWaitTimeout), "Timed out waiting for second set of items to be received."); await changeFeedProcessorBuilder.StopAsync(); // Verify we processed all items (including when using the V3 leases) Assert.True(!expectedIds.Except(receivedIds).Any() && expectedIds.Count == expectedIds.Count); // Verify the after-migration leases have both PartitionId and LeaseToken with the same value using (DocumentClient client = new DocumentClient(this.LeaseCollectionInfo.Uri, this.LeaseCollectionInfo.MasterKey, this.LeaseCollectionInfo.ConnectionPolicy)) { Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.LeaseCollectionInfo.DatabaseName, this.LeaseCollectionInfo.CollectionName); IDocumentQuery <JObject> query = client.CreateDocumentQuery <JObject>(collectionUri, "SELECT * FROM c").AsDocumentQuery(); while (query.HasMoreResults) { foreach (JObject lease in await query.ExecuteNextAsync()) { string leaseId = lease.Value <string>("id"); if (leaseId.Contains(".info")) { // These are the store initialization marks continue; } Assert.NotNull(lease.Value <string>("PartitionId")); Assert.Null(lease.Value <string>("LeaseToken")); } } } }
public async Task WriteToDocDB_SouldGeneratesIncomingEvent_IfStartingFromDefaultPoint() { string sessionId = Guid.NewGuid().ToString(); IntegrationTestsHelper.GetConfigurationSettings(out DocumentCollectionInfo feedCollectionInfo, out DocumentCollectionInfo leaseCollectionInfo, out int feedOfferThroughput, out int leaseOfferThroughput); feedCollectionInfo.CollectionName = Guid.NewGuid().ToString(); leaseCollectionInfo.CollectionName = feedCollectionInfo.CollectionName + "-lease"; using (var feedClient = new DocumentClient( feedCollectionInfo.Uri, feedCollectionInfo.MasterKey, feedCollectionInfo.ConnectionPolicy)) { await IntegrationTestsHelper.CreateDocumentCollectionAsync(feedClient, feedCollectionInfo.DatabaseName, new DocumentCollection { Id = feedCollectionInfo.CollectionName }, 400); using (var client = new DocumentClient(leaseCollectionInfo.Uri, leaseCollectionInfo.MasterKey, leaseCollectionInfo.ConnectionPolicy)) { await IntegrationTestsHelper.CreateDocumentCollectionAsync( client, leaseCollectionInfo.DatabaseName, new DocumentCollection { Id = leaseCollectionInfo.CollectionName }, 400); } TaskCompletionSource <bool> documentReceived = new TaskCompletionSource <bool>(); Task Process(IChangeFeedObserverContext changeFeedObserverContext, IReadOnlyList <Document> readOnlyList) { if (readOnlyList.Any(d => d.GetPropertyValue <string>("payload") == sessionId)) { documentReceived.SetResult(true); } return(Task.CompletedTask); } FeedProcessing.IChangeFeedObserverFactory observerFactory = new DelegatingMemoryObserverFactory( ctx => Task.CompletedTask, (ctx, reason) => Task.CompletedTask, Process); IChangeFeedProcessor changeFeedProcessor = await new ChangeFeedProcessorBuilder() .WithObserverFactory(observerFactory) .WithHostName("smoke_test") .WithFeedCollection(feedCollectionInfo) .WithLeaseCollection(leaseCollectionInfo) .BuildAsync(); await changeFeedProcessor.StartAsync().ConfigureAwait(false); try { var generateDocsTask = await Task.Factory.StartNew(async() => { while (!documentReceived.Task.IsCompleted) { var document = new Document(); document.SetPropertyValue("payload", sessionId); var collectionUri = UriFactory.CreateDocumentCollectionUri( feedCollectionInfo.DatabaseName, feedCollectionInfo.CollectionName); await feedClient.CreateDocumentAsync(collectionUri, document); await Task.Delay(TimeSpan.FromMilliseconds(100)); } }); await documentReceived.Task; await generateDocsTask; } finally { await changeFeedProcessor.StopAsync(); } } }