private static async Task <IChangeFeedProcessor> StartChangeFeedProcessorAsync( string dataCollectionName, DocumentClient client, string dbName, string serverUri) { var leasesCollectionName = dataCollectionName + ".leases"; await client.CreateDocumentCollectionIfNotExistsAsync(UriFactory.CreateDatabaseUri(dbName), new DocumentCollection { Id = leasesCollectionName }); IChangeFeedProcessor processor = await new ChangeFeedProcessorBuilder() .WithObserver <ConsoleObserver>() .WithHostName("console_app_host") .WithFeedDocumentClient(client) .WithFeedCollection(new DocumentCollectionInfo { CollectionName = dataCollectionName, DatabaseName = dbName, Uri = new Uri(serverUri) }) .WithLeaseDocumentClient(client) .WithLeaseCollection(new DocumentCollectionInfo { CollectionName = leasesCollectionName, DatabaseName = dbName }) .WithProcessorOptions(new ChangeFeedProcessorOptions() { StartFromBeginning = true }) .BuildAsync(); await processor.StartAsync().ConfigureAwait(false); return(processor); }
public async Task StartAsync() { var feedCollectionInfo = new DocumentCollectionInfo { DatabaseName = _database, CollectionName = _eventContainer, Uri = new Uri(_endpointUri), MasterKey = _authKey }; var leaseCollectionInfo = new DocumentCollectionInfo { DatabaseName = _database, CollectionName = _leaseContainer, Uri = new Uri(_endpointUri), MasterKey = _authKey }; var viewRepository = new CosmosDBViewRepository(_endpointUri, _authKey, _database); var builder = new ChangeFeedProcessorBuilder(); _changeFeedProcessor = await builder .WithHostName("ProjectionHost") .WithFeedCollection(feedCollectionInfo) .WithLeaseCollection(leaseCollectionInfo) .WithObserverFactory(new EventObserverFactory(_projections, viewRepository)) .WithProcessorOptions(new ChangeFeedProcessorOptions { StartFromBeginning = true }) .BuildAsync(); await _changeFeedProcessor.StartAsync(); }
internal virtual async Task StartProcessorAsync() { if (_host == null) { _host = await _hostBuilder.BuildAsync(); } await _host.StartAsync(); }
public async Task Schema_DefaultsToNoLeaseToken() { TestObserverFactory observerFactory = new TestObserverFactory( openProcessor: null, (FeedProcessing.IChangeFeedObserverContext context, IReadOnlyList <Document> docs) => { return(Task.CompletedTask); }); IChangeFeedProcessor changeFeedProcessorBuilder = await new ChangeFeedProcessorBuilder() .WithObserverFactory(observerFactory) .WithHostName("smoke_test") .WithFeedCollection(this.MonitoredCollectionInfo) .WithLeaseCollection(this.LeaseCollectionInfo) .BuildAsync(); await changeFeedProcessorBuilder.StartAsync(); await this.WaitUntilLeaseStoreIsInitializedAsync(new CancellationTokenSource(5000).Token); await changeFeedProcessorBuilder.StopAsync(); // Verify that no leases have LeaseToken (V3 contract) int leasesProcessed = 0; using (DocumentClient client = new DocumentClient(this.LeaseCollectionInfo.Uri, this.LeaseCollectionInfo.MasterKey, this.LeaseCollectionInfo.ConnectionPolicy)) { Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.LeaseCollectionInfo.DatabaseName, this.LeaseCollectionInfo.CollectionName); IDocumentQuery <JObject> query = client.CreateDocumentQuery <JObject>(collectionUri, "SELECT * FROM c").AsDocumentQuery(); while (query.HasMoreResults) { foreach (JObject lease in await query.ExecuteNextAsync()) { string leaseId = lease.Value <string>("id"); if (leaseId.Contains(".info")) { // These are the store initialization marks continue; } Assert.NotNull(lease.Value <string>("PartitionId")); Assert.Null(lease.Value <string>("LeaseToken")); leasesProcessed++; } } } Assert.True(leasesProcessed > 0); }
private async void OnStarted() // private void OnStarted() { _logger.LogInformation("OnStarted has been called."); // Perform post-startup activities here _logger.LogInformation("Log directory is " + localLogdirectory); string hostName = Guid.NewGuid().ToString(); DocumentCollectionInfo documentCollectionInfo = new DocumentCollectionInfo { Uri = new Uri(_configuration["az_cosmos_uri"]), MasterKey = _configuration["az_cosmos_key"], DatabaseName = _configuration["az_cosmos_db_name"], CollectionName = _configuration["az_cosmos_collection_name"] }; DocumentCollectionInfo leaseCollectionInfo = new DocumentCollectionInfo { Uri = new Uri(_configuration["az_cosmos_uri"]), MasterKey = _configuration["az_cosmos_key"], DatabaseName = _configuration["az_cosmos_db_name"], CollectionName = _configuration["az_cosmos_lease_collection_name"] }; DocumentFeedObserverFactory docObserverFactory = new DocumentFeedObserverFactory(); ChangeFeedOptions feedOptions = new ChangeFeedOptions(); feedOptions.StartFromBeginning = true; ChangeFeedProcessorOptions feedProcessorOptions = new ChangeFeedProcessorOptions(); feedProcessorOptions.LeaseRenewInterval = TimeSpan.FromSeconds(15); this.builder .WithHostName(hostName) .WithFeedCollection(documentCollectionInfo) .WithLeaseCollection(leaseCollectionInfo) .WithProcessorOptions(feedProcessorOptions) .WithObserverFactory(new DocumentFeedObserverFactory()); //; // .WithObserver<DocumentFeedObserver>(); //or just pass a observer iChangeFeedProcessor = await this.builder.BuildAsync(); await iChangeFeedProcessor.StartAsync(); }
public override async Task StartAsync(CancellationToken cancellationToken) { var hostName = Environment.MachineName; var builder = new ChangeFeedProcessorBuilder() .WithHostName(hostName) .WithFeedCollection(_feedCollection) .WithLeaseCollection(_leaseCollection) .WithObserverFactory(this); _processor = await builder.BuildAsync(); await _processor.StartAsync(); _logger.LogInformation($"Change feed processor started at {hostName}..."); await base.StartAsync(cancellationToken); }
public async Task Schema_OnV2MigrationMaintainLeaseToken() { const int batchSize = 10; int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo); int openedCount = 0; ManualResetEvent allObserversStarted = new ManualResetEvent(false); List <int> expectedIds = Enumerable.Range(0, batchSize * 2).ToList(); ManualResetEvent firstSetOfResultsProcessed = new ManualResetEvent(false); ManualResetEvent secondSetOfResultsProcessed = new ManualResetEvent(false); List <int> receivedIds = new List <int>(); TestObserverFactory observerFactory = new TestObserverFactory( context => { int newCount = Interlocked.Increment(ref openedCount); if (newCount == partitionCount) { allObserversStarted.Set(); } return(Task.CompletedTask); }, (FeedProcessing.IChangeFeedObserverContext context, IReadOnlyList <Document> docs) => { foreach (Document doc in docs) { receivedIds.Add(int.Parse(doc.Id)); } if (receivedIds.Count == batchSize) { firstSetOfResultsProcessed.Set(); } if (receivedIds.Count == batchSize * 2) { secondSetOfResultsProcessed.Set(); } return(Task.CompletedTask); }); IChangeFeedProcessor changeFeedProcessorBuilder = await new ChangeFeedProcessorBuilder() .WithObserverFactory(observerFactory) .WithHostName("smoke_test") .WithFeedCollection(this.MonitoredCollectionInfo) .WithLeaseCollection(this.LeaseCollectionInfo) .BuildAsync(); await changeFeedProcessorBuilder.StartAsync(); await this.WaitUntilLeaseStoreIsInitializedAsync(new CancellationTokenSource(5000).Token); // Inserting some documents using (DocumentClient client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy)) { Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName); foreach (int id in expectedIds.Take(10)) { await client.CreateDocumentAsync(collectionUri, new { id = id.ToString() }); } } Assert.True(firstSetOfResultsProcessed.WaitOne(IntegrationTest.changeWaitTimeout), "Timed out waiting for first set of items to be received."); await changeFeedProcessorBuilder.StopAsync(); // At this point we have leases for V2, so we will simulate V3 by manually adding LeaseToken and removing PartitionId using (DocumentClient client = new DocumentClient(this.LeaseCollectionInfo.Uri, this.LeaseCollectionInfo.MasterKey, this.LeaseCollectionInfo.ConnectionPolicy)) { Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.LeaseCollectionInfo.DatabaseName, this.LeaseCollectionInfo.CollectionName); IDocumentQuery <JObject> query = client.CreateDocumentQuery <JObject>(collectionUri, "SELECT * FROM c").AsDocumentQuery(); while (query.HasMoreResults) { foreach (JObject lease in await query.ExecuteNextAsync()) { string leaseId = lease.Value <string>("id"); if (leaseId.Contains(".info")) { // These are the store initialization marks continue; } // create the LeaseToken property lease.Add("LeaseToken", lease.Value <string>("PartitionId")); lease.Remove("PartitionId"); await client.UpsertDocumentAsync(collectionUri, lease); } } } // Now all leases are V3 leases, start another processor that should migrate to V2 schema and maintain LeaseToken for compatibility openedCount = 0; allObserversStarted.Reset(); changeFeedProcessorBuilder = await new ChangeFeedProcessorBuilder() .WithObserverFactory(observerFactory) .WithHostName("smoke_test") .WithFeedCollection(this.MonitoredCollectionInfo) .WithLeaseCollection(this.LeaseCollectionInfo) .BuildAsync(); await changeFeedProcessorBuilder.StartAsync(); Assert.True(allObserversStarted.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout), "Timed out waiting for observres to start"); // Create the rest of the documents using (DocumentClient client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy)) { Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName); foreach (int id in expectedIds.TakeLast(10)) { await client.CreateDocumentAsync(collectionUri, new { id = id.ToString() }); } } Assert.True(secondSetOfResultsProcessed.WaitOne(IntegrationTest.changeWaitTimeout), "Timed out waiting for second set of items to be received."); await changeFeedProcessorBuilder.StopAsync(); // Verify we processed all items (including when using the V3 leases) Assert.True(!expectedIds.Except(receivedIds).Any() && expectedIds.Count == expectedIds.Count); // Verify the after-migration leases have both PartitionId and LeaseToken with the same value using (DocumentClient client = new DocumentClient(this.LeaseCollectionInfo.Uri, this.LeaseCollectionInfo.MasterKey, this.LeaseCollectionInfo.ConnectionPolicy)) { Uri collectionUri = UriFactory.CreateDocumentCollectionUri(this.LeaseCollectionInfo.DatabaseName, this.LeaseCollectionInfo.CollectionName); IDocumentQuery <JObject> query = client.CreateDocumentQuery <JObject>(collectionUri, "SELECT * FROM c").AsDocumentQuery(); while (query.HasMoreResults) { foreach (JObject lease in await query.ExecuteNextAsync()) { string leaseId = lease.Value <string>("id"); if (leaseId.Contains(".info")) { // These are the store initialization marks continue; } Assert.NotNull(lease.Value <string>("PartitionId")); Assert.Null(lease.Value <string>("LeaseToken")); } } } }
public async Task WriteToDocDB_SouldGeneratesIncomingEvent_IfStartingFromDefaultPoint() { string sessionId = Guid.NewGuid().ToString(); IntegrationTestsHelper.GetConfigurationSettings(out DocumentCollectionInfo feedCollectionInfo, out DocumentCollectionInfo leaseCollectionInfo, out int feedOfferThroughput, out int leaseOfferThroughput); feedCollectionInfo.CollectionName = Guid.NewGuid().ToString(); leaseCollectionInfo.CollectionName = feedCollectionInfo.CollectionName + "-lease"; using (var feedClient = new DocumentClient( feedCollectionInfo.Uri, feedCollectionInfo.MasterKey, feedCollectionInfo.ConnectionPolicy)) { await IntegrationTestsHelper.CreateDocumentCollectionAsync(feedClient, feedCollectionInfo.DatabaseName, new DocumentCollection { Id = feedCollectionInfo.CollectionName }, 400); using (var client = new DocumentClient(leaseCollectionInfo.Uri, leaseCollectionInfo.MasterKey, leaseCollectionInfo.ConnectionPolicy)) { await IntegrationTestsHelper.CreateDocumentCollectionAsync( client, leaseCollectionInfo.DatabaseName, new DocumentCollection { Id = leaseCollectionInfo.CollectionName }, 400); } TaskCompletionSource <bool> documentReceived = new TaskCompletionSource <bool>(); Task Process(IChangeFeedObserverContext changeFeedObserverContext, IReadOnlyList <Document> readOnlyList) { if (readOnlyList.Any(d => d.GetPropertyValue <string>("payload") == sessionId)) { documentReceived.SetResult(true); } return(Task.CompletedTask); } FeedProcessing.IChangeFeedObserverFactory observerFactory = new DelegatingMemoryObserverFactory( ctx => Task.CompletedTask, (ctx, reason) => Task.CompletedTask, Process); IChangeFeedProcessor changeFeedProcessor = await new ChangeFeedProcessorBuilder() .WithObserverFactory(observerFactory) .WithHostName("smoke_test") .WithFeedCollection(feedCollectionInfo) .WithLeaseCollection(leaseCollectionInfo) .BuildAsync(); await changeFeedProcessor.StartAsync().ConfigureAwait(false); try { var generateDocsTask = await Task.Factory.StartNew(async() => { while (!documentReceived.Task.IsCompleted) { var document = new Document(); document.SetPropertyValue("payload", sessionId); var collectionUri = UriFactory.CreateDocumentCollectionUri( feedCollectionInfo.DatabaseName, feedCollectionInfo.CollectionName); await feedClient.CreateDocumentAsync(collectionUri, document); await Task.Delay(TimeSpan.FromMilliseconds(100)); } }); await documentReceived.Task; await generateDocsTask; } finally { await changeFeedProcessor.StopAsync(); } } }
public async Task <IChangeFeedProcessor> RunChangeFeedHostAsync() { string hostName = Guid.NewGuid().ToString(); Trace.TraceInformation("Host name {0}", hostName); // monitored collection info var documentCollectionLocation = new DocumentCollectionInfo { Uri = new Uri(this.config.MonitoredUri), MasterKey = this.config.MonitoredSecretKey, DatabaseName = this.config.MonitoredDbName, CollectionName = this.config.MonitoredCollectionName }; var policy = new ConnectionPolicy() { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp }; policy.PreferredLocations.Add("North Europe"); // lease collection info var leaseCollectionLocation = new DocumentCollectionInfo { Uri = new Uri(this.config.LeaseUri), MasterKey = this.config.LeaseSecretKey, DatabaseName = this.config.LeaseDbName, CollectionName = this.config.LeaseCollectionName, ConnectionPolicy = policy }; // destination collection info var destCollInfo = new DocumentCollectionInfo { Uri = new Uri(this.config.DestUri), MasterKey = this.config.DestSecretKey, DatabaseName = this.config.DestDbName, CollectionName = this.config.DestCollectionName }; var processorOptions = new ChangeFeedProcessorOptions(); if (config.DataAgeInHours.HasValue) { if (config.DataAgeInHours.Value >= 0) { processorOptions.StartTime = DateTime.UtcNow.Subtract(TimeSpan.FromHours(config.DataAgeInHours.Value)); } } else { processorOptions.StartFromBeginning = true; } processorOptions.LeaseRenewInterval = TimeSpan.FromSeconds(30); Trace.TraceInformation("Processor options Starts from Beginning - {0}, Lease renew interval - {1}", processorOptions.StartFromBeginning, processorOptions.LeaseRenewInterval.ToString()); processorOptions.MaxItemCount = 1000; var destClient = new DocumentClient(destCollInfo.Uri, destCollInfo.MasterKey, new ConnectionPolicy() { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp }, ConsistencyLevel.Eventual); var docTransformer = new DefaultDocumentTransformer(); BlobContainerClient containerClient = null; if (!String.IsNullOrEmpty(config.BlobConnectionString)) { BlobServiceClient blobServiceClient = new BlobServiceClient(config.BlobConnectionString); containerClient = blobServiceClient.GetBlobContainerClient(config.BlobContainerName); await containerClient.CreateIfNotExistsAsync(); } var docObserverFactory = new DocumentFeedObserverFactory(config.SourcePartitionKeys, config.TargetPartitionKey, destClient, destCollInfo, docTransformer, containerClient); changeFeedProcessor = await new ChangeFeedProcessorBuilder() .WithObserverFactory(docObserverFactory) .WithHostName(hostName) .WithFeedCollection(documentCollectionLocation) .WithLeaseCollection(leaseCollectionLocation) .WithProcessorOptions(processorOptions) .WithFeedDocumentClient(new DocumentClient(documentCollectionLocation.Uri, documentCollectionLocation.MasterKey, policy, ConsistencyLevel.Eventual)) .BuildAsync(); await changeFeedProcessor.StartAsync().ConfigureAwait(false); return(changeFeedProcessor); }