public static async Task StartChangeFeedHost() { var config = ServiceLocator.GetService <IConfiguration>(); string hostName = config.MonitoredCollectionName + "_monitor_" + Guid.NewGuid().ToString(); DocumentCollectionInfo documentCollectionLocation = new DocumentCollectionInfo { Uri = config.DatabaseAccountUri, MasterKey = config.DatabaseAccountKey, DatabaseName = config.MonitoredDatabaseName, CollectionName = config.MonitoredCollectionName }; DocumentCollectionInfo leaseCollectionLocation = new DocumentCollectionInfo { Uri = config.DatabaseAccountUri, MasterKey = config.DatabaseAccountKey, DatabaseName = config.ChangeTrackingDatabaseName, CollectionName = config.ChangeTrackingLeaseCollectionName }; Console.WriteLine("Main program: Creating ChangeFeedEventHost..."); ChangeFeedOptions feedOptions = new ChangeFeedOptions(); ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions(); ChangeFeedEventHost host = new ChangeFeedEventHost(hostName, documentCollectionLocation, leaseCollectionLocation, feedOptions, feedHostOptions); await host.RegisterObserverAsync <DocumentFeedObserver>(); Console.WriteLine("Main program: press Enter to stop..."); Console.ReadLine(); await host.UnregisterObserversAsync(); }
/// <summary> /// Registers change feed observer to update changes read on change feed to destination /// collection. Deregisters change feed observer and closes process when enter key is pressed /// </summary> /// <returns>A Task to allow asynchronous execution</returns> public async Task RunChangeFeedHostAsync() { string hostName = Guid.NewGuid().ToString(); // monitored collection info DocumentCollectionInfo documentCollectionLocation = new DocumentCollectionInfo { Uri = new Uri(this.monitoredUri), MasterKey = this.monitoredSecretKey, DatabaseName = this.monitoredDbName, CollectionName = this.monitoredCollectionName }; // lease collection info DocumentCollectionInfo leaseCollectionLocation = new DocumentCollectionInfo { Uri = new Uri(this.leaseUri), MasterKey = this.leaseSecretKey, DatabaseName = this.leaseDbName, CollectionName = this.leaseCollectionName }; // destination collection info DocumentCollectionInfo destCollInfo = new DocumentCollectionInfo { Uri = new Uri(this.destUri), MasterKey = this.destSecretKey, DatabaseName = this.destDbName, CollectionName = this.destCollectionName }; // Customizable change feed option and host options ChangeFeedOptions feedOptions = new ChangeFeedOptions(); // ie customize StartFromBeginning so change feed reads from beginning // can customize MaxItemCount, PartitonKeyRangeId, RequestContinuation, SessionToken and StartFromBeginning feedOptions.StartFromBeginning = true; ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions(); // ie. customizing lease renewal interval to 15 seconds // can customize LeaseRenewInterval, LeaseAcquireInterval, LeaseExpirationInterval, FeedPollDelay feedHostOptions.LeaseRenewInterval = TimeSpan.FromSeconds(15); using (DocumentClient destClient = new DocumentClient(destCollInfo.Uri, destCollInfo.MasterKey)) { DocumentFeedObserverFactory docObserverFactory = new DocumentFeedObserverFactory(destClient, destCollInfo); ChangeFeedEventHost host = new ChangeFeedEventHost(hostName, documentCollectionLocation, leaseCollectionLocation, feedOptions, feedHostOptions); await host.RegisterObserverFactoryAsync(docObserverFactory); Console.WriteLine("Running... Press enter to stop."); Console.ReadLine(); await host.UnregisterObserversAsync(); } }
public async Task CountDocumentsInCollection_TwoHosts() { await this.InitializeDocumentsAsync(); int partitionKeyRangeCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo); Assert.True(partitionKeyRangeCount > 1, "Prerequisite failed: expected monitored collection with at least 2 partitions."); int processedCount = 0; var allDocsProcessed = new ManualResetEvent(false); var observerFactory = new TestObserverFactory( (ChangeFeedObserverContext context, IReadOnlyList <Document> docs) => { int newCount = Interlocked.Add(ref processedCount, docs.Count); if (newCount == documentCount) { allDocsProcessed.Set(); } return(Task.CompletedTask); }); var host1 = new ChangeFeedEventHost( Guid.NewGuid().ToString(), this.MonitoredCollectionInfo, this.LeaseCollectionInfo, new ChangeFeedOptions { StartFromBeginning = true }, new ChangeFeedHostOptions { MaxPartitionCount = partitionKeyRangeCount / 2 }); await host1.RegisterObserverFactoryAsync(observerFactory); var host2 = new ChangeFeedEventHost( Guid.NewGuid().ToString(), this.MonitoredCollectionInfo, this.LeaseCollectionInfo, new ChangeFeedOptions { StartFromBeginning = true }, new ChangeFeedHostOptions { MaxPartitionCount = partitionKeyRangeCount - partitionKeyRangeCount / 2 }); await host2.RegisterObserverFactoryAsync(observerFactory); allDocsProcessed.WaitOne(changeWaitTimeout + changeWaitTimeout); try { Assert.True(documentCount == processedCount, $"Wrong processedCount {documentCount} {processedCount}"); } finally { await host1.UnregisterObserversAsync(); await host2.UnregisterObserversAsync(); } }
public async Task WhenLeasesHaveContinuationTokenNullReturn0() { // Cleanup the test collection to avoid other tests' documents causing issues with StartFromBeginning await this.ResetTestCollection(); int documentCount = 1; int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.ClassData.monitoredCollectionInfo); int openedCount = 0, processedCount = 0; var allObserversStarted = new ManualResetEvent(false); var allDocsProcessed = new ManualResetEvent(false); var observerFactory = new TestObserverFactory( context => { int newCount = Interlocked.Increment(ref openedCount); if (newCount == partitionCount) { allObserversStarted.Set(); } return(Task.CompletedTask); }, null, (ChangeFeedObserverContext context, IReadOnlyList <Document> docs) => { int newCount = Interlocked.Add(ref processedCount, docs.Count); if (newCount == documentCount) { allDocsProcessed.Set(); } return(Task.CompletedTask); }); var hostName = Guid.NewGuid().ToString(); // We create a host to initialize the leases with ContinuationToken null var host = new ChangeFeedEventHost( hostName, this.ClassData.monitoredCollectionInfo, this.LeaseCollectionInfo, new ChangeFeedOptions { StartFromBeginning = false }, new ChangeFeedHostOptions()); // Initialize leases await host.RegisterObserverFactoryAsync(observerFactory); // Stop host, this leaves the leases with ContinuationToken null state await host.UnregisterObserversAsync(); // Since the leases have ContinuationToken null state, the estimator will use StartFromBeginning and pick-up the changes that happened from the start long estimation = await host.GetEstimatedRemainingWork(); Assert.Equal(0, estimation); }
/// <summary> /// Registers change feed observer to update changes read on change feed to destination /// collection. Deregisters change feed observer and closes process when enter key is pressed /// </summary> /// <returns>A Task to allow asynchronous execution</returns> public async Task RunChangeFeed() { string hostName = Guid.NewGuid().ToString(); // monitored collection info DocumentCollectionInfo starshipCollectionInfo = new DocumentCollectionInfo { Uri = new Uri(ConfigurationManager.AppSettings["SqlApi.EndpointUrl"]), MasterKey = ConfigurationManager.AppSettings["SqlApi.AuthorizationKey"], DatabaseName = ConfigurationManager.AppSettings["SqlApi.DatabaseName"], CollectionName = ConfigurationManager.AppSettings["SqlApi.StarshipCollectionName"] }; // lease collection info DocumentCollectionInfo leaseCollectionInfo = new DocumentCollectionInfo { Uri = new Uri(ConfigurationManager.AppSettings["SqlApi.EndpointUrl"]), MasterKey = ConfigurationManager.AppSettings["SqlApi.AuthorizationKey"], DatabaseName = ConfigurationManager.AppSettings["SqlApi.DatabaseName"], CollectionName = ConfigurationManager.AppSettings["SqlApi.LeaseCollectionName"] }; await CreateCollectionIfNotExistsAsync(starshipCollectionInfo); await CreateCollectionIfNotExistsAsync(leaseCollectionInfo); // Customizable change feed option and host options ChangeFeedOptions feedOptions = new ChangeFeedOptions(); // ie customize StartFromBeginning so change feed reads from beginning // can customize MaxItemCount, PartitonKeyRangeId, RequestContinuation, SessionToken and StartFromBeginning feedOptions.StartFromBeginning = true; ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions(); // ie. customizing lease renewal interval to 15 seconds // can customize LeaseRenewInterval, LeaseAcquireInterval, LeaseExpirationInterval, FeedPollDelay feedHostOptions.LeaseRenewInterval = TimeSpan.FromSeconds(15); using (DocumentClient client = new DocumentClient(starshipCollectionInfo.Uri, starshipCollectionInfo.MasterKey)) { DocumentFeedObserverFactory docObserverFactory = new DocumentFeedObserverFactory(client, starshipCollectionInfo); ChangeFeedEventHost host = new ChangeFeedEventHost(hostName, starshipCollectionInfo, leaseCollectionInfo, feedOptions, feedHostOptions); await host.RegisterObserverFactoryAsync(docObserverFactory).ConfigureAwait(false); Console.WriteLine("Running... Press enter to stop."); Console.ReadLine(); await host.UnregisterObserversAsync(); } }
public async Task CountDocumentsInCollection_ProcessChangesThrows() { await this.InitializeDocumentsAsync(); int processedCount = 0; var allDocsProcessed = new ManualResetEvent(false); bool isFirstChangeNotification = false; // Make sure there was at least one throw. int throwCount = 0; var observerFactory = new TestObserverFactory((ChangeFeedObserverContext context, IReadOnlyList <Document> docs) => { bool shouldThrow = (isFirstChangeNotification || new Random().Next(0, 1) == 1) && throwCount < 10; isFirstChangeNotification = false; if (shouldThrow) { Interlocked.Increment(ref throwCount); throw new Exception("Error injection exception from observer!"); } int newCount = Interlocked.Add(ref processedCount, docs.Count); if (newCount == documentCount) { allDocsProcessed.Set(); } return(Task.CompletedTask); }); var host = new ChangeFeedEventHost( Guid.NewGuid().ToString(), this.MonitoredCollectionInfo, this.LeaseCollectionInfo, new ChangeFeedOptions { StartFromBeginning = true }, new ChangeFeedHostOptions()); await host.RegisterObserverFactoryAsync(observerFactory); allDocsProcessed.WaitOne(changeWaitTimeout + changeWaitTimeout + changeWaitTimeout); try { Assert.Equal(documentCount, processedCount); } finally { await host.UnregisterObserversAsync(); } }
public async Task CountDocumentsInCollection_NormalCase() { await this.InitializeDocumentsAsync(); int partitionKeyRangeCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo); int openedCount = 0, closedCount = 0, processedCount = 0; var allDocsProcessed = new ManualResetEvent(false); var observerFactory = new TestObserverFactory( context => { Interlocked.Increment(ref openedCount); return(Task.CompletedTask); }, (context, reason) => { Interlocked.Increment(ref closedCount); return(Task.CompletedTask); }, (ChangeFeedObserverContext context, IReadOnlyList <Document> docs) => { int newCount = Interlocked.Add(ref processedCount, docs.Count); if (newCount == documentCount) { allDocsProcessed.Set(); } return(Task.CompletedTask); }); var host = new ChangeFeedEventHost( Guid.NewGuid().ToString(), this.MonitoredCollectionInfo, this.LeaseCollectionInfo, new ChangeFeedOptions { StartFromBeginning = true }, new ChangeFeedHostOptions()); await host.RegisterObserverFactoryAsync(observerFactory); allDocsProcessed.WaitOne(changeWaitTimeout + changeWaitTimeout); try { Assert.True(partitionKeyRangeCount == openedCount, "Wrong openedCount"); Assert.True(documentCount == processedCount, $"Wrong processedCount {documentCount} {processedCount}"); } finally { await host.UnregisterObserversAsync(); } Assert.True(partitionKeyRangeCount == closedCount, "Wrong closedCount"); }
private static async Task Run() { ChangeFeedOptions feedOptions = new ChangeFeedOptions { StartFromBeginning = true }; // Can customize LeaseRenewInterval, LeaseAcquireInterval, LeaseExpirationInterval, FeedPollDelay ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions { LeaseRenewInterval = TimeSpan.FromSeconds(15) }; DocumentCollectionInfo documentCollectionLocation = new DocumentCollectionInfo { Uri = new Uri(Connection.EndpointUrl), CollectionName = "FamilyCollection", // ConnectionPolicy = new ConnectionPolicy{}, DatabaseName = "FamilyDB", MasterKey = Connection.PrimaryKey }; ChangeFeedEventHost changeFeedEventHost = new ChangeFeedEventHost("hostName", documentCollectionLocation, documentCollectionLocation, feedOptions, feedHostOptions); ChangeFeedObserverFactory changeFeedObserverFactory = new ChangeFeedObserverFactory(); await changeFeedEventHost.RegisterObserverFactoryAsync(changeFeedObserverFactory); await changeFeedEventHost.UnregisterObserversAsync(); }
public async Task StopAtFullSpeed() { int partitionKeyRangeCount = await Helper.GetPartitionCount(this.ClassData.monitoredCollectionInfo); int openedCount = 0, closedCount = 0, processedCount = 0; var quarterDocsProcessed = new ManualResetEvent(false); var observerFactory = new TestObserverFactory( context => { Interlocked.Increment(ref openedCount); return(Task.CompletedTask); }, (context, reason) => { Interlocked.Increment(ref closedCount); return(Task.CompletedTask); }, (ChangeFeedObserverContext context, IReadOnlyList <Document> docs) => { int newCount = Interlocked.Add(ref processedCount, docs.Count); if (newCount >= documentCount / 4) { quarterDocsProcessed.Set(); } return(Task.CompletedTask); }); var host = new ChangeFeedEventHost( Guid.NewGuid().ToString(), this.ClassData.monitoredCollectionInfo, this.LeaseCollectionInfo, new ChangeFeedOptions { StartFromBeginning = true, MaxItemCount = 2 }, new ChangeFeedHostOptions()); await host.RegisterObserverFactoryAsync(observerFactory); quarterDocsProcessed.WaitOne(Debugger.IsAttached ? changeWaitTimeout + changeWaitTimeout : changeWaitTimeout); await host.UnregisterObserversAsync(); Assert.AreEqual(partitionKeyRangeCount, openedCount, "Wrong closedCount"); Assert.AreEqual(partitionKeyRangeCount, closedCount, "Wrong closedCount"); }
public async Task TestStartTime() { var collectionUri = UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName); using (var client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy)) { await client.CreateDocumentAsync(collectionUri, JsonConvert.DeserializeObject("{\"id\": \"doc1\"}")); // In worst case (long transaction, heavy load, the atomicity of StartTime is 5 sec). // For this case (different transactions) it's OK to wait timestamp precision time. await Task.Delay(TimeSpan.FromSeconds(1)); DateTime timeInBeweeen = DateTime.Now; await Task.Delay(TimeSpan.FromSeconds(1)); await client.CreateDocumentAsync(collectionUri, JsonConvert.DeserializeObject("{\"id\": \"doc2\"}")); int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo); var allDocsProcessed = new ManualResetEvent(false); var processedDocs = new List <Document>(); var observerFactory = new TestObserverFactory( null, null, (context, docs) => { processedDocs.AddRange(docs); foreach (var doc in docs) { if (doc.Id == "doc2") { allDocsProcessed.Set(); } } return(Task.CompletedTask); }); var host = new ChangeFeedEventHost( Guid.NewGuid().ToString(), this.MonitoredCollectionInfo, this.LeaseCollectionInfo, new ChangeFeedOptions { StartTime = timeInBeweeen }, new ChangeFeedHostOptions()); await host.RegisterObserverFactoryAsync(observerFactory); var isStartOk = allDocsProcessed.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout); try { Assert.True(isStartOk, "Timed out waiting for docs to process"); Assert.True(1 == processedDocs.Count, "Wrong processed count"); Assert.True("doc2" == processedDocs[0].Id, "Wrong doc.id"); } finally { await host.UnregisterObserversAsync(); } } }
public async Task CountAddedDocuments() { int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo); int openedCount = 0, processedCount = 0; var allObserversStarted = new ManualResetEvent(false); var allDocsProcessed = new ManualResetEvent(false); var observerFactory = new TestObserverFactory( context => { int newCount = Interlocked.Increment(ref openedCount); if (newCount == partitionCount) { allObserversStarted.Set(); } return(Task.CompletedTask); }, null, (ChangeFeedObserverContext context, IReadOnlyList <Document> docs) => { int newCount = Interlocked.Add(ref processedCount, docs.Count); if (newCount == documentCount) { allDocsProcessed.Set(); } return(Task.CompletedTask); }); var host = new ChangeFeedEventHost( Guid.NewGuid().ToString(), this.MonitoredCollectionInfo, this.LeaseCollectionInfo, new ChangeFeedOptions { StartFromBeginning = false }, new ChangeFeedHostOptions()); await host.RegisterObserverFactoryAsync(observerFactory); var isStartOk = allObserversStarted.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout); Assert.True(isStartOk, "Timed out waiting for observres to start"); using (var client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy)) { await IntegrationTestsHelper.CreateDocumentsAsync( client, UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName), documentCount); } allDocsProcessed.WaitOne(IntegrationTest.changeWaitTimeout); try { Assert.True(documentCount == processedCount, "Wrong processedCount"); } finally { await host.UnregisterObserversAsync(); } }
public async Task TestReducePageSizeScenario() { // Use different colleciton: we need 1-partition collection to make sure all docs get to same partition. var databaseUri = UriFactory.CreateDatabaseUri(this.MonitoredCollectionInfo.DatabaseName); DocumentCollectionInfo monitoredCollectionInfo = new DocumentCollectionInfo(this.MonitoredCollectionInfo); monitoredCollectionInfo.CollectionName = this.MonitoredCollectionInfo.CollectionName + "_" + Guid.NewGuid().ToString(); var collectionUri = UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, monitoredCollectionInfo.CollectionName); var monitoredCollection = new DocumentCollection { Id = monitoredCollectionInfo.CollectionName }; using (var client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy)) { await client.CreateDocumentCollectionAsync(databaseUri, monitoredCollection, new RequestOptions { OfferThroughput = 10000 }); try { // Create some docs to make sure that one separate response is returned for 1st execute of query before retries. // These are to make sure continuation token is passed along during retries. var sproc = new StoredProcedure { Id = "createTwoDocs", Body = @"function(startIndex) { for (var i = 0; i < 2; ++i) __.createDocument( __.getSelfLink(), { id: 'doc' + (i + startIndex).toString(), value: 'y'.repeat(1500000) }, err => { if (err) throw err;} );}" }; var sprocUri = UriFactory.CreateStoredProcedureUri(this.MonitoredCollectionInfo.DatabaseName, monitoredCollection.Id, sproc.Id); await client.CreateStoredProcedureAsync(collectionUri, sproc); await client.ExecuteStoredProcedureAsync <object>(sprocUri, 0); // Create 3 docs each 1.5MB. All 3 do not fit into MAX_RESPONSE_SIZE (4 MB). 2nd and 3rd are in same transaction. var content = string.Format("{{\"id\": \"doc2\", \"value\": \"{0}\"}}", new string('x', 1500000)); await client.CreateDocumentAsync(collectionUri, JsonConvert.DeserializeObject(content)); await client.ExecuteStoredProcedureAsync <object>(sprocUri, 3); var allDocsProcessed = new ManualResetEvent(false); int processedDocCount = 0; string accumulator = string.Empty; var observerFactory = new TestObserverFactory( null, null, (context, docs) => { processedDocCount += docs.Count; foreach (var doc in docs) { accumulator += doc.Id.ToString() + "."; } if (processedDocCount == 5) { allDocsProcessed.Set(); } return(Task.CompletedTask); }); var host = new ChangeFeedEventHost( Guid.NewGuid().ToString(), monitoredCollectionInfo, this.LeaseCollectionInfo, new ChangeFeedOptions { StartFromBeginning = true, MaxItemCount = 6 }, new ChangeFeedHostOptions()); await host.RegisterObserverFactoryAsync(observerFactory); var isStartOk = allDocsProcessed.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout); try { Assert.True(isStartOk, "Timed out waiting for docs to process"); Assert.Equal("doc0.doc1.doc2.doc3.doc4.", accumulator); } finally { await host.UnregisterObserversAsync(); } } finally { await client.DeleteDocumentCollectionAsync(collectionUri); } } }
/// <summary> /// Registers change feed observer to update changes read on change feed to destination /// collection. Deregisters change feed observer and closes process when enter key is pressed /// </summary> /// <returns>A Task to allow asynchronous execution</returns> public async Task Start(CosmosDBQueueConsumerSettings settings, CancellationTokenSource cts = default) { if (this.started) { throw new InvalidOperationException("Already started"); } this.settings = settings; await Utils.CreateCollectionIfNotExists(this.settings.QueueCollectionDefinition); await Utils.CreateCollectionIfNotExists(this.settings.LeaseCollectionDefinition); if (this.OnMessage == null) { throw new InvalidOperationException($"Set a {nameof(OnMessage)} before start consuming queue items"); } this.workerId = this.settings.WorkerId ?? Guid.NewGuid().ToString(); // queue collection info var documentCollectionLocation = new DocumentCollectionInfo { Uri = new Uri(this.settings.QueueCollectionDefinition.Endpoint), MasterKey = this.settings.QueueCollectionDefinition.SecretKey, DatabaseName = this.settings.QueueCollectionDefinition.DbName, CollectionName = this.settings.QueueCollectionDefinition.CollectionName }; // lease collection info DocumentCollectionInfo leaseCollectionLocation = new DocumentCollectionInfo { Uri = new Uri(this.settings.LeaseCollectionDefinition.Endpoint), MasterKey = this.settings.LeaseCollectionDefinition.SecretKey, DatabaseName = this.settings.LeaseCollectionDefinition.DbName, CollectionName = this.settings.LeaseCollectionDefinition.CollectionName, }; this.queueCollectionClient = new DocumentClient(new Uri(this.settings.QueueCollectionDefinition.Endpoint), this.settings.QueueCollectionDefinition.SecretKey); // Customizable change feed option and host options ChangeFeedOptions feedOptions = new ChangeFeedOptions { StartFromBeginning = true }; ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions { LeaseRenewInterval = TimeSpan.FromSeconds(15), LeasePrefix = string.Concat(this.workerId, "_"), }; this.logger?.LogTrace($"Starting worker {this.workerId}"); ChangeFeedEventHost host = new ChangeFeedEventHost(this.workerId, documentCollectionLocation, leaseCollectionLocation, feedOptions, feedHostOptions); await host.RegisterObserverFactoryAsync(this); if (cts != null) { cts.Token.Register(async() => { await host.UnregisterObserversAsync(); }); } this.started = true; }
public async Task CountPendingDocuments() { int documentCount = 1; int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo); int openedCount = 0, processedCount = 0; var allObserversStarted = new ManualResetEvent(false); var allDocsProcessed = new ManualResetEvent(false); var observerFactory = new TestObserverFactory( context => { int newCount = Interlocked.Increment(ref openedCount); if (newCount == partitionCount) { allObserversStarted.Set(); } return(Task.CompletedTask); }, null, (ChangeFeedObserverContext context, IReadOnlyList <Document> docs) => { int newCount = Interlocked.Add(ref processedCount, docs.Count); if (newCount == documentCount) { allDocsProcessed.Set(); } return(Task.CompletedTask); }); var hostName = Guid.NewGuid().ToString(); var host = new ChangeFeedEventHost( hostName, this.MonitoredCollectionInfo, this.LeaseCollectionInfo, new ChangeFeedOptions { StartFromBeginning = false }, new ChangeFeedHostOptions()); // Initialize leases await host.RegisterObserverFactoryAsync(observerFactory); // Verify that 0 is returned on empty collection long estimation = await host.GetEstimatedRemainingWork(); Assert.Equal(0, estimation); using (var client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy)) { await IntegrationTestsHelper.CreateDocumentsAsync( client, UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName), 1); var isStartOk = allObserversStarted.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout); Assert.True(isStartOk, "Timed out waiting for observer to start"); allDocsProcessed.WaitOne(IntegrationTest.changeWaitTimeout); // Halt the processor temporarily await host.UnregisterObserversAsync(); estimation = await host.GetEstimatedRemainingWork(); Assert.Equal(0, estimation); await IntegrationTestsHelper.CreateDocumentsAsync( client, UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName), 1); estimation = await host.GetEstimatedRemainingWork(); Assert.Equal(1, estimation); await IntegrationTestsHelper.CreateDocumentsAsync( client, UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName), 10); estimation = await host.GetEstimatedRemainingWork(); Assert.Equal(11, estimation); // Create a new host to process pending changes var newHost = new ChangeFeedEventHost( hostName, this.MonitoredCollectionInfo, this.LeaseCollectionInfo, new ChangeFeedOptions { StartFromBeginning = false }, new ChangeFeedHostOptions()); openedCount = 0; processedCount = 0; allObserversStarted.Reset(); allDocsProcessed.Reset(); await newHost.RegisterObserverFactoryAsync(observerFactory); isStartOk = allObserversStarted.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout); Assert.True(isStartOk, "Timed out waiting for observer to start"); allDocsProcessed.WaitOne(IntegrationTest.changeWaitTimeout); try { estimation = await newHost.GetEstimatedRemainingWork(); Assert.Equal(0, estimation); } finally { await newHost.UnregisterObserversAsync(); } } }
public Task StopAsync(CancellationToken cancellationToken) { return(host.UnregisterObserversAsync()); }
private static async Task ProcessChangeFeedWithChangeFeedProcessor(DocumentClient client, Database db, DocumentCollection personCol) { // Use the ChangeFeed Processor which manages to read changes accross partitions. // This requires another CosmosDB collection so that ChangeFeedProcessor can store it's lease information! // https://docs.microsoft.com/en-us/azure/cosmos-db/change-feed#change-feed-processor #region Change Feed Processor // Create LeaseCollection DocumentCollection leasecol = await CosDB.CreateOrGetCollection(client, db, "leasecol", 400, null, null, false, true); // Customizable change feed option and host options ChangeFeedOptions feedOptions = new ChangeFeedOptions() { // ie customize StartFromBeginning so change feed reads from beginning // can customize MaxItemCount, PartitonKeyRangeId, RequestContinuation, SessionToken and StartFromBeginning StartFromBeginning = true }; ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions() { // ie. customizing lease renewal interval to 15 seconds // can customize LeaseRenewInterval, LeaseAcquireInterval, LeaseExpirationInterval, FeedPollDelay LeaseRenewInterval = TimeSpan.FromSeconds(15) }; string hostName = "UniqueHostName01"; DocumentCollectionInfo documentCollectionLocation = new DocumentCollectionInfo() { CollectionName = personCol.Id, ConnectionPolicy = ConnectionPolicy.Default, DatabaseName = db.Id, MasterKey = Config.Account_DemoBuild_Mongo_Key, Uri = new Uri(Config.Account_DemoBuild_Mongo) }; DocumentCollectionInfo leaseCollectionLocation = new DocumentCollectionInfo() { CollectionName = "leasecol", ConnectionPolicy = ConnectionPolicy.Default, DatabaseName = db.Id, MasterKey = Config.Account_DemoBuild_Mongo_Key, Uri = new Uri(Config.Account_DemoBuild_Mongo) }; DocumentFeedObserverFactory docObserverFactory = new DocumentFeedObserverFactory(); ChangeFeedEventHost host = new ChangeFeedEventHost(hostName, documentCollectionLocation, leaseCollectionLocation, feedOptions, feedHostOptions); await host.RegisterObserverFactoryAsync(docObserverFactory); Console.WriteLine("Waiting 2-3 seconds before adding more documents..."); Thread.Sleep(2500); Random randGen = new Random((int)DateTime.Now.Ticks); List <string> cities = new List <string>() { "Vienna", "Paris", "Frankfurt", "Prag", "Seattle" }; for (int c = 0; c < 10000; c++) // with this long running sample, the Lease is probably lost { string city = cities[randGen.Next() % cities.Count]; await client.CreateDocumentAsync(personCol.SelfLink, new Person() { name = Guid.NewGuid().ToString("D"), city = city, label = "person" }); await client.CreateDocumentAsync(personCol.SelfLink, new Person() { name = Guid.NewGuid().ToString("D"), city = city, label = "person" }); await client.CreateDocumentAsync(personCol.SelfLink, new Person() { name = Guid.NewGuid().ToString("D"), city = city, label = "person" }); } Console.WriteLine("Observer is still running... Modify or add documents with DocumentStudio. Press enter to stop."); Console.ReadLine(); await host.UnregisterObserversAsync(); #endregion }