public async Task CountDocumentsInCollection_NormalCase()
{
int partitionKeyRangeCount = await Helper.GetPartitionCount(this.ClassData.monitoredCollectionInfo);
int openedCount = 0, closedCount = 0, processedCount = 0;
var allDocsProcessed = new ManualResetEvent(false);
var observerFactory = new TestObserverFactory(
context => { Interlocked.Increment(ref openedCount); return(Task.CompletedTask); },
(context, reason) => { Interlocked.Increment(ref closedCount); return(Task.CompletedTask); },
(ChangeFeedObserverContext context, IReadOnlyList <Document> docs) =>
{
int newCount = Interlocked.Add(ref processedCount, docs.Count);
if (newCount == documentCount)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
});
var host = new ChangeFeedEventHost(
Guid.NewGuid().ToString(),
this.ClassData.monitoredCollectionInfo,
this.LeaseCollectionInfo,
new ChangeFeedOptions {
StartFromBeginning = true
},
new ChangeFeedHostOptions());
await host.RegisterObserverFactoryAsync(observerFactory);
allDocsProcessed.WaitOne(Debugger.IsAttached ? changeWaitTimeout + changeWaitTimeout : changeWaitTimeout);
try
{
Assert.AreEqual(partitionKeyRangeCount, openedCount, "Wrong openedCount");
Assert.AreEqual(documentCount, processedCount, "Wrong processedCount");
}
finally
{
await host.UnregisterObserversAsync();
}
Assert.AreEqual(partitionKeyRangeCount, closedCount, "Wrong closedCount");
}
static async Task Main(string[] args)
{
var config = new ConfigurationBuilder().AddEnvironmentVariables("GraphProjection_").Build();
var docCollection = new DocumentCollectionInfo()
{
CollectionName = config["SourceCollection"],
Uri = new Uri(config["Uri"]),
MasterKey = config["MasterKey"],
DatabaseName = config["SourceDatabase"]
};
var auxCollection = new DocumentCollectionInfo()
{
CollectionName = config["CheckpointCollection"],
Uri = new Uri(config["Uri"]),
MasterKey = config["MasterKey"],
DatabaseName = config["CheckpointDatabase"]
};
var host = new ChangeFeedEventHost(
"GraphProjectionObserver",
docCollection,
auxCollection,
new ChangeFeedOptions {
StartFromBeginning = true
},
new ChangeFeedHostOptions()
);
var typeMap = new ConfigurableSerializationTypeMap()
.RegisterTypes(
typeof(Program).GetTypeInfo().Assembly,
t => t.Namespace.EndsWith("Events"),
t => t.Name);
var client = new DocumentClient(new Uri(config["GraphUri"]), config["GraphMasterKey"]);
var collection = await client.ReadDocumentCollectionAsync(UriFactory.CreateDocumentCollectionUri("Support", "SupportGraph"));
await host.RegisterObserverFactoryAsync(new GraphProjectionObserverFactory(client, collection, typeMap));
Console.WriteLine("Running - press any key to exit");
Console.Read();
}
public async Task WhenNoLeasesExistReturn1()
{
// Cleanup the test collection to avoid other tests' documents causing issues with StartFromBeginning
await this.ResetTestCollection();
var hostName = Guid.NewGuid().ToString();
var host = new ChangeFeedEventHost(
hostName,
this.ClassData.monitoredCollectionInfo,
this.LeaseCollectionInfo,
new ChangeFeedOptions {
StartFromBeginning = false
},
new ChangeFeedHostOptions());
// Verify that 1 is returned on an uninitialized collection
long estimation = await host.GetEstimatedRemainingWork();
Assert.Equal(1, estimation);
}
public async Task StartAsync(CancellationToken cancellationToken)
{
int previousStatus = Interlocked.CompareExchange(ref this._listenerStatus, ListenerRegistering, ListenerNotRegistered);
if (previousStatus == ListenerRegistering)
{
throw new InvalidOperationException("The listener is already starting.");
}
else if (previousStatus == ListenerRegistered)
{
throw new InvalidOperationException("The listener has already started.");
}
this.InitializeHost();
try
{
await RegisterObserverFactoryAsync();
Interlocked.CompareExchange(ref this._listenerStatus, ListenerRegistered, ListenerRegistering);
}
catch (Exception ex)
{
// Reset to NotRegistered
this._listenerStatus = ListenerNotRegistered;
// Throw a custom error if NotFound.
if (ex is DocumentClientException docEx && docEx.StatusCode == HttpStatusCode.NotFound)
{
// Throw a custom error so that it's easier to decipher.
string message = $"Either the source collection '{_monitorCollection.CollectionName}' (in database '{_monitorCollection.DatabaseName}') or the lease collection '{_leaseCollection.CollectionName}' (in database '{_leaseCollection.DatabaseName}') does not exist. Both collections must exist before the listener starts. To automatically create the lease collection, set '{nameof(CosmosDBTriggerAttribute.CreateLeaseCollectionIfNotExists)}' to 'true'.";
this._host = null;
throw new InvalidOperationException(message, ex);
}
throw;
}
}
private static async Task Run()
{
ChangeFeedOptions feedOptions = new ChangeFeedOptions {
StartFromBeginning = true
};
// Can customize LeaseRenewInterval, LeaseAcquireInterval, LeaseExpirationInterval, FeedPollDelay
ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions {
LeaseRenewInterval = TimeSpan.FromSeconds(15)
};
DocumentCollectionInfo documentCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(Connection.EndpointUrl),
CollectionName = "FamilyCollection",
// ConnectionPolicy = new ConnectionPolicy{},
DatabaseName = "FamilyDB",
MasterKey = Connection.PrimaryKey
};
ChangeFeedEventHost changeFeedEventHost = new ChangeFeedEventHost("hostName", documentCollectionLocation, documentCollectionLocation, feedOptions, feedHostOptions);
ChangeFeedObserverFactory changeFeedObserverFactory = new ChangeFeedObserverFactory();
await changeFeedEventHost.RegisterObserverFactoryAsync(changeFeedObserverFactory);
await changeFeedEventHost.UnregisterObserversAsync();
}
public async Task StopAtFullSpeed()
{
int partitionKeyRangeCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo);
int openedCount = 0, closedCount = 0, processedCount = 0;
var quarterDocsProcessed = new ManualResetEvent(false);
var observerFactory = new TestObserverFactory(
context => { Interlocked.Increment(ref openedCount); return(Task.CompletedTask); },
(context, reason) => { Interlocked.Increment(ref closedCount); return(Task.CompletedTask); },
(ChangeFeedObserverContext context, IReadOnlyList <Document> docs) =>
{
int newCount = Interlocked.Add(ref processedCount, docs.Count);
if (newCount >= documentCount / 4)
{
quarterDocsProcessed.Set();
}
return(Task.CompletedTask);
});
var host = new ChangeFeedEventHost(
Guid.NewGuid().ToString(),
this.MonitoredCollectionInfo,
this.LeaseCollectionInfo,
new ChangeFeedOptions {
StartFromBeginning = true, MaxItemCount = 2
},
new ChangeFeedHostOptions());
await host.RegisterObserverFactoryAsync(observerFactory);
quarterDocsProcessed.WaitOne(changeWaitTimeout + changeWaitTimeout);
await host.UnregisterObserversAsync();
Assert.True(partitionKeyRangeCount == openedCount, "Wrong closedCount");
Assert.True(partitionKeyRangeCount == closedCount, "Wrong closedCount");
}
public async Task TestStartTime()
{
var collectionUri = UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName);
using (var client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy))
{
await client.CreateDocumentAsync(collectionUri, JsonConvert.DeserializeObject("{\"id\": \"doc1\"}"));
// In worst case (long transaction, heavy load, the atomicity of StartTime is 5 sec).
// For this case (different transactions) it's OK to wait timestamp precision time.
await Task.Delay(TimeSpan.FromSeconds(1));
DateTime timeInBeweeen = DateTime.Now;
await Task.Delay(TimeSpan.FromSeconds(1));
await client.CreateDocumentAsync(collectionUri, JsonConvert.DeserializeObject("{\"id\": \"doc2\"}"));
int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo);
var allDocsProcessed = new ManualResetEvent(false);
var processedDocs = new List <Document>();
var observerFactory = new TestObserverFactory(
null,
null,
(context, docs) =>
{
processedDocs.AddRange(docs);
foreach (var doc in docs)
{
if (doc.Id == "doc2")
{
allDocsProcessed.Set();
}
}
return(Task.CompletedTask);
});
var host = new ChangeFeedEventHost(
Guid.NewGuid().ToString(),
this.MonitoredCollectionInfo,
this.LeaseCollectionInfo,
new ChangeFeedOptions {
StartTime = timeInBeweeen
},
new ChangeFeedHostOptions());
await host.RegisterObserverFactoryAsync(observerFactory);
var isStartOk = allDocsProcessed.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout);
try
{
Assert.True(isStartOk, "Timed out waiting for docs to process");
Assert.True(1 == processedDocs.Count, "Wrong processed count");
Assert.True("doc2" == processedDocs[0].Id, "Wrong doc.id");
}
finally
{
await host.UnregisterObserversAsync();
}
}
}
public async Task CountAddedDocuments()
{
int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo);
int openedCount = 0, processedCount = 0;
var allObserversStarted = new ManualResetEvent(false);
var allDocsProcessed = new ManualResetEvent(false);
var observerFactory = new TestObserverFactory(
context =>
{
int newCount = Interlocked.Increment(ref openedCount);
if (newCount == partitionCount)
{
allObserversStarted.Set();
}
return(Task.CompletedTask);
},
null,
(ChangeFeedObserverContext context, IReadOnlyList <Document> docs) =>
{
int newCount = Interlocked.Add(ref processedCount, docs.Count);
if (newCount == documentCount)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
});
var host = new ChangeFeedEventHost(
Guid.NewGuid().ToString(),
this.MonitoredCollectionInfo,
this.LeaseCollectionInfo,
new ChangeFeedOptions {
StartFromBeginning = false
},
new ChangeFeedHostOptions());
await host.RegisterObserverFactoryAsync(observerFactory);
var isStartOk = allObserversStarted.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout);
Assert.True(isStartOk, "Timed out waiting for observres to start");
using (var client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy))
{
await IntegrationTestsHelper.CreateDocumentsAsync(
client,
UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName),
documentCount);
}
allDocsProcessed.WaitOne(IntegrationTest.changeWaitTimeout);
try
{
Assert.True(documentCount == processedCount, "Wrong processedCount");
}
finally
{
await host.UnregisterObserversAsync();
}
}
public async Task TestReducePageSizeScenario()
{
// Use different colleciton: we need 1-partition collection to make sure all docs get to same partition.
var databaseUri = UriFactory.CreateDatabaseUri(this.MonitoredCollectionInfo.DatabaseName);
DocumentCollectionInfo monitoredCollectionInfo = new DocumentCollectionInfo(this.MonitoredCollectionInfo);
monitoredCollectionInfo.CollectionName = this.MonitoredCollectionInfo.CollectionName + "_" + Guid.NewGuid().ToString();
var collectionUri = UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, monitoredCollectionInfo.CollectionName);
var monitoredCollection = new DocumentCollection {
Id = monitoredCollectionInfo.CollectionName
};
using (var client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy))
{
await client.CreateDocumentCollectionAsync(databaseUri, monitoredCollection, new RequestOptions { OfferThroughput = 10000 });
try
{
// Create some docs to make sure that one separate response is returned for 1st execute of query before retries.
// These are to make sure continuation token is passed along during retries.
var sproc = new StoredProcedure
{
Id = "createTwoDocs",
Body = @"function(startIndex) { for (var i = 0; i < 2; ++i) __.createDocument(
__.getSelfLink(),
{ id: 'doc' + (i + startIndex).toString(), value: 'y'.repeat(1500000) },
err => { if (err) throw err;}
);}"
};
var sprocUri = UriFactory.CreateStoredProcedureUri(this.MonitoredCollectionInfo.DatabaseName, monitoredCollection.Id, sproc.Id);
await client.CreateStoredProcedureAsync(collectionUri, sproc);
await client.ExecuteStoredProcedureAsync <object>(sprocUri, 0);
// Create 3 docs each 1.5MB. All 3 do not fit into MAX_RESPONSE_SIZE (4 MB). 2nd and 3rd are in same transaction.
var content = string.Format("{{\"id\": \"doc2\", \"value\": \"{0}\"}}", new string('x', 1500000));
await client.CreateDocumentAsync(collectionUri, JsonConvert.DeserializeObject(content));
await client.ExecuteStoredProcedureAsync <object>(sprocUri, 3);
var allDocsProcessed = new ManualResetEvent(false);
int processedDocCount = 0;
string accumulator = string.Empty;
var observerFactory = new TestObserverFactory(
null,
null,
(context, docs) =>
{
processedDocCount += docs.Count;
foreach (var doc in docs)
{
accumulator += doc.Id.ToString() + ".";
}
if (processedDocCount == 5)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
});
var host = new ChangeFeedEventHost(
Guid.NewGuid().ToString(),
monitoredCollectionInfo,
this.LeaseCollectionInfo,
new ChangeFeedOptions {
StartFromBeginning = true, MaxItemCount = 6
},
new ChangeFeedHostOptions());
await host.RegisterObserverFactoryAsync(observerFactory);
var isStartOk = allDocsProcessed.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout);
try
{
Assert.True(isStartOk, "Timed out waiting for docs to process");
Assert.Equal("doc0.doc1.doc2.doc3.doc4.", accumulator);
}
finally
{
await host.UnregisterObserversAsync();
}
}
finally
{
await client.DeleteDocumentCollectionAsync(collectionUri);
}
}
}
/// <summary>
/// Registers change feed observer to update changes read on change feed to destination
/// collection. Deregisters change feed observer and closes process when enter key is pressed
/// </summary>
/// <returns>A Task to allow asynchronous execution</returns>
public async Task Start(CosmosDBQueueConsumerSettings settings, CancellationTokenSource cts = default)
{
if (this.started)
{
throw new InvalidOperationException("Already started");
}
this.settings = settings;
await Utils.CreateCollectionIfNotExists(this.settings.QueueCollectionDefinition);
await Utils.CreateCollectionIfNotExists(this.settings.LeaseCollectionDefinition);
if (this.OnMessage == null)
{
throw new InvalidOperationException($"Set a {nameof(OnMessage)} before start consuming queue items");
}
this.workerId = this.settings.WorkerId ?? Guid.NewGuid().ToString();
// queue collection info
var documentCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(this.settings.QueueCollectionDefinition.Endpoint),
MasterKey = this.settings.QueueCollectionDefinition.SecretKey,
DatabaseName = this.settings.QueueCollectionDefinition.DbName,
CollectionName = this.settings.QueueCollectionDefinition.CollectionName
};
// lease collection info
DocumentCollectionInfo leaseCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(this.settings.LeaseCollectionDefinition.Endpoint),
MasterKey = this.settings.LeaseCollectionDefinition.SecretKey,
DatabaseName = this.settings.LeaseCollectionDefinition.DbName,
CollectionName = this.settings.LeaseCollectionDefinition.CollectionName,
};
this.queueCollectionClient = new DocumentClient(new Uri(this.settings.QueueCollectionDefinition.Endpoint), this.settings.QueueCollectionDefinition.SecretKey);
// Customizable change feed option and host options
ChangeFeedOptions feedOptions = new ChangeFeedOptions
{
StartFromBeginning = true
};
ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions
{
LeaseRenewInterval = TimeSpan.FromSeconds(15),
LeasePrefix = string.Concat(this.workerId, "_"),
};
this.logger?.LogTrace($"Starting worker {this.workerId}");
ChangeFeedEventHost host = new ChangeFeedEventHost(this.workerId, documentCollectionLocation, leaseCollectionLocation, feedOptions, feedHostOptions);
await host.RegisterObserverFactoryAsync(this);
if (cts != null)
{
cts.Token.Register(async() =>
{
await host.UnregisterObserversAsync();
});
}
this.started = true;
}
public async Task CountPendingDocuments()
{
int documentCount = 1;
int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo);
int openedCount = 0, processedCount = 0;
var allObserversStarted = new ManualResetEvent(false);
var allDocsProcessed = new ManualResetEvent(false);
var observerFactory = new TestObserverFactory(
context =>
{
int newCount = Interlocked.Increment(ref openedCount);
if (newCount == partitionCount)
{
allObserversStarted.Set();
}
return(Task.CompletedTask);
},
null,
(ChangeFeedObserverContext context, IReadOnlyList <Document> docs) =>
{
int newCount = Interlocked.Add(ref processedCount, docs.Count);
if (newCount == documentCount)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
});
var hostName = Guid.NewGuid().ToString();
var host = new ChangeFeedEventHost(
hostName,
this.MonitoredCollectionInfo,
this.LeaseCollectionInfo,
new ChangeFeedOptions {
StartFromBeginning = false
},
new ChangeFeedHostOptions());
// Initialize leases
await host.RegisterObserverFactoryAsync(observerFactory);
// Verify that 0 is returned on empty collection
long estimation = await host.GetEstimatedRemainingWork();
Assert.Equal(0, estimation);
using (var client = new DocumentClient(this.MonitoredCollectionInfo.Uri, this.MonitoredCollectionInfo.MasterKey, this.MonitoredCollectionInfo.ConnectionPolicy))
{
await IntegrationTestsHelper.CreateDocumentsAsync(
client,
UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName),
1);
var isStartOk = allObserversStarted.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout);
Assert.True(isStartOk, "Timed out waiting for observer to start");
allDocsProcessed.WaitOne(IntegrationTest.changeWaitTimeout);
// Halt the processor temporarily
await host.UnregisterObserversAsync();
estimation = await host.GetEstimatedRemainingWork();
Assert.Equal(0, estimation);
await IntegrationTestsHelper.CreateDocumentsAsync(
client,
UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName),
1);
estimation = await host.GetEstimatedRemainingWork();
Assert.Equal(1, estimation);
await IntegrationTestsHelper.CreateDocumentsAsync(
client,
UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName),
10);
estimation = await host.GetEstimatedRemainingWork();
Assert.Equal(11, estimation);
// Create a new host to process pending changes
var newHost = new ChangeFeedEventHost(
hostName,
this.MonitoredCollectionInfo,
this.LeaseCollectionInfo,
new ChangeFeedOptions {
StartFromBeginning = false
},
new ChangeFeedHostOptions());
openedCount = 0;
processedCount = 0;
allObserversStarted.Reset();
allDocsProcessed.Reset();
await newHost.RegisterObserverFactoryAsync(observerFactory);
isStartOk = allObserversStarted.WaitOne(IntegrationTest.changeWaitTimeout + IntegrationTest.changeWaitTimeout);
Assert.True(isStartOk, "Timed out waiting for observer to start");
allDocsProcessed.WaitOne(IntegrationTest.changeWaitTimeout);
try
{
estimation = await newHost.GetEstimatedRemainingWork();
Assert.Equal(0, estimation);
}
finally
{
await newHost.UnregisterObserversAsync();
}
}
}
public async Task WhenLeasesHaveContinuationTokenNullStartFromBeginning()
{
int documentCount = 1;
int partitionCount = await IntegrationTestsHelper.GetPartitionCount(this.MonitoredCollectionInfo);
int openedCount = 0, processedCount = 0;
var allObserversStarted = new ManualResetEvent(false);
var allDocsProcessed = new ManualResetEvent(false);
var observerFactory = new TestObserverFactory(
context =>
{
int newCount = Interlocked.Increment(ref openedCount);
if (newCount == partitionCount)
{
allObserversStarted.Set();
}
return(Task.CompletedTask);
},
null,
(ChangeFeedObserverContext context, IReadOnlyList <Document> docs) =>
{
int newCount = Interlocked.Add(ref processedCount, docs.Count);
if (newCount == documentCount)
{
allDocsProcessed.Set();
}
return(Task.CompletedTask);
});
var hostName = Guid.NewGuid().ToString();
// We create a host to initialize the leases with ContinuationToken null
var host = new ChangeFeedEventHost(
hostName,
this.MonitoredCollectionInfo,
this.LeaseCollectionInfo,
new ChangeFeedOptions {
StartFromBeginning = false
},
new ChangeFeedHostOptions());
// Initialize leases
await host.RegisterObserverFactoryAsync(observerFactory);
// Stop host, this leaves the leases with ContinuationToken null state
await host.UnregisterObserversAsync();
using (var client = new DocumentClient(
this.MonitoredCollectionInfo.Uri,
this.MonitoredCollectionInfo.MasterKey,
this.MonitoredCollectionInfo.ConnectionPolicy))
{
// Insert documents
await IntegrationTestsHelper.CreateDocumentsAsync(
client,
UriFactory.CreateDocumentCollectionUri(this.MonitoredCollectionInfo.DatabaseName, this.MonitoredCollectionInfo.CollectionName),
10);
// Since the leases have ContinuationToken null state, the estimator will use StartFromBeginning and pick-up the changes that happened from the start
long estimation = await host.GetEstimatedRemainingWork();
Assert.Equal(10, estimation);
}
}
/// <summary>
/// Registers change feed observer to update changes read on change feed to destination
/// collection. Deregisters change feed observer and closes process when enter key is pressed
/// </summary>
/// <returns>A Task to allow asynchronous execution</returns>
public async Task RunChangeFeedHostAsync()
{
// newRecord(8888);
string hostName = Guid.NewGuid().ToString();
// monitored collection info
DocumentCollectionInfo documentCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(this.monitoredUri),
MasterKey = this.monitoredSecretKey,
DatabaseName = this.monitoredDbName,
CollectionName = this.monitoredCollectionName
};
// lease collection info
DocumentCollectionInfo leaseCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(this.leaseUri),
MasterKey = this.leaseSecretKey,
DatabaseName = this.leaseDbName,
CollectionName = this.leaseCollectionName
};
// destination collection info
DocumentCollectionInfo destCollInfo = new DocumentCollectionInfo
{
Uri = new Uri(this.destUri),
MasterKey = this.destSecretKey,
DatabaseName = this.destDbName,
CollectionName = this.destCollectionName
};
// Customizable change feed option and host options
ChangeFeedOptions feedOptions = new ChangeFeedOptions();
// ie customize StartFromBeginning so change feed reads from beginning
// can customize MaxItemCount, PartitonKeyRangeId, RequestContinuation, SessionToken and StartFromBeginning
feedOptions.StartFromBeginning = true;
ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions();
// ie. customizing lease renewal interval to 15 seconds
// can customize LeaseRenewInterval, LeaseAcquireInterval, LeaseExpirationInterval, FeedPollDelay
feedHostOptions.LeaseRenewInterval = TimeSpan.FromSeconds(15);
feedHostOptions.LeaseRenewInterval = TimeSpan.FromMilliseconds(100);
feedHostOptions.LeaseAcquireInterval = TimeSpan.FromMilliseconds(100);
feedHostOptions.FeedPollDelay = TimeSpan.FromMilliseconds(100);
using (DocumentClient destClient = new DocumentClient(destCollInfo.Uri, destCollInfo.MasterKey))
{
DocumentFeedObserverFactory docObserverFactory = new DocumentFeedObserverFactory(destClient, destCollInfo);
ChangeFeedEventHost host = new ChangeFeedEventHost(hostName, documentCollectionLocation, leaseCollectionLocation, feedOptions, feedHostOptions);
await host.RegisterObserverFactoryAsync(docObserverFactory);
Console.WriteLine("Running... Press enter to stop.");
Console.ReadLine();
// do not unreg for now
// await host.UnregisterObserversAsync();
}
}
private static async Task ProcessChangeFeedWithChangeFeedProcessor(DocumentClient client, Database db, DocumentCollection personCol)
{
// Use the ChangeFeed Processor which manages to read changes accross partitions.
// This requires another CosmosDB collection so that ChangeFeedProcessor can store it's lease information!
// https://docs.microsoft.com/en-us/azure/cosmos-db/change-feed#change-feed-processor
#region Change Feed Processor
// Create LeaseCollection
DocumentCollection leasecol = await CosDB.CreateOrGetCollection(client, db, "leasecol", 400, null, null, false, true);
// Customizable change feed option and host options
ChangeFeedOptions feedOptions = new ChangeFeedOptions()
{
// ie customize StartFromBeginning so change feed reads from beginning
// can customize MaxItemCount, PartitonKeyRangeId, RequestContinuation, SessionToken and StartFromBeginning
StartFromBeginning = true
};
ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions()
{
// ie. customizing lease renewal interval to 15 seconds
// can customize LeaseRenewInterval, LeaseAcquireInterval, LeaseExpirationInterval, FeedPollDelay
LeaseRenewInterval = TimeSpan.FromSeconds(15)
};
string hostName = "UniqueHostName01";
DocumentCollectionInfo documentCollectionLocation = new DocumentCollectionInfo()
{
CollectionName = personCol.Id,
ConnectionPolicy = ConnectionPolicy.Default,
DatabaseName = db.Id,
MasterKey = Config.Account_DemoBuild_Mongo_Key,
Uri = new Uri(Config.Account_DemoBuild_Mongo)
};
DocumentCollectionInfo leaseCollectionLocation = new DocumentCollectionInfo()
{
CollectionName = "leasecol",
ConnectionPolicy = ConnectionPolicy.Default,
DatabaseName = db.Id,
MasterKey = Config.Account_DemoBuild_Mongo_Key,
Uri = new Uri(Config.Account_DemoBuild_Mongo)
};
DocumentFeedObserverFactory docObserverFactory = new DocumentFeedObserverFactory();
ChangeFeedEventHost host = new ChangeFeedEventHost(hostName,
documentCollectionLocation,
leaseCollectionLocation,
feedOptions, feedHostOptions);
await host.RegisterObserverFactoryAsync(docObserverFactory);
Console.WriteLine("Waiting 2-3 seconds before adding more documents...");
Thread.Sleep(2500);
Random randGen = new Random((int)DateTime.Now.Ticks);
List <string> cities = new List <string>()
{
"Vienna", "Paris", "Frankfurt", "Prag", "Seattle"
};
for (int c = 0; c < 10000; c++) // with this long running sample, the Lease is probably lost
{
string city = cities[randGen.Next() % cities.Count];
await client.CreateDocumentAsync(personCol.SelfLink, new Person()
{
name = Guid.NewGuid().ToString("D"), city = city, label = "person"
});
await client.CreateDocumentAsync(personCol.SelfLink, new Person()
{
name = Guid.NewGuid().ToString("D"), city = city, label = "person"
});
await client.CreateDocumentAsync(personCol.SelfLink, new Person()
{
name = Guid.NewGuid().ToString("D"), city = city, label = "person"
});
}
Console.WriteLine("Observer is still running... Modify or add documents with DocumentStudio. Press enter to stop.");
Console.ReadLine();
await host.UnregisterObserversAsync();
#endregion
}