private ChangeFeedProcessorOptions BuildProcessorOptions(CosmosDBTriggerAttribute attribute)
{
ChangeFeedHostOptions leasesOptions = _options.LeaseOptions;
ChangeFeedProcessorOptions processorOptions = new ChangeFeedProcessorOptions
{
LeasePrefix = ResolveAttributeValue(attribute.LeaseCollectionPrefix) ?? leasesOptions.LeasePrefix,
FeedPollDelay = ResolveTimeSpanFromMilliseconds(nameof(CosmosDBTriggerAttribute.FeedPollDelay), leasesOptions.FeedPollDelay, attribute.FeedPollDelay),
LeaseAcquireInterval = ResolveTimeSpanFromMilliseconds(nameof(CosmosDBTriggerAttribute.LeaseAcquireInterval), leasesOptions.LeaseAcquireInterval, attribute.LeaseAcquireInterval),
LeaseExpirationInterval = ResolveTimeSpanFromMilliseconds(nameof(CosmosDBTriggerAttribute.LeaseExpirationInterval), leasesOptions.LeaseExpirationInterval, attribute.LeaseExpirationInterval),
LeaseRenewInterval = ResolveTimeSpanFromMilliseconds(nameof(CosmosDBTriggerAttribute.LeaseRenewInterval), leasesOptions.LeaseRenewInterval, attribute.LeaseRenewInterval),
CheckpointFrequency = leasesOptions.CheckpointFrequency ?? new CheckpointFrequency()
};
if (attribute.CheckpointInterval > 0)
{
processorOptions.CheckpointFrequency.TimeInterval = TimeSpan.FromMilliseconds(attribute.CheckpointInterval);
}
if (attribute.CheckpointDocumentCount > 0)
{
processorOptions.CheckpointFrequency.ProcessedDocumentCount = attribute.CheckpointDocumentCount;
}
return(processorOptions);
}
public static async Task StartChangeFeedHost()
{
var config = ServiceLocator.GetService <IConfiguration>();
string hostName = config.MonitoredCollectionName + "_monitor_" + Guid.NewGuid().ToString();
DocumentCollectionInfo documentCollectionLocation = new DocumentCollectionInfo
{
Uri = config.DatabaseAccountUri,
MasterKey = config.DatabaseAccountKey,
DatabaseName = config.MonitoredDatabaseName,
CollectionName = config.MonitoredCollectionName
};
DocumentCollectionInfo leaseCollectionLocation = new DocumentCollectionInfo
{
Uri = config.DatabaseAccountUri,
MasterKey = config.DatabaseAccountKey,
DatabaseName = config.ChangeTrackingDatabaseName,
CollectionName = config.ChangeTrackingLeaseCollectionName
};
Console.WriteLine("Main program: Creating ChangeFeedEventHost...");
ChangeFeedOptions feedOptions = new ChangeFeedOptions();
ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions();
ChangeFeedEventHost host = new ChangeFeedEventHost(hostName, documentCollectionLocation, leaseCollectionLocation, feedOptions, feedHostOptions);
await host.RegisterObserverAsync <DocumentFeedObserver>();
Console.WriteLine("Main program: press Enter to stop...");
Console.ReadLine();
await host.UnregisterObserversAsync();
}
public async Task StartAsync_Retries()
{
var mockExecutor = new Mock <ITriggeredFunctionExecutor>();
var collInfo = new DocumentCollectionInfo {
Uri = new Uri("http://fakeaccount"), MasterKey = "c29tZV9rZXk=", DatabaseName = "FakeDb", CollectionName = "FakeColl"
};
var leaseInfo = new DocumentCollectionInfo {
Uri = new Uri("http://fakeaccount"), MasterKey = "c29tZV9rZXk=", DatabaseName = "FakeDb", CollectionName = "leases"
};
var hostOptions = new ChangeFeedHostOptions();
var feedOptions = new ChangeFeedOptions();
var listener = new MockListener(mockExecutor.Object, collInfo, leaseInfo, hostOptions, feedOptions, NullLogger.Instance);
// Ensure that we can call StartAsync() multiple times to retry if there is an error.
for (int i = 0; i < 3; i++)
{
var ex = await Assert.ThrowsAnyAsync <InvalidOperationException>(() => listener.StartAsync(CancellationToken.None));
Assert.Equal("Failed to register!", ex.Message);
}
// This should succeed
await listener.StartAsync(CancellationToken.None);
}
public CosmosDBTriggerBinding(ParameterInfo parameter, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, ChangeFeedHostOptions leaseHostOptions)
{
_documentCollectionLocation = documentCollectionLocation;
_leaseCollectionLocation = leaseCollectionLocation;
_leaseHostOptions = leaseHostOptions;
_parameter = parameter;
}
/// <summary>
/// Registers change feed observer to update changes read on change feed to destination
/// collection. Deregisters change feed observer and closes process when enter key is pressed
/// </summary>
/// <returns>A Task to allow asynchronous execution</returns>
public async Task RunChangeFeedHostAsync()
{
string hostName = Guid.NewGuid().ToString();
// monitored collection info
DocumentCollectionInfo documentCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(this.monitoredUri),
MasterKey = this.monitoredSecretKey,
DatabaseName = this.monitoredDbName,
CollectionName = this.monitoredCollectionName
};
// lease collection info
DocumentCollectionInfo leaseCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(this.leaseUri),
MasterKey = this.leaseSecretKey,
DatabaseName = this.leaseDbName,
CollectionName = this.leaseCollectionName
};
// destination collection info
DocumentCollectionInfo destCollInfo = new DocumentCollectionInfo
{
Uri = new Uri(this.destUri),
MasterKey = this.destSecretKey,
DatabaseName = this.destDbName,
CollectionName = this.destCollectionName
};
// Customizable change feed option and host options
ChangeFeedOptions feedOptions = new ChangeFeedOptions();
// ie customize StartFromBeginning so change feed reads from beginning
// can customize MaxItemCount, PartitonKeyRangeId, RequestContinuation, SessionToken and StartFromBeginning
feedOptions.StartFromBeginning = true;
ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions();
// ie. customizing lease renewal interval to 15 seconds
// can customize LeaseRenewInterval, LeaseAcquireInterval, LeaseExpirationInterval, FeedPollDelay
feedHostOptions.LeaseRenewInterval = TimeSpan.FromSeconds(15);
using (DocumentClient destClient = new DocumentClient(destCollInfo.Uri, destCollInfo.MasterKey))
{
DocumentFeedObserverFactory docObserverFactory = new DocumentFeedObserverFactory(destClient, destCollInfo);
ChangeFeedEventHost host = new ChangeFeedEventHost(hostName, documentCollectionLocation, leaseCollectionLocation, feedOptions, feedHostOptions);
await host.RegisterObserverFactoryAsync(docObserverFactory);
Console.WriteLine("Running... Press enter to stop.");
Console.ReadLine();
await host.UnregisterObserversAsync();
}
}
public CosmosDBTriggerBinding(ParameterInfo parameter, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, ChangeFeedHostOptions leaseHostOptions)
{
_bindingDataProvider = BindingDataProvider.FromType(parameter.ParameterType);
_documentCollectionLocation = documentCollectionLocation;
_leaseCollectionLocation = leaseCollectionLocation;
_leaseHostOptions = leaseHostOptions;
_parameter = parameter;
}
public CosmosDBTriggerBinding(ParameterInfo parameter, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, ChangeFeedHostOptions leaseHostOptions, int?maxItemsPerCall)
{
_documentCollectionLocation = documentCollectionLocation;
_leaseCollectionLocation = leaseCollectionLocation;
_leaseHostOptions = leaseHostOptions;
_parameter = parameter;
_maxItemsPerCall = maxItemsPerCall;
}
public CosmosDBTriggerListener(ITriggeredFunctionExecutor executor, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, ChangeFeedHostOptions leaseHostOptions, ChangeFeedOptions changeFeedOptions, ILogger logger)
{
this._logger = logger;
this._executor = executor;
this._hostName = Guid.NewGuid().ToString();
this._monitorCollection = documentCollectionLocation;
this._leaseCollection = leaseCollectionLocation;
this._leaseHostOptions = leaseHostOptions;
this._changeFeedOptions = changeFeedOptions;
}
/// <summary>
/// Registers change feed observer to update changes read on change feed to destination
/// collection. Deregisters change feed observer and closes process when enter key is pressed
/// </summary>
/// <returns>A Task to allow asynchronous execution</returns>
public async Task RunChangeFeed()
{
string hostName = Guid.NewGuid().ToString();
// monitored collection info
DocumentCollectionInfo starshipCollectionInfo = new DocumentCollectionInfo
{
Uri = new Uri(ConfigurationManager.AppSettings["SqlApi.EndpointUrl"]),
MasterKey = ConfigurationManager.AppSettings["SqlApi.AuthorizationKey"],
DatabaseName = ConfigurationManager.AppSettings["SqlApi.DatabaseName"],
CollectionName = ConfigurationManager.AppSettings["SqlApi.StarshipCollectionName"]
};
// lease collection info
DocumentCollectionInfo leaseCollectionInfo = new DocumentCollectionInfo
{
Uri = new Uri(ConfigurationManager.AppSettings["SqlApi.EndpointUrl"]),
MasterKey = ConfigurationManager.AppSettings["SqlApi.AuthorizationKey"],
DatabaseName = ConfigurationManager.AppSettings["SqlApi.DatabaseName"],
CollectionName = ConfigurationManager.AppSettings["SqlApi.LeaseCollectionName"]
};
await CreateCollectionIfNotExistsAsync(starshipCollectionInfo);
await CreateCollectionIfNotExistsAsync(leaseCollectionInfo);
// Customizable change feed option and host options
ChangeFeedOptions feedOptions = new ChangeFeedOptions();
// ie customize StartFromBeginning so change feed reads from beginning
// can customize MaxItemCount, PartitonKeyRangeId, RequestContinuation, SessionToken and StartFromBeginning
feedOptions.StartFromBeginning = true;
ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions();
// ie. customizing lease renewal interval to 15 seconds
// can customize LeaseRenewInterval, LeaseAcquireInterval, LeaseExpirationInterval, FeedPollDelay
feedHostOptions.LeaseRenewInterval = TimeSpan.FromSeconds(15);
using (DocumentClient client = new DocumentClient(starshipCollectionInfo.Uri, starshipCollectionInfo.MasterKey))
{
DocumentFeedObserverFactory docObserverFactory = new DocumentFeedObserverFactory(client, starshipCollectionInfo);
ChangeFeedEventHost host = new ChangeFeedEventHost(hostName, starshipCollectionInfo, leaseCollectionInfo, feedOptions, feedHostOptions);
await host.RegisterObserverFactoryAsync(docObserverFactory).ConfigureAwait(false);
Console.WriteLine("Running... Press enter to stop.");
Console.ReadLine();
await host.UnregisterObserversAsync();
}
}
public void ValidateLegacyOptionsAreUsed()
{
DateTime startTime = DateTime.Now;
var feedOptions = new ChangeFeedOptions
{
MaxItemCount = 1,
StartFromBeginning = true,
StartTime = startTime,
RequestContinuation = "RequestContinuation",
SessionToken = "SessionToken",
};
var checkpointFrequency = new CheckpointFrequency {
ExplicitCheckpoint = true
};
var hostOptions = new ChangeFeedHostOptions
{
LeaseRenewInterval = TimeSpan.FromSeconds(2),
LeaseAcquireInterval = TimeSpan.FromSeconds(3),
LeaseExpirationInterval = TimeSpan.FromSeconds(4),
FeedPollDelay = TimeSpan.FromSeconds(5),
CheckpointFrequency = checkpointFrequency,
LeasePrefix = "LeasePrefix",
MinPartitionCount = 6,
MaxPartitionCount = 7,
QueryPartitionsMaxBatchSize = 8,
};
var processorOptions = ChangeFeedEventHost.CreateProcessorOptions(feedOptions, hostOptions);
Assert.Equal(1, processorOptions.MaxItemCount);
Assert.True(processorOptions.StartFromBeginning);
Assert.Equal(startTime, processorOptions.StartTime);
Assert.Equal("RequestContinuation", processorOptions.RequestContinuation);
Assert.Equal("SessionToken", processorOptions.SessionToken);
Assert.Equal(TimeSpan.FromSeconds(2), processorOptions.LeaseRenewInterval);
Assert.Equal(TimeSpan.FromSeconds(3), processorOptions.LeaseAcquireInterval);
Assert.Equal(TimeSpan.FromSeconds(4), processorOptions.LeaseExpirationInterval);
Assert.Equal(TimeSpan.FromSeconds(5), processorOptions.FeedPollDelay);
Assert.Equal(checkpointFrequency, processorOptions.CheckpointFrequency);
Assert.Equal("LeasePrefix", processorOptions.LeasePrefix);
Assert.Equal(6, processorOptions.MinPartitionCount);
Assert.Equal(7, processorOptions.MaxPartitionCount);
Assert.Equal(8, processorOptions.QueryPartitionsMaxBatchSize);
}
private async Task StartChangeFeedHost()
{
ConnectionPolicy ConnectionPolicy = new ConnectionPolicy
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp,
RequestTimeout = new TimeSpan(1, 0, 0),
MaxConnectionLimit = 1000,
RetryOptions = new RetryOptions
{
MaxRetryAttemptsOnThrottledRequests = 10,
MaxRetryWaitTimeInSeconds = 60
}
};
//Customizable change feed option and host options
ChangeFeedOptions feedOptions = new ChangeFeedOptions();
// ie customize StartFromBeginning so change feed reads from beginning
// can customize MaxItemCount, PartitonKeyRangeId, RequestContinuation, SessionToken and StartFromBeginning
feedOptions.StartTime = this.Config.ChangeFeedStartTime;
var feedHostOptions = new ChangeFeedHostOptions
{
// ie. customizing lease renewal interval to 15 seconds
// can customize LeaseRenewInterval, LeaseAcquireInterval, LeaseExpirationInterval, FeedPollDelay
LeaseRenewInterval = TimeSpan.FromSeconds(15)
};
string hostName = Guid.NewGuid().ToString();
DocumentFeedObserverFactory docObserverFactory = new DocumentFeedObserverFactory(this.Config);
this.ChangeFeedEventHost = new ChangeFeedEventHost(hostName, Config.SourceDocumentCollectionInfo, this.Config.LeaseDocumentCollectionInfo, feedOptions, feedHostOptions);
await this.ChangeFeedEventHost.RegisterObserverFactoryAsync(docObserverFactory);
Console.WriteLine("Running... Press enter to stop.");
}
private static async Task Run()
{
ChangeFeedOptions feedOptions = new ChangeFeedOptions {
StartFromBeginning = true
};
// Can customize LeaseRenewInterval, LeaseAcquireInterval, LeaseExpirationInterval, FeedPollDelay
ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions {
LeaseRenewInterval = TimeSpan.FromSeconds(15)
};
DocumentCollectionInfo documentCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(Connection.EndpointUrl),
CollectionName = "FamilyCollection",
// ConnectionPolicy = new ConnectionPolicy{},
DatabaseName = "FamilyDB",
MasterKey = Connection.PrimaryKey
};
ChangeFeedEventHost changeFeedEventHost = new ChangeFeedEventHost("hostName", documentCollectionLocation, documentCollectionLocation, feedOptions, feedHostOptions);
ChangeFeedObserverFactory changeFeedObserverFactory = new ChangeFeedObserverFactory();
await changeFeedEventHost.RegisterObserverFactoryAsync(changeFeedObserverFactory);
await changeFeedEventHost.UnregisterObserversAsync();
}
public Task <ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context)
{
// Tries to parse the context parameters and see if it belongs to this [CosmosDBTrigger] binder
if (context == null)
{
throw new ArgumentNullException("context");
}
ParameterInfo parameter = context.Parameter;
CosmosDBTriggerAttribute attribute = parameter.GetCustomAttribute <CosmosDBTriggerAttribute>(inherit: false);
if (attribute == null)
{
return(Task.FromResult <ITriggerBinding>(null));
}
DocumentCollectionInfo documentCollectionLocation;
DocumentCollectionInfo leaseCollectionLocation;
ChangeFeedHostOptions leaseHostOptions = ResolveLeaseOptions(attribute);
try
{
string triggerConnectionString = ResolveAttributeConnectionString(attribute);
DocumentDBConnectionString triggerConnection = new DocumentDBConnectionString(triggerConnectionString);
if (triggerConnection.ServiceEndpoint == null)
{
throw new InvalidOperationException("The connection string for the monitored collection is in an invalid format, please use AccountEndpoint=XXXXXX;AccountKey=XXXXXX;.");
}
DocumentDBConnectionString leasesConnection = new DocumentDBConnectionString(ResolveAttributeLeasesConnectionString(attribute, triggerConnectionString));
if (triggerConnection.ServiceEndpoint == null)
{
throw new InvalidOperationException("The connection string for the leases collection is in an invalid format, please use AccountEndpoint=XXXXXX;AccountKey=XXXXXX;.");
}
documentCollectionLocation = new DocumentCollectionInfo
{
Uri = triggerConnection.ServiceEndpoint,
MasterKey = triggerConnection.AuthKey,
DatabaseName = attribute.DatabaseName,
CollectionName = attribute.CollectionName
};
leaseCollectionLocation = new DocumentCollectionInfo
{
Uri = leasesConnection.ServiceEndpoint,
MasterKey = leasesConnection.AuthKey,
DatabaseName = attribute.LeaseDatabaseName,
CollectionName = attribute.LeaseCollectionName
};
if (documentCollectionLocation.Uri.Equals(leaseCollectionLocation.Uri) &&
documentCollectionLocation.DatabaseName.Equals(leaseCollectionLocation.DatabaseName) &&
documentCollectionLocation.CollectionName.Equals(leaseCollectionLocation.CollectionName))
{
throw new InvalidOperationException("The monitored collection cannot be the same as the collection storing the leases.");
}
}
catch (Exception ex)
{
throw new InvalidOperationException(string.Format("Cannot create Collection Information for {0} in database {1} with lease {2} in database {3} : {4}", attribute.CollectionName, attribute.DatabaseName, attribute.LeaseCollectionName, attribute.LeaseDatabaseName, ex.Message), ex);
}
return(Task.FromResult <ITriggerBinding>(new CosmosDBTriggerBinding(parameter, documentCollectionLocation, leaseCollectionLocation, leaseHostOptions)));
}
public CosmosDBTriggerAttributeBindingProvider(string monitorConnectionString, string leasesConnectionString, ChangeFeedHostOptions leasesOptions = null)
{
_monitorConnectionString = monitorConnectionString;
_leasesConnectionString = leasesConnectionString;
_leasesOptions = leasesOptions ?? new ChangeFeedHostOptions();
}
public CosmosDBTriggerListener(ITriggeredFunctionExecutor executor, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, ChangeFeedHostOptions leaseHostOptions, int?maxItemCount)
{
this.executor = executor;
string hostName = Guid.NewGuid().ToString();
monitorCollection = documentCollectionLocation;
leaseCollection = leaseCollectionLocation;
ChangeFeedOptions changeFeedOptions = new ChangeFeedOptions();
if (maxItemCount.HasValue)
{
changeFeedOptions.MaxItemCount = maxItemCount;
}
this.host = new ChangeFeedEventHost(hostName, documentCollectionLocation, leaseCollectionLocation, changeFeedOptions, leaseHostOptions);
}
public async Task <ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context)
{
// Tries to parse the context parameters and see if it belongs to this [CosmosDBTrigger] binder
if (context == null)
{
throw new ArgumentNullException("context");
}
ParameterInfo parameter = context.Parameter;
CosmosDBTriggerAttribute attribute = parameter.GetCustomAttribute <CosmosDBTriggerAttribute>(inherit: false);
if (attribute == null)
{
return(null);
}
_monitorConnectionString = _nameResolver.Resolve(CosmosDBConfiguration.AzureWebJobsCosmosDBConnectionStringName);
_leasesConnectionString = _nameResolver.Resolve(CosmosDBConfiguration.AzureWebJobsCosmosDBConnectionStringName);
DocumentCollectionInfo documentCollectionLocation;
DocumentCollectionInfo leaseCollectionLocation;
ChangeFeedHostOptions leaseHostOptions = ResolveLeaseOptions(attribute);
int?maxItemCount = null;
if (attribute.MaxItemsPerInvocation > 0)
{
maxItemCount = attribute.MaxItemsPerInvocation;
}
try
{
string triggerConnectionString = ResolveAttributeConnectionString(attribute);
CosmosDBConnectionString triggerConnection = new CosmosDBConnectionString(triggerConnectionString);
if (triggerConnection.ServiceEndpoint == null)
{
throw new InvalidOperationException("The connection string for the monitored collection is in an invalid format, please use AccountEndpoint=XXXXXX;AccountKey=XXXXXX;.");
}
string leasesConnectionString = ResolveAttributeLeasesConnectionString(attribute, triggerConnectionString);
CosmosDBConnectionString leasesConnection = new CosmosDBConnectionString(leasesConnectionString);
if (leasesConnection.ServiceEndpoint == null)
{
throw new InvalidOperationException("The connection string for the leases collection is in an invalid format, please use AccountEndpoint=XXXXXX;AccountKey=XXXXXX;.");
}
documentCollectionLocation = new DocumentCollectionInfo
{
Uri = triggerConnection.ServiceEndpoint,
MasterKey = triggerConnection.AuthKey,
DatabaseName = ResolveAttributeValue(attribute.DatabaseName),
CollectionName = ResolveAttributeValue(attribute.CollectionName)
};
documentCollectionLocation.ConnectionPolicy.UserAgentSuffix = CosmosDBTriggerUserAgentSuffix;
leaseCollectionLocation = new DocumentCollectionInfo
{
Uri = leasesConnection.ServiceEndpoint,
MasterKey = leasesConnection.AuthKey,
DatabaseName = ResolveAttributeValue(attribute.LeaseDatabaseName),
CollectionName = ResolveAttributeValue(attribute.LeaseCollectionName)
};
leaseCollectionLocation.ConnectionPolicy.UserAgentSuffix = CosmosDBTriggerUserAgentSuffix;
if (string.IsNullOrEmpty(documentCollectionLocation.DatabaseName) ||
string.IsNullOrEmpty(documentCollectionLocation.CollectionName) ||
string.IsNullOrEmpty(leaseCollectionLocation.DatabaseName) ||
string.IsNullOrEmpty(leaseCollectionLocation.CollectionName))
{
throw new InvalidOperationException("Cannot establish database and collection values. If you are using environment and configuration values, please ensure these are correctly set.");
}
if (documentCollectionLocation.Uri.Equals(leaseCollectionLocation.Uri) &&
documentCollectionLocation.DatabaseName.Equals(leaseCollectionLocation.DatabaseName) &&
documentCollectionLocation.CollectionName.Equals(leaseCollectionLocation.CollectionName))
{
throw new InvalidOperationException("The monitored collection cannot be the same as the collection storing the leases.");
}
if (attribute.CreateLeaseCollectionIfNotExists)
{
// Not disposing this because it might be reused on other Trigger since Triggers could share lease collection
ICosmosDBService service = _config.GetService(leasesConnectionString);
await CosmosDBUtility.CreateDatabaseAndCollectionIfNotExistAsync(service, leaseCollectionLocation.DatabaseName, leaseCollectionLocation.CollectionName, null, attribute.LeasesCollectionThroughput);
}
}
catch (Exception ex)
{
throw new InvalidOperationException(string.Format("Cannot create Collection Information for {0} in database {1} with lease {2} in database {3} : {4}", attribute.CollectionName, attribute.DatabaseName, attribute.LeaseCollectionName, attribute.LeaseDatabaseName, ex.Message), ex);
}
return(new CosmosDBTriggerBinding(parameter, documentCollectionLocation, leaseCollectionLocation, leaseHostOptions, maxItemCount));
}
/// <summary>
/// Registers change feed observer to update changes read on change feed to destination
/// collection. Deregisters change feed observer and closes process when enter key is pressed
/// </summary>
/// <returns>A Task to allow asynchronous execution</returns>
public async Task Start(CosmosDBQueueConsumerSettings settings, CancellationTokenSource cts = default)
{
if (this.started)
{
throw new InvalidOperationException("Already started");
}
this.settings = settings;
await Utils.CreateCollectionIfNotExists(this.settings.QueueCollectionDefinition);
await Utils.CreateCollectionIfNotExists(this.settings.LeaseCollectionDefinition);
if (this.OnMessage == null)
{
throw new InvalidOperationException($"Set a {nameof(OnMessage)} before start consuming queue items");
}
this.workerId = this.settings.WorkerId ?? Guid.NewGuid().ToString();
// queue collection info
var documentCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(this.settings.QueueCollectionDefinition.Endpoint),
MasterKey = this.settings.QueueCollectionDefinition.SecretKey,
DatabaseName = this.settings.QueueCollectionDefinition.DbName,
CollectionName = this.settings.QueueCollectionDefinition.CollectionName
};
// lease collection info
DocumentCollectionInfo leaseCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(this.settings.LeaseCollectionDefinition.Endpoint),
MasterKey = this.settings.LeaseCollectionDefinition.SecretKey,
DatabaseName = this.settings.LeaseCollectionDefinition.DbName,
CollectionName = this.settings.LeaseCollectionDefinition.CollectionName,
};
this.queueCollectionClient = new DocumentClient(new Uri(this.settings.QueueCollectionDefinition.Endpoint), this.settings.QueueCollectionDefinition.SecretKey);
// Customizable change feed option and host options
ChangeFeedOptions feedOptions = new ChangeFeedOptions
{
StartFromBeginning = true
};
ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions
{
LeaseRenewInterval = TimeSpan.FromSeconds(15),
LeasePrefix = string.Concat(this.workerId, "_"),
};
this.logger?.LogTrace($"Starting worker {this.workerId}");
ChangeFeedEventHost host = new ChangeFeedEventHost(this.workerId, documentCollectionLocation, leaseCollectionLocation, feedOptions, feedHostOptions);
await host.RegisterObserverFactoryAsync(this);
if (cts != null)
{
cts.Token.Register(async() =>
{
await host.UnregisterObserversAsync();
});
}
this.started = true;
}
public MockListener(ITriggeredFunctionExecutor executor, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, ChangeFeedHostOptions leaseHostOptions, ChangeFeedOptions changeFeedOptions, ILogger logger)
: base(executor, documentCollectionLocation, leaseCollectionLocation, leaseHostOptions, changeFeedOptions, logger)
{
}
public CosmosDBTriggerAttributeBindingProvider(INameResolver nameResolver, ChangeFeedHostOptions leasesOptions = null)
{
_nameResolver = nameResolver;
_leasesOptions = leasesOptions ?? new ChangeFeedHostOptions();
}
public CosmosDBTriggerListener(ITriggeredFunctionExecutor executor, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, ChangeFeedHostOptions leaseHostOptions)
{
this.executor = executor;
string hostName = Guid.NewGuid().ToString();
monitorCollection = documentCollectionLocation;
leaseCollection = leaseCollectionLocation;
this.host = new ChangeFeedEventHost(hostName, documentCollectionLocation, leaseCollectionLocation, new ChangeFeedOptions(), leaseHostOptions);
}
private static async Task ProcessChangeFeedWithChangeFeedProcessor(DocumentClient client, Database db, DocumentCollection personCol)
{
// Use the ChangeFeed Processor which manages to read changes accross partitions.
// This requires another CosmosDB collection so that ChangeFeedProcessor can store it's lease information!
// https://docs.microsoft.com/en-us/azure/cosmos-db/change-feed#change-feed-processor
#region Change Feed Processor
// Create LeaseCollection
DocumentCollection leasecol = await CosDB.CreateOrGetCollection(client, db, "leasecol", 400, null, null, false, true);
// Customizable change feed option and host options
ChangeFeedOptions feedOptions = new ChangeFeedOptions()
{
// ie customize StartFromBeginning so change feed reads from beginning
// can customize MaxItemCount, PartitonKeyRangeId, RequestContinuation, SessionToken and StartFromBeginning
StartFromBeginning = true
};
ChangeFeedHostOptions feedHostOptions = new ChangeFeedHostOptions()
{
// ie. customizing lease renewal interval to 15 seconds
// can customize LeaseRenewInterval, LeaseAcquireInterval, LeaseExpirationInterval, FeedPollDelay
LeaseRenewInterval = TimeSpan.FromSeconds(15)
};
string hostName = "UniqueHostName01";
DocumentCollectionInfo documentCollectionLocation = new DocumentCollectionInfo()
{
CollectionName = personCol.Id,
ConnectionPolicy = ConnectionPolicy.Default,
DatabaseName = db.Id,
MasterKey = Config.Account_DemoBuild_Mongo_Key,
Uri = new Uri(Config.Account_DemoBuild_Mongo)
};
DocumentCollectionInfo leaseCollectionLocation = new DocumentCollectionInfo()
{
CollectionName = "leasecol",
ConnectionPolicy = ConnectionPolicy.Default,
DatabaseName = db.Id,
MasterKey = Config.Account_DemoBuild_Mongo_Key,
Uri = new Uri(Config.Account_DemoBuild_Mongo)
};
DocumentFeedObserverFactory docObserverFactory = new DocumentFeedObserverFactory();
ChangeFeedEventHost host = new ChangeFeedEventHost(hostName,
documentCollectionLocation,
leaseCollectionLocation,
feedOptions, feedHostOptions);
await host.RegisterObserverFactoryAsync(docObserverFactory);
Console.WriteLine("Waiting 2-3 seconds before adding more documents...");
Thread.Sleep(2500);
Random randGen = new Random((int)DateTime.Now.Ticks);
List <string> cities = new List <string>()
{
"Vienna", "Paris", "Frankfurt", "Prag", "Seattle"
};
for (int c = 0; c < 10000; c++) // with this long running sample, the Lease is probably lost
{
string city = cities[randGen.Next() % cities.Count];
await client.CreateDocumentAsync(personCol.SelfLink, new Person()
{
name = Guid.NewGuid().ToString("D"), city = city, label = "person"
});
await client.CreateDocumentAsync(personCol.SelfLink, new Person()
{
name = Guid.NewGuid().ToString("D"), city = city, label = "person"
});
await client.CreateDocumentAsync(personCol.SelfLink, new Person()
{
name = Guid.NewGuid().ToString("D"), city = city, label = "person"
});
}
Console.WriteLine("Observer is still running... Modify or add documents with DocumentStudio. Press enter to stop.");
Console.ReadLine();
await host.UnregisterObserversAsync();
#endregion
}
