/// <summary>
/// Indicates whether change feed in the Azure Cosmos DB service should start from beginning.
/// By default it's start from current time.
/// </summary>
/// <remarks>
/// This is only used when:
/// (1) Lease store is not initialized and is ignored if a lease exists and has continuation token.
/// (2) StartContinuation is not specified.
/// (3) StartTime is not specified.
/// </remarks>
/// <returns>The instance of <see cref="ChangeFeedProcessorBuilder"/> to use.</returns>
internal virtual ChangeFeedProcessorBuilder WithStartFromBeginning()
{
this.changeFeedProcessorOptions = this.changeFeedProcessorOptions ?? new ChangeFeedProcessorOptions();
this.changeFeedProcessorOptions.StartFromBeginning = true;
return(this);
}
/// <summary>
/// Sets the start request session continuation token to start looking for changes after.
/// </summary>
/// <remarks>
/// This is only used when lease store is not initialized and is ignored if a lease exists and has continuation token.
/// If this is specified, both StartTime and StartFromBeginning are ignored.
/// </remarks>
/// <returns>The instance of <see cref="ChangeFeedProcessorBuilder"/> to use.</returns>
internal virtual ChangeFeedProcessorBuilder WithSessionContinuationToken(string startContinuation)
{
this.changeFeedProcessorOptions = this.changeFeedProcessorOptions ?? new ChangeFeedProcessorOptions();
this.changeFeedProcessorOptions.StartContinuation = startContinuation;
return(this);
}
private void InitializeDefaultOptions()
{
this.changeFeedProcessorOptions = this.changeFeedProcessorOptions ?? new ChangeFeedProcessorOptions();
}
public MockListener(ITriggeredFunctionExecutor executor, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, ChangeFeedProcessorOptions processorOptions, ICosmosDBService monitoredCosmosDBService, ICosmosDBService leasesCosmosDBService, ILogger logger)
: base(executor, documentCollectionLocation, leaseCollectionLocation, processorOptions, monitoredCosmosDBService, leasesCosmosDBService, logger)
{
}
public async Task <ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context)
{
// Tries to parse the context parameters and see if it belongs to this [CosmosDBTrigger] binder
if (context == null)
{
throw new ArgumentNullException("context");
}
ParameterInfo parameter = context.Parameter;
CosmosDBTriggerAttribute attribute = parameter.GetCustomAttribute <CosmosDBTriggerAttribute>(inherit: false);
if (attribute == null)
{
return(null);
}
ConnectionMode?desiredConnectionMode = _options.ConnectionMode;
Protocol? desiredConnectionProtocol = _options.Protocol;
DocumentCollectionInfo documentCollectionLocation;
DocumentCollectionInfo leaseCollectionLocation;
ChangeFeedProcessorOptions processorOptions = BuildProcessorOptions(attribute);
processorOptions.StartFromBeginning = attribute.StartFromBeginning;
if (attribute.MaxItemsPerInvocation > 0)
{
processorOptions.MaxItemCount = attribute.MaxItemsPerInvocation;
}
ICosmosDBService monitoredCosmosDBService;
ICosmosDBService leaseCosmosDBService;
try
{
string triggerConnectionString = ResolveAttributeConnectionString(attribute);
CosmosDBConnectionString triggerConnection = new CosmosDBConnectionString(triggerConnectionString);
if (triggerConnection.ServiceEndpoint == null)
{
throw new InvalidOperationException("The connection string for the monitored collection is in an invalid format, please use AccountEndpoint=XXXXXX;AccountKey=XXXXXX;.");
}
string leasesConnectionString = ResolveAttributeLeasesConnectionString(attribute);
CosmosDBConnectionString leasesConnection = new CosmosDBConnectionString(leasesConnectionString);
if (leasesConnection.ServiceEndpoint == null)
{
throw new InvalidOperationException("The connection string for the leases collection is in an invalid format, please use AccountEndpoint=XXXXXX;AccountKey=XXXXXX;.");
}
documentCollectionLocation = new DocumentCollectionInfo
{
Uri = triggerConnection.ServiceEndpoint,
MasterKey = triggerConnection.AuthKey,
DatabaseName = ResolveAttributeValue(attribute.DatabaseName),
CollectionName = ResolveAttributeValue(attribute.CollectionName)
};
documentCollectionLocation.ConnectionPolicy.UserAgentSuffix = CosmosDBTriggerUserAgentSuffix;
if (desiredConnectionMode.HasValue)
{
documentCollectionLocation.ConnectionPolicy.ConnectionMode = desiredConnectionMode.Value;
}
if (desiredConnectionProtocol.HasValue)
{
documentCollectionLocation.ConnectionPolicy.ConnectionProtocol = desiredConnectionProtocol.Value;
}
leaseCollectionLocation = new DocumentCollectionInfo
{
Uri = leasesConnection.ServiceEndpoint,
MasterKey = leasesConnection.AuthKey,
DatabaseName = ResolveAttributeValue(attribute.LeaseDatabaseName),
CollectionName = ResolveAttributeValue(attribute.LeaseCollectionName)
};
leaseCollectionLocation.ConnectionPolicy.UserAgentSuffix = CosmosDBTriggerUserAgentSuffix;
if (desiredConnectionMode.HasValue)
{
leaseCollectionLocation.ConnectionPolicy.ConnectionMode = desiredConnectionMode.Value;
}
if (desiredConnectionProtocol.HasValue)
{
leaseCollectionLocation.ConnectionPolicy.ConnectionProtocol = desiredConnectionProtocol.Value;
}
string resolvedPreferredLocations = ResolveAttributeValue(attribute.PreferredLocations);
foreach (var location in CosmosDBUtility.ParsePreferredLocations(resolvedPreferredLocations))
{
documentCollectionLocation.ConnectionPolicy.PreferredLocations.Add(location);
leaseCollectionLocation.ConnectionPolicy.PreferredLocations.Add(location);
}
if (string.IsNullOrEmpty(documentCollectionLocation.DatabaseName) ||
string.IsNullOrEmpty(documentCollectionLocation.CollectionName) ||
string.IsNullOrEmpty(leaseCollectionLocation.DatabaseName) ||
string.IsNullOrEmpty(leaseCollectionLocation.CollectionName))
{
throw new InvalidOperationException("Cannot establish database and collection values. If you are using environment and configuration values, please ensure these are correctly set.");
}
if (documentCollectionLocation.Uri.Equals(leaseCollectionLocation.Uri) &&
documentCollectionLocation.DatabaseName.Equals(leaseCollectionLocation.DatabaseName) &&
documentCollectionLocation.CollectionName.Equals(leaseCollectionLocation.CollectionName))
{
throw new InvalidOperationException("The monitored collection cannot be the same as the collection storing the leases.");
}
monitoredCosmosDBService = _configProvider.GetService(triggerConnectionString, resolvedPreferredLocations);
leaseCosmosDBService = _configProvider.GetService(leasesConnectionString, resolvedPreferredLocations);
if (attribute.CreateLeaseCollectionIfNotExists)
{
await CreateLeaseCollectionIfNotExistsAsync(leaseCosmosDBService, leaseCollectionLocation.DatabaseName, leaseCollectionLocation.CollectionName, attribute.LeasesCollectionThroughput);
}
}
catch (Exception ex)
{
throw new InvalidOperationException(string.Format("Cannot create Collection Information for {0} in database {1} with lease {2} in database {3} : {4}", attribute.CollectionName, attribute.DatabaseName, attribute.LeaseCollectionName, attribute.LeaseDatabaseName, ex.Message), ex);
}
return(new CosmosDBTriggerBinding(
parameter,
documentCollectionLocation,
leaseCollectionLocation,
processorOptions,
monitoredCosmosDBService,
leaseCosmosDBService,
_logger));
}
public async Task <IChangeFeedProcessor> RunChangeFeedHostAsync()
{
string hostName = Guid.NewGuid().ToString();
Trace.TraceInformation("Host name {0}", hostName);
// monitored collection info
var documentCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(this.config.MonitoredUri),
MasterKey = this.config.MonitoredSecretKey,
DatabaseName = this.config.MonitoredDbName,
CollectionName = this.config.MonitoredCollectionName
};
var policy = new ConnectionPolicy()
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp
};
policy.PreferredLocations.Add("North Europe");
// lease collection info
var leaseCollectionLocation = new DocumentCollectionInfo
{
Uri = new Uri(this.config.LeaseUri),
MasterKey = this.config.LeaseSecretKey,
DatabaseName = this.config.LeaseDbName,
CollectionName = this.config.LeaseCollectionName,
ConnectionPolicy = policy
};
// destination collection info
var destCollInfo = new DocumentCollectionInfo
{
Uri = new Uri(this.config.DestUri),
MasterKey = this.config.DestSecretKey,
DatabaseName = this.config.DestDbName,
CollectionName = this.config.DestCollectionName
};
var processorOptions = new ChangeFeedProcessorOptions();
if (config.DataAgeInHours.HasValue)
{
if (config.DataAgeInHours.Value >= 0)
{
processorOptions.StartTime = DateTime.UtcNow.Subtract(TimeSpan.FromHours(config.DataAgeInHours.Value));
}
}
else
{
processorOptions.StartFromBeginning = true;
}
processorOptions.LeaseRenewInterval = TimeSpan.FromSeconds(30);
Trace.TraceInformation("Processor options Starts from Beginning - {0}, Lease renew interval - {1}",
processorOptions.StartFromBeginning,
processorOptions.LeaseRenewInterval.ToString());
processorOptions.MaxItemCount = 1000;
var destClient = new DocumentClient(destCollInfo.Uri, destCollInfo.MasterKey,
new ConnectionPolicy()
{
ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp
},
ConsistencyLevel.Eventual);
var docTransformer = new DefaultDocumentTransformer();
BlobContainerClient containerClient = null;
if (!String.IsNullOrEmpty(config.BlobConnectionString))
{
BlobServiceClient blobServiceClient = new BlobServiceClient(config.BlobConnectionString);
containerClient = blobServiceClient.GetBlobContainerClient(config.BlobContainerName);
await containerClient.CreateIfNotExistsAsync();
}
var docObserverFactory = new DocumentFeedObserverFactory(config.SourcePartitionKeys, config.TargetPartitionKey, destClient, destCollInfo, docTransformer, containerClient);
changeFeedProcessor = await new ChangeFeedProcessorBuilder()
.WithObserverFactory(docObserverFactory)
.WithHostName(hostName)
.WithFeedCollection(documentCollectionLocation)
.WithLeaseCollection(leaseCollectionLocation)
.WithProcessorOptions(processorOptions)
.WithFeedDocumentClient(new DocumentClient(documentCollectionLocation.Uri, documentCollectionLocation.MasterKey, policy, ConsistencyLevel.Eventual))
.BuildAsync();
await changeFeedProcessor.StartAsync().ConfigureAwait(false);
return(changeFeedProcessor);
}
public CosmosStoreTriggerBinding(ParameterInfo parameter, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, IDocumentClient monitoredDocumentClient, IDocumentClient leaseDocumentClient, ChangeFeedProcessorOptions processorOptions, ILogger logger)
{
DocumentCollectionLocation = documentCollectionLocation;
LeaseCollectionLocation = leaseCollectionLocation;
_monitoredDocumentClient = monitoredDocumentClient;
_leaseDocumentClient = leaseDocumentClient;
_parameter = parameter;
ChangeFeedProcessorOptions = processorOptions;
_logger = logger;
}
public CosmosStoreTriggerListener(ITriggeredFunctionExecutor executor, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, ChangeFeedProcessorOptions processorOptions, IDocumentClient monitoredDocumentClient, IDocumentClient leaseDocumentClient, ILogger logger)
{
_logger = logger;
_executor = executor;
_hostName = Guid.NewGuid().ToString();
_monitoredDocumentClient = monitoredDocumentClient;
_leaseDocumentClient = leaseDocumentClient;
_monitorCollection = documentCollectionLocation;
_leaseCollection = leaseCollectionLocation;
_processorOptions = processorOptions;
}
