/// <summary> /// Indicates whether change feed in the Azure Cosmos DB service should start from beginning. /// By default it's start from current time. /// </summary> /// <remarks> /// This is only used when: /// (1) Lease store is not initialized and is ignored if a lease exists and has continuation token. /// (2) StartContinuation is not specified. /// (3) StartTime is not specified. /// </remarks> /// <returns>The instance of <see cref="ChangeFeedProcessorBuilder"/> to use.</returns> internal virtual ChangeFeedProcessorBuilder WithStartFromBeginning() { this.changeFeedProcessorOptions = this.changeFeedProcessorOptions ?? new ChangeFeedProcessorOptions(); this.changeFeedProcessorOptions.StartFromBeginning = true; return(this); }
/// <summary> /// Sets the start request session continuation token to start looking for changes after. /// </summary> /// <remarks> /// This is only used when lease store is not initialized and is ignored if a lease exists and has continuation token. /// If this is specified, both StartTime and StartFromBeginning are ignored. /// </remarks> /// <returns>The instance of <see cref="ChangeFeedProcessorBuilder"/> to use.</returns> internal virtual ChangeFeedProcessorBuilder WithSessionContinuationToken(string startContinuation) { this.changeFeedProcessorOptions = this.changeFeedProcessorOptions ?? new ChangeFeedProcessorOptions(); this.changeFeedProcessorOptions.StartContinuation = startContinuation; return(this); }
private void InitializeDefaultOptions() { this.changeFeedProcessorOptions = this.changeFeedProcessorOptions ?? new ChangeFeedProcessorOptions(); }
public MockListener(ITriggeredFunctionExecutor executor, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, ChangeFeedProcessorOptions processorOptions, ICosmosDBService monitoredCosmosDBService, ICosmosDBService leasesCosmosDBService, ILogger logger) : base(executor, documentCollectionLocation, leaseCollectionLocation, processorOptions, monitoredCosmosDBService, leasesCosmosDBService, logger) { }
public async Task <ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context) { // Tries to parse the context parameters and see if it belongs to this [CosmosDBTrigger] binder if (context == null) { throw new ArgumentNullException("context"); } ParameterInfo parameter = context.Parameter; CosmosDBTriggerAttribute attribute = parameter.GetCustomAttribute <CosmosDBTriggerAttribute>(inherit: false); if (attribute == null) { return(null); } ConnectionMode?desiredConnectionMode = _options.ConnectionMode; Protocol? desiredConnectionProtocol = _options.Protocol; DocumentCollectionInfo documentCollectionLocation; DocumentCollectionInfo leaseCollectionLocation; ChangeFeedProcessorOptions processorOptions = BuildProcessorOptions(attribute); processorOptions.StartFromBeginning = attribute.StartFromBeginning; if (attribute.MaxItemsPerInvocation > 0) { processorOptions.MaxItemCount = attribute.MaxItemsPerInvocation; } ICosmosDBService monitoredCosmosDBService; ICosmosDBService leaseCosmosDBService; try { string triggerConnectionString = ResolveAttributeConnectionString(attribute); CosmosDBConnectionString triggerConnection = new CosmosDBConnectionString(triggerConnectionString); if (triggerConnection.ServiceEndpoint == null) { throw new InvalidOperationException("The connection string for the monitored collection is in an invalid format, please use AccountEndpoint=XXXXXX;AccountKey=XXXXXX;."); } string leasesConnectionString = ResolveAttributeLeasesConnectionString(attribute); CosmosDBConnectionString leasesConnection = new CosmosDBConnectionString(leasesConnectionString); if (leasesConnection.ServiceEndpoint == null) { throw new InvalidOperationException("The connection string for the leases collection is in an invalid format, please use AccountEndpoint=XXXXXX;AccountKey=XXXXXX;."); } documentCollectionLocation = new DocumentCollectionInfo { Uri = triggerConnection.ServiceEndpoint, MasterKey = triggerConnection.AuthKey, DatabaseName = ResolveAttributeValue(attribute.DatabaseName), CollectionName = ResolveAttributeValue(attribute.CollectionName) }; documentCollectionLocation.ConnectionPolicy.UserAgentSuffix = CosmosDBTriggerUserAgentSuffix; if (desiredConnectionMode.HasValue) { documentCollectionLocation.ConnectionPolicy.ConnectionMode = desiredConnectionMode.Value; } if (desiredConnectionProtocol.HasValue) { documentCollectionLocation.ConnectionPolicy.ConnectionProtocol = desiredConnectionProtocol.Value; } leaseCollectionLocation = new DocumentCollectionInfo { Uri = leasesConnection.ServiceEndpoint, MasterKey = leasesConnection.AuthKey, DatabaseName = ResolveAttributeValue(attribute.LeaseDatabaseName), CollectionName = ResolveAttributeValue(attribute.LeaseCollectionName) }; leaseCollectionLocation.ConnectionPolicy.UserAgentSuffix = CosmosDBTriggerUserAgentSuffix; if (desiredConnectionMode.HasValue) { leaseCollectionLocation.ConnectionPolicy.ConnectionMode = desiredConnectionMode.Value; } if (desiredConnectionProtocol.HasValue) { leaseCollectionLocation.ConnectionPolicy.ConnectionProtocol = desiredConnectionProtocol.Value; } string resolvedPreferredLocations = ResolveAttributeValue(attribute.PreferredLocations); foreach (var location in CosmosDBUtility.ParsePreferredLocations(resolvedPreferredLocations)) { documentCollectionLocation.ConnectionPolicy.PreferredLocations.Add(location); leaseCollectionLocation.ConnectionPolicy.PreferredLocations.Add(location); } if (string.IsNullOrEmpty(documentCollectionLocation.DatabaseName) || string.IsNullOrEmpty(documentCollectionLocation.CollectionName) || string.IsNullOrEmpty(leaseCollectionLocation.DatabaseName) || string.IsNullOrEmpty(leaseCollectionLocation.CollectionName)) { throw new InvalidOperationException("Cannot establish database and collection values. If you are using environment and configuration values, please ensure these are correctly set."); } if (documentCollectionLocation.Uri.Equals(leaseCollectionLocation.Uri) && documentCollectionLocation.DatabaseName.Equals(leaseCollectionLocation.DatabaseName) && documentCollectionLocation.CollectionName.Equals(leaseCollectionLocation.CollectionName)) { throw new InvalidOperationException("The monitored collection cannot be the same as the collection storing the leases."); } monitoredCosmosDBService = _configProvider.GetService(triggerConnectionString, resolvedPreferredLocations); leaseCosmosDBService = _configProvider.GetService(leasesConnectionString, resolvedPreferredLocations); if (attribute.CreateLeaseCollectionIfNotExists) { await CreateLeaseCollectionIfNotExistsAsync(leaseCosmosDBService, leaseCollectionLocation.DatabaseName, leaseCollectionLocation.CollectionName, attribute.LeasesCollectionThroughput); } } catch (Exception ex) { throw new InvalidOperationException(string.Format("Cannot create Collection Information for {0} in database {1} with lease {2} in database {3} : {4}", attribute.CollectionName, attribute.DatabaseName, attribute.LeaseCollectionName, attribute.LeaseDatabaseName, ex.Message), ex); } return(new CosmosDBTriggerBinding( parameter, documentCollectionLocation, leaseCollectionLocation, processorOptions, monitoredCosmosDBService, leaseCosmosDBService, _logger)); }
public async Task <IChangeFeedProcessor> RunChangeFeedHostAsync() { string hostName = Guid.NewGuid().ToString(); Trace.TraceInformation("Host name {0}", hostName); // monitored collection info var documentCollectionLocation = new DocumentCollectionInfo { Uri = new Uri(this.config.MonitoredUri), MasterKey = this.config.MonitoredSecretKey, DatabaseName = this.config.MonitoredDbName, CollectionName = this.config.MonitoredCollectionName }; var policy = new ConnectionPolicy() { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp }; policy.PreferredLocations.Add("North Europe"); // lease collection info var leaseCollectionLocation = new DocumentCollectionInfo { Uri = new Uri(this.config.LeaseUri), MasterKey = this.config.LeaseSecretKey, DatabaseName = this.config.LeaseDbName, CollectionName = this.config.LeaseCollectionName, ConnectionPolicy = policy }; // destination collection info var destCollInfo = new DocumentCollectionInfo { Uri = new Uri(this.config.DestUri), MasterKey = this.config.DestSecretKey, DatabaseName = this.config.DestDbName, CollectionName = this.config.DestCollectionName }; var processorOptions = new ChangeFeedProcessorOptions(); if (config.DataAgeInHours.HasValue) { if (config.DataAgeInHours.Value >= 0) { processorOptions.StartTime = DateTime.UtcNow.Subtract(TimeSpan.FromHours(config.DataAgeInHours.Value)); } } else { processorOptions.StartFromBeginning = true; } processorOptions.LeaseRenewInterval = TimeSpan.FromSeconds(30); Trace.TraceInformation("Processor options Starts from Beginning - {0}, Lease renew interval - {1}", processorOptions.StartFromBeginning, processorOptions.LeaseRenewInterval.ToString()); processorOptions.MaxItemCount = 1000; var destClient = new DocumentClient(destCollInfo.Uri, destCollInfo.MasterKey, new ConnectionPolicy() { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp }, ConsistencyLevel.Eventual); var docTransformer = new DefaultDocumentTransformer(); BlobContainerClient containerClient = null; if (!String.IsNullOrEmpty(config.BlobConnectionString)) { BlobServiceClient blobServiceClient = new BlobServiceClient(config.BlobConnectionString); containerClient = blobServiceClient.GetBlobContainerClient(config.BlobContainerName); await containerClient.CreateIfNotExistsAsync(); } var docObserverFactory = new DocumentFeedObserverFactory(config.SourcePartitionKeys, config.TargetPartitionKey, destClient, destCollInfo, docTransformer, containerClient); changeFeedProcessor = await new ChangeFeedProcessorBuilder() .WithObserverFactory(docObserverFactory) .WithHostName(hostName) .WithFeedCollection(documentCollectionLocation) .WithLeaseCollection(leaseCollectionLocation) .WithProcessorOptions(processorOptions) .WithFeedDocumentClient(new DocumentClient(documentCollectionLocation.Uri, documentCollectionLocation.MasterKey, policy, ConsistencyLevel.Eventual)) .BuildAsync(); await changeFeedProcessor.StartAsync().ConfigureAwait(false); return(changeFeedProcessor); }
public CosmosStoreTriggerBinding(ParameterInfo parameter, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, IDocumentClient monitoredDocumentClient, IDocumentClient leaseDocumentClient, ChangeFeedProcessorOptions processorOptions, ILogger logger) { DocumentCollectionLocation = documentCollectionLocation; LeaseCollectionLocation = leaseCollectionLocation; _monitoredDocumentClient = monitoredDocumentClient; _leaseDocumentClient = leaseDocumentClient; _parameter = parameter; ChangeFeedProcessorOptions = processorOptions; _logger = logger; }
public CosmosStoreTriggerListener(ITriggeredFunctionExecutor executor, DocumentCollectionInfo documentCollectionLocation, DocumentCollectionInfo leaseCollectionLocation, ChangeFeedProcessorOptions processorOptions, IDocumentClient monitoredDocumentClient, IDocumentClient leaseDocumentClient, ILogger logger) { _logger = logger; _executor = executor; _hostName = Guid.NewGuid().ToString(); _monitoredDocumentClient = monitoredDocumentClient; _leaseDocumentClient = leaseDocumentClient; _monitorCollection = documentCollectionLocation; _leaseCollection = leaseCollectionLocation; _processorOptions = processorOptions; }