private static async Task <string> GetCollectionResourceIdAsync(IChangeFeedDocumentClient documentClient, DocumentCollectionInfo collectionLocation) { Logger.InfoFormat("Reading collection: '{0}'", collectionLocation.CollectionName); DocumentCollection documentCollection = await documentClient.GetDocumentCollectionAsync(collectionLocation).ConfigureAwait(false); return(documentCollection.ResourceId); }
private async Task <IPartitionManager> BuildPartitionManagerAsync(ILeaseManager leaseManager) { this.leaseDocumentClient = this.leaseDocumentClient ?? this.leaseCollectionLocation.CreateDocumentClient(); DocumentCollection leaseCollection = await this.leaseDocumentClient.GetDocumentCollectionAsync(this.leaseCollectionLocation).ConfigureAwait(false); string leaseStoreCollectionLink = leaseCollection.SelfLink; string collectionSelfLink = this.feedCollectionLocation.GetCollectionSelfLink(); var factory = new CheckpointerObserverFactory(this.observerFactory, this.changeFeedProcessorOptions.CheckpointFrequency); var synchronizer = new PartitionSynchronizer(this.feedDocumentClient, collectionSelfLink, leaseManager, this.changeFeedProcessorOptions.DegreeOfParallelism, this.changeFeedProcessorOptions.QueryPartitionsMaxBatchSize); var leaseStore = new LeaseStore(this.leaseDocumentClient, this.leaseCollectionLocation, this.GetLeasePrefix(), leaseStoreCollectionLink); var bootstrapper = new Bootstrapper(synchronizer, leaseStore, this.lockTime, this.sleepTime); var partitionObserverFactory = new PartitionSupervisorFactory( factory, leaseManager, this.partitionProcessorFactory ?? new PartitionProcessorFactory(this.feedDocumentClient, this.changeFeedProcessorOptions, leaseManager, collectionSelfLink), this.changeFeedProcessorOptions); var partitionController = new PartitionController(leaseManager, partitionObserverFactory, synchronizer); if (this.loadBalancingStrategy == null) { this.loadBalancingStrategy = new EqualPartitionsBalancingStrategy(this.hostName, this.changeFeedProcessorOptions.MinPartitionCount, this.changeFeedProcessorOptions.MaxPartitionCount, this.changeFeedProcessorOptions.LeaseExpirationInterval); } var partitionLoadBalancer = new PartitionLoadBalancer(partitionController, leaseManager, this.loadBalancingStrategy, this.changeFeedProcessorOptions.LeaseAcquireInterval); return(new PartitionManager(bootstrapper, partitionController, partitionLoadBalancer)); }
public async Task SplitPartitionAsync_ShouldReturnOnlyNewLeases_IfSplitWasAlreadyPerformed() { const string lastKnowToken = "last know token"; IEnumerable <PartitionKeyRange> keyRanges = new[] { new PartitionKeyRange { Id = "20", Parents = new Collection <string>(new[] { "10" }) }, new PartitionKeyRange { Id = "30", Parents = new Collection <string>(new[] { "10" }) } }; var lease = Mock.Of <ILease>(l => l.PartitionId == "10" && l.ContinuationToken == lastKnowToken); var keyRangeResponse = Mock.Of <IFeedResponse <PartitionKeyRange> >(r => r.GetEnumerator() == keyRanges.GetEnumerator()); IChangeFeedDocumentClient documentClient = Mock.Of <IChangeFeedDocumentClient>(c => c.ReadPartitionKeyRangeFeedAsync(It.IsAny <string>(), It.IsAny <FeedOptions>()) == Task.FromResult(keyRangeResponse)); var lease20 = Mock.Of <ILease>(); ILeaseManager leaseManager = Mock.Of <ILeaseManager>(m => m.CreateLeaseIfNotExistAsync("20", lastKnowToken) == Task.FromResult(lease20) && m.CreateLeaseIfNotExistAsync("30", lastKnowToken) == Task.FromResult <ILease>(null)); var sut = new PartitionSynchronizer(documentClient, "collectionlink", leaseManager, 1, int.MaxValue); IEnumerable <ILease> result = await sut.SplitPartitionAsync(lease); Assert.NotNull(result); Assert.Equal(new[] { lease20 }, result); }
public PartitionProcessorFactory( IChangeFeedDocumentClient documentClient, ChangeFeedProcessorOptions changeFeedProcessorOptions, string collectionSelfLink, IHealthMonitor healthMonitor) { if (documentClient == null) { throw new ArgumentNullException(nameof(documentClient)); } if (changeFeedProcessorOptions == null) { throw new ArgumentNullException(nameof(changeFeedProcessorOptions)); } if (collectionSelfLink == null) { throw new ArgumentNullException(nameof(collectionSelfLink)); } if (healthMonitor == null) { throw new ArgumentNullException(nameof(healthMonitor)); } this.documentClient = documentClient; this.changeFeedProcessorOptions = changeFeedProcessorOptions; this.collectionSelfLink = collectionSelfLink; this.healthMonitor = healthMonitor; }
public async Task <ILeaseManager> BuildAsync() { if (this.leaseCollectionLocation == null) { throw new InvalidOperationException(nameof(this.leaseCollectionLocation) + " was not specified"); } this.leaseDocumentClient = this.leaseDocumentClient ?? this.leaseCollectionLocation.CreateDocumentClient(); if (this.leaseStoreCollectionLink == null) { DocumentCollection documentCollection = await this.leaseDocumentClient.GetDocumentCollectionAsync(this.leaseCollectionLocation).ConfigureAwait(false); this.leaseStoreCollectionLink = documentCollection.SelfLink; } var updater = new DocumentServiceLeaseUpdater(this.leaseDocumentClient); var documentServiceLeaseManager = new DocumentServiceLeaseManager( this.leaseDocumentClient, updater, this.leaseCollectionLocation, this.leasePrefix, this.leaseStoreCollectionLink, this.hostName); return(documentServiceLeaseManager); }
public LeaseStore(IChangeFeedDocumentClient client, DocumentCollectionInfo leaseStoreCollectionInfo, string containerNamePrefix, string leaseStoreCollectionLink) { this.client = client; this.leaseStoreCollectionInfo = leaseStoreCollectionInfo; this.containerNamePrefix = containerNamePrefix; this.leaseStoreCollectionLink = leaseStoreCollectionLink; }
private DocumentServiceLeaseStoreManager CreateLeaseStoreManager( IChangeFeedDocumentClient documentClient, string hostName, IDocumentServiceLeaseUpdater leaseUpdater = null, IRequestOptionsFactory requestOptionsFactory = null) { var settings = new DocumentServiceLeaseStoreManagerSettings { LeaseCollectionInfo = collectionInfo, ContainerNamePrefix = storeNamePrefix, LeaseCollectionLink = collectionLink, HostName = hostName, }; // Make sure we test both constructors. return(leaseUpdater != null ? new DocumentServiceLeaseStoreManager( settings, documentClient, requestOptionsFactory ?? Mock.Of <IRequestOptionsFactory>(), leaseUpdater) : new DocumentServiceLeaseStoreManager( settings, documentClient, requestOptionsFactory ?? Mock.Of <IRequestOptionsFactory>())); }
public PartitionProcessorFactory( IChangeFeedDocumentClient documentClient, ChangeFeedProcessorOptions changeFeedProcessorOptions, ILeaseCheckpointer leaseCheckpointer, string collectionSelfLink) { if (documentClient == null) { throw new ArgumentNullException(nameof(documentClient)); } if (changeFeedProcessorOptions == null) { throw new ArgumentNullException(nameof(changeFeedProcessorOptions)); } if (leaseCheckpointer == null) { throw new ArgumentNullException(nameof(leaseCheckpointer)); } if (collectionSelfLink == null) { throw new ArgumentNullException(nameof(collectionSelfLink)); } this.documentClient = documentClient; this.changeFeedProcessorOptions = changeFeedProcessorOptions; this.leaseCheckpointer = leaseCheckpointer; this.collectionSelfLink = collectionSelfLink; }
public DocumentServiceLeaseStoreManager( DocumentServiceLeaseStoreManagerSettings settings, IChangeFeedDocumentClient leaseDocumentClient, IRequestOptionsFactory requestOptionsFactory) : this(settings, leaseDocumentClient, requestOptionsFactory, new DocumentServiceLeaseUpdater(leaseDocumentClient)) { }
public RemainingWorkEstimator( ILeaseContainer leaseContainer, IChangeFeedDocumentClient feedDocumentClient, string collectionSelfLink, int degreeOfParallelism) { if (leaseContainer == null) { throw new ArgumentNullException(nameof(leaseContainer)); } if (string.IsNullOrEmpty(collectionSelfLink)) { throw new ArgumentNullException(nameof(collectionSelfLink)); } if (feedDocumentClient == null) { throw new ArgumentNullException(nameof(feedDocumentClient)); } if (degreeOfParallelism < 1) { throw new ArgumentException("Degree of parallelism is out of range", nameof(degreeOfParallelism)); } this.leaseContainer = leaseContainer; this.collectionSelfLink = collectionSelfLink; this.feedDocumentClient = feedDocumentClient; this.degreeOfParallelism = degreeOfParallelism; }
private static async Task <string> GetDatabaseResourceIdAsync(IChangeFeedDocumentClient documentClient, DocumentCollectionInfo collectionLocation) { Logger.InfoFormat("Reading database: '{0}'", collectionLocation.DatabaseName); Uri databaseUri = UriFactory.CreateDatabaseUri(collectionLocation.DatabaseName); var response = await documentClient.ReadDatabaseAsync(databaseUri, null).ConfigureAwait(false); return(response.Resource.ResourceId); }
public DocumentServiceLeaseUpdater(IChangeFeedDocumentClient client) { if (client == null) { throw new ArgumentNullException(nameof(client)); } this.client = client; }
private async Task InitializeCollectionPropertiesForBuildAsync() { this.feedDocumentClient = this.feedDocumentClient ?? this.feedCollectionLocation.CreateDocumentClient(); this.changeFeedProcessorOptions = this.changeFeedProcessorOptions ?? new ChangeFeedProcessorOptions(); this.databaseResourceId = this.databaseResourceId ?? await GetDatabaseResourceIdAsync(this.feedDocumentClient, this.feedCollectionLocation).ConfigureAwait(false); this.collectionResourceId = this.collectionResourceId ?? await GetCollectionResourceIdAsync(this.feedDocumentClient, this.feedCollectionLocation).ConfigureAwait(false); }
public PartitionSynchronizer(IChangeFeedDocumentClient documentClient, string collectionSelfLink, ILeaseManager leaseManager, int degreeOfParallelism, int maxBatchSize) { this.documentClient = documentClient; this.collectionSelfLink = collectionSelfLink; this.leaseManager = leaseManager; this.degreeOfParallelism = degreeOfParallelism; this.maxBatchSize = maxBatchSize; }
private void SetupReplaceConflict(IChangeFeedDocumentClient client, ILease updatedLease) { Mock.Get(client) .Setup(c => c.ReplaceDocumentAsync( documentUri, updatedLease, It.Is <RequestOptions>(options => options.AccessCondition.Type == AccessConditionType.IfMatch && options.AccessCondition.Condition == eTag1))) .ThrowsAsync(DocumentExceptionHelpers.CreatePreconditionFailedException()); }
/// <summary> /// Sets an existing <see cref="DocumentClient"/> to be used to read from the monitored collection. /// </summary> /// <param name="feedDocumentClient">The instance of <see cref="DocumentClient"/> to use.</param> /// <returns>The instance of <see cref="ChangeFeedProcessorBuilder"/> to use.</returns> public ChangeFeedProcessorBuilder WithFeedDocumentClient(DocumentClient feedDocumentClient) { if (feedDocumentClient == null) { throw new ArgumentNullException(nameof(feedDocumentClient)); } this.feedDocumentClient = new ChangeFeedDocumentClient(feedDocumentClient); return(this); }
/// <summary> /// Sets an existing <see cref="IChangeFeedDocumentClient"/> to be used to read from the leases collection. /// </summary> /// <param name="leaseDocumentClient">The instance of <see cref="IChangeFeedDocumentClient"/> to use.</param> /// <returns>The instance of <see cref="ChangeFeedProcessorBuilder"/> to use.</returns> public ChangeFeedProcessorBuilder WithLeaseDocumentClient(IChangeFeedDocumentClient leaseDocumentClient) { if (leaseDocumentClient == null) { throw new ArgumentNullException(nameof(leaseDocumentClient)); } this.leaseDocumentClient = leaseDocumentClient; return(this); }
public DocumentServiceLeaseStoreManagerBuilder WithLeaseDocumentClient(IChangeFeedDocumentClient leaseDocumentClient) { if (leaseDocumentClient == null) { throw new ArgumentNullException(nameof(leaseDocumentClient)); } this.client = leaseDocumentClient; return(this); }
public PartitionProcessorFactoryTests() { this.collectionSelfLink = this.fixture.Create <string>(); this.docClient = Mock.Of <IChangeFeedDocumentClient>(); Mock.Get(this.docClient) .Setup(ex => ex.CreateDocumentChangeFeedQuery(this.collectionSelfLink, It.IsAny <ChangeFeedOptions>())) .Returns(Mock.Of <IChangeFeedDocumentQuery <Document> >()); this.observer = Mock.Of <IChangeFeedObserver>(); }
public static async Task <DocumentCollection> GetDocumentCollectionAsync( this IChangeFeedDocumentClient client, DocumentCollectionInfo collectionInfo) { Uri collectionUri = UriFactory.CreateDocumentCollectionUri(collectionInfo.DatabaseName, collectionInfo.CollectionName); IResourceResponse <DocumentCollection> response = await client.ReadDocumentCollectionAsync(collectionUri, new RequestOptions()).ConfigureAwait(false); return(response.Resource); }
private static void SetupReadDocument(string storedContinuationToken, IChangeFeedDocumentClient documentClient) { Document storedLeaseDocument = new Document(); storedLeaseDocument.SetPropertyValue("id", leaseId); storedLeaseDocument.SetPropertyValue("PartitionId", partitionId); storedLeaseDocument.SetPropertyValue("ContinuationToken", storedContinuationToken); Mock.Get(documentClient) .Setup(c => c.ReadDocumentAsync(documentUri)) .ReturnsAsync(new ResourceResponse <Document>(storedLeaseDocument)); }
private static void SetupQuery(IChangeFeedDocumentClient documentClient, Mock <IDocumentQuery <Document> > queryMock) { Mock.Get(documentClient) .Setup(c => c.CreateDocumentQuery <Document>(collectionLink, It.Is <SqlQuerySpec>(spec => spec.QueryText == "SELECT * FROM c WHERE STARTSWITH(c.id, @PartitionLeasePrefix)" && spec.Parameters.Count == 1 && spec.Parameters[0].Name == "@PartitionLeasePrefix" && (string)spec.Parameters[0].Value == storeNamePrefix + ".." ))) .Returns(queryMock.As <IQueryable <Document> >().Object); }
public RemainingWorkEstimatorTests() { var document = new Document(); document.SetPropertyValue("_lsn", "10"); documents = new List <Document> { document }; feedResponse = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse) .Setup(response => response.SessionToken) .Returns("0:15"); Mock.Get(feedResponse) .Setup(response => response.Count) .Returns(documents.Count); Mock.Get(feedResponse) .Setup(response => response.GetEnumerator()) .Returns(documents.GetEnumerator()); lease = Mock.Of <ILease>(); Mock.Get(lease) .Setup(l => l.PartitionId) .Returns("partitionId"); leaseManager = Mock.Of <ILeaseManager>(); Mock.Get(leaseManager) .Setup(manager => manager.ListAllLeasesAsync()) .ReturnsAsync(new List <ILease>() { lease }); documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >(); Mock.Get(documentQuery) .Setup(query => query.HasMoreResults) .Returns(false); Mock.Get(documentQuery) .Setup(query => query.ExecuteNextAsync <Document>(It.IsAny <CancellationToken>())) .ReturnsAsync(() => feedResponse) .Callback(() => cancellationTokenSource.Cancel()); docClient = Mock.Of <IChangeFeedDocumentClient>(); Mock.Get(docClient) .Setup(ex => ex.CreateDocumentChangeFeedQuery(collectionSelfLink, It.IsAny <ChangeFeedOptions>())) .Returns(documentQuery); remainingWorkEstimator = new RemainingWorkEstimator(leaseManager, docClient, collectionSelfLink); }
public DocumentServiceLeaseStore( IChangeFeedDocumentClient client, DocumentCollectionInfo leaseCollectionInfo, string containerNamePrefix, string leaseCollectionLink, IRequestOptionsFactory requestOptionsFactory) { this.client = client; this.leaseStoreCollectionInfo = leaseCollectionInfo; this.containerNamePrefix = containerNamePrefix; this.leaseCollectionLink = leaseCollectionLink; this.requestOptionsFactory = requestOptionsFactory; }
public static async Task <Document> TryCreateDocumentAsync(this IChangeFeedDocumentClient client, string collectionLink, object document) { try { IResourceResponse <Document> response = await client.CreateDocumentAsync(collectionLink, document).ConfigureAwait(false); return(response.Resource); } catch (DocumentClientException ex) when(ex.StatusCode == HttpStatusCode.Conflict) { return(null); // Ignore -- document already exists. } }
/// <summary> /// Initializes a new instance of the <see cref="DocumentServiceLeaseStoreManager"/> class. /// </summary> /// <remarks> /// Internal only for testing purposes, otherwise would be private. /// </remarks> internal DocumentServiceLeaseStoreManager( DocumentServiceLeaseStoreManagerSettings settings, IChangeFeedDocumentClient leaseDocumentClient, IRequestOptionsFactory requestOptionsFactory, IDocumentServiceLeaseUpdater leaseUpdater) // For testing purposes only. { if (settings == null) { throw new ArgumentNullException(nameof(settings)); } if (settings.LeaseCollectionInfo == null) { throw new ArgumentNullException(nameof(settings.LeaseCollectionInfo)); } if (settings.ContainerNamePrefix == null) { throw new ArgumentNullException(nameof(settings.ContainerNamePrefix)); } if (settings.LeaseCollectionLink == null) { throw new ArgumentNullException(nameof(settings.LeaseCollectionLink)); } if (string.IsNullOrEmpty(settings.HostName)) { throw new ArgumentNullException(nameof(settings.HostName)); } if (leaseDocumentClient == null) { throw new ArgumentNullException(nameof(leaseDocumentClient)); } if (requestOptionsFactory == null) { throw new ArgumentException(nameof(requestOptionsFactory)); } if (leaseUpdater == null) { throw new ArgumentException(nameof(leaseUpdater)); } this.settings = settings; this.client = leaseDocumentClient; this.requestOptionsFactory = requestOptionsFactory; this.leaseUpdater = leaseUpdater; this.leaseStore = new DocumentServiceLeaseStore( this.client, this.settings.LeaseCollectionInfo, this.settings.ContainerNamePrefix, this.settings.LeaseCollectionLink, this.requestOptionsFactory); }
public PartitionExceptionsTests() { processorSettings = new ProcessorSettings { CollectionSelfLink = "selfLink", FeedPollDelay = TimeSpan.FromMilliseconds(16), MaxItemCount = 5, PartitionKeyRangeId = "keyRangeId", RequestContinuation = "initialToken" }; var document = new Document(); documents = new List <Document> { document }; feedResponse = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse) .Setup(response => response.Count) .Returns(documents.Count); Mock.Get(feedResponse) .Setup(response => response.ResponseContinuation) .Returns("token"); Mock.Get(feedResponse) .Setup(response => response.GetEnumerator()) .Returns(documents.GetEnumerator()); documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >(); Mock.Get(documentQuery) .Setup(query => query.HasMoreResults) .Returns(false); docClient = Mock.Of <IChangeFeedDocumentClient>(); Mock.Get(docClient) .Setup(ex => ex.CreateDocumentChangeFeedQuery(processorSettings.CollectionSelfLink, It.IsAny <ChangeFeedOptions>())) .Returns(documentQuery); observer = Mock.Of <IChangeFeedObserver>(); Mock.Get(observer) .Setup(feedObserver => feedObserver .ProcessChangesAsync(It.IsAny <ChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(false)) .Callback(cancellationTokenSource.Cancel); var checkPointer = new Mock <IPartitionCheckpointer>(); partitionProcessor = new PartitionProcessor(observer, docClient, processorSettings, checkPointer.Object); }
public static async Task <Document> TryDeleteDocumentAsync( this IChangeFeedDocumentClient client, Uri documentUri, RequestOptions requestOptions = null) { try { IResourceResponse <Document> response = await client.DeleteDocumentAsync(documentUri, requestOptions).ConfigureAwait(false); return(response.Resource); } catch (DocumentClientException ex) when(ex.StatusCode == HttpStatusCode.NotFound) { return(null); // Ignore -- document not found. } }
public PartitionProcessor(IChangeFeedObserver observer, IChangeFeedDocumentClient documentClient, ProcessorSettings settings, IPartitionCheckpointer checkpointer) { this.observer = observer; this.settings = settings; this.checkpointer = checkpointer; this.options = new ChangeFeedOptions { MaxItemCount = settings.MaxItemCount, PartitionKeyRangeId = settings.PartitionKeyRangeId, SessionToken = settings.SessionToken, StartFromBeginning = settings.StartFromBeginning, RequestContinuation = settings.RequestContinuation, StartTime = settings.StartTime, }; this.query = documentClient.CreateDocumentChangeFeedQuery(settings.CollectionSelfLink, this.options); }
public static async Task <bool> TryCreateDocumentAsync(this IChangeFeedDocumentClient client, string collectionLink, object document) { try { await client.CreateDocumentAsync(collectionLink, document).ConfigureAwait(false); return(true); } catch (DocumentClientException ex) { if (ex.StatusCode != HttpStatusCode.Conflict) { throw; } } return(false); }