private async Task <List <PartitionKeyRange> > EnumPartitionKeyRangesAsync() { string containerUri = this.container.LinkUri.ToString(); string partitionKeyRangesPath = string.Format(CultureInfo.InvariantCulture, "{0}/pkranges", containerUri); IFeedResponse <PartitionKeyRange> response = null; List <PartitionKeyRange> partitionKeyRanges = new List <PartitionKeyRange>(); do { FeedOptions feedOptions = new FeedOptions { MaxItemCount = this.maxBatchSize, RequestContinuation = response?.ResponseContinuation, }; response = await this.container.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(containerUri, feedOptions).ConfigureAwait(false); IEnumerator <PartitionKeyRange> enumerator = response.GetEnumerator(); while (enumerator.MoveNext()) { partitionKeyRanges.Add(enumerator.Current); } }while (!string.IsNullOrEmpty(response.ResponseContinuation)); return(partitionKeyRanges); }
public AutoCheckPointTests() { changeFeedObserver = Mock.Of <IChangeFeedObserver>(); partitionCheckpointer = Mock.Of <IPartitionCheckpointer>(); Mock.Get(partitionCheckpointer) .Setup(checkPointer => checkPointer.CheckpointPartitionAsync(It.IsAny <string>())) .Returns(Task.CompletedTask); checkpointFrequency = new CheckpointFrequency(); sut = new AutoCheckpointer(checkpointFrequency, changeFeedObserver); documents = Mock.Of <IReadOnlyList <Document> >(); feedResponse = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse) .Setup(response => response.Count) .Returns(documents.Count); Mock.Get(feedResponse) .Setup(response => response.ResponseContinuation) .Returns("token"); Mock.Get(feedResponse) .Setup(response => response.GetEnumerator()) .Returns(documents.GetEnumerator()); observerContext = Mock.Of <IChangeFeedObserverContext>(); Mock.Get(observerContext) .Setup(context => context.CheckpointAsync()) .Returns(partitionCheckpointer.CheckpointPartitionAsync("token")); }
private async Task <long> GetRemainingWorkAsync(ILease existingLease) { ChangeFeedOptions options = new ChangeFeedOptions { MaxItemCount = 1, PartitionKeyRangeId = existingLease.PartitionId, RequestContinuation = existingLease.ContinuationToken, StartFromBeginning = string.IsNullOrEmpty(existingLease.ContinuationToken), }; IChangeFeedDocumentQuery <Document> query = this.feedDocumentClient.CreateDocumentChangeFeedQuery(this.collectionSelfLink, options); IFeedResponse <Document> response = null; try { response = await query.ExecuteNextAsync <Document>().ConfigureAwait(false); long parsedLSNFromSessionToken = TryConvertToNumber(ExtractLsnFromSessionToken(response.SessionToken)); long lastQueryLSN = response.Count > 0 ? TryConvertToNumber(GetFirstDocument(response).GetPropertyValue <string>(LSNPropertyName)) - 1 : parsedLSNFromSessionToken; if (lastQueryLSN < 0) { return(1); } long partitionRemainingWork = parsedLSNFromSessionToken - lastQueryLSN; return(partitionRemainingWork < 0 ? 0 : partitionRemainingWork); } catch (Exception clientException) { Logger.WarnException($"GetEstimateWork > exception: partition '{existingLease.PartitionId}'", clientException); throw; } }
public static ResponseInformation FromSubscriptionReadResponse(IFeedResponse <Document> response) { return(new ResponseInformation { CurrentResourceQuotaUsage = response.CurrentResourceQuotaUsage, MaxResourceQuota = response.MaxResourceQuota, RequestCharge = response.RequestCharge, ResponseHeaders = response.ResponseHeaders }); }
public static ResponseInformation FromReadResponse(string requestIdentifier, IFeedResponse <DocumentDbStorageEvent> response) { return(new ResponseInformation { RequestIdentifier = requestIdentifier, CurrentResourceQuotaUsage = response.CurrentResourceQuotaUsage, MaxResourceQuota = response.MaxResourceQuota, RequestCharge = response.RequestCharge, ResponseHeaders = response.ResponseHeaders }); }
private static Document GetFirstDocument(IFeedResponse <Document> response) { using (IEnumerator <Document> e = response.GetEnumerator()) { while (e.MoveNext()) { return(e.Current); } } return(null); }
public RemainingWorkEstimatorTests() { var document = new Document(); document.SetPropertyValue("_lsn", "10"); documents = new List <Document> { document }; feedResponse = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse) .Setup(response => response.SessionToken) .Returns("0:15"); Mock.Get(feedResponse) .Setup(response => response.Count) .Returns(documents.Count); Mock.Get(feedResponse) .Setup(response => response.GetEnumerator()) .Returns(documents.GetEnumerator()); lease = Mock.Of <ILease>(); Mock.Get(lease) .Setup(l => l.PartitionId) .Returns("partitionId"); leaseManager = Mock.Of <ILeaseManager>(); Mock.Get(leaseManager) .Setup(manager => manager.ListAllLeasesAsync()) .ReturnsAsync(new List <ILease>() { lease }); documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >(); Mock.Get(documentQuery) .Setup(query => query.HasMoreResults) .Returns(false); Mock.Get(documentQuery) .Setup(query => query.ExecuteNextAsync <Document>(It.IsAny <CancellationToken>())) .ReturnsAsync(() => feedResponse) .Callback(() => cancellationTokenSource.Cancel()); docClient = Mock.Of <IChangeFeedDocumentClient>(); Mock.Get(docClient) .Setup(ex => ex.CreateDocumentChangeFeedQuery(collectionSelfLink, It.IsAny <ChangeFeedOptions>())) .Returns(documentQuery); remainingWorkEstimator = new RemainingWorkEstimator(leaseManager, docClient, collectionSelfLink); }
public async Task <long> GetEstimatedRemainingWork() { long remainingWork = 0; bool hasAtLeastOneLease = false; ChangeFeedOptions options = new ChangeFeedOptions { MaxItemCount = 1, }; foreach (ILease existingLease in await this.leaseManager.ListAllLeasesAsync().ConfigureAwait(false)) { hasAtLeastOneLease = true; options.PartitionKeyRangeId = existingLease.PartitionId; options.RequestContinuation = existingLease.ContinuationToken; options.StartFromBeginning = string.IsNullOrEmpty(existingLease.ContinuationToken); IChangeFeedDocumentQuery <Document> query = this.feedDocumentClient.CreateDocumentChangeFeedQuery(this.collectionSelfLink, options); IFeedResponse <Document> response = null; try { response = await query.ExecuteNextAsync <Document>().ConfigureAwait(false); long parsedLSNFromSessionToken = TryConvertToNumber(ExtractLSNFromSessionToken(response.SessionToken)); long lastQueryLSN = response.Count > 0 ? TryConvertToNumber(GetFirstDocument(response).GetPropertyValue <string>(LSNPropertyName)) - 1 : parsedLSNFromSessionToken; if (lastQueryLSN < 0) { // Could not parse LSN from document, we cannot determine the amount of changes but since the query returned 1 document, we know it's at least 1 remainingWork += 1; continue; } long partitionRemainingWork = parsedLSNFromSessionToken - lastQueryLSN; remainingWork += partitionRemainingWork < 0 ? 0 : partitionRemainingWork; } catch (DocumentClientException clientException) { Logger.WarnException("GetEstimateWork > exception: partition '{0}'", clientException, existingLease.PartitionId); } } if (!hasAtLeastOneLease) { return(1); } return(remainingWork); }
public PartitionExceptionsTests() { processorSettings = new ProcessorSettings { CollectionSelfLink = "selfLink", FeedPollDelay = TimeSpan.FromMilliseconds(16), MaxItemCount = 5, PartitionKeyRangeId = "keyRangeId", RequestContinuation = "initialToken" }; var document = new Document(); documents = new List <Document> { document }; feedResponse = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse) .Setup(response => response.Count) .Returns(documents.Count); Mock.Get(feedResponse) .Setup(response => response.ResponseContinuation) .Returns("token"); Mock.Get(feedResponse) .Setup(response => response.GetEnumerator()) .Returns(documents.GetEnumerator()); documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >(); Mock.Get(documentQuery) .Setup(query => query.HasMoreResults) .Returns(false); docClient = Mock.Of <IChangeFeedDocumentClient>(); Mock.Get(docClient) .Setup(ex => ex.CreateDocumentChangeFeedQuery(processorSettings.CollectionSelfLink, It.IsAny <ChangeFeedOptions>())) .Returns(documentQuery); observer = Mock.Of <IChangeFeedObserver>(); Mock.Get(observer) .Setup(feedObserver => feedObserver .ProcessChangesAsync(It.IsAny <ChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(false)) .Callback(cancellationTokenSource.Cancel); var checkPointer = new Mock <IPartitionCheckpointer>(); partitionProcessor = new PartitionProcessor(observer, docClient, processorSettings, checkPointer.Object); }
private Task DispatchChanges(IFeedResponse <Document> response, CancellationToken cancellationToken) { IChangeFeedObserverContext context = new ChangeFeedObserverContext(this.settings.PartitionKeyRangeId, response, this.checkpointer); var docs = new List <Document>(response.Count); using (IEnumerator <Document> e = response.GetEnumerator()) { while (e.MoveNext()) { docs.Add(e.Current); } } return(this.observer.ProcessChangesAsync(context, docs, cancellationToken)); }
private Task DispatchChanges(IFeedResponse <T> response, CancellationToken cancellationToken) { ChangeFeedObserverContext context = new ChangeFeedObserverContextCore <T>(this.settings.LeaseToken, response, this.checkpointer); List <T> docs = new List <T>(response.Count); using (IEnumerator <T> e = response.GetEnumerator()) { while (e.MoveNext()) { docs.Add(e.Current); } } return(this.observer.ProcessChangesAsync(context, docs, cancellationToken)); }
public PartitionProcessorTests() { processorSettings = new ProcessorSettings { CollectionSelfLink = "selfLink", FeedPollDelay = TimeSpan.FromMilliseconds(16), MaxItemCount = 5, PartitionKeyRangeId = "keyRangeId", StartContinuation = "initialToken" }; var document = new Document(); documents = new List <Document> { document }; feedResponse = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse) .Setup(response => response.Count) .Returns(documents.Count); Mock.Get(feedResponse) .Setup(response => response.ResponseContinuation) .Returns("token"); Mock.Get(feedResponse) .Setup(response => response.GetEnumerator()) .Returns(documents.GetEnumerator()); documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >(); Mock.Get(documentQuery) .Setup(query => query.HasMoreResults) .Returns(false); Mock.Get(documentQuery) .Setup(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token))) .ReturnsAsync(() => feedResponse) .Callback(() => cancellationTokenSource.Cancel()); observer = Mock.Of <IChangeFeedObserver>(); var checkPointer = new Mock <IPartitionCheckpointer>(); sut = new PartitionProcessor( new ObserverExceptionWrappingChangeFeedObserverDecorator(observer), documentQuery, new ChangeFeedOptions(), processorSettings, checkPointer.Object); }
private async Task <string> ProcessBatch(CancellationToken cancellation) { string lastContinuation; do { IFeedResponse <Document> response = await this.query.ExecuteNextAsync <Document>(cancellation).ConfigureAwait(false); lastContinuation = response.ResponseContinuation; if (response.Count > 0) { await this.DispatchChanges(response, cancellation).ConfigureAwait(false); } }while (this.query.HasMoreResults && !cancellation.IsCancellationRequested); return(lastContinuation); }
public async Task Estimate_ShouldReturnZero_WhenEmpty() { var document = new Document(); documents = new List <Document> { }; feedResponse = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse) .Setup(response => response.SessionToken) .Returns("0:15"); Mock.Get(feedResponse) .Setup(response => response.Count) .Returns(documents.Count); Mock.Get(feedResponse) .Setup(response => response.GetEnumerator()) .Returns(documents.GetEnumerator()); long pendingWork = await remainingWorkEstimator.GetEstimatedRemainingWork(); Assert.Equal(0, pendingWork); }
internal ChangeFeedObserverContext(string partitionId, IFeedResponse <Document> feedResponse, IPartitionCheckpointer checkpointer) { this.PartitionKeyRangeId = partitionId; this.FeedResponse = feedResponse; this.checkpointer = checkpointer; }
public PartitionChainingTests() { processorSettings = new ProcessorSettings { CollectionSelfLink = "selfLink", MaxItemCount = 5, FeedPollDelay = TimeSpan.FromMilliseconds(16), PartitionKeyRangeId = "keyRangeId", RequestContinuation = "initialToken" }; var document = new Document(); batch1 = new List <Document> { document }; document = new Document(); batch2 = new List <Document> { document }; feedResponse1 = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse1) .Setup(response => response.Count) .Returns(batch1.Count); Mock.Get(feedResponse1) .SetupSequence(response => response.ResponseContinuation) .Returns("token1"); Mock.Get(feedResponse1) .SetupSequence(response => response.GetEnumerator()) .Returns(batch1.GetEnumerator()); feedResponse2 = Mock.Of <IFeedResponse <Document> >(); Mock.Get(feedResponse2) .Setup(response => response.Count) .Returns(batch2.Count); Mock.Get(feedResponse2) .SetupSequence(response => response.ResponseContinuation) .Returns("token2"); Mock.Get(feedResponse2) .SetupSequence(response => response.GetEnumerator()) .Returns(batch2.GetEnumerator()); documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >(); Mock.Get(documentQuery) .SetupSequence(query => query.HasMoreResults) .Returns(true) .Returns(false); Mock.Get(documentQuery) .SetupSequence(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token))) .Returns(Task.FromResult(feedResponse1)) .Returns(Task.FromResult(feedResponse2)); docClient = Mock.Of <IChangeFeedDocumentClient>(); Mock.Get(docClient) .Setup(ex => ex.CreateDocumentChangeFeedQuery(processorSettings.CollectionSelfLink, It.IsAny <ChangeFeedOptions>())) .Returns(documentQuery); observer = Mock.Of <IChangeFeedObserver>(); var checkPointer = new Mock <IPartitionCheckpointer>(); partitionProcessor = new PartitionProcessor(observer, docClient, processorSettings, checkPointer.Object); var i = 0; Mock.Get(observer) .Setup(feedObserver => feedObserver .ProcessChangesAsync(It.IsAny <ChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(false)) .Callback(() => { if (++i == 2) { cancellationTokenSource.Cancel(); } }); }
public async Task RunAsync(CancellationToken cancellationToken) { string lastContinuation = this.settings.StartContinuation; while (!cancellationToken.IsCancellationRequested) { TimeSpan delay = this.settings.FeedPollDelay; try { do { IFeedResponse <Document> response = await this.query.ExecuteNextAsync <Document>(cancellationToken).ConfigureAwait(false); lastContinuation = response.ResponseContinuation; if (response.Count > 0) { await this.DispatchChanges(response, cancellationToken).ConfigureAwait(false); } }while (this.query.HasMoreResults && !cancellationToken.IsCancellationRequested); if (this.options.MaxItemCount != this.settings.MaxItemCount) { this.options.MaxItemCount = this.settings.MaxItemCount; // Reset after successful execution. } } catch (DocumentClientException clientException) { this.logger.WarnException("exception: partition '{0}'", clientException, this.settings.PartitionKeyRangeId); DocDbError docDbError = ExceptionClassifier.ClassifyClientException(clientException); switch (docDbError) { case DocDbError.PartitionNotFound: throw new PartitionNotFoundException("Partition not found.", lastContinuation); case DocDbError.PartitionSplit: throw new PartitionSplitException("Partition split.", lastContinuation); case DocDbError.Undefined: throw; case DocDbError.TransientError: // Retry on transient (429) errors break; case DocDbError.MaxItemCountTooLarge: if (!this.options.MaxItemCount.HasValue) { this.options.MaxItemCount = DefaultMaxItemCount; } else if (this.options.MaxItemCount <= 1) { this.logger.ErrorFormat("Cannot reduce maxItemCount further as it's already at {0}.", this.options.MaxItemCount); throw; } this.options.MaxItemCount /= 2; this.logger.WarnFormat("Reducing maxItemCount, new value: {0}.", this.options.MaxItemCount); break; default: this.logger.Fatal($"Unrecognized DocDbError enum value {docDbError}"); Debug.Fail($"Unrecognized DocDbError enum value {docDbError}"); throw; } if (clientException.RetryAfter != TimeSpan.Zero) { delay = clientException.RetryAfter; } } catch (TaskCanceledException canceledException) { if (cancellationToken.IsCancellationRequested) { throw; } this.logger.WarnException("exception: partition '{0}'", canceledException, this.settings.PartitionKeyRangeId); // ignore as it is caused by DocumentDB client } await Task.Delay(delay, cancellationToken).ConfigureAwait(false); } }
internal ChangeFeedObserverContextCore(string leaseToken, IFeedResponse <T> feedResponse, PartitionCheckpointer checkpointer) { this.LeaseToken = leaseToken; this.FeedResponse = feedResponse; this.checkpointer = checkpointer; }
public async Task ProcessResponse <T>(IFeedResponse <T> feedResponse) { await ProcessResponse(feedResponse.SessionToken, feedResponse.RequestCharge, statusCode : null); }