private async Task <long> GetRemainingWorkAsync(ILease existingLease)
{
ChangeFeedOptions options = new ChangeFeedOptions
{
MaxItemCount = 1,
PartitionKeyRangeId = existingLease.PartitionId,
RequestContinuation = existingLease.ContinuationToken,
StartFromBeginning = string.IsNullOrEmpty(existingLease.ContinuationToken),
};
IChangeFeedDocumentQuery <Document> query = this.feedDocumentClient.CreateDocumentChangeFeedQuery(this.collectionSelfLink, options);
IFeedResponse <Document> response = null;
try
{
response = await query.ExecuteNextAsync <Document>().ConfigureAwait(false);
long parsedLSNFromSessionToken = TryConvertToNumber(ExtractLsnFromSessionToken(response.SessionToken));
long lastQueryLSN = response.Count > 0
? TryConvertToNumber(GetFirstDocument(response).GetPropertyValue <string>(LSNPropertyName)) - 1
: parsedLSNFromSessionToken;
if (lastQueryLSN < 0)
{
return(1);
}
long partitionRemainingWork = parsedLSNFromSessionToken - lastQueryLSN;
return(partitionRemainingWork < 0 ? 0 : partitionRemainingWork);
}
catch (Exception clientException)
{
Logger.WarnException($"GetEstimateWork > exception: partition '{existingLease.PartitionId}'", clientException);
throw;
}
}
public ChangeFeedQueryTimeoutDecorator(IChangeFeedDocumentQuery <Document> query, IHealthMonitor monitor, TimeSpan timeout, ILease lease)
{
this.query = query;
this.monitor = monitor;
this.timeout = timeout;
this.lease = lease;
}
public PartitionProcessor(IChangeFeedObserver observer, IChangeFeedDocumentQuery <Document> query, ChangeFeedOptions options, ProcessorSettings settings, IPartitionCheckpointer checkpointer)
{
this.observer = observer;
this.settings = settings;
this.checkpointer = checkpointer;
this.options = options;
this.query = query;
}
public RemainingWorkEstimatorTests()
{
var document = new Document();
document.SetPropertyValue("_lsn", "10");
documents = new List <Document> {
document
};
feedResponse = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse)
.Setup(response => response.SessionToken)
.Returns("0:15");
Mock.Get(feedResponse)
.Setup(response => response.Count)
.Returns(documents.Count);
Mock.Get(feedResponse)
.Setup(response => response.GetEnumerator())
.Returns(documents.GetEnumerator());
lease = Mock.Of <ILease>();
Mock.Get(lease)
.Setup(l => l.PartitionId)
.Returns("partitionId");
leaseManager = Mock.Of <ILeaseManager>();
Mock.Get(leaseManager)
.Setup(manager => manager.ListAllLeasesAsync())
.ReturnsAsync(new List <ILease>()
{
lease
});
documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.Setup(query => query.HasMoreResults)
.Returns(false);
Mock.Get(documentQuery)
.Setup(query => query.ExecuteNextAsync <Document>(It.IsAny <CancellationToken>()))
.ReturnsAsync(() => feedResponse)
.Callback(() => cancellationTokenSource.Cancel());
docClient = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(docClient)
.Setup(ex => ex.CreateDocumentChangeFeedQuery(collectionSelfLink, It.IsAny <ChangeFeedOptions>()))
.Returns(documentQuery);
remainingWorkEstimator = new RemainingWorkEstimator(leaseManager, docClient, collectionSelfLink);
}
public async Task <long> GetEstimatedRemainingWork()
{
long remainingWork = 0;
bool hasAtLeastOneLease = false;
ChangeFeedOptions options = new ChangeFeedOptions
{
MaxItemCount = 1,
};
foreach (ILease existingLease in await this.leaseManager.ListAllLeasesAsync().ConfigureAwait(false))
{
hasAtLeastOneLease = true;
options.PartitionKeyRangeId = existingLease.PartitionId;
options.RequestContinuation = existingLease.ContinuationToken;
options.StartFromBeginning = string.IsNullOrEmpty(existingLease.ContinuationToken);
IChangeFeedDocumentQuery <Document> query = this.feedDocumentClient.CreateDocumentChangeFeedQuery(this.collectionSelfLink, options);
IFeedResponse <Document> response = null;
try
{
response = await query.ExecuteNextAsync <Document>().ConfigureAwait(false);
long parsedLSNFromSessionToken = TryConvertToNumber(ExtractLSNFromSessionToken(response.SessionToken));
long lastQueryLSN = response.Count > 0 ?
TryConvertToNumber(GetFirstDocument(response).GetPropertyValue <string>(LSNPropertyName)) - 1
: parsedLSNFromSessionToken;
if (lastQueryLSN < 0)
{
// Could not parse LSN from document, we cannot determine the amount of changes but since the query returned 1 document, we know it's at least 1
remainingWork += 1;
continue;
}
long partitionRemainingWork = parsedLSNFromSessionToken - lastQueryLSN;
remainingWork += partitionRemainingWork < 0 ? 0 : partitionRemainingWork;
}
catch (DocumentClientException clientException)
{
Logger.WarnException("GetEstimateWork > exception: partition '{0}'", clientException, existingLease.PartitionId);
}
}
if (!hasAtLeastOneLease)
{
return(1);
}
return(remainingWork);
}
public PartitionExceptionsTests()
{
processorSettings = new ProcessorSettings
{
CollectionSelfLink = "selfLink",
FeedPollDelay = TimeSpan.FromMilliseconds(16),
MaxItemCount = 5,
PartitionKeyRangeId = "keyRangeId",
RequestContinuation = "initialToken"
};
var document = new Document();
documents = new List <Document> {
document
};
feedResponse = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse)
.Setup(response => response.Count)
.Returns(documents.Count);
Mock.Get(feedResponse)
.Setup(response => response.ResponseContinuation)
.Returns("token");
Mock.Get(feedResponse)
.Setup(response => response.GetEnumerator())
.Returns(documents.GetEnumerator());
documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.Setup(query => query.HasMoreResults)
.Returns(false);
docClient = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(docClient)
.Setup(ex => ex.CreateDocumentChangeFeedQuery(processorSettings.CollectionSelfLink, It.IsAny <ChangeFeedOptions>()))
.Returns(documentQuery);
observer = Mock.Of <IChangeFeedObserver>();
Mock.Get(observer)
.Setup(feedObserver => feedObserver
.ProcessChangesAsync(It.IsAny <ChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(false))
.Callback(cancellationTokenSource.Cancel);
var checkPointer = new Mock <IPartitionCheckpointer>();
partitionProcessor = new PartitionProcessor(observer, docClient, processorSettings, checkPointer.Object);
}
public PartitionProcessorTests()
{
processorSettings = new ProcessorSettings
{
CollectionSelfLink = "selfLink",
FeedPollDelay = TimeSpan.FromMilliseconds(16),
MaxItemCount = 5,
PartitionKeyRangeId = "keyRangeId",
StartContinuation = "initialToken"
};
var document = new Document();
documents = new List <Document> {
document
};
feedResponse = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse)
.Setup(response => response.Count)
.Returns(documents.Count);
Mock.Get(feedResponse)
.Setup(response => response.ResponseContinuation)
.Returns("token");
Mock.Get(feedResponse)
.Setup(response => response.GetEnumerator())
.Returns(documents.GetEnumerator());
documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.Setup(query => query.HasMoreResults)
.Returns(false);
Mock.Get(documentQuery)
.Setup(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)))
.ReturnsAsync(() => feedResponse)
.Callback(() => cancellationTokenSource.Cancel());
observer = Mock.Of <IChangeFeedObserver>();
var checkPointer = new Mock <IPartitionCheckpointer>();
sut = new PartitionProcessor(
new ObserverExceptionWrappingChangeFeedObserverDecorator(observer),
documentQuery,
new ChangeFeedOptions(),
processorSettings,
checkPointer.Object);
}
public PartitionProcessor(IChangeFeedObserver observer, IChangeFeedDocumentClient documentClient, ProcessorSettings settings, IPartitionCheckpointer checkpointer)
{
this.observer = observer;
this.settings = settings;
this.checkpointer = checkpointer;
this.options = new ChangeFeedOptions
{
MaxItemCount = settings.MaxItemCount,
PartitionKeyRangeId = settings.PartitionKeyRangeId,
SessionToken = settings.SessionToken,
StartFromBeginning = settings.StartFromBeginning,
RequestContinuation = settings.RequestContinuation,
StartTime = settings.StartTime,
};
this.query = documentClient.CreateDocumentChangeFeedQuery(settings.CollectionSelfLink, this.options);
}
public PartitionChainingTests()
{
processorSettings = new ProcessorSettings
{
CollectionSelfLink = "selfLink",
MaxItemCount = 5,
FeedPollDelay = TimeSpan.FromMilliseconds(16),
PartitionKeyRangeId = "keyRangeId",
RequestContinuation = "initialToken"
};
var document = new Document();
batch1 = new List <Document> {
document
};
document = new Document();
batch2 = new List <Document> {
document
};
feedResponse1 = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse1)
.Setup(response => response.Count)
.Returns(batch1.Count);
Mock.Get(feedResponse1)
.SetupSequence(response => response.ResponseContinuation)
.Returns("token1");
Mock.Get(feedResponse1)
.SetupSequence(response => response.GetEnumerator())
.Returns(batch1.GetEnumerator());
feedResponse2 = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse2)
.Setup(response => response.Count)
.Returns(batch2.Count);
Mock.Get(feedResponse2)
.SetupSequence(response => response.ResponseContinuation)
.Returns("token2");
Mock.Get(feedResponse2)
.SetupSequence(response => response.GetEnumerator())
.Returns(batch2.GetEnumerator());
documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.SetupSequence(query => query.HasMoreResults)
.Returns(true)
.Returns(false);
Mock.Get(documentQuery)
.SetupSequence(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)))
.Returns(Task.FromResult(feedResponse1))
.Returns(Task.FromResult(feedResponse2));
docClient = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(docClient)
.Setup(ex => ex.CreateDocumentChangeFeedQuery(processorSettings.CollectionSelfLink, It.IsAny <ChangeFeedOptions>()))
.Returns(documentQuery);
observer = Mock.Of <IChangeFeedObserver>();
var checkPointer = new Mock <IPartitionCheckpointer>();
partitionProcessor = new PartitionProcessor(observer, docClient, processorSettings, checkPointer.Object);
var i = 0;
Mock.Get(observer)
.Setup(feedObserver => feedObserver
.ProcessChangesAsync(It.IsAny <ChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(false))
.Callback(() =>
{
if (++i == 2)
{
cancellationTokenSource.Cancel();
}
});
}
public QoSMeteringChangeFeedDocumentQuery(IChangeFeedDocumentQuery <Document> inner, string partitionRangeId, IQoSMeteringReporter meter)
{
_inner = inner ?? throw new ArgumentNullException(nameof(inner));
_partitionRangeId = partitionRangeId ?? throw new ArgumentNullException(nameof(partitionRangeId));
_meter = meter ?? throw new ArgumentNullException(nameof(meter));
}
