public void ExtractLsnFromSessionToken_ShouldParseNewSessionTokenWithMultipleRegionalLsn()
{
string newTokenWithRegionalLsn = "0:-1#12345#Region1=1#Region2=2";
string expectedLsn = "12345";
Assert.Equal(expectedLsn, RemainingWorkEstimator.ExtractLsnFromSessionToken(newTokenWithRegionalLsn));
}
public void ExtractLsnFromSessionToken_ShouldParseNewSessionToken()
{
string newToken = "0:-1#12345";
string expectedLsn = "12345";
Assert.Equal(expectedLsn, RemainingWorkEstimator.ExtractLsnFromSessionToken(newToken));
}
private FeedEstimator BuildFeedEstimator()
{
if (this.remainingWorkEstimator == null)
{
Func <string, string, bool, FeedIterator> feedCreator = (string partitionKeyRangeId, string continuationToken, bool startFromBeginning) =>
{
return(ResultSetIteratorUtils.BuildResultSetIterator(
partitionKeyRangeId: partitionKeyRangeId,
continuationToken: continuationToken,
maxItemCount: 1,
cosmosContainer: this.monitoredContainer,
startTime: null,
startFromBeginning: string.IsNullOrEmpty(continuationToken)));
};
this.remainingWorkEstimator = new RemainingWorkEstimatorCore(
this.documentServiceLeaseStoreManager.LeaseContainer,
feedCreator,
this.monitoredContainer.ClientContext.Client.ClientOptions?.GatewayModeMaxConnectionLimit ?? 1);
}
ChangeFeedEstimatorDispatcher estimatorDispatcher = new ChangeFeedEstimatorDispatcher(this.initialEstimateDelegate, this.estimatorPeriod);
return(new FeedEstimatorCore(estimatorDispatcher, this.remainingWorkEstimator));
}
internal ChangeFeedEstimatorCore(
Func <long, CancellationToken, Task> initialEstimateDelegate,
TimeSpan?estimatorPeriod,
RemainingWorkEstimator remainingWorkEstimator) : this(initialEstimateDelegate, estimatorPeriod)
{
this.remainingWorkEstimator = remainingWorkEstimator;
}
public void ExtractLsnFromSessionToken_ShouldParseOldSessionToken()
{
string oldToken = "0:12345";
string expectedLsn = "12345";
Assert.Equal(expectedLsn, RemainingWorkEstimator.ExtractLsnFromSessionToken(oldToken));
}
internal ChangeFeedEstimatorCore(
ChangesEstimationHandler initialEstimateDelegate,
TimeSpan?estimatorPeriod,
RemainingWorkEstimator remainingWorkEstimator)
: this(initialEstimateDelegate, estimatorPeriod)
{
this.remainingWorkEstimator = remainingWorkEstimator;
}
public async Task EstimateTotal_ShouldReturnOne_IfNoLeaseDocumentsCreated()
{
IReadOnlyList <ILease> leases = new List <ILease>(0);
var sut = new RemainingWorkEstimator(
Mock.Of <ILeaseContainer>(m => m.GetAllLeasesAsync() == Task.FromResult(leases)),
Mock.Of <IChangeFeedDocumentClient>(),
collectionSelfLink,
1);
long pendingWork = await sut.GetEstimatedRemainingWork();
Assert.Equal(1, pendingWork);
}
public RemainingWorkEstimatorTests()
{
var document = new Document();
document.SetPropertyValue("_lsn", "10");
documents = new List <Document> {
document
};
feedResponse = Mock.Of <IFeedResponse <Document> >();
Mock.Get(feedResponse)
.Setup(response => response.SessionToken)
.Returns("0:15");
Mock.Get(feedResponse)
.Setup(response => response.Count)
.Returns(documents.Count);
Mock.Get(feedResponse)
.Setup(response => response.GetEnumerator())
.Returns(documents.GetEnumerator());
lease = Mock.Of <ILease>();
Mock.Get(lease)
.Setup(l => l.PartitionId)
.Returns("partitionId");
leaseManager = Mock.Of <ILeaseManager>();
Mock.Get(leaseManager)
.Setup(manager => manager.ListAllLeasesAsync())
.ReturnsAsync(new List <ILease>()
{
lease
});
documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.Setup(query => query.HasMoreResults)
.Returns(false);
Mock.Get(documentQuery)
.Setup(query => query.ExecuteNextAsync <Document>(It.IsAny <CancellationToken>()))
.ReturnsAsync(() => feedResponse)
.Callback(() => cancellationTokenSource.Cancel());
docClient = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(docClient)
.Setup(ex => ex.CreateDocumentChangeFeedQuery(collectionSelfLink, It.IsAny <ChangeFeedOptions>()))
.Returns(documentQuery);
remainingWorkEstimator = new RemainingWorkEstimator(leaseManager, docClient, collectionSelfLink);
}
public async Task EstimateTotal_ShouldPropagateExecption_IfUnknownExceptionHappened()
{
IReadOnlyList <ILease> leases = new List <ILease>(0);
var leaseContainer = new Mock <ILeaseContainer>();
leaseContainer.Setup(m => m.GetAllLeasesAsync()).ThrowsAsync(new InvalidOperationException());
var sut = new RemainingWorkEstimator(
leaseContainer.Object,
Mock.Of <IChangeFeedDocumentClient>(),
collectionSelfLink,
1);
await Assert.ThrowsAsync <InvalidOperationException>(sut.GetEstimatedRemainingWork);
}
private FeedEstimator BuildFeedEstimator()
{
if (this.remainingWorkEstimator == null)
{
this.remainingWorkEstimator = new RemainingWorkEstimatorCore(
this.documentServiceLeaseStoreManager.LeaseContainer,
this.monitoredContainer,
this.monitoredContainer.ClientContext.Client.Configuration?.MaxConnectionLimit ?? 1);
}
ChangeFeedEstimatorDispatcher estimatorDispatcher = new ChangeFeedEstimatorDispatcher(this.initialEstimateDelegate, this.estimatorPeriod);
return(new FeedEstimatorCore(estimatorDispatcher, this.remainingWorkEstimator));
}
public async Task EstimateTotal_ShouldReturnPendingWork_IfOnePartition()
{
IReadOnlyList <ILease> leases = new List <ILease> {
Mock.Of <ILease>(l => l.PartitionId == "1" && l.ContinuationToken == "100")
};
var sut = new RemainingWorkEstimator(
Mock.Of <ILeaseContainer>(m => m.GetAllLeasesAsync() == Task.FromResult(leases)),
Mock.Of <IChangeFeedDocumentClient>()
.SetupQueryResponse("1", "100", "101", "1:106"),
collectionSelfLink,
1);
long pendingWork = await sut.GetEstimatedRemainingWork();
Assert.Equal(6, pendingWork);
}
public async Task EstimatePerPartition_ShouldReturnEmpty_WhenNothingSucceeds()
{
IReadOnlyList <ILease> leases = new List <ILease>
{
Mock.Of <ILease>(l => l.PartitionId == "1" && l.ContinuationToken == "100")
};
var sut = new RemainingWorkEstimator(
Mock.Of <ILeaseContainer>(m => m.GetAllLeasesAsync() == Task.FromResult(leases)),
Mock.Of <IChangeFeedDocumentClient>()
.SetupQueryResponseFailure("1", "100"),
collectionSelfLink,
1);
var pendingWork = await sut.GetEstimatedRemainingWorkPerPartitionAsync();
Assert.Empty(pendingWork);
}
public async Task EstimateTotal_ShouldRethrowException_IfEstimatorPartitionWorkThrowsUnknonwException()
{
IReadOnlyList <ILease> leases = new List <ILease>
{
Mock.Of <ILease>(l => l.PartitionId == "1" && l.ContinuationToken == "100"),
Mock.Of <ILease>(l => l.PartitionId == "2" && l.ContinuationToken == "200")
};
var sut = new RemainingWorkEstimator(
Mock.Of <ILeaseContainer>(m => m.GetAllLeasesAsync() == Task.FromResult(leases)),
Mock.Of <IChangeFeedDocumentClient>()
.SetupQueryResponse("1", "100", "101", "1:103")
.SetupQueryResponseFailure("2", "200", new InvalidOperationException()),
collectionSelfLink,
1);
await Assert.ThrowsAsync <InvalidOperationException>(sut.GetEstimatedRemainingWork);
}
public async Task EstimateForOwnedPartition_ShouldReturnPendingWork_IfOnePartition()
{
IEnumerable <ILease> leases = new List <ILease> {
Mock.Of <ILease>(l => l.PartitionId == "1" && l.ContinuationToken == "100")
};
var sut = new RemainingWorkEstimator(
Mock.Of <ILeaseContainer>(m => m.GetOwnedLeasesAsync() == Task.FromResult(leases)),
Mock.Of <IChangeFeedDocumentClient>()
.SetupQueryResponse("1", "100", "101", "1:106"),
collectionSelfLink,
1);
var pendingWork = await sut.GetEstimatedRemainingWorkForOwnedPartitionsAsync();
Assert.Contains(pendingWork, work => work.PartitionKeyRangeId == "1" && work.RemainingWork == 6);
Assert.Single(pendingWork);
}
public async Task EstimatePerPartition_ShouldReturnZero_WhenEmptyResponse()
{
IReadOnlyList <ILease> leases = new List <ILease>
{
Mock.Of <ILease>(l => l.PartitionId == "1" && l.ContinuationToken == "100")
};
var sut = new RemainingWorkEstimator(
Mock.Of <ILeaseContainer>(m => m.GetAllLeasesAsync() == Task.FromResult(leases)),
Mock.Of <IChangeFeedDocumentClient>()
.SetupQueryResponse("1", "100", null, "1:100"),
collectionSelfLink,
1);
var pendingWork = await sut.GetEstimatedRemainingWorkPerPartitionAsync();
Assert.Contains(pendingWork, work => work.PartitionKeyRangeId == "1" && work.RemainingWork == 0);
Assert.Single(pendingWork);
}
/// <summary>
/// Builds a new instance of the <see cref="IRemainingWorkEstimator"/> to estimate pending work with the specified configuration.
/// </summary>
/// <returns>An instance of <see cref="IRemainingWorkEstimator"/>.</returns>
public async Task <IRemainingWorkEstimator> BuildEstimatorAsync()
{
if (this.feedCollectionLocation == null)
{
throw new InvalidOperationException(nameof(this.feedCollectionLocation) + " was not specified");
}
if (this.leaseCollectionLocation == null)
{
throw new InvalidOperationException(nameof(this.leaseCollectionLocation) + " was not specified");
}
await this.InitializeCollectionPropertiesForBuildAsync().ConfigureAwait(false);
ILeaseManager leaseManager = await this.GetLeaseManagerAsync().ConfigureAwait(false);
IRemainingWorkEstimator remainingWorkEstimator = new RemainingWorkEstimator(leaseManager, this.feedDocumentClient, this.feedCollectionLocation.GetCollectionSelfLink());
return(remainingWorkEstimator);
}
public async Task EstimatePerPartition_ShouldRunInParallel_IfDegreeOfParallelismIsTwo()
{
IReadOnlyList <ILease> leases = new List <ILease>
{
Mock.Of <ILease>(l => l.PartitionId == "1" && l.ContinuationToken == "100"),
Mock.Of <ILease>(l => l.PartitionId == "2" && l.ContinuationToken == "200")
};
TaskCompletionSource <bool> cts1 = new TaskCompletionSource <bool>();
TaskCompletionSource <bool> cts2 = new TaskCompletionSource <bool>();
TaskCompletionSource <bool> ctsAll = new TaskCompletionSource <bool>();
var sut = new RemainingWorkEstimator(
Mock.Of <ILeaseContainer>(m => m.GetAllLeasesAsync() == Task.FromResult(leases)),
Mock.Of <IChangeFeedDocumentClient>()
.SetupQueryResponse("1", "100", "101", "1:106", async r =>
{
cts1.SetResult(true);
await ctsAll.Task;
return(r);
})
.SetupQueryResponse("2", "200", "201", "2:-1#201", async r =>
{
cts2.SetResult(true);
await ctsAll.Task;
return(r);
}),
collectionSelfLink,
2);
var workPerPartitionTask = sut.GetEstimatedRemainingWorkPerPartitionAsync();
await Task.WhenAll(cts1.Task, cts2.Task);
ctsAll.SetResult(true);
var pendingWork = await workPerPartitionTask;
Assert.Contains(pendingWork, work => work.PartitionKeyRangeId == "1" && work.RemainingWork == 6);
Assert.Contains(pendingWork, work => work.PartitionKeyRangeId == "2" && work.RemainingWork == 1);
Assert.Equal(2, pendingWork.Count);
}
public async Task EstimatePerPartition_ShouldReturnPendingWork_IfMultiplePartitions()
{
IReadOnlyList <ILease> leases = new List <ILease>
{
Mock.Of <ILease>(l => l.PartitionId == "1" && l.ContinuationToken == "100"),
Mock.Of <ILease>(l => l.PartitionId == "2" && l.ContinuationToken == "200")
};
var sut = new RemainingWorkEstimator(
Mock.Of <ILeaseContainer>(m => m.GetAllLeasesAsync() == Task.FromResult(leases)),
Mock.Of <IChangeFeedDocumentClient>()
.SetupQueryResponse("1", "100", "101", "1:106")
.SetupQueryResponse("2", "200", "201", "2:201"),
collectionSelfLink,
1);
var pendingWork = await sut.GetEstimatedRemainingWorkPerPartitionAsync();
Assert.Contains(pendingWork, work => work.PartitionKeyRangeId == "1" && work.RemainingWork == 6);
Assert.Contains(pendingWork, work => work.PartitionKeyRangeId == "2" && work.RemainingWork == 1);
Assert.Equal(2, pendingWork.Count);
}
/// <summary>
/// Builds a new instance of the <see cref="IRemainingWorkEstimator"/> to estimate pending work with the specified configuration.
/// </summary>
/// <returns>An instance of <see cref="IRemainingWorkEstimator"/>.</returns>
public async Task <IRemainingWorkEstimator> BuildEstimatorAsync()
{
if (this.feedCollectionLocation == null)
{
throw new InvalidOperationException(nameof(this.feedCollectionLocation) + " was not specified");
}
if (this.leaseCollectionLocation == null && this.LeaseStoreManager == null)
{
throw new InvalidOperationException($"Either {nameof(this.leaseCollectionLocation)} or {nameof(this.LeaseStoreManager)} must be specified");
}
await this.InitializeCollectionPropertiesForBuildAsync().ConfigureAwait(false);
var leaseStoreManager = await this.GetLeaseStoreManagerAsync(this.leaseCollectionLocation, true).ConfigureAwait(false);
IRemainingWorkEstimator remainingWorkEstimator = new RemainingWorkEstimator(
leaseStoreManager,
this.feedDocumentClient,
this.feedCollectionLocation.GetCollectionSelfLink(),
this.feedCollectionLocation.ConnectionPolicy.MaxConnectionLimit);
return(remainingWorkEstimator);
}
public FeedEstimatorCore(ChangeFeedEstimatorDispatcher dispatcher, RemainingWorkEstimator remainingWorkEstimator)
{
this.dispatcher = dispatcher;
this.remainingWorkEstimator = remainingWorkEstimator;
this.monitoringDelay = dispatcher.DispatchPeriod ?? FeedEstimatorCore.defaultMonitoringDelay;
}
