public async Task UpdateLeaseAsync_ShouldThrowLeaseLostException_WhenConflictAfterAllRetries()
{
var client = Mock.Of <IChangeFeedDocumentClient>();
var updater = new DocumentServiceLeaseUpdater(client);
ILease oldLease = CreateLease();
const int retryCount = 5;
var getDocumentSequence = Mock.Get(client)
.SetupSequence(c => c.ReadDocumentAsync(documentUri));
for (int i = 0; i <= retryCount; i++)
{
string eTag = i.ToString();
Mock.Get(client)
.Setup(c => c.ReplaceDocumentAsync(
documentUri,
It.Is <ILease>(lease => lease.ConcurrencyToken == eTag),
It.Is <RequestOptions>(options => options.AccessCondition.Type == AccessConditionType.IfMatch && options.AccessCondition.Condition == eTag)))
.ThrowsAsync(DocumentExceptionHelpers.CreatePreconditionFailedException());
getDocumentSequence = getDocumentSequence.ReturnsAsync(CreateLeaseResponse(eTag));
}
int callbackInvokeCount = 0;
Exception exception = await Record.ExceptionAsync(async() => await updater.UpdateLeaseAsync(oldLease, documentUri, serverLease =>
{
callbackInvokeCount++;
if (serverLease.ConcurrencyToken == null)
{
return(CreateLease("0"));
}
return(CreateLease((int.Parse(serverLease.ConcurrencyToken) + 1).ToString()));
}));
Assert.IsAssignableFrom <LeaseLostException>(exception);
Assert.Equal(retryCount + 1, callbackInvokeCount);
}
public async Task UpdateLeaseAsync_ShouldThrowLeaseLostException_WhenConflictAndRetriesDisabled()
{
var client = Mock.Of <IChangeFeedDocumentClient>();
var updater = new DocumentServiceLeaseUpdater(client);
const string etag = "1";
ILease oldLease = CreateLease();
Mock.Get(client)
.Setup(c => c.ReplaceDocumentAsync(
documentUri,
It.Is <ILease>(lease => lease.ConcurrencyToken == etag),
It.Is <RequestOptions>(options => options.AccessCondition.Type == AccessConditionType.IfMatch && options.AccessCondition.Condition == etag),
default(CancellationToken)))
.ThrowsAsync(DocumentExceptionHelpers.CreatePreconditionFailedException());
Mock.Get(client)
.Setup(c => c.ReadDocumentAsync(documentUri, null, default(CancellationToken)))
.ReturnsAsync(CreateLeaseResponse(etag));
int callbackInvokeCount = 0;
Exception exception = await Record.ExceptionAsync(async() => await updater.UpdateLeaseAsync(oldLease, documentUri, null, serverLease =>
{
callbackInvokeCount++;
return(CreateLease(etag));
}, false));
Assert.IsAssignableFrom <LeaseLostException>(exception);
Assert.Equal(1, callbackInvokeCount);
}
public async Task Run_ShouldThrow_IfObserverThrowsDocumentClientException()
{
// If the user code throws a DCE, we should bubble it up to stop the Observer and not treat it as a DCE from the Feed Query
Mock.Get(documentQuery)
.Reset();
Mock.Get(documentQuery)
.Setup(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)))
.ReturnsAsync(feedResponse)
.Callback(() => cancellationTokenSource.Cancel());
Mock.Get(observer)
.Setup(feedObserver => feedObserver
.ProcessChangesAsync(It.IsAny <IChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>()))
.Throws(DocumentExceptionHelpers.CreateRequestRateTooLargeException());
Exception exception = await Record.ExceptionAsync(() => sut.RunAsync(cancellationTokenSource.Token));
Assert.IsAssignableFrom <ObserverException>(exception);
Assert.IsAssignableFrom <DocumentClientException>(exception.InnerException);
Mock.Get(documentQuery)
.Verify(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)), Times.Once);
Mock.Get(observer)
.Verify(feedObserver => feedObserver
.ProcessChangesAsync(
It.Is <IChangeFeedObserverContext>(context => context.PartitionKeyRangeId == processorSettings.PartitionKeyRangeId),
It.Is <IReadOnlyList <Document> >(list => list.SequenceEqual(documents)),
It.IsAny <CancellationToken>()),
Times.Once);
}
public async Task InitializeAsync_ShouldReleaseLock_OnException()
{
var synchronizer = Mock.Of <IPartitionSynchronizer>();
Mock.Get(synchronizer)
.Setup(s => s.CreateMissingLeasesAsync())
.ThrowsAsync(DocumentExceptionHelpers.CreateConflictException());
var leaseStore = Mock.Of <ILeaseStore>();
Mock.Get(leaseStore)
.Setup(store => store.IsInitializedAsync())
.ReturnsAsync(false);
Mock.Get(leaseStore)
.Setup(store => store.AcquireInitializationLockAsync(lockTime))
.ReturnsAsync(true);
Mock.Get(leaseStore)
.Setup(store => store.ReleaseInitializationLockAsync())
.ReturnsAsync(true);
var bootstrapper = new Bootstrapper(synchronizer, leaseStore, lockTime, sleepTime);
Exception exception = await Record.ExceptionAsync(async() => await bootstrapper.InitializeAsync());
Assert.IsAssignableFrom <DocumentClientException>(exception);
Mock.Get(leaseStore).VerifyAll();
}
private void SetupReplaceConflict(IChangeFeedDocumentClient client, ILease updatedLease)
{
Mock.Get(client)
.Setup(c => c.ReplaceDocumentAsync(
documentUri,
updatedLease,
It.Is <RequestOptions>(options => options.AccessCondition.Type == AccessConditionType.IfMatch && options.AccessCondition.Condition == eTag1)))
.ThrowsAsync(DocumentExceptionHelpers.CreatePreconditionFailedException());
}
public async Task Run_ShouldThrowSplit_IfPartitionSplitting()
{
Mock.Get(documentQuery)
.SetupSequence(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)))
.Throws(DocumentExceptionHelpers.CreateException("Microsoft.Azure.Documents.GoneException", 1007))
.ReturnsAsync(feedResponse);
await Assert.ThrowsAsync <PartitionSplitException>(() => partitionProcessor.RunAsync(cancellationTokenSource.Token));
}
public async Task Run_ShouldRethrowReadSessionNotAvailable()
{
Mock.Get(documentQuery)
.SetupSequence(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)))
.Throws(DocumentExceptionHelpers.CreateException("Microsoft.Azure.Documents.NotFoundException", (int)SubStatusCode.ReadSessionNotAvailable));
Exception exception = await Record.ExceptionAsync(() => partitionProcessor.RunAsync(cancellationTokenSource.Token));
Assert.IsAssignableFrom <ReadSessionNotAvailableException>(exception);
}
public async Task Run_ShouldReThrow_IfUnknownNotFoundSubcode()
{
Mock.Get(documentQuery)
.SetupSequence(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)))
.Throws(DocumentExceptionHelpers.CreateException("Microsoft.Azure.Documents.NotFoundException", 1002))
.ReturnsAsync(feedResponse);
Exception exception = await Record.ExceptionAsync(() => partitionProcessor.RunAsync(cancellationTokenSource.Token));
Assert.IsAssignableFrom <DocumentClientException>(exception);
}
public async Task MarkInitializedAsync_ShouldThrow_IfMarkerThrows()
{
var client = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(client)
.Setup(c => c.CreateDocumentAsync(It.IsAny <string>(), It.IsAny <object>(), null, false, default(CancellationToken)))
.ThrowsAsync(DocumentExceptionHelpers.CreateRequestRateTooLargeException());
var leaseStore = new DocumentServiceLeaseStore(client, collectionInfo, containerNamePrefix, leaseCollectionLink, Mock.Of <IRequestOptionsFactory>());
Exception exception = await Record.ExceptionAsync(async() => await leaseStore.MarkInitializedAsync());
Assert.IsAssignableFrom <DocumentClientException>(exception);
}
public async Task IsInitializedAsync_ShouldReturnFalse_IfNoDocument()
{
var client = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(client)
.Setup(c => c.ReadDocumentAsync(It.Is <Uri>(uri => uri.ToString().EndsWith(storeMarker))))
.ThrowsAsync(DocumentExceptionHelpers.CreateNotFoundException());
var leaseStore = new LeaseStore(client, collectionInfo, containerNamePrefix, leaseStoreCollectionLink);
bool isInited = await leaseStore.IsInitializedAsync();
Assert.False(isInited);
}
public async Task LockInitializationAsync_ShouldThrow_IfLockThrows()
{
var client = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(client)
.Setup(c => c.CreateDocumentAsync(It.IsAny <string>(), It.IsAny <object>()))
.ThrowsAsync(DocumentExceptionHelpers.CreateRequestRateTooLargeException());
var leaseStore = new LeaseStore(client, collectionInfo, containerNamePrefix, leaseStoreCollectionLink);
Exception exception = await Record.ExceptionAsync(() => leaseStore.LockInitializationAsync(lockTime));
Assert.IsAssignableFrom <DocumentClientException>(exception);
}
public async Task ReleaseInitializationLockAsync_ShouldThrow_IfLockThrows()
{
var client = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(client)
.Setup(c => c.DeleteDocumentAsync(It.IsAny <Uri>(), It.IsAny <RequestOptions>(), default(CancellationToken)))
.ThrowsAsync(DocumentExceptionHelpers.CreateRequestRateTooLargeException());
var leaseStore = new DocumentServiceLeaseStore(client, collectionInfo, containerNamePrefix, leaseCollectionLink, Mock.Of <IRequestOptionsFactory>());
Exception exception = await Record.ExceptionAsync(() => leaseStore.ReleaseInitializationLockAsync());
Assert.IsAssignableFrom <DocumentClientException>(exception);
}
public async Task ReleaseAsync_ReThrows_WhenOtherDocumentException()
{
var documentClient = Mock.Of <IChangeFeedDocumentClient>();
var cachedLease = CreateCachedLease(owner);
var leaseStoreManager = CreateLeaseStoreManager(documentClient, owner);
Mock.Get(documentClient)
.Setup(c => c.ReadDocumentAsync(It.IsAny <Uri>(), null, default(CancellationToken)))
.ThrowsAsync(DocumentExceptionHelpers.CreateConflictException());
var exception = await Record.ExceptionAsync(async() => await leaseStoreManager.ReleaseAsync(cachedLease));
Assert.IsAssignableFrom <DocumentClientException>(exception);
}
public async Task ReleaseAsync_ThrowsLeaseLost_WhenDocumentIsNotFound()
{
var documentClient = Mock.Of <IChangeFeedDocumentClient>();
var cachedLease = CreateCachedLease(owner);
var leaseUpdater = Mock.Of <IDocumentServiceLeaseUpdater>();
var leaseManager = CreateLeaseManager(documentClient, leaseUpdater, owner);
Mock.Get(documentClient)
.Setup(c => c.ReadDocumentAsync(It.IsAny <Uri>()))
.ThrowsAsync(DocumentExceptionHelpers.CreateNotFoundException());
var exception = await Record.ExceptionAsync(async() => await leaseManager.ReleaseAsync(cachedLease));
Assert.IsAssignableFrom <LeaseLostException>(exception);
}
public async Task MarkInitializedAsync_ShouldSucceed_IfMarkerConflicts()
{
var client = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(client)
.Setup(c => c.CreateDocumentAsync(It.IsAny <string>(), It.IsAny <object>()))
.ThrowsAsync(DocumentExceptionHelpers.CreateConflictException());
var leaseStore = new LeaseStore(client, collectionInfo, containerNamePrefix, leaseStoreCollectionLink);
await leaseStore.MarkInitializedAsync();
Mock.Get(client)
.Verify(c =>
c.CreateDocumentAsync(leaseStoreCollectionLink, It.Is <Document>(d => d.Id == storeMarker)),
Times.Once);
}
public async Task DeleteAsync_Throws_IfDeleteFails()
{
var documentClient = Mock.Of <IChangeFeedDocumentClient>();
var cachedLease = CreateCachedLease(owner);
var leaseStoreManager = CreateLeaseStoreManager(documentClient, owner);
Mock.Get(documentClient)
.Setup(c => c.DeleteDocumentAsync(documentUri, null, default(CancellationToken)))
.ThrowsAsync(DocumentExceptionHelpers.CreateConflictException())
.Verifiable();
var exception = await Record.ExceptionAsync(async() => await leaseStoreManager.DeleteAsync(cachedLease));
Assert.IsAssignableFrom <DocumentClientException>(exception);
}
public async Task DeleteAsync_ReturnsSuccess_IfDocumentDoesNotExist()
{
var documentClient = Mock.Of <IChangeFeedDocumentClient>();
var cachedLease = CreateCachedLease(owner);
var leaseStoreManager = CreateLeaseStoreManager(documentClient, owner);
Mock.Get(documentClient)
.Setup(c => c.DeleteDocumentAsync(documentUri, null, default(CancellationToken)))
.ThrowsAsync(DocumentExceptionHelpers.CreateNotFoundException())
.Verifiable();
await leaseStoreManager.DeleteAsync(cachedLease);
Mock.Get(documentClient).VerifyAll();
}
public async Task ReleaseInitializationLockAsync_ShouldReturnFalse_IfLockNotFound()
{
var client = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(client)
.Setup(c => c.DeleteDocumentAsync(It.IsAny <Uri>(), It.IsAny <RequestOptions>(), default(CancellationToken)))
.ThrowsAsync(DocumentExceptionHelpers.CreateNotFoundException());
var leaseStore = new DocumentServiceLeaseStore(client, collectionInfo, containerNamePrefix, leaseCollectionLink, Mock.Of <IRequestOptionsFactory>());
bool isLockFoundAndReleased = await leaseStore.ReleaseInitializationLockAsync();
Assert.False(isLockFoundAndReleased);
Mock.Get(client)
.Verify(c => c.DeleteDocumentAsync(It.Is <Uri>(uri => uri.OriginalString.EndsWith("prefix.lock")), It.IsAny <RequestOptions>(), default(CancellationToken)), Times.Once);
}
public async Task CreateLeaseIfNotExistAsync_Throws_WhenCreateFails()
{
var documentClient = Mock.Of <IChangeFeedDocumentClient>();
var leaseStoreManager = CreateLeaseStoreManager(documentClient, owner);
Mock.Get(documentClient)
.Setup(c => c.CreateDocumentAsync(collectionLink, It.Is <DocumentServiceLease>(d =>
d.PartitionId == partitionId &&
d.ContinuationToken == continuationToken &&
d.Id == $"{storeNamePrefix}..{partitionId}"), null, false, default(CancellationToken)))
.ThrowsAsync(DocumentExceptionHelpers.CreateNotFoundException());
var exception = await Record.ExceptionAsync(async() => await leaseStoreManager.CreateLeaseIfNotExistAsync(partitionId, continuationToken));
Assert.IsAssignableFrom <DocumentClientException>(exception);
}
public async Task BuildPassesPartitionKey_WhenLeaseCollectionIsPartitionedById()
{
var leaseCollection = MockHelpers.CreateCollection(
"collectionId",
"collectionRid",
new PartitionKeyDefinition {
Paths = { "/id" }
},
collectionLink);
var lease = Mock.Of <ILease>();
Mock.Get(lease)
.SetupGet(l => l.Id)
.Returns("leaseId");
var leaseClient = this.CreateMockDocumentClient(collection);
Mock.Get(leaseClient)
.Setup(c => c.ReadDocumentCollectionAsync(
It.IsAny <Uri>(),
It.IsAny <RequestOptions>()))
.ReturnsAsync(new ResourceResponse <DocumentCollection>(leaseCollection));
Mock.Get(leaseClient)
.Setup(c => c.ReadDocumentAsync(
It.IsAny <Uri>(),
It.IsAny <RequestOptions>(),
It.IsAny <CancellationToken>()))
.Callback((Uri uri, RequestOptions options, CancellationToken token) =>
{
if (new PartitionKey(lease.Id).Equals(options.PartitionKey))
{
throw DocumentExceptionHelpers.CreateNotFoundException(); // Success code path: cause lease lost.
}
throw new Exception("Failure");
});
this.builder
.WithFeedDocumentClient(this.CreateMockDocumentClient())
.WithLeaseDocumentClient(leaseClient)
.WithObserverFactory(Mock.Of <IChangeFeedObserverFactory>());
await this.builder.BuildAsync();
Exception exception = await Record.ExceptionAsync(() => this.builder.LeaseStoreManager.ReleaseAsync(lease));
Assert.Equal(typeof(LeaseLostException), exception.GetType());
}
public async Task DeleteAsync_ReturnsSuccess_IfDocumentDoesNotExist()
{
var documentClient = Mock.Of <IChangeFeedDocumentClient>();
var cachedLease = CreateCachedLease(owner);
var leaseUpdater = Mock.Of <IDocumentServiceLeaseUpdater>();
var leaseManager = CreateLeaseManager(documentClient, leaseUpdater, owner);
Mock.Get(documentClient)
.Setup(c => c.DeleteDocumentAsync(documentUri))
.ThrowsAsync(DocumentExceptionHelpers.CreateNotFoundException())
.Verifiable();
await leaseManager.DeleteAsync(cachedLease);
Mock.Get(documentClient).VerifyAll();
}
public async Task LockInitializationAsync_ShouldReturnFalse_IfLockConflicts()
{
var client = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(client)
.Setup(c => c.CreateDocumentAsync(It.IsAny <string>(), It.IsAny <object>()))
.ThrowsAsync(DocumentExceptionHelpers.CreateConflictException());
var leaseStore = new LeaseStore(client, collectionInfo, containerNamePrefix, leaseStoreCollectionLink);
bool isLocked = await leaseStore.LockInitializationAsync(lockTime);
Assert.False(isLocked);
Mock.Get(client)
.Verify(c =>
c.CreateDocumentAsync(leaseStoreCollectionLink, It.Is <Document>(d => d.TimeToLive == (int)lockTime.TotalSeconds && d.Id == "prefix.lock")),
Times.Once);
}
public async Task CreateLeaseIfNotExistAsync_ReturnsNull_WhenDocumentExists()
{
var documentClient = Mock.Of <IChangeFeedDocumentClient>();
var leaseStoreManager = CreateLeaseStoreManager(documentClient, owner);
Mock.Get(documentClient)
.Setup(c => c.CreateDocumentAsync(collectionLink, It.Is <DocumentServiceLease>(d =>
d.PartitionId == partitionId &&
d.ContinuationToken == continuationToken &&
d.Id == $"{storeNamePrefix}..{partitionId}"), null, false, default(CancellationToken)))
.ThrowsAsync(DocumentExceptionHelpers.CreateConflictException())
.Verifiable();
var lease = await leaseStoreManager.CreateLeaseIfNotExistAsync(partitionId, continuationToken);
Mock.Get(documentClient).VerifyAll();
Assert.Null(lease);
}
public async Task Run_ShouldRetry_IfThrottled()
{
Mock.Get(documentQuery)
.SetupSequence(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)))
.Throws(DocumentExceptionHelpers.CreateRequestRateTooLargeException())
.ReturnsAsync(feedResponse);
await Assert.ThrowsAsync <TaskCanceledException>(() => partitionProcessor.RunAsync(cancellationTokenSource.Token));
Mock.Get(documentQuery)
.Verify(query => query.ExecuteNextAsync <Document>(It.Is <CancellationToken>(token => token == cancellationTokenSource.Token)), Times.Exactly(2));
Mock.Get(observer)
.Verify(feedObserver => feedObserver
.ProcessChangesAsync(
It.Is <ChangeFeedObserverContext>(context => context.PartitionKeyRangeId == processorSettings.PartitionKeyRangeId),
It.Is <IReadOnlyList <Document> >(list => list.SequenceEqual(documents)),
It.IsAny <CancellationToken>()),
Times.Once);
}
public async Task ProcessChangesAsync_ShouldThrow_IfObserverThrowsDocumentClientException()
{
Mock.Get(observer)
.SetupSequence(feedObserver => feedObserver
.ProcessChangesAsync(It.IsAny <IChangeFeedObserverContext>(), It.IsAny <IReadOnlyList <Document> >(), It.IsAny <CancellationToken>()))
.Throws(DocumentExceptionHelpers.CreateRequestRateTooLargeException());
Exception exception = await Record.ExceptionAsync(() => observerWrapper.ProcessChangesAsync(this.changeFeedObserverContext, this.documents, cancellationTokenSource.Token));
Assert.IsAssignableFrom <ObserverException>(exception);
Assert.IsAssignableFrom <DocumentClientException>(exception.InnerException);
Mock.Get(observer)
.Verify(feedObserver => feedObserver
.ProcessChangesAsync(It.IsAny <IChangeFeedObserverContext>(),
It.Is <IReadOnlyList <Document> >(list => list.SequenceEqual(documents)),
It.IsAny <CancellationToken>()
),
Times.Once);
}
public static IChangeFeedDocumentClient SetupQueryResponseFailure(
this IChangeFeedDocumentClient client,
string partitionKeyRangeId,
string token,
Exception exception = null)
{
var documentQuery = Mock.Of <IChangeFeedDocumentQuery <Document> >();
Mock.Get(documentQuery)
.Setup(q => q.ExecuteNextAsync <Document>(It.IsAny <CancellationToken>()))
.ThrowsAsync(exception ?? DocumentExceptionHelpers.CreateNotFoundException());
Mock.Get(client)
.Setup(c => c.CreateDocumentChangeFeedQuery(
It.IsAny <string>(),
It.Is <ChangeFeedOptions>(o => o.PartitionKeyRangeId == partitionKeyRangeId && o.RequestContinuation == token)))
.Returns(documentQuery);
return(client);
}
public async Task AcquireInitializationLockAsync_ShouldReturnFalse_IfLockConflicts()
{
var client = Mock.Of <IChangeFeedDocumentClient>();
Mock.Get(client)
.Setup(c => c.CreateDocumentAsync(It.IsAny <string>(), It.IsAny <object>(), null, false, default(CancellationToken)))
.ThrowsAsync(DocumentExceptionHelpers.CreateConflictException());
var leaseStore = new DocumentServiceLeaseStore(client, collectionInfo, containerNamePrefix, leaseCollectionLink, Mock.Of <IRequestOptionsFactory>());
bool isLocked = await leaseStore.AcquireInitializationLockAsync(lockTime);
Assert.False(isLocked);
Mock.Get(client)
.Verify(c =>
c.CreateDocumentAsync(leaseCollectionLink, It.Is <Document>(d => d.TimeToLive == (int)lockTime.TotalSeconds && d.Id == "prefix.lock"),
null,
false,
default(CancellationToken)),
Times.Once);
}
public async Task UpdateLeaseAsync_ShouldRethrow_WhenConflictAndReadReturnsOtherException()
{
Exception exception = await TestReadException(DocumentExceptionHelpers.CreatePreconditionFailedException());
Assert.IsAssignableFrom <DocumentClientException>(exception);
}
public async Task UpdateLeaseAsync_ShouldRethrow_WhenReplaceReturnsOtherError()
{
Exception exception = await TestReplaceException(DocumentExceptionHelpers.CreateException("Microsoft.Azure.Documents.GoneException", 1));
Assert.IsAssignableFrom <DocumentClientException>(exception);
}
public async Task UpdateLeaseAsync_ShouldThrowLostLeaseException_WhenReplaceReturnsConflict()
{
Exception exception = await TestReplaceException(DocumentExceptionHelpers.CreateConflictException());
Assert.IsAssignableFrom <LeaseLostException>(exception);
}
