/// <summary>
/// Handle a Partition Gone response and decide what to do based on the type of lease.
/// </summary>
/// <returns>Returns the list of leases to create and a boolean that indicates whether or not to remove the current lease.</returns>
public override async Task <(IEnumerable <DocumentServiceLease>, bool)> HandlePartitionGoneAsync(DocumentServiceLease lease)
{
if (lease == null)
{
throw new ArgumentNullException(nameof(lease));
}
string leaseToken = lease.CurrentLeaseToken;
string lastContinuationToken = lease.ContinuationToken;
DefaultTrace.TraceInformation("Lease {0} is gone due to split or merge", leaseToken);
IReadOnlyList <PartitionKeyRange> overlappingRanges = await this.partitionKeyRangeCache.TryGetOverlappingRangesAsync(
this.containerRid,
((FeedRangeEpk)lease.FeedRange).Range,
NoOpTrace.Singleton,
forceRefresh : true);
if (overlappingRanges.Count == 0)
{
DefaultTrace.TraceError("Lease {0} is gone but we failed to find at least one child range", leaseToken);
throw new InvalidOperationException();
}
return(lease switch
{
DocumentServiceLeaseCoreEpk feedRangeBaseLease => await this.HandlePartitionGoneAsync(leaseToken, lastContinuationToken, feedRangeBaseLease, overlappingRanges),
_ => await this.HandlePartitionGoneAsync(leaseToken, lastContinuationToken, (DocumentServiceLeaseCore)lease, overlappingRanges)
});
public async Task HandlePartitionGoneAsync_EpkBasedLease_Merge()
{
string continuation = Guid.NewGuid().ToString();
Documents.Routing.Range <string> range = new Documents.Routing.Range <string>("AA", "EE", true, false);
DocumentServiceLeaseCoreEpk lease = new DocumentServiceLeaseCoreEpk()
{
LeaseToken = "AA-BB",
ContinuationToken = continuation,
Owner = Guid.NewGuid().ToString(),
FeedRange = new FeedRangeEpk(range)
};
Mock <Routing.PartitionKeyRangeCache> pkRangeCache = new Mock <Routing.PartitionKeyRangeCache>(
Mock.Of <Documents.IAuthorizationTokenProvider>(),
Mock.Of <Documents.IStoreModel>(),
Mock.Of <Common.CollectionCache>());
List <Documents.PartitionKeyRange> resultingRanges = new List <Documents.PartitionKeyRange>()
{
new Documents.PartitionKeyRange()
{
Id = "1", MinInclusive = "", MaxExclusive = "FF"
},
};
pkRangeCache.Setup(p => p.TryGetOverlappingRangesAsync(
It.IsAny <string>(),
It.Is <Documents.Routing.Range <string> >(r => r.Min == range.Min && r.Max == range.Max),
It.IsAny <ITrace>(),
true))
.ReturnsAsync(resultingRanges);
Mock <DocumentServiceLeaseManager> leaseManager = new Mock <DocumentServiceLeaseManager>();
PartitionSynchronizerCore partitionSynchronizerCore = new PartitionSynchronizerCore(
Mock.Of <ContainerInternal>(),
Mock.Of <DocumentServiceLeaseContainer>(),
leaseManager.Object,
1,
pkRangeCache.Object,
Guid.NewGuid().ToString());
(IEnumerable <DocumentServiceLease> addedLeases, bool shouldDelete) = await partitionSynchronizerCore.HandlePartitionGoneAsync(lease);
Assert.IsFalse(shouldDelete);
Assert.AreEqual(lease, addedLeases.First());
leaseManager.Verify(l => l.CreateLeaseIfNotExistAsync(
It.IsAny <Documents.PartitionKeyRange>(),
It.IsAny <string>()), Times.Never);
leaseManager.Verify(l => l.CreateLeaseIfNotExistAsync(
It.IsAny <FeedRangeEpk>(),
It.IsAny <string>()), Times.Never);
}
public async Task CreatesEPKBasedLease(int factoryType)
{
RequestOptionsFactory requestOptionsFactory = GetRequestOptionsFactory(factoryType);
string continuation = Guid.NewGuid().ToString();
DocumentServiceLeaseStoreManagerOptions options = new DocumentServiceLeaseStoreManagerOptions
{
HostName = Guid.NewGuid().ToString()
};
FeedRangeEpk feedRangeEpk = new FeedRangeEpk(new Documents.Routing.Range <string>("AA", "BB", true, false));
Mock <DocumentServiceLeaseUpdater> mockUpdater = new Mock <DocumentServiceLeaseUpdater>();
Mock <ContainerInternal> mockedContainer = new Mock <ContainerInternal>();
mockedContainer.Setup(c => c.CreateItemStreamAsync(
It.IsAny <Stream>(),
It.IsAny <PartitionKey>(),
It.IsAny <ItemRequestOptions>(),
It.IsAny <CancellationToken>())).ReturnsAsync((Stream stream, PartitionKey partitionKey, ItemRequestOptions options, CancellationToken token) => new ResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = stream
});
DocumentServiceLeaseManagerCosmos documentServiceLeaseManagerCosmos = new DocumentServiceLeaseManagerCosmos(
Mock.Of <ContainerInternal>(),
mockedContainer.Object,
mockUpdater.Object,
options,
requestOptionsFactory);
DocumentServiceLease afterAcquire = await documentServiceLeaseManagerCosmos.CreateLeaseIfNotExistAsync(feedRangeEpk, continuation);
DocumentServiceLeaseCoreEpk epkBasedLease = (DocumentServiceLeaseCoreEpk)afterAcquire;
Assert.IsNotNull(epkBasedLease);
Assert.AreEqual(continuation, afterAcquire.ContinuationToken);
Assert.AreEqual(feedRangeEpk.Range.Min, ((FeedRangeEpk)epkBasedLease.FeedRange).Range.Min);
Assert.AreEqual(feedRangeEpk.Range.Max, ((FeedRangeEpk)epkBasedLease.FeedRange).Range.Max);
ValidateRequestOptionsFactory(requestOptionsFactory, epkBasedLease);
}
public void ValidateJsonSerialization_EPKLease()
{
DocumentServiceLeaseCoreEpk originalLease = new DocumentServiceLeaseCoreEpk
{
LeaseId = "id",
ETag = "etag",
LeaseToken = "0",
Owner = "owner",
ContinuationToken = "continuation",
Timestamp = DateTime.Now - TimeSpan.FromSeconds(5),
LeasePartitionKey = "partitionKey",
Properties = new Dictionary <string, string> {
{ "key", "value" }
},
FeedRange = new FeedRangeEpk(new Documents.Routing.Range <string>("AA", "BB", true, false))
};
string serialized = JsonConvert.SerializeObject(originalLease);
DocumentServiceLease documentServiceLease = JsonConvert.DeserializeObject <DocumentServiceLease>(serialized);
if (documentServiceLease is DocumentServiceLeaseCoreEpk documentServiceLeaseCore)
{
Assert.AreEqual(originalLease.LeaseId, documentServiceLeaseCore.LeaseId);
Assert.AreEqual(originalLease.ETag, documentServiceLeaseCore.ETag);
Assert.AreEqual(originalLease.LeaseToken, documentServiceLeaseCore.LeaseToken);
Assert.AreEqual(originalLease.Owner, documentServiceLeaseCore.Owner);
Assert.AreEqual(originalLease.PartitionKey, documentServiceLeaseCore.PartitionKey);
Assert.AreEqual(originalLease.ContinuationToken, documentServiceLeaseCore.ContinuationToken);
Assert.AreEqual(originalLease.Timestamp, documentServiceLeaseCore.Timestamp);
Assert.AreEqual(originalLease.Properties["key"], documentServiceLeaseCore.Properties["key"]);
Assert.AreEqual(originalLease.FeedRange.ToJsonString(), documentServiceLeaseCore.FeedRange.ToJsonString());
}
else
{
Assert.Fail();
}
}
public async Task ShouldUseFeedRangeEpk()
{
int itemCount = 5;
string pkRangeId = "0";
DateTime startTime = DateTime.UtcNow;
Documents.Routing.Range <string> range = new Documents.Routing.Range <string>("AA", "BB", true, false);
FeedRangeEpk feedRange = new FeedRangeEpk(range);
DocumentServiceLeaseCoreEpk documentServiceLeaseCore = new DocumentServiceLeaseCoreEpk()
{
LeaseToken = pkRangeId,
FeedRange = feedRange
};
Mock <ContainerInternal> containerMock = new Mock <ContainerInternal>();
Mock <CosmosClientContext> mockContext = new Mock <CosmosClientContext>();
mockContext.Setup(c => c.ProcessResourceOperationStreamAsync(
It.IsAny <string>(),
It.Is <Documents.ResourceType>(rt => rt == Documents.ResourceType.Document),
It.Is <Documents.OperationType>(rt => rt == Documents.OperationType.ReadFeed),
It.Is <ChangeFeedRequestOptions>(cfo => cfo.PageSizeHint == itemCount),
It.Is <ContainerInternal>(o => o == containerMock.Object),
It.Is <FeedRange>(fr => fr is FeedRangeEpk),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <ITrace>(),
It.IsAny <CancellationToken>()
)
).ReturnsAsync(new ResponseMessage(System.Net.HttpStatusCode.OK));
containerMock.Setup(c => c.ClientContext).Returns(mockContext.Object);
containerMock.Setup(c => c.LinkUri).Returns("http://localhot");
MockDocumentClient mockDocumentClient = new MockDocumentClient();
mockContext.Setup(c => c.DocumentClient).Returns(mockDocumentClient);
ChangeFeedPartitionKeyResultSetIteratorCore iterator = ChangeFeedPartitionKeyResultSetIteratorCore.Create(
lease: documentServiceLeaseCore,
continuationToken: null,
maxItemCount: itemCount,
container: containerMock.Object,
startTime: startTime,
startFromBeginning: false);
ResponseMessage response = await iterator.ReadNextAsync();
Assert.AreEqual(System.Net.HttpStatusCode.OK, response.StatusCode);
mockContext.Verify(c => c.ProcessResourceOperationStreamAsync(
It.IsAny <string>(),
It.Is <Documents.ResourceType>(rt => rt == Documents.ResourceType.Document),
It.Is <Documents.OperationType>(rt => rt == Documents.OperationType.ReadFeed),
It.Is <ChangeFeedRequestOptions>(cfo => cfo.PageSizeHint == itemCount),
It.Is <ContainerInternal>(o => o == containerMock.Object),
It.Is <FeedRange>(fr => fr is FeedRangeEpk),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <ITrace>(),
It.IsAny <CancellationToken>()
), Times.Once);
}
public async Task HandlePartitionGoneAsync_EpkBasedLease_Split()
{
string continuation = Guid.NewGuid().ToString();
Documents.Routing.Range <string> range = new Documents.Routing.Range <string>("AA", "EE", true, false);
DocumentServiceLeaseCoreEpk lease = new DocumentServiceLeaseCoreEpk()
{
LeaseToken = "AA-BB",
ContinuationToken = continuation,
Owner = Guid.NewGuid().ToString(),
FeedRange = new FeedRangeEpk(range)
};
Mock <Routing.PartitionKeyRangeCache> pkRangeCache = new Mock <Routing.PartitionKeyRangeCache>(
Mock.Of <Documents.IAuthorizationTokenProvider>(),
Mock.Of <Documents.IStoreModel>(),
Mock.Of <Common.CollectionCache>());
List <Documents.PartitionKeyRange> resultingRanges = new List <Documents.PartitionKeyRange>()
{
new Documents.PartitionKeyRange()
{
Id = "1", MinInclusive = "", MaxExclusive = "BB"
},
new Documents.PartitionKeyRange()
{
Id = "2", MinInclusive = "BB", MaxExclusive = "DD"
},
new Documents.PartitionKeyRange()
{
Id = "3", MinInclusive = "DD", MaxExclusive = "FF"
},
};
pkRangeCache.Setup(p => p.TryGetOverlappingRangesAsync(
It.IsAny <string>(),
It.Is <Documents.Routing.Range <string> >(r => r.Min == range.Min && r.Max == range.Max),
It.IsAny <ITrace>(),
It.Is <bool>(b => b == true)))
.ReturnsAsync(resultingRanges);
Mock <DocumentServiceLeaseManager> leaseManager = new Mock <DocumentServiceLeaseManager>();
PartitionSynchronizerCore partitionSynchronizerCore = new PartitionSynchronizerCore(
Mock.Of <ContainerInternal>(),
Mock.Of <DocumentServiceLeaseContainer>(),
leaseManager.Object,
1,
pkRangeCache.Object,
Guid.NewGuid().ToString());
await partitionSynchronizerCore.HandlePartitionGoneAsync(lease);
leaseManager.Verify(l => l.CreateLeaseIfNotExistAsync(
It.IsAny <Documents.PartitionKeyRange>(),
It.IsAny <string>()), Times.Never);
leaseManager.Verify(l => l.CreateLeaseIfNotExistAsync(
It.IsAny <FeedRangeEpk>(),
It.IsAny <string>()), Times.Exactly(3));
leaseManager.Verify(l => l.CreateLeaseIfNotExistAsync(
It.Is <FeedRangeEpk>(epk => epk.Range.Min == range.Min && epk.Range.Max == resultingRanges[0].MaxExclusive),
It.Is <string>(c => c == continuation)), Times.Once);
leaseManager.Verify(l => l.CreateLeaseIfNotExistAsync(
It.Is <FeedRangeEpk>(epk => epk.Range.Min == resultingRanges[1].MinInclusive && epk.Range.Max == resultingRanges[1].MaxExclusive),
It.Is <string>(c => c == continuation)), Times.Once);
leaseManager.Verify(l => l.CreateLeaseIfNotExistAsync(
It.Is <FeedRangeEpk>(epk => epk.Range.Min == resultingRanges[2].MinInclusive && epk.Range.Max == range.Max),
It.Is <string>(c => c == continuation)), Times.Once);
}
public async Task ShouldUseFeedRangeEpk()
{
int itemCount = 5;
string pkRangeId = "0";
DateTime startTime = DateTime.UtcNow;
Documents.Routing.Range <string> range = new Documents.Routing.Range <string>("AA", "BB", true, false);
FeedRangeEpk feedRange = new FeedRangeEpk(range);
DocumentServiceLeaseCoreEpk documentServiceLeaseCore = new DocumentServiceLeaseCoreEpk()
{
LeaseToken = pkRangeId,
FeedRange = feedRange
};
Mock <ContainerInternal> containerMock = new Mock <ContainerInternal>();
Mock <CosmosClientContext> mockContext = new Mock <CosmosClientContext>();
mockContext.Setup(x => x.OperationHelperAsync <ResponseMessage>(
It.Is <string>(str => str.Contains("Change Feed Processor")),
It.IsAny <RequestOptions>(),
It.IsAny <Func <ITrace, Task <ResponseMessage> > >(),
It.IsAny <Func <ResponseMessage, OpenTelemetryAttributes> >(),
It.Is <TraceComponent>(tc => tc == TraceComponent.ChangeFeed),
It.IsAny <TraceLevel>()))
.Returns <string, RequestOptions, Func <ITrace, Task <ResponseMessage> >, Func <ResponseMessage, OpenTelemetryAttributes>, TraceComponent, TraceLevel>(
(operationName, requestOptions, func, oTelFunc, comp, level) =>
{
using (ITrace trace = Trace.GetRootTrace(operationName, comp, level))
{
return(func(trace));
}
});
mockContext.Setup(c => c.ProcessResourceOperationStreamAsync(
It.IsAny <string>(),
It.Is <Documents.ResourceType>(rt => rt == Documents.ResourceType.Document),
It.Is <Documents.OperationType>(rt => rt == Documents.OperationType.ReadFeed),
It.Is <ChangeFeedRequestOptions>(cfo => cfo.PageSizeHint == itemCount),
It.Is <ContainerInternal>(o => o == containerMock.Object),
It.Is <FeedRange>(fr => fr is FeedRangeEpk),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.Is <ITrace>(t => !(t is NoOpTrace)),
It.IsAny <CancellationToken>()
)
).ReturnsAsync(new ResponseMessage(System.Net.HttpStatusCode.OK));
containerMock.Setup(c => c.ClientContext).Returns(mockContext.Object);
containerMock.Setup(c => c.LinkUri).Returns("http://localhot");
MockDocumentClient mockDocumentClient = new MockDocumentClient();
mockContext.Setup(c => c.DocumentClient).Returns(mockDocumentClient);
ChangeFeedPartitionKeyResultSetIteratorCore iterator = ChangeFeedPartitionKeyResultSetIteratorCore.Create(
lease: documentServiceLeaseCore,
continuationToken: null,
maxItemCount: itemCount,
container: containerMock.Object,
startTime: startTime,
startFromBeginning: false);
ResponseMessage response = await iterator.ReadNextAsync();
Assert.AreEqual(System.Net.HttpStatusCode.OK, response.StatusCode);
mockContext.Verify(c => c.ProcessResourceOperationStreamAsync(
It.IsAny <string>(),
It.Is <Documents.ResourceType>(rt => rt == Documents.ResourceType.Document),
It.Is <Documents.OperationType>(rt => rt == Documents.OperationType.ReadFeed),
It.Is <ChangeFeedRequestOptions>(cfo => cfo.PageSizeHint == itemCount),
It.Is <ContainerInternal>(o => o == containerMock.Object),
It.Is <FeedRange>(fr => fr is FeedRangeEpk),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.Is <ITrace>(t => !(t is NoOpTrace)),
It.IsAny <CancellationToken>()
), Times.Once);
}
