public async Task StartFromBeginingTestAsync()
{
IDocumentContainer documentContainer = await this.CreateDocumentContainerAsync(numItems : 10);
List <FeedRangeEpk> ranges = await documentContainer.GetFeedRangesAsync(NoOpTrace.Singleton, cancellationToken : default);
// Should get back the all the documents inserted so far
ChangeFeedState resumeState;
{
TryCatch <ChangeFeedPage> monadicChangeFeedPage = await documentContainer.MonadicChangeFeedAsync(
ChangeFeedState.Beginning(),
ranges[0],
pageSize : int.MaxValue,
trace : NoOpTrace.Singleton,
cancellationToken : default);
Assert.IsTrue(monadicChangeFeedPage.Succeeded);
resumeState = monadicChangeFeedPage.Result.State;
}
// No more changes left
{
TryCatch <ChangeFeedPage> monadicChangeFeedPage = await documentContainer.MonadicChangeFeedAsync(
resumeState,
ranges[0],
pageSize : 10,
trace : NoOpTrace.Singleton,
cancellationToken : default);
Assert.IsTrue(monadicChangeFeedPage.Succeeded);
Assert.IsTrue(monadicChangeFeedPage.Result is ChangeFeedNotModifiedPage);
}
}
public async Task SomeChangesAsync()
{
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems : 1);
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
ChangeFeedMode.Incremental,
new ChangeFeedRequestOptions(),
new CrossFeedRangeState <ChangeFeedState>(
new FeedRangeState <ChangeFeedState>[]
{
new FeedRangeState <ChangeFeedState>(FeedRangeEpk.FullRange, ChangeFeedState.Beginning())
}),
cancellationToken: default);
// First page should be true and skip the 304 not modified
Assert.IsTrue(await enumerator.MoveNextAsync());
Assert.IsTrue(enumerator.Current.Succeeded);
Assert.IsTrue(enumerator.Current.Result.Page is ChangeFeedSuccessPage);
// Second page should surface up the 304
Assert.IsTrue(await enumerator.MoveNextAsync());
Assert.IsTrue(enumerator.Current.Succeeded);
Assert.IsTrue(enumerator.Current.Result.Page is ChangeFeedNotModifiedPage);
}
public void Beginning()
{
ChangeFeedState beginning = ChangeFeedState.Beginning();
CosmosElement cosmosElement = ChangeFeedStateCosmosElementSerializer.ToCosmosElement(beginning);
TryCatch <ChangeFeedState> monadicState = ChangeFeedStateCosmosElementSerializer.MonadicFromCosmosElement(cosmosElement);
Assert.IsTrue(monadicState.Succeeded);
Assert.IsTrue(monadicState.Result is ChangeFeedStateBeginning);
}
public async Task EmptyContainerTestAsync()
{
IDocumentContainer documentContainer = await this.CreateDocumentContainerAsync(numItems : 0);
List <FeedRangeEpk> ranges = await documentContainer.GetFeedRangesAsync(NoOpTrace.Singleton, cancellationToken : default);
TryCatch <ChangeFeedPage> monadicChangeFeedPage = await documentContainer.MonadicChangeFeedAsync(
feedRangeState : new FeedRangeState <ChangeFeedState>(ranges[0], ChangeFeedState.Beginning()),
changeFeedPaginationOptions : new ChangeFeedPaginationOptions(ChangeFeedMode.Incremental, pageSizeHint: 10),
trace : NoOpTrace.Singleton,
cancellationToken : default);
public async Task MonadicChangeFeedAsync_ChangeFeedMode_Incremental()
{
Mock <ContainerInternal> container = new Mock <ContainerInternal>();
Mock <CosmosClientContext> context = new Mock <CosmosClientContext>();
container.Setup(m => m.ClientContext).Returns(context.Object);
Func <Action <RequestMessage>, bool> validateEnricher = (Action <RequestMessage> enricher) =>
{
RequestMessage requestMessage = new RequestMessage();
enricher(requestMessage);
Assert.AreEqual(HttpConstants.A_IMHeaderValues.IncrementalFeed, requestMessage.Headers[HttpConstants.HttpHeaders.A_IM]);
return(true);
};
ResponseMessage response = new ResponseMessage(System.Net.HttpStatusCode.NotModified);
response.Headers.ETag = Guid.NewGuid().ToString();
response.Headers.ActivityId = Guid.NewGuid().ToString();
response.Headers.RequestCharge = 1;
context.SetupSequence(c => c.ProcessResourceOperationStreamAsync(
It.IsAny <string>(),
It.Is <ResourceType>(rt => rt == ResourceType.Document),
It.Is <OperationType>(rt => rt == OperationType.ReadFeed),
It.IsAny <RequestOptions>(),
It.Is <ContainerInternal>(o => o == container.Object),
It.IsAny <FeedRangeInternal>(),
It.IsAny <Stream>(),
It.Is <Action <RequestMessage> >(enricher => validateEnricher(enricher)),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <ITrace>(),
It.IsAny <CancellationToken>()
)
).ReturnsAsync(response);
NetworkAttachedDocumentContainer networkAttachedDocumentContainer = new NetworkAttachedDocumentContainer(
container.Object,
Mock.Of <CosmosQueryClient>(),
Mock.Of <CosmosDiagnosticsContext>());
await networkAttachedDocumentContainer.MonadicChangeFeedAsync(
state : ChangeFeedState.Beginning(),
feedRange : new FeedRangePartitionKeyRange("0"),
pageSize : 10,
changeFeedMode : ChangeFeedMode.Incremental,
jsonSerializationFormat : null,
trace : NoOpTrace.Singleton,
cancellationToken : default);
public async Task TargetMultipleLogicalPartitionKeys()
{
int batchSize = 25;
string pkToRead1 = "pkToRead1";
string pkToRead2 = "pkToRead2";
string otherPK = "otherPK";
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead1));
}
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead2));
}
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(otherPK));
}
// Create one start state for each logical partition key.
List <FeedRangeState <ChangeFeedState> > feedRangeStates = new List <FeedRangeState <ChangeFeedState> >();
IReadOnlyList <string> partitionKeysToTarget = new List <string>()
{
pkToRead1,
pkToRead2
};
foreach (string partitionKeyToTarget in partitionKeysToTarget)
{
feedRangeStates.Add(
new FeedRangeState <ChangeFeedState>(
(FeedRangeInternal)FeedRange.FromPartitionKey(
new Cosmos.PartitionKey(partitionKeyToTarget)),
ChangeFeedState.Beginning()));
}
// Use the list composition property of the constructor to merge them in to a single state.
ChangeFeedCrossFeedRangeState multipleLogicalPartitionKeyState = new ChangeFeedCrossFeedRangeState(feedRangeStates.ToImmutableArray());
IAsyncEnumerable <TryCatch <ChangeFeedPage> > asyncEnumerable = this.Container.GetChangeFeedAsyncEnumerable(multipleLogicalPartitionKeyState, ChangeFeedMode.Incremental);
(int totalCount, ChangeFeedCrossFeedRangeState _) = await PartialDrainAsync(asyncEnumerable);
Assert.AreEqual(2 * batchSize, totalCount);
}
public async Task EmptyContainerTestAsync()
{
IDocumentContainer documentContainer = await this.CreateDocumentContainerAsync(numItems : 0);
List <FeedRangeEpk> ranges = await documentContainer.GetFeedRangesAsync(NoOpTrace.Singleton, cancellationToken : default);
TryCatch <ChangeFeedPage> monadicChangeFeedPage = await documentContainer.MonadicChangeFeedAsync(
ChangeFeedState.Beginning(),
ranges[0],
pageSize : 10,
trace : NoOpTrace.Singleton,
cancellationToken : default);
Assert.IsTrue(monadicChangeFeedPage.Succeeded);
Assert.IsTrue(monadicChangeFeedPage.Result is ChangeFeedNotModifiedPage);
}
public static ChangeFeedCrossFeedRangeState CreateFromBeginning(FeedRange feedRange)
{
if (!(feedRange is FeedRangeInternal feedRangeInternal))
{
throw new ArgumentException($"{nameof(feedRange)} needs to be a {nameof(FeedRangeInternal)}.");
}
if (feedRange.Equals(FeedRangeEpk.FullRange))
{
return(FullRangeStatesSingletons.Beginning);
}
return(new ChangeFeedCrossFeedRangeState(
new List <FeedRangeState <ChangeFeedState> >()
{
new FeedRangeState <ChangeFeedState>(feedRangeInternal, ChangeFeedState.Beginning())
}));
}
public async Task NoChangesAsync()
{
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems: 0);
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState<ChangeFeedState>(
new FeedRangeState<ChangeFeedState>[]
{
new FeedRangeState<ChangeFeedState>(FeedRangeEpk.FullRange, ChangeFeedState.Beginning())
}),
ChangeFeedPaginationOptions.Default,
cancellationToken: default);
Assert.IsTrue(await enumerator.MoveNextAsync());
Assert.IsTrue(enumerator.Current.Succeeded);
Assert.IsTrue(enumerator.Current.Result.Page is ChangeFeedNotModifiedPage);
Assert.IsNotNull(enumerator.Current.Result.State);
}
public async Task StartFromBeginningAsync(bool useContinuations)
{
int numItems = 100;
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems);
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState<ChangeFeedState>(
new FeedRangeState<ChangeFeedState>[]
{
new FeedRangeState<ChangeFeedState>(FeedRangeEpk.FullRange, ChangeFeedState.Beginning())
}),
ChangeFeedPaginationOptions.Default,
cancellationToken: default);
(int globalCount, double _) = await (useContinuations
? DrainWithUntilNotModifiedWithContinuationTokens(documentContainer, enumerator)
: DrainUntilNotModifedAsync(enumerator));
Assert.AreEqual(numItems, globalCount);
}
public ChangeFeedIteratorCore(
IDocumentContainer documentContainer,
ChangeFeedRequestOptions changeFeedRequestOptions,
ChangeFeedStartFrom changeFeedStartFrom)
{
if (changeFeedStartFrom == null)
{
throw new ArgumentNullException(nameof(changeFeedStartFrom));
}
this.documentContainer = documentContainer ?? throw new ArgumentNullException(nameof(documentContainer));
this.changeFeedRequestOptions = changeFeedRequestOptions ?? new ChangeFeedRequestOptions();
this.lazyMonadicEnumerator = new AsyncLazy <TryCatch <CrossPartitionChangeFeedAsyncEnumerator> >(
valueFactory: async(cancellationToken) =>
{
if (changeFeedStartFrom is ChangeFeedStartFromContinuation startFromContinuation)
{
TryCatch <CosmosElement> monadicParsedToken = CosmosElement.Monadic.Parse(startFromContinuation.Continuation);
if (monadicParsedToken.Failed)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse continuation token: {startFromContinuation.Continuation}.",
innerException: monadicParsedToken.Exception)));
}
TryCatch <VersionedAndRidCheckedCompositeToken> monadicVersionedToken = VersionedAndRidCheckedCompositeToken
.MonadicCreateFromCosmosElement(monadicParsedToken.Result);
if (monadicVersionedToken.Failed)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse continuation token: {startFromContinuation.Continuation}.",
innerException: monadicVersionedToken.Exception)));
}
VersionedAndRidCheckedCompositeToken versionedAndRidCheckedCompositeToken = monadicVersionedToken.Result;
if (versionedAndRidCheckedCompositeToken.VersionNumber == VersionedAndRidCheckedCompositeToken.Version.V1)
{
// Need to migrate continuation token
if (!(versionedAndRidCheckedCompositeToken.ContinuationToken is CosmosArray cosmosArray))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse get array continuation token: {startFromContinuation.Continuation}.")));
}
List <CosmosElement> changeFeedTokensV2 = new List <CosmosElement>();
foreach (CosmosElement arrayItem in cosmosArray)
{
if (!(arrayItem is CosmosObject cosmosObject))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse get object in composite continuation: {startFromContinuation.Continuation}.")));
}
if (!cosmosObject.TryGetValue("min", out CosmosString min))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse start of range: {cosmosObject}.")));
}
if (!cosmosObject.TryGetValue("max", out CosmosString max))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse end of range: {cosmosObject}.")));
}
if (!cosmosObject.TryGetValue("token", out CosmosElement token))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse token: {cosmosObject}.")));
}
FeedRangeEpk feedRangeEpk = new FeedRangeEpk(new Documents.Routing.Range <string>(
min: min.Value,
max: max.Value,
isMinInclusive: true,
isMaxInclusive: false));
ChangeFeedState state = token is CosmosNull ? ChangeFeedState.Beginning() : ChangeFeedStateContinuation.Continuation(token);
FeedRangeState <ChangeFeedState> feedRangeState = new FeedRangeState <ChangeFeedState>(feedRangeEpk, state);
changeFeedTokensV2.Add(ChangeFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
CosmosArray changeFeedTokensArrayV2 = CosmosArray.Create(changeFeedTokensV2);
versionedAndRidCheckedCompositeToken = new VersionedAndRidCheckedCompositeToken(
VersionedAndRidCheckedCompositeToken.Version.V2,
changeFeedTokensArrayV2,
versionedAndRidCheckedCompositeToken.Rid);
}
if (versionedAndRidCheckedCompositeToken.VersionNumber != VersionedAndRidCheckedCompositeToken.Version.V2)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Wrong version number: {versionedAndRidCheckedCompositeToken.VersionNumber}.")));
}
string collectionRid = await documentContainer.GetResourceIdentifierAsync(cancellationToken);
if (versionedAndRidCheckedCompositeToken.Rid != collectionRid)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"rids mismatched. Expected: {collectionRid} but got {versionedAndRidCheckedCompositeToken.Rid}.")));
}
changeFeedStartFrom = ChangeFeedStartFrom.ContinuationToken(versionedAndRidCheckedCompositeToken.ContinuationToken.ToString());
}
TryCatch <ChangeFeedCrossFeedRangeState> monadicChangeFeedCrossFeedRangeState = changeFeedStartFrom.Accept(ChangeFeedStateFromToChangeFeedCrossFeedRangeState.Singleton);
if (monadicChangeFeedCrossFeedRangeState.Failed)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Could not convert to {nameof(ChangeFeedCrossFeedRangeState)}.",
innerException: monadicChangeFeedCrossFeedRangeState.Exception)));
}
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
changeFeedRequestOptions,
new CrossFeedRangeState <ChangeFeedState>(monadicChangeFeedCrossFeedRangeState.Result.FeedRangeStates),
cancellationToken: default);
TryCatch <CrossPartitionChangeFeedAsyncEnumerator> monadicEnumerator = TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromResult(enumerator);
return(monadicEnumerator);
});
this.hasMoreResults = true;
}
