private static async Task <int> DrainWithUntilNotModifiedWithContinuationTokens(
IDocumentContainer documentContainer,
CrossPartitionChangeFeedAsyncEnumerator enumerator)
{
List <CosmosElement> globalChanges = new List <CosmosElement>();
while (true)
{
if (!await enumerator.MoveNextAsync())
{
throw new InvalidOperationException();
}
Assert.IsTrue(enumerator.Current.Succeeded);
if (!(enumerator.Current.Result.Page is ChangeFeedSuccessPage changeFeedSuccessPage))
{
break;
}
CosmosArray changes = GetChanges(changeFeedSuccessPage.Content);
globalChanges.AddRange(changes);
enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
new ChangeFeedRequestOptions(),
enumerator.Current.Result.State,
cancellationToken: default);
}
return(globalChanges.Count);
}
public async Task StartFromTimeAsync(bool useContinuations)
{
int numItems = 100;
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems);
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState<ChangeFeedState>(
new FeedRangeState<ChangeFeedState>[]
{
new FeedRangeState<ChangeFeedState>(FeedRangeEpk.FullRange, ChangeFeedState.Time(DateTime.UtcNow))
}),
ChangeFeedPaginationOptions.Default,
cancellationToken: default);
for (int i = 0; i < numItems; i++)
{
// Insert an item
CosmosObject item = CosmosObject.Parse($"{{\"pk\" : {i} }}");
while (true)
{
TryCatch<Record> monadicCreateRecord = await documentContainer.MonadicCreateItemAsync(item, cancellationToken: default);
if (monadicCreateRecord.Succeeded)
{
break;
}
}
}
(int globalCount, double _) = await (useContinuations
? DrainWithUntilNotModifiedWithContinuationTokens(documentContainer, enumerator)
: DrainUntilNotModifedAsync(enumerator));
Assert.AreEqual(numItems, globalCount);
}
public async Task SomeChangesAsync()
{
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems : 1);
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
ChangeFeedMode.Incremental,
new ChangeFeedRequestOptions(),
new CrossFeedRangeState <ChangeFeedState>(
new FeedRangeState <ChangeFeedState>[]
{
new FeedRangeState <ChangeFeedState>(FeedRangeEpk.FullRange, ChangeFeedState.Beginning())
}),
cancellationToken: default);
// First page should be true and skip the 304 not modified
Assert.IsTrue(await enumerator.MoveNextAsync());
Assert.IsTrue(enumerator.Current.Succeeded);
Assert.IsTrue(enumerator.Current.Result.Page is ChangeFeedSuccessPage);
// Second page should surface up the 304
Assert.IsTrue(await enumerator.MoveNextAsync());
Assert.IsTrue(enumerator.Current.Succeeded);
Assert.IsTrue(enumerator.Current.Result.Page is ChangeFeedNotModifiedPage);
}
public IAsyncEnumerator <TryCatch <ChangeFeedPage> > GetAsyncEnumerator(CancellationToken cancellationToken = default)
{
CrossFeedRangeState <ChangeFeedState> innerState = new CrossFeedRangeState <ChangeFeedState>(this.state.FeedRangeStates);
CrossPartitionChangeFeedAsyncEnumerator innerEnumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
this.documentContainer,
this.changeFeedRequestOptions,
innerState,
cancellationToken);
return(new ChangeFeedCrossFeedRangeAsyncEnumerator(innerEnumerator));
}
public async Task NoChangesAsync()
{
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems: 0);
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState<ChangeFeedState>(
new FeedRangeState<ChangeFeedState>[]
{
new FeedRangeState<ChangeFeedState>(FeedRangeEpk.FullRange, ChangeFeedState.Beginning())
}),
ChangeFeedPaginationOptions.Default,
cancellationToken: default);
Assert.IsTrue(await enumerator.MoveNextAsync());
Assert.IsTrue(enumerator.Current.Succeeded);
Assert.IsTrue(enumerator.Current.Result.Page is ChangeFeedNotModifiedPage);
Assert.IsNotNull(enumerator.Current.Result.State);
}
public async Task StartFromBeginningAsync(bool useContinuations)
{
int numItems = 100;
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems);
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState<ChangeFeedState>(
new FeedRangeState<ChangeFeedState>[]
{
new FeedRangeState<ChangeFeedState>(FeedRangeEpk.FullRange, ChangeFeedState.Beginning())
}),
ChangeFeedPaginationOptions.Default,
cancellationToken: default);
(int globalCount, double _) = await (useContinuations
? DrainWithUntilNotModifiedWithContinuationTokens(documentContainer, enumerator)
: DrainUntilNotModifedAsync(enumerator));
Assert.AreEqual(numItems, globalCount);
}
public async Task ChangeFeedAsync()
{
int numItems = 100;
IDocumentContainer documentContainer = await this.CreateDocumentContainerAsync(numItems);
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
ChangeFeedMode.Incremental,
new ChangeFeedRequestOptions()
{
PageSizeHint = int.MaxValue
},
new CrossFeedRangeState <ChangeFeedState>(
ChangeFeedCrossFeedRangeState.CreateFromBeginning().FeedRangeStates),
cancellationToken: default);
int numChildren = 0;
Trace rootTrace;
using (rootTrace = Trace.GetRootTrace("Cross Partition Change Feed"))
{
while (await enumerator.MoveNextAsync(rootTrace))
{
numChildren++;
if (enumerator.Current.Result.Page is ChangeFeedNotModifiedPage)
{
break;
}
}
}
string traceString = TraceWriter.TraceToText(rootTrace);
Console.WriteLine(traceString);
Assert.AreEqual(numChildren, rootTrace.Children.Count);
}
public ChangeFeedIteratorCore(
IDocumentContainer documentContainer,
ChangeFeedRequestOptions changeFeedRequestOptions,
ChangeFeedStartFrom changeFeedStartFrom)
{
if (changeFeedStartFrom == null)
{
throw new ArgumentNullException(nameof(changeFeedStartFrom));
}
this.documentContainer = documentContainer ?? throw new ArgumentNullException(nameof(documentContainer));
this.changeFeedRequestOptions = changeFeedRequestOptions ?? new ChangeFeedRequestOptions();
this.lazyMonadicEnumerator = new AsyncLazy <TryCatch <CrossPartitionChangeFeedAsyncEnumerator> >(
valueFactory: async(cancellationToken) =>
{
if (changeFeedStartFrom is ChangeFeedStartFromContinuation startFromContinuation)
{
TryCatch <CosmosElement> monadicParsedToken = CosmosElement.Monadic.Parse(startFromContinuation.Continuation);
if (monadicParsedToken.Failed)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse continuation token: {startFromContinuation.Continuation}.",
innerException: monadicParsedToken.Exception)));
}
TryCatch <VersionedAndRidCheckedCompositeToken> monadicVersionedToken = VersionedAndRidCheckedCompositeToken
.MonadicCreateFromCosmosElement(monadicParsedToken.Result);
if (monadicVersionedToken.Failed)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse continuation token: {startFromContinuation.Continuation}.",
innerException: monadicVersionedToken.Exception)));
}
VersionedAndRidCheckedCompositeToken versionedAndRidCheckedCompositeToken = monadicVersionedToken.Result;
if (versionedAndRidCheckedCompositeToken.VersionNumber == VersionedAndRidCheckedCompositeToken.Version.V1)
{
// Need to migrate continuation token
if (!(versionedAndRidCheckedCompositeToken.ContinuationToken is CosmosArray cosmosArray))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse get array continuation token: {startFromContinuation.Continuation}.")));
}
List <CosmosElement> changeFeedTokensV2 = new List <CosmosElement>();
foreach (CosmosElement arrayItem in cosmosArray)
{
if (!(arrayItem is CosmosObject cosmosObject))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse get object in composite continuation: {startFromContinuation.Continuation}.")));
}
if (!cosmosObject.TryGetValue("min", out CosmosString min))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse start of range: {cosmosObject}.")));
}
if (!cosmosObject.TryGetValue("max", out CosmosString max))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse end of range: {cosmosObject}.")));
}
if (!cosmosObject.TryGetValue("token", out CosmosElement token))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse token: {cosmosObject}.")));
}
FeedRangeEpk feedRangeEpk = new FeedRangeEpk(new Documents.Routing.Range <string>(
min: min.Value,
max: max.Value,
isMinInclusive: true,
isMaxInclusive: false));
ChangeFeedState state = token is CosmosNull ? ChangeFeedState.Beginning() : ChangeFeedStateContinuation.Continuation(token);
FeedRangeState <ChangeFeedState> feedRangeState = new FeedRangeState <ChangeFeedState>(feedRangeEpk, state);
changeFeedTokensV2.Add(ChangeFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
CosmosArray changeFeedTokensArrayV2 = CosmosArray.Create(changeFeedTokensV2);
versionedAndRidCheckedCompositeToken = new VersionedAndRidCheckedCompositeToken(
VersionedAndRidCheckedCompositeToken.Version.V2,
changeFeedTokensArrayV2,
versionedAndRidCheckedCompositeToken.Rid);
}
if (versionedAndRidCheckedCompositeToken.VersionNumber != VersionedAndRidCheckedCompositeToken.Version.V2)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Wrong version number: {versionedAndRidCheckedCompositeToken.VersionNumber}.")));
}
string collectionRid = await documentContainer.GetResourceIdentifierAsync(cancellationToken);
if (versionedAndRidCheckedCompositeToken.Rid != collectionRid)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"rids mismatched. Expected: {collectionRid} but got {versionedAndRidCheckedCompositeToken.Rid}.")));
}
changeFeedStartFrom = ChangeFeedStartFrom.ContinuationToken(versionedAndRidCheckedCompositeToken.ContinuationToken.ToString());
}
TryCatch <ChangeFeedCrossFeedRangeState> monadicChangeFeedCrossFeedRangeState = changeFeedStartFrom.Accept(ChangeFeedStateFromToChangeFeedCrossFeedRangeState.Singleton);
if (monadicChangeFeedCrossFeedRangeState.Failed)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Could not convert to {nameof(ChangeFeedCrossFeedRangeState)}.",
innerException: monadicChangeFeedCrossFeedRangeState.Exception)));
}
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
changeFeedRequestOptions,
new CrossFeedRangeState <ChangeFeedState>(monadicChangeFeedCrossFeedRangeState.Result.FeedRangeStates),
cancellationToken: default);
TryCatch <CrossPartitionChangeFeedAsyncEnumerator> monadicEnumerator = TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromResult(enumerator);
return(monadicEnumerator);
});
this.hasMoreResults = true;
}
public async Task ScenariosAsync()
{
List <Input> inputs = new List <Input>();
int startLineNumber;
int endLineNumber;
//----------------------------------------------------------------
// ReadFeed
//----------------------------------------------------------------
{
startLineNumber = GetLineNumber();
int numItems = 100;
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems);
CrossPartitionReadFeedAsyncEnumerator enumerator = CrossPartitionReadFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState <ReadFeedState>(ReadFeedCrossFeedRangeState.CreateFromBeginning().FeedRangeStates),
new ReadFeedPaginationOptions(pageSizeHint: 10),
cancellationToken: default);
int numChildren = 1; // One extra since we need to read one past the last user page to get the null continuation.
TraceForBaselineTesting rootTrace;
using (rootTrace = TraceForBaselineTesting.GetRootTrace())
{
while (await enumerator.MoveNextAsync(rootTrace))
{
numChildren++;
}
}
Assert.AreEqual(numChildren, rootTrace.Children.Count);
endLineNumber = GetLineNumber();
inputs.Add(new Input("ReadFeed", rootTrace, startLineNumber, endLineNumber));
}
//----------------------------------------------------------------
//----------------------------------------------------------------
// ChangeFeed
//----------------------------------------------------------------
{
startLineNumber = GetLineNumber();
int numItems = 100;
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems);
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState <ChangeFeedState>(
ChangeFeedCrossFeedRangeState.CreateFromBeginning().FeedRangeStates),
new ChangeFeedPaginationOptions(
ChangeFeedMode.Incremental,
pageSizeHint: int.MaxValue),
cancellationToken: default);
int numChildren = 0;
TraceForBaselineTesting rootTrace;
using (rootTrace = TraceForBaselineTesting.GetRootTrace())
{
while (await enumerator.MoveNextAsync(rootTrace))
{
numChildren++;
if (enumerator.Current.Result.Page is ChangeFeedNotModifiedPage)
{
break;
}
}
}
Assert.AreEqual(numChildren, rootTrace.Children.Count);
endLineNumber = GetLineNumber();
inputs.Add(new Input("ChangeFeed", rootTrace, startLineNumber, endLineNumber));
}
//----------------------------------------------------------------
//----------------------------------------------------------------
// Query
//----------------------------------------------------------------
{
startLineNumber = GetLineNumber();
int numItems = 100;
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems);
IQueryPipelineStage pipelineStage = CreatePipeline(documentContainer, "SELECT * FROM c", pageSize: 10);
TraceForBaselineTesting rootTrace;
int numChildren = 1; // One extra since we need to read one past the last user page to get the null continuation.
using (rootTrace = TraceForBaselineTesting.GetRootTrace())
{
while (await pipelineStage.MoveNextAsync(rootTrace))
{
numChildren++;
}
}
Assert.AreEqual(numChildren, rootTrace.Children.Count);
endLineNumber = GetLineNumber();
inputs.Add(new Input("Query", rootTrace, startLineNumber, endLineNumber));
}
//----------------------------------------------------------------
this.ExecuteTestSuite(inputs);
}