public async Task TestScaleUpAndScaleDown()
{
int batchSize = 25;
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey: true);
int totalCount = 0;
// Start draining as 1 iterator
IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(
ReadFeedCrossFeedRangeState.CreateFromBeginning(),
new QueryRequestOptions()
{
MaxItemCount = 1,
});
(int localCount, ReadFeedCrossFeedRangeState? state) = await DrainOnePageAsync(asyncEnumerable);
totalCount += localCount;
// Continue draining as two iterators
if (!state.Value.TrySplit(out ReadFeedCrossFeedRangeState first, out ReadFeedCrossFeedRangeState second))
{
Assert.Fail("Failed to split");
}
IAsyncEnumerable <TryCatch <ReadFeedPage> > firstEnumerable = this.Container.GetReadFeedAsyncEnumerable(
first,
new QueryRequestOptions()
{
MaxItemCount = 1,
});
(int leftCount, ReadFeedCrossFeedRangeState? firstResumeState) = await DrainOnePageAsync(firstEnumerable);
totalCount += leftCount;
IAsyncEnumerable <TryCatch <ReadFeedPage> > secondEnumerable = this.Container.GetReadFeedAsyncEnumerable(
second,
new QueryRequestOptions()
{
MaxItemCount = 1,
});
(int rightCount, ReadFeedCrossFeedRangeState? secondResumeState) = await DrainOnePageAsync(secondEnumerable);
totalCount += rightCount;
// Finish draining again as a single enumerator
ReadFeedCrossFeedRangeState mergedState = firstResumeState.Value.Merge(secondResumeState.Value);
IAsyncEnumerable <TryCatch <ReadFeedPage> > mergedEnumerable = this.Container.GetReadFeedAsyncEnumerable(mergedState);
(int mergedCount, ReadFeedCrossFeedRangeState? _) = await DrainAllAsync(mergedEnumerable);
totalCount += mergedCount;
Assert.AreEqual(batchSize, totalCount);
}
public async Task TestCancellationToken()
{
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.Cancel();
IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(
ReadFeedCrossFeedRangeState.CreateFromBeginning());
await foreach (TryCatch <ReadFeedPage> monadicPage in asyncEnumerable.WithCancellation(cancellationTokenSource.Token))
{
monadicPage.ThrowIfFailed();
}
}
public async Task ParallelizeAcrossFeedRanges()
{
int batchSize = 25;
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey: true);
// Create one start state for each physical partition.
List <ReadFeedCrossFeedRangeState> startStates = new List <ReadFeedCrossFeedRangeState>();
IReadOnlyList <FeedRange> feedRanges = await this.Container.GetFeedRangesAsync();
foreach (FeedRange feedRange in feedRanges)
{
startStates.Add(ReadFeedCrossFeedRangeState.CreateFromBeginning(feedRange));
}
// Create an independant enumerable for each of those start states.
List <IAsyncEnumerable <TryCatch <ReadFeedPage> > > asyncEnumerables = new List <IAsyncEnumerable <TryCatch <ReadFeedPage> > >();
foreach (ReadFeedCrossFeedRangeState state in startStates)
{
IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(state);
asyncEnumerables.Add(asyncEnumerable);
}
int totalCount = 0;
foreach (IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable in asyncEnumerables)
{
// This part can be done in parallel on the same machine or on different machines,
// since they are independant enumerables.
(int totalCount, ReadFeedCrossFeedRangeState? state)countAndState = await DrainAllAsync(asyncEnumerable);
totalCount += countAndState.totalCount;
}
Assert.AreEqual(batchSize, totalCount);
}
public async Task DrainUsingState()
{
int batchSize = 25;
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey : true);
int totalCount = 0;
ReadFeedCrossFeedRangeState?state = ReadFeedCrossFeedRangeState.CreateFromBeginning();
do
{
IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(
state.Value,
new QueryRequestOptions()
{
MaxItemCount = 1,
});
(int localCount, ReadFeedCrossFeedRangeState? newState) = await DrainOnePageAsync(asyncEnumerable);
totalCount += localCount;
state = newState;
}while (state.HasValue);
Assert.AreEqual(batchSize, totalCount);
}
public ReadFeedIteratorCore(
IDocumentContainer documentContainer,
string continuationToken,
ReadFeedPaginationOptions readFeedPaginationOptions,
QueryRequestOptions queryRequestOptions,
CancellationToken cancellationToken)
{
this.queryRequestOptions = queryRequestOptions;
readFeedPaginationOptions ??= ReadFeedPaginationOptions.Default;
if (!string.IsNullOrEmpty(continuationToken))
{
bool isNewArrayFormat = (continuationToken.Length >= 2) && (continuationToken[0] == '[') && (continuationToken[continuationToken.Length - 1] == ']');
if (!isNewArrayFormat)
{
// One of the two older formats
if (!FeedRangeContinuation.TryParse(continuationToken, out FeedRangeContinuation feedRangeContinuation))
{
// Backward compatible with old format
feedRangeContinuation = new FeedRangeCompositeContinuation(
containerRid: string.Empty,
FeedRangeEpk.FullRange,
new List <Documents.Routing.Range <string> >()
{
new Documents.Routing.Range <string>(
Documents.Routing.PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey,
Documents.Routing.PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey,
isMinInclusive: true,
isMaxInclusive: false)
},
continuationToken);
}
// need to massage it a little
List <CosmosElement> feedRangeStates = new List <CosmosElement>();
string oldContinuationFormat = feedRangeContinuation.ToString();
if (feedRangeContinuation.FeedRange is FeedRangePartitionKey feedRangePartitionKey)
{
CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat);
CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"];
if (continuations.Count != 1)
{
throw new InvalidOperationException("Expected only one continuation for partition key queries");
}
CosmosElement continuation = continuations[0];
CosmosObject continuationObject = (CosmosObject)continuation;
CosmosElement token = continuationObject["token"];
ReadFeedState state;
if (token is CosmosNull)
{
state = ReadFeedState.Beginning();
}
else
{
CosmosString tokenAsString = (CosmosString)token;
state = ReadFeedState.Continuation(CosmosElement.Parse(tokenAsString.Value));
}
FeedRangeState <ReadFeedState> feedRangeState = new FeedRangeState <ReadFeedState>(feedRangePartitionKey, state);
feedRangeStates.Add(ReadFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
else
{
CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat);
CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"];
foreach (CosmosElement continuation in continuations)
{
CosmosObject continuationObject = (CosmosObject)continuation;
CosmosObject rangeObject = (CosmosObject)continuationObject["range"];
string min = ((CosmosString)rangeObject["min"]).Value;
string max = ((CosmosString)rangeObject["max"]).Value;
CosmosElement token = continuationObject["token"];
FeedRangeInternal feedRange = new FeedRangeEpk(new Documents.Routing.Range <string>(min, max, isMinInclusive: true, isMaxInclusive: false));
ReadFeedState state;
if (token is CosmosNull)
{
state = ReadFeedState.Beginning();
}
else
{
CosmosString tokenAsString = (CosmosString)token;
state = ReadFeedState.Continuation(CosmosElement.Parse(tokenAsString.Value));
}
FeedRangeState <ReadFeedState> feedRangeState = new FeedRangeState <ReadFeedState>(feedRange, state);
feedRangeStates.Add(ReadFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
}
CosmosArray cosmosArrayContinuationTokens = CosmosArray.Create(feedRangeStates);
continuationToken = cosmosArrayContinuationTokens.ToString();
}
}
TryCatch <ReadFeedCrossFeedRangeState> monadicReadFeedState;
if (continuationToken == null)
{
FeedRange feedRange;
if ((this.queryRequestOptions != null) && this.queryRequestOptions.PartitionKey.HasValue)
{
feedRange = new FeedRangePartitionKey(this.queryRequestOptions.PartitionKey.Value);
}
else if ((this.queryRequestOptions != null) && (this.queryRequestOptions.FeedRange != null))
{
feedRange = this.queryRequestOptions.FeedRange;
}
else
{
feedRange = FeedRangeEpk.FullRange;
}
monadicReadFeedState = TryCatch <ReadFeedCrossFeedRangeState> .FromResult(ReadFeedCrossFeedRangeState.CreateFromBeginning(feedRange));
}
else
{
monadicReadFeedState = ReadFeedCrossFeedRangeState.Monadic.Parse(continuationToken);
}
if (monadicReadFeedState.Failed)
{
this.monadicEnumerator = TryCatch <CrossPartitionReadFeedAsyncEnumerator> .FromException(monadicReadFeedState.Exception);
}
else
{
this.monadicEnumerator = TryCatch <CrossPartitionReadFeedAsyncEnumerator> .FromResult(
CrossPartitionReadFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState <ReadFeedState>(monadicReadFeedState.Result.FeedRangeStates),
readFeedPaginationOptions,
cancellationToken));
}
this.hasMoreResults = true;
}
