internal static TryCatch <ReadFeedCrossFeedRangeState> CreateFromCosmosElement(CosmosElement cosmosElement)
{
if (cosmosElement == null)
{
throw new ArgumentNullException(nameof(cosmosElement));
}
if (!(cosmosElement is CosmosArray cosmosArray))
{
return(TryCatch <ReadFeedCrossFeedRangeState> .FromException(
new FormatException(
$"Expected array: {cosmosElement}")));
}
FeedRangeState <ReadFeedState>[] feedRangeStates = new FeedRangeState <ReadFeedState> [cosmosArray.Count];
int i = 0;
foreach (CosmosElement arrayItem in cosmosArray)
{
TryCatch <FeedRangeState <ReadFeedState> > monadicFeedRangeState = ReadFeedFeedRangeStateSerializer.Monadic.CreateFromCosmosElement(arrayItem);
if (monadicFeedRangeState.Failed)
{
return(TryCatch <ReadFeedCrossFeedRangeState> .FromException(monadicFeedRangeState.Exception));
}
feedRangeStates[i++] = monadicFeedRangeState.Result;
}
ReadFeedCrossFeedRangeState crossFeedRangeState = new ReadFeedCrossFeedRangeState(feedRangeStates.AsMemory());
return(TryCatch <ReadFeedCrossFeedRangeState> .FromResult(crossFeedRangeState));
}
public async Task SerializeAndDeserializeContinuationToken()
{
int batchSize = 25;
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey : true);
int totalCount = 0;
IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(
ReadFeedCrossFeedRangeState.CreateFromBeginning());
(int localCount, ReadFeedCrossFeedRangeState? state) = await DrainOnePageAsync(asyncEnumerable);
totalCount += localCount;
// Serialize the state and send it over the wire for your user to resume execution.
string continuationToken = state.Value.ToString();
// Deserialize the state that the user came back with to resume from.
ReadFeedCrossFeedRangeState parsedState = ReadFeedCrossFeedRangeState.Parse(continuationToken);
asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(parsedState);
(localCount, _) = await DrainAllAsync(asyncEnumerable);
totalCount += localCount;
Assert.AreEqual(batchSize, totalCount);
}
public ReadFeedCrossFeedRangeState Merge(ReadFeedCrossFeedRangeState first)
{
Memory <FeedRangeState <ReadFeedState> > mergedRange = CrossFeedRangeStateSplitterAndMerger.Merge <ReadFeedState>(
this.FeedRangeStates,
first.FeedRangeStates);
return(new ReadFeedCrossFeedRangeState(mergedRange));
}
public async Task TestScaleUpAndScaleDown()
{
int batchSize = 25;
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey: true);
int totalCount = 0;
// Start draining as 1 iterator
IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(
ReadFeedCrossFeedRangeState.CreateFromBeginning(),
new QueryRequestOptions()
{
MaxItemCount = 1,
});
(int localCount, ReadFeedCrossFeedRangeState? state) = await DrainOnePageAsync(asyncEnumerable);
totalCount += localCount;
// Continue draining as two iterators
if (!state.Value.TrySplit(out ReadFeedCrossFeedRangeState first, out ReadFeedCrossFeedRangeState second))
{
Assert.Fail("Failed to split");
}
IAsyncEnumerable <TryCatch <ReadFeedPage> > firstEnumerable = this.Container.GetReadFeedAsyncEnumerable(
first,
new QueryRequestOptions()
{
MaxItemCount = 1,
});
(int leftCount, ReadFeedCrossFeedRangeState? firstResumeState) = await DrainOnePageAsync(firstEnumerable);
totalCount += leftCount;
IAsyncEnumerable <TryCatch <ReadFeedPage> > secondEnumerable = this.Container.GetReadFeedAsyncEnumerable(
second,
new QueryRequestOptions()
{
MaxItemCount = 1,
});
(int rightCount, ReadFeedCrossFeedRangeState? secondResumeState) = await DrainOnePageAsync(secondEnumerable);
totalCount += rightCount;
// Finish draining again as a single enumerator
ReadFeedCrossFeedRangeState mergedState = firstResumeState.Value.Merge(secondResumeState.Value);
IAsyncEnumerable <TryCatch <ReadFeedPage> > mergedEnumerable = this.Container.GetReadFeedAsyncEnumerable(mergedState);
(int mergedCount, ReadFeedCrossFeedRangeState? _) = await DrainAllAsync(mergedEnumerable);
totalCount += mergedCount;
Assert.AreEqual(batchSize, totalCount);
}
public ReadFeedCrossFeedRangeAsyncEnumerable(
IDocumentContainer documentContainer,
ReadFeedCrossFeedRangeState state,
ReadFeedPaginationOptions readFeedPaginationOptions)
{
this.documentContainer = documentContainer ?? throw new ArgumentNullException(nameof(documentContainer));
this.state = state;
this.readFeedPaginationOptions = readFeedPaginationOptions;
}
public ReadFeedCrossFeedRangeAsyncEnumerable(
IDocumentContainer documentContainer,
QueryRequestOptions requestOptions,
ReadFeedCrossFeedRangeState state)
{
this.documentContainer = documentContainer ?? throw new ArgumentNullException(nameof(documentContainer));
this.requestOptions = requestOptions;
this.state = state;
}
public async Task TestCancellationToken()
{
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.Cancel();
IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(
ReadFeedCrossFeedRangeState.CreateFromBeginning());
await foreach (TryCatch <ReadFeedPage> monadicPage in asyncEnumerable.WithCancellation(cancellationTokenSource.Token))
{
monadicPage.ThrowIfFailed();
}
}
public async Task TargetMultipleLogicalPartitionKeys()
{
int batchSize = 25;
string pkToRead1 = "pkToRead1";
string pkToRead2 = "pkToRead2";
string otherPK = "otherPK";
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: pkToRead1));
}
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: pkToRead2));
}
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: otherPK));
}
// Create one start state for each logical partition key.
List <FeedRangeState <ReadFeedState> > feedRangeStates = new List <FeedRangeState <ReadFeedState> >();
IReadOnlyList <string> partitionKeysToTarget = new List <string>()
{
pkToRead1,
pkToRead2
};
foreach (string partitionKeyToTarget in partitionKeysToTarget)
{
feedRangeStates.Add(
new FeedRangeState <ReadFeedState>(
(FeedRangeInternal)FeedRange.FromPartitionKey(
new Cosmos.PartitionKey(partitionKeyToTarget)),
ReadFeedState.Beginning()));
}
// Use the list composition property of the constructor to merge them in to a single state.
ReadFeedCrossFeedRangeState multipleLogicalPartitionKeyState = new ReadFeedCrossFeedRangeState(feedRangeStates.ToImmutableArray());
IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(multipleLogicalPartitionKeyState);
(int totalCount, ReadFeedCrossFeedRangeState? _) = await DrainAllAsync(asyncEnumerable);
Assert.AreEqual(2 * batchSize, totalCount);
}
public bool TrySplit(out ReadFeedCrossFeedRangeState first, out ReadFeedCrossFeedRangeState second)
{
if (!CrossFeedRangeStateSplitterAndMerger.TrySplit(
this.FeedRangeStates,
out ReadOnlyMemory <FeedRangeState <ReadFeedState> > firstRange,
out ReadOnlyMemory <FeedRangeState <ReadFeedState> > secondRange))
{
first = default;
second = default;
return(false);
}
first = new ReadFeedCrossFeedRangeState(firstRange);
second = new ReadFeedCrossFeedRangeState(secondRange);
return(true);
}
public static bool TryParse(string text, out ReadFeedCrossFeedRangeState state)
{
if (text == null)
{
throw new ArgumentNullException(nameof(text));
}
TryCatch <ReadFeedCrossFeedRangeState> monadicParse = Monadic.Parse(text);
if (monadicParse.Failed)
{
state = default;
return(false);
}
state = monadicParse.Result;
return(true);
}
public async Task ParallelizeAcrossFeedRanges()
{
int batchSize = 25;
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey: true);
// Create one start state for each physical partition.
List <ReadFeedCrossFeedRangeState> startStates = new List <ReadFeedCrossFeedRangeState>();
IReadOnlyList <FeedRange> feedRanges = await this.Container.GetFeedRangesAsync();
foreach (FeedRange feedRange in feedRanges)
{
startStates.Add(ReadFeedCrossFeedRangeState.CreateFromBeginning(feedRange));
}
// Create an independant enumerable for each of those start states.
List <IAsyncEnumerable <TryCatch <ReadFeedPage> > > asyncEnumerables = new List <IAsyncEnumerable <TryCatch <ReadFeedPage> > >();
foreach (ReadFeedCrossFeedRangeState state in startStates)
{
IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(state);
asyncEnumerables.Add(asyncEnumerable);
}
int totalCount = 0;
foreach (IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable in asyncEnumerables)
{
// This part can be done in parallel on the same machine or on different machines,
// since they are independant enumerables.
(int totalCount, ReadFeedCrossFeedRangeState? state)countAndState = await DrainAllAsync(asyncEnumerable);
totalCount += countAndState.totalCount;
}
Assert.AreEqual(batchSize, totalCount);
}
public async Task DrainUsingState()
{
int batchSize = 25;
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey : true);
int totalCount = 0;
ReadFeedCrossFeedRangeState?state = ReadFeedCrossFeedRangeState.CreateFromBeginning();
do
{
IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(
state.Value,
new QueryRequestOptions()
{
MaxItemCount = 1,
});
(int localCount, ReadFeedCrossFeedRangeState? newState) = await DrainOnePageAsync(asyncEnumerable);
totalCount += localCount;
state = newState;
}while (state.HasValue);
Assert.AreEqual(batchSize, totalCount);
}
public ReadFeedIteratorCore(
IDocumentContainer documentContainer,
string continuationToken,
ReadFeedPaginationOptions readFeedPaginationOptions,
QueryRequestOptions queryRequestOptions,
CancellationToken cancellationToken)
{
this.queryRequestOptions = queryRequestOptions;
readFeedPaginationOptions ??= ReadFeedPaginationOptions.Default;
if (!string.IsNullOrEmpty(continuationToken))
{
bool isNewArrayFormat = (continuationToken.Length >= 2) && (continuationToken[0] == '[') && (continuationToken[continuationToken.Length - 1] == ']');
if (!isNewArrayFormat)
{
// One of the two older formats
if (!FeedRangeContinuation.TryParse(continuationToken, out FeedRangeContinuation feedRangeContinuation))
{
// Backward compatible with old format
feedRangeContinuation = new FeedRangeCompositeContinuation(
containerRid: string.Empty,
FeedRangeEpk.FullRange,
new List <Documents.Routing.Range <string> >()
{
new Documents.Routing.Range <string>(
Documents.Routing.PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey,
Documents.Routing.PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey,
isMinInclusive: true,
isMaxInclusive: false)
},
continuationToken);
}
// need to massage it a little
List <CosmosElement> feedRangeStates = new List <CosmosElement>();
string oldContinuationFormat = feedRangeContinuation.ToString();
if (feedRangeContinuation.FeedRange is FeedRangePartitionKey feedRangePartitionKey)
{
CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat);
CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"];
if (continuations.Count != 1)
{
throw new InvalidOperationException("Expected only one continuation for partition key queries");
}
CosmosElement continuation = continuations[0];
CosmosObject continuationObject = (CosmosObject)continuation;
CosmosElement token = continuationObject["token"];
ReadFeedState state;
if (token is CosmosNull)
{
state = ReadFeedState.Beginning();
}
else
{
CosmosString tokenAsString = (CosmosString)token;
state = ReadFeedState.Continuation(CosmosElement.Parse(tokenAsString.Value));
}
FeedRangeState <ReadFeedState> feedRangeState = new FeedRangeState <ReadFeedState>(feedRangePartitionKey, state);
feedRangeStates.Add(ReadFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
else
{
CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat);
CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"];
foreach (CosmosElement continuation in continuations)
{
CosmosObject continuationObject = (CosmosObject)continuation;
CosmosObject rangeObject = (CosmosObject)continuationObject["range"];
string min = ((CosmosString)rangeObject["min"]).Value;
string max = ((CosmosString)rangeObject["max"]).Value;
CosmosElement token = continuationObject["token"];
FeedRangeInternal feedRange = new FeedRangeEpk(new Documents.Routing.Range <string>(min, max, isMinInclusive: true, isMaxInclusive: false));
ReadFeedState state;
if (token is CosmosNull)
{
state = ReadFeedState.Beginning();
}
else
{
CosmosString tokenAsString = (CosmosString)token;
state = ReadFeedState.Continuation(CosmosElement.Parse(tokenAsString.Value));
}
FeedRangeState <ReadFeedState> feedRangeState = new FeedRangeState <ReadFeedState>(feedRange, state);
feedRangeStates.Add(ReadFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
}
CosmosArray cosmosArrayContinuationTokens = CosmosArray.Create(feedRangeStates);
continuationToken = cosmosArrayContinuationTokens.ToString();
}
}
TryCatch <ReadFeedCrossFeedRangeState> monadicReadFeedState;
if (continuationToken == null)
{
FeedRange feedRange;
if ((this.queryRequestOptions != null) && this.queryRequestOptions.PartitionKey.HasValue)
{
feedRange = new FeedRangePartitionKey(this.queryRequestOptions.PartitionKey.Value);
}
else if ((this.queryRequestOptions != null) && (this.queryRequestOptions.FeedRange != null))
{
feedRange = this.queryRequestOptions.FeedRange;
}
else
{
feedRange = FeedRangeEpk.FullRange;
}
monadicReadFeedState = TryCatch <ReadFeedCrossFeedRangeState> .FromResult(ReadFeedCrossFeedRangeState.CreateFromBeginning(feedRange));
}
else
{
monadicReadFeedState = ReadFeedCrossFeedRangeState.Monadic.Parse(continuationToken);
}
if (monadicReadFeedState.Failed)
{
this.monadicEnumerator = TryCatch <CrossPartitionReadFeedAsyncEnumerator> .FromException(monadicReadFeedState.Exception);
}
else
{
this.monadicEnumerator = TryCatch <CrossPartitionReadFeedAsyncEnumerator> .FromResult(
CrossPartitionReadFeedAsyncEnumerator.Create(
documentContainer,
new CrossFeedRangeState <ReadFeedState>(monadicReadFeedState.Result.FeedRangeStates),
readFeedPaginationOptions,
cancellationToken));
}
this.hasMoreResults = true;
}
