internal static TryCatch <ReadFeedCrossFeedRangeState> CreateFromCosmosElement(CosmosElement cosmosElement) { if (cosmosElement == null) { throw new ArgumentNullException(nameof(cosmosElement)); } if (!(cosmosElement is CosmosArray cosmosArray)) { return(TryCatch <ReadFeedCrossFeedRangeState> .FromException( new FormatException( $"Expected array: {cosmosElement}"))); } FeedRangeState <ReadFeedState>[] feedRangeStates = new FeedRangeState <ReadFeedState> [cosmosArray.Count]; int i = 0; foreach (CosmosElement arrayItem in cosmosArray) { TryCatch <FeedRangeState <ReadFeedState> > monadicFeedRangeState = ReadFeedFeedRangeStateSerializer.Monadic.CreateFromCosmosElement(arrayItem); if (monadicFeedRangeState.Failed) { return(TryCatch <ReadFeedCrossFeedRangeState> .FromException(monadicFeedRangeState.Exception)); } feedRangeStates[i++] = monadicFeedRangeState.Result; } ReadFeedCrossFeedRangeState crossFeedRangeState = new ReadFeedCrossFeedRangeState(feedRangeStates.AsMemory()); return(TryCatch <ReadFeedCrossFeedRangeState> .FromResult(crossFeedRangeState)); }
public async Task SerializeAndDeserializeContinuationToken() { int batchSize = 25; await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey : true); int totalCount = 0; IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable( ReadFeedCrossFeedRangeState.CreateFromBeginning()); (int localCount, ReadFeedCrossFeedRangeState? state) = await DrainOnePageAsync(asyncEnumerable); totalCount += localCount; // Serialize the state and send it over the wire for your user to resume execution. string continuationToken = state.Value.ToString(); // Deserialize the state that the user came back with to resume from. ReadFeedCrossFeedRangeState parsedState = ReadFeedCrossFeedRangeState.Parse(continuationToken); asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(parsedState); (localCount, _) = await DrainAllAsync(asyncEnumerable); totalCount += localCount; Assert.AreEqual(batchSize, totalCount); }
public ReadFeedCrossFeedRangeState Merge(ReadFeedCrossFeedRangeState first) { Memory <FeedRangeState <ReadFeedState> > mergedRange = CrossFeedRangeStateSplitterAndMerger.Merge <ReadFeedState>( this.FeedRangeStates, first.FeedRangeStates); return(new ReadFeedCrossFeedRangeState(mergedRange)); }
public async Task TestScaleUpAndScaleDown() { int batchSize = 25; await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey: true); int totalCount = 0; // Start draining as 1 iterator IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable( ReadFeedCrossFeedRangeState.CreateFromBeginning(), new QueryRequestOptions() { MaxItemCount = 1, }); (int localCount, ReadFeedCrossFeedRangeState? state) = await DrainOnePageAsync(asyncEnumerable); totalCount += localCount; // Continue draining as two iterators if (!state.Value.TrySplit(out ReadFeedCrossFeedRangeState first, out ReadFeedCrossFeedRangeState second)) { Assert.Fail("Failed to split"); } IAsyncEnumerable <TryCatch <ReadFeedPage> > firstEnumerable = this.Container.GetReadFeedAsyncEnumerable( first, new QueryRequestOptions() { MaxItemCount = 1, }); (int leftCount, ReadFeedCrossFeedRangeState? firstResumeState) = await DrainOnePageAsync(firstEnumerable); totalCount += leftCount; IAsyncEnumerable <TryCatch <ReadFeedPage> > secondEnumerable = this.Container.GetReadFeedAsyncEnumerable( second, new QueryRequestOptions() { MaxItemCount = 1, }); (int rightCount, ReadFeedCrossFeedRangeState? secondResumeState) = await DrainOnePageAsync(secondEnumerable); totalCount += rightCount; // Finish draining again as a single enumerator ReadFeedCrossFeedRangeState mergedState = firstResumeState.Value.Merge(secondResumeState.Value); IAsyncEnumerable <TryCatch <ReadFeedPage> > mergedEnumerable = this.Container.GetReadFeedAsyncEnumerable(mergedState); (int mergedCount, ReadFeedCrossFeedRangeState? _) = await DrainAllAsync(mergedEnumerable); totalCount += mergedCount; Assert.AreEqual(batchSize, totalCount); }
public ReadFeedCrossFeedRangeAsyncEnumerable( IDocumentContainer documentContainer, ReadFeedCrossFeedRangeState state, ReadFeedPaginationOptions readFeedPaginationOptions) { this.documentContainer = documentContainer ?? throw new ArgumentNullException(nameof(documentContainer)); this.state = state; this.readFeedPaginationOptions = readFeedPaginationOptions; }
public ReadFeedCrossFeedRangeAsyncEnumerable( IDocumentContainer documentContainer, QueryRequestOptions requestOptions, ReadFeedCrossFeedRangeState state) { this.documentContainer = documentContainer ?? throw new ArgumentNullException(nameof(documentContainer)); this.requestOptions = requestOptions; this.state = state; }
public async Task TestCancellationToken() { CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); cancellationTokenSource.Cancel(); IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable( ReadFeedCrossFeedRangeState.CreateFromBeginning()); await foreach (TryCatch <ReadFeedPage> monadicPage in asyncEnumerable.WithCancellation(cancellationTokenSource.Token)) { monadicPage.ThrowIfFailed(); } }
public async Task TargetMultipleLogicalPartitionKeys() { int batchSize = 25; string pkToRead1 = "pkToRead1"; string pkToRead2 = "pkToRead2"; string otherPK = "otherPK"; for (int i = 0; i < batchSize; i++) { await this.Container.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: pkToRead1)); } for (int i = 0; i < batchSize; i++) { await this.Container.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: pkToRead2)); } for (int i = 0; i < batchSize; i++) { await this.Container.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: otherPK)); } // Create one start state for each logical partition key. List <FeedRangeState <ReadFeedState> > feedRangeStates = new List <FeedRangeState <ReadFeedState> >(); IReadOnlyList <string> partitionKeysToTarget = new List <string>() { pkToRead1, pkToRead2 }; foreach (string partitionKeyToTarget in partitionKeysToTarget) { feedRangeStates.Add( new FeedRangeState <ReadFeedState>( (FeedRangeInternal)FeedRange.FromPartitionKey( new Cosmos.PartitionKey(partitionKeyToTarget)), ReadFeedState.Beginning())); } // Use the list composition property of the constructor to merge them in to a single state. ReadFeedCrossFeedRangeState multipleLogicalPartitionKeyState = new ReadFeedCrossFeedRangeState(feedRangeStates.ToImmutableArray()); IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(multipleLogicalPartitionKeyState); (int totalCount, ReadFeedCrossFeedRangeState? _) = await DrainAllAsync(asyncEnumerable); Assert.AreEqual(2 * batchSize, totalCount); }
public bool TrySplit(out ReadFeedCrossFeedRangeState first, out ReadFeedCrossFeedRangeState second) { if (!CrossFeedRangeStateSplitterAndMerger.TrySplit( this.FeedRangeStates, out ReadOnlyMemory <FeedRangeState <ReadFeedState> > firstRange, out ReadOnlyMemory <FeedRangeState <ReadFeedState> > secondRange)) { first = default; second = default; return(false); } first = new ReadFeedCrossFeedRangeState(firstRange); second = new ReadFeedCrossFeedRangeState(secondRange); return(true); }
public static bool TryParse(string text, out ReadFeedCrossFeedRangeState state) { if (text == null) { throw new ArgumentNullException(nameof(text)); } TryCatch <ReadFeedCrossFeedRangeState> monadicParse = Monadic.Parse(text); if (monadicParse.Failed) { state = default; return(false); } state = monadicParse.Result; return(true); }
public async Task ParallelizeAcrossFeedRanges() { int batchSize = 25; await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey: true); // Create one start state for each physical partition. List <ReadFeedCrossFeedRangeState> startStates = new List <ReadFeedCrossFeedRangeState>(); IReadOnlyList <FeedRange> feedRanges = await this.Container.GetFeedRangesAsync(); foreach (FeedRange feedRange in feedRanges) { startStates.Add(ReadFeedCrossFeedRangeState.CreateFromBeginning(feedRange)); } // Create an independant enumerable for each of those start states. List <IAsyncEnumerable <TryCatch <ReadFeedPage> > > asyncEnumerables = new List <IAsyncEnumerable <TryCatch <ReadFeedPage> > >(); foreach (ReadFeedCrossFeedRangeState state in startStates) { IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable(state); asyncEnumerables.Add(asyncEnumerable); } int totalCount = 0; foreach (IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable in asyncEnumerables) { // This part can be done in parallel on the same machine or on different machines, // since they are independant enumerables. (int totalCount, ReadFeedCrossFeedRangeState? state)countAndState = await DrainAllAsync(asyncEnumerable); totalCount += countAndState.totalCount; } Assert.AreEqual(batchSize, totalCount); }
public async Task DrainUsingState() { int batchSize = 25; await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey : true); int totalCount = 0; ReadFeedCrossFeedRangeState?state = ReadFeedCrossFeedRangeState.CreateFromBeginning(); do { IAsyncEnumerable <TryCatch <ReadFeedPage> > asyncEnumerable = this.Container.GetReadFeedAsyncEnumerable( state.Value, new QueryRequestOptions() { MaxItemCount = 1, }); (int localCount, ReadFeedCrossFeedRangeState? newState) = await DrainOnePageAsync(asyncEnumerable); totalCount += localCount; state = newState; }while (state.HasValue); Assert.AreEqual(batchSize, totalCount); }
public ReadFeedIteratorCore( IDocumentContainer documentContainer, string continuationToken, ReadFeedPaginationOptions readFeedPaginationOptions, QueryRequestOptions queryRequestOptions, CancellationToken cancellationToken) { this.queryRequestOptions = queryRequestOptions; readFeedPaginationOptions ??= ReadFeedPaginationOptions.Default; if (!string.IsNullOrEmpty(continuationToken)) { bool isNewArrayFormat = (continuationToken.Length >= 2) && (continuationToken[0] == '[') && (continuationToken[continuationToken.Length - 1] == ']'); if (!isNewArrayFormat) { // One of the two older formats if (!FeedRangeContinuation.TryParse(continuationToken, out FeedRangeContinuation feedRangeContinuation)) { // Backward compatible with old format feedRangeContinuation = new FeedRangeCompositeContinuation( containerRid: string.Empty, FeedRangeEpk.FullRange, new List <Documents.Routing.Range <string> >() { new Documents.Routing.Range <string>( Documents.Routing.PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey, Documents.Routing.PartitionKeyInternal.MaximumExclusiveEffectivePartitionKey, isMinInclusive: true, isMaxInclusive: false) }, continuationToken); } // need to massage it a little List <CosmosElement> feedRangeStates = new List <CosmosElement>(); string oldContinuationFormat = feedRangeContinuation.ToString(); if (feedRangeContinuation.FeedRange is FeedRangePartitionKey feedRangePartitionKey) { CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat); CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"]; if (continuations.Count != 1) { throw new InvalidOperationException("Expected only one continuation for partition key queries"); } CosmosElement continuation = continuations[0]; CosmosObject continuationObject = (CosmosObject)continuation; CosmosElement token = continuationObject["token"]; ReadFeedState state; if (token is CosmosNull) { state = ReadFeedState.Beginning(); } else { CosmosString tokenAsString = (CosmosString)token; state = ReadFeedState.Continuation(CosmosElement.Parse(tokenAsString.Value)); } FeedRangeState <ReadFeedState> feedRangeState = new FeedRangeState <ReadFeedState>(feedRangePartitionKey, state); feedRangeStates.Add(ReadFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState)); } else { CosmosObject cosmosObject = CosmosObject.Parse(oldContinuationFormat); CosmosArray continuations = (CosmosArray)cosmosObject["Continuation"]; foreach (CosmosElement continuation in continuations) { CosmosObject continuationObject = (CosmosObject)continuation; CosmosObject rangeObject = (CosmosObject)continuationObject["range"]; string min = ((CosmosString)rangeObject["min"]).Value; string max = ((CosmosString)rangeObject["max"]).Value; CosmosElement token = continuationObject["token"]; FeedRangeInternal feedRange = new FeedRangeEpk(new Documents.Routing.Range <string>(min, max, isMinInclusive: true, isMaxInclusive: false)); ReadFeedState state; if (token is CosmosNull) { state = ReadFeedState.Beginning(); } else { CosmosString tokenAsString = (CosmosString)token; state = ReadFeedState.Continuation(CosmosElement.Parse(tokenAsString.Value)); } FeedRangeState <ReadFeedState> feedRangeState = new FeedRangeState <ReadFeedState>(feedRange, state); feedRangeStates.Add(ReadFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState)); } } CosmosArray cosmosArrayContinuationTokens = CosmosArray.Create(feedRangeStates); continuationToken = cosmosArrayContinuationTokens.ToString(); } } TryCatch <ReadFeedCrossFeedRangeState> monadicReadFeedState; if (continuationToken == null) { FeedRange feedRange; if ((this.queryRequestOptions != null) && this.queryRequestOptions.PartitionKey.HasValue) { feedRange = new FeedRangePartitionKey(this.queryRequestOptions.PartitionKey.Value); } else if ((this.queryRequestOptions != null) && (this.queryRequestOptions.FeedRange != null)) { feedRange = this.queryRequestOptions.FeedRange; } else { feedRange = FeedRangeEpk.FullRange; } monadicReadFeedState = TryCatch <ReadFeedCrossFeedRangeState> .FromResult(ReadFeedCrossFeedRangeState.CreateFromBeginning(feedRange)); } else { monadicReadFeedState = ReadFeedCrossFeedRangeState.Monadic.Parse(continuationToken); } if (monadicReadFeedState.Failed) { this.monadicEnumerator = TryCatch <CrossPartitionReadFeedAsyncEnumerator> .FromException(monadicReadFeedState.Exception); } else { this.monadicEnumerator = TryCatch <CrossPartitionReadFeedAsyncEnumerator> .FromResult( CrossPartitionReadFeedAsyncEnumerator.Create( documentContainer, new CrossFeedRangeState <ReadFeedState>(monadicReadFeedState.Result.FeedRangeStates), readFeedPaginationOptions, cancellationToken)); } this.hasMoreResults = true; }