public ChangeFeedIteratorCore(
IDocumentContainer documentContainer,
ChangeFeedRequestOptions changeFeedRequestOptions,
ChangeFeedStartFrom changeFeedStartFrom)
{
if (changeFeedStartFrom == null)
{
throw new ArgumentNullException(nameof(changeFeedStartFrom));
}
this.documentContainer = documentContainer ?? throw new ArgumentNullException(nameof(documentContainer));
this.changeFeedRequestOptions = changeFeedRequestOptions ?? new ChangeFeedRequestOptions();
this.lazyMonadicEnumerator = new AsyncLazy <TryCatch <CrossPartitionChangeFeedAsyncEnumerator> >(
valueFactory: async(cancellationToken) =>
{
if (changeFeedStartFrom is ChangeFeedStartFromContinuation startFromContinuation)
{
TryCatch <CosmosElement> monadicParsedToken = CosmosElement.Monadic.Parse(startFromContinuation.Continuation);
if (monadicParsedToken.Failed)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse continuation token: {startFromContinuation.Continuation}.",
innerException: monadicParsedToken.Exception)));
}
TryCatch <VersionedAndRidCheckedCompositeToken> monadicVersionedToken = VersionedAndRidCheckedCompositeToken
.MonadicCreateFromCosmosElement(monadicParsedToken.Result);
if (monadicVersionedToken.Failed)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse continuation token: {startFromContinuation.Continuation}.",
innerException: monadicVersionedToken.Exception)));
}
VersionedAndRidCheckedCompositeToken versionedAndRidCheckedCompositeToken = monadicVersionedToken.Result;
if (versionedAndRidCheckedCompositeToken.VersionNumber == VersionedAndRidCheckedCompositeToken.Version.V1)
{
// Need to migrate continuation token
if (!(versionedAndRidCheckedCompositeToken.ContinuationToken is CosmosArray cosmosArray))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse get array continuation token: {startFromContinuation.Continuation}.")));
}
List <CosmosElement> changeFeedTokensV2 = new List <CosmosElement>();
foreach (CosmosElement arrayItem in cosmosArray)
{
if (!(arrayItem is CosmosObject cosmosObject))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse get object in composite continuation: {startFromContinuation.Continuation}.")));
}
if (!cosmosObject.TryGetValue("min", out CosmosString min))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse start of range: {cosmosObject}.")));
}
if (!cosmosObject.TryGetValue("max", out CosmosString max))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse end of range: {cosmosObject}.")));
}
if (!cosmosObject.TryGetValue("token", out CosmosElement token))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse token: {cosmosObject}.")));
}
FeedRangeEpk feedRangeEpk = new FeedRangeEpk(new Documents.Routing.Range <string>(
min: min.Value,
max: max.Value,
isMinInclusive: true,
isMaxInclusive: false));
ChangeFeedState state = token is CosmosNull ? ChangeFeedState.Beginning() : ChangeFeedStateContinuation.Continuation(token);
FeedRangeState <ChangeFeedState> feedRangeState = new FeedRangeState <ChangeFeedState>(feedRangeEpk, state);
changeFeedTokensV2.Add(ChangeFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
CosmosArray changeFeedTokensArrayV2 = CosmosArray.Create(changeFeedTokensV2);
versionedAndRidCheckedCompositeToken = new VersionedAndRidCheckedCompositeToken(
VersionedAndRidCheckedCompositeToken.Version.V2,
changeFeedTokensArrayV2,
versionedAndRidCheckedCompositeToken.Rid);
}
if (versionedAndRidCheckedCompositeToken.VersionNumber != VersionedAndRidCheckedCompositeToken.Version.V2)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Wrong version number: {versionedAndRidCheckedCompositeToken.VersionNumber}.")));
}
string collectionRid = await documentContainer.GetResourceIdentifierAsync(cancellationToken);
if (versionedAndRidCheckedCompositeToken.Rid != collectionRid)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"rids mismatched. Expected: {collectionRid} but got {versionedAndRidCheckedCompositeToken.Rid}.")));
}
changeFeedStartFrom = ChangeFeedStartFrom.ContinuationToken(versionedAndRidCheckedCompositeToken.ContinuationToken.ToString());
}
TryCatch <ChangeFeedCrossFeedRangeState> monadicChangeFeedCrossFeedRangeState = changeFeedStartFrom.Accept(ChangeFeedStateFromToChangeFeedCrossFeedRangeState.Singleton);
if (monadicChangeFeedCrossFeedRangeState.Failed)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Could not convert to {nameof(ChangeFeedCrossFeedRangeState)}.",
innerException: monadicChangeFeedCrossFeedRangeState.Exception)));
}
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
changeFeedRequestOptions,
new CrossFeedRangeState <ChangeFeedState>(monadicChangeFeedCrossFeedRangeState.Result.FeedRangeStates),
cancellationToken: default);
TryCatch <CrossPartitionChangeFeedAsyncEnumerator> monadicEnumerator = TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromResult(enumerator);
return(monadicEnumerator);
});
this.hasMoreResults = true;
}
public async Task ChangeFeed_FeedRange_FromV2SDK()
{
ContainerResponse largerContainer = await this.database.CreateContainerAsync(
new ContainerProperties(id : Guid.NewGuid().ToString(), partitionKeyPath : "/status"),
throughput : 20000,
cancellationToken : this.cancellationToken);
ContainerCore container = (ContainerInlineCore)largerContainer;
int expected = 100;
int count = 0;
await this.CreateRandomItems(container, expected, randomPartitionKey : true);
IReadOnlyList <FeedRange> feedRanges = await container.GetFeedRangesAsync();
List <string> continuations = new List <string>();
// First do one request to construct the old model information based on Etag
foreach (FeedRange feedRange in feedRanges)
{
IEnumerable <string> pkRangeIds = await container.GetPartitionKeyRangesAsync(feedRange);
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
StartTime = DateTime.MinValue.ToUniversalTime(), MaxItemCount = 1
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(feedRange: feedRange, changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
if (firstResponse.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(firstResponse.Content).Data;
count += response.Count;
}
// Construct the continuation's range, using PKRangeId + ETag
List <dynamic> ct = new List <dynamic>()
{
new { min = string.Empty, max = string.Empty, token = firstResponse.Headers.ETag }
};
// Extract Etag and manually construct the continuation
dynamic oldContinuation = new { V = 0, PKRangeId = pkRangeIds.First(), Continuation = ct };
continuations.Add(JsonConvert.SerializeObject(oldContinuation));
}
// Now start the new iterators with the constructed continuations from migration
foreach (string continuation in continuations)
{
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
MaxItemCount = 100
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(continuationToken: continuation, changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
if (firstResponse.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(firstResponse.Content).Data;
count += response.Count;
string migratedContinuation = firstResponse.ContinuationToken;
Assert.IsTrue(FeedRangeContinuation.TryParse(migratedContinuation, out FeedRangeContinuation feedRangeContinuation));
Assert.IsTrue(feedRangeContinuation.FeedRange is FeedRangeEPK);
}
}
Assert.AreEqual(expected, count);
}
public async Task ChangeFeed_FeedRange_FromV0Token()
{
ContainerResponse largerContainer = await this.database.CreateContainerAsync(
new ContainerProperties(id : Guid.NewGuid().ToString(), partitionKeyPath : "/status"),
throughput : 20000,
cancellationToken : this.cancellationToken);
ContainerInternal container = (ContainerInlineCore)largerContainer;
int expected = 100;
int count = 0;
await this.CreateRandomItems((ContainerCore)container, expected, randomPartitionKey : true);
IReadOnlyList <FeedRange> feedRanges = await container.GetFeedRangesAsync();
List <string> continuations = new List <string>();
// First do one request to construct the old model information based on Etag
foreach (FeedRange feedRange in feedRanges)
{
IEnumerable <string> pkRangeIds = await container.GetPartitionKeyRangesAsync(feedRange);
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
PageSizeHint = 1
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(
changeFeedStartFrom: ChangeFeedStartFrom.Beginning(feedRange),
changeFeedMode: ChangeFeedMode.Incremental,
changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
FeedRangeEpk FeedRangeEpk = feedRange as FeedRangeEpk;
// Construct the continuation's range, using PKRangeId + ETag
List <dynamic> ct = new List <dynamic>()
{
new
{
min = FeedRangeEpk.Range.Min,
max = FeedRangeEpk.Range.Max,
token = (string)null
}
};
// Extract Etag and manually construct the continuation
dynamic oldContinuation = new
{
V = 0,
Rid = await container.GetCachedRIDAsync(cancellationToken : this.cancellationToken),
Continuation = ct
};
continuations.Add(JsonConvert.SerializeObject(oldContinuation));
}
// Now start the new iterators with the constructed continuations from migration
foreach (string continuation in continuations)
{
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
PageSizeHint = 100,
EmitOldContinuationToken = true,
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(
changeFeedStartFrom: ChangeFeedStartFrom.ContinuationToken(continuation),
changeFeedMode: ChangeFeedMode.Incremental,
changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
if (firstResponse.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(firstResponse.Content).Data;
count += response.Count;
string migratedContinuation = firstResponse.ContinuationToken;
Assert.IsTrue(FeedRangeContinuation.TryParse(migratedContinuation, out FeedRangeContinuation feedRangeContinuation));
Assert.IsTrue(feedRangeContinuation.FeedRange is FeedRangeEpk);
}
}
Assert.AreEqual(expected, count);
}
public override FeedIterator <T> GetChangeFeedIterator <T>(ChangeFeedStartFrom changeFeedStartFrom, ChangeFeedMode changeFeedMode,
ChangeFeedRequestOptions changeFeedRequestOptions = null) =>
throw new NotImplementedException();
public async Task ChangeFeedIteratorCore_NoFetchNext()
{
int pkRangesCount = (await this.LargerContainer.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(this.LargerContainer.LinkUri)).Count;
int expected = 25;
int iterations = 0;
await this.CreateRandomItems(this.LargerContainer, expected, randomPartitionKey : true);
ContainerInternal itemsCore = this.LargerContainer;
string continuation = null;
int count = 0;
while (true)
{
ChangeFeedRequestOptions requestOptions;
if (continuation == null)
{
requestOptions = new ChangeFeedRequestOptions()
{
From = ChangeFeedRequestOptions.StartFrom.CreateFromBeginning(),
};
}
else
{
requestOptions = new ChangeFeedRequestOptions()
{
From = ChangeFeedRequestOptions.StartFrom.CreateFromContinuation(continuation),
};
}
ChangeFeedIteratorCore feedIterator = itemsCore.GetChangeFeedStreamIterator(changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
using (ResponseMessage responseMessage = await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (responseMessage.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
count += response.Count;
}
else
{
if (responseMessage.StatusCode != HttpStatusCode.NotModified)
{
Assert.Fail(responseMessage.ErrorMessage);
}
}
continuation = responseMessage.ContinuationToken;
}
if (count.Equals(expected))
{
break;
}
if (iterations++ > pkRangesCount)
{
Assert.Fail("" +
"Feed does not contain all elements even after looping through PK ranges. " +
"Either the continuation is not moving forward or there is some state problem.");
}
}
}
