internal CosmosChangeFeedResultSetIteratorCoreMock(
ContainerCore container,
ChangeFeedStartFrom changeFeedStartFrom,
ChangeFeedRequestOptions options) : base(
clientContext: container.ClientContext,
container: container,
changeFeedStartFrom: changeFeedStartFrom,
options: options)
{
List <CompositeContinuationToken> compositeContinuationTokens = new List <CompositeContinuationToken>()
{
new CompositeContinuationToken()
{
Token = null,
Range = new Documents.Routing.Range <string>("A", "B", true, false)
}
};
string serialized = JsonConvert.SerializeObject(compositeContinuationTokens);
this.compositeContinuationToken = StandByFeedContinuationToken.CreateAsync("containerRid", serialized, (string containerRid, Documents.Routing.Range <string> ranges, bool forceRefresh) =>
{
IReadOnlyList <Documents.PartitionKeyRange> filteredRanges = new List <Documents.PartitionKeyRange>()
{
new Documents.PartitionKeyRange()
{
MinInclusive = "A", MaxExclusive = "B", Id = "0"
}
};
if (forceRefresh)
{
this.HasCalledForceRefresh = true;
}
return(Task.FromResult(filteredRanges));
}).Result;
}
public async Task ChangeFeedIteratorCore_UpdatesContinuation_On304()
{
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems : 0);
ChangeFeedIteratorCore changeFeedIteratorCore = new ChangeFeedIteratorCore(
documentContainer,
ChangeFeedMode.Incremental,
new ChangeFeedRequestOptions(),
ChangeFeedStartFrom.Beginning(),
this.MockClientContext());
ResponseMessage responseMessage = await changeFeedIteratorCore.ReadNextAsync();
Assert.AreEqual(HttpStatusCode.NotModified, responseMessage.StatusCode);
string continuationToken = responseMessage.Headers.ContinuationToken;
ResponseMessage responseMessage2 = await changeFeedIteratorCore.ReadNextAsync();
Assert.AreEqual(HttpStatusCode.NotModified, responseMessage.StatusCode);
string continuationToken2 = responseMessage2.Headers.ContinuationToken;
Assert.AreNotEqual(continuationToken, continuationToken2);
}
public async Task SomeChangesAsync()
{
IDocumentContainer documentContainer = await CreateDocumentContainerAsync(numItems : 1);
TryCatch <CrossPartitionChangeFeedAsyncEnumerator> monadicEnumerator = CrossPartitionChangeFeedAsyncEnumerator.MonadicCreate(
documentContainer,
new ChangeFeedRequestOptions(),
ChangeFeedStartFrom.Beginning(),
cancellationToken: default);
Assert.IsTrue(monadicEnumerator.Succeeded);
CrossPartitionChangeFeedAsyncEnumerator enumerator = monadicEnumerator.Result;
// First page should be true and skip the 304 not modified
Assert.IsTrue(await enumerator.MoveNextAsync());
Assert.IsTrue(enumerator.Current.Succeeded);
Assert.IsTrue(enumerator.Current.Result is ChangeFeedSuccessPage);
// Second page should surface up the 304
Assert.IsTrue(await enumerator.MoveNextAsync());
Assert.IsTrue(enumerator.Current.Succeeded);
Assert.IsTrue(enumerator.Current.Result is ChangeFeedNotModifiedPage);
}
public async Task StandByFeedIterator_WithInexistentRange()
{
// Add some random range, this will force the failure
List <CompositeContinuationToken> corruptedTokens = new List <CompositeContinuationToken>
{
new CompositeContinuationToken()
{
Range = new Documents.Routing.Range <string>("whatever", "random", true, false),
Token = "oops"
}
};
string corruptedTokenSerialized = JsonConvert.SerializeObject(corruptedTokens);
ContainerInternal itemsCore = this.Container;
FeedIterator setIteratorNew =
itemsCore.GetStandByFeedIterator(
ChangeFeedStartFrom.ContinuationToken(corruptedTokenSerialized));
_ = await setIteratorNew.ReadNextAsync(this.cancellationToken);
Assert.Fail("Should have thrown.");
}
public override FeedIterator GetChangeFeedStreamIterator(
ChangeFeedStartFrom changeFeedStartFrom,
ChangeFeedMode changeFeedMode,
ChangeFeedRequestOptions changeFeedRequestOptions = null)
{
ChangeFeedRequestOptions clonedchangeFeedRequestOptions;
if (changeFeedRequestOptions != null)
{
clonedchangeFeedRequestOptions = (ChangeFeedRequestOptions)changeFeedRequestOptions.ShallowCopy();
}
else
{
clonedchangeFeedRequestOptions = new ChangeFeedRequestOptions();
}
return(new EncryptionFeedIterator(
this.container.GetChangeFeedStreamIterator(
changeFeedStartFrom,
changeFeedMode,
clonedchangeFeedRequestOptions),
this,
clonedchangeFeedRequestOptions));
}
public async Task ChangeFeedIteratorCore_WithFullFidelityReadFromBeginning()
{
ContainerProperties properties = new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: ChangeFeedIteratorCoreTests.PartitionKey);
properties.ChangeFeedPolicy.FullFidelityRetention = TimeSpan.FromMinutes(5);
ContainerResponse response = await this.database.CreateContainerAsync(
properties,
cancellationToken : this.cancellationToken);
ContainerInternal container = (ContainerInternal)response;
int totalDocuments = 10;
await this.CreateRandomItems(container, totalDocuments, randomPartitionKey : true);
// FF does not work with StartFromBeginning currently, capture error
FeedIterator <ToDoActivityWithMetadata> fullFidelityIterator = container.GetChangeFeedIterator <ToDoActivityWithMetadata>(
ChangeFeedStartFrom.Beginning(),
ChangeFeedMode.FullFidelity);
CosmosException cosmosException = await Assert.ThrowsExceptionAsync <CosmosException>(() => fullFidelityIterator.ReadNextAsync());
Assert.AreEqual(HttpStatusCode.BadRequest, cosmosException.StatusCode, "Full Fidelity Change Feed does not work with StartFromBeginning currently.");
Assert.IsTrue(cosmosException.Message.Contains("FullFidelity Change Feed must have valid If-None-Match header."));
}
public async Task ChangeFeedIteratorCore_StartTime()
{
int totalCount = 0;
int batchSize = 25;
ContainerInternal itemsCore = await this.InitializeContainerAsync();
await this.CreateRandomItems(itemsCore, batchSize, randomPartitionKey : true);
await Task.Delay(1000);
DateTime now = DateTime.UtcNow;
await Task.Delay(1000);
await this.CreateRandomItems(itemsCore, batchSize, randomPartitionKey : true);
FeedIterator feedIterator = itemsCore.GetChangeFeedStreamIterator(
ChangeFeedStartFrom.Time(now),
ChangeFeedMode.Incremental);
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (!responseMessage.IsSuccessStatusCode)
{
break;
}
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
}
Assert.AreEqual(totalCount, batchSize);
}
public async Task ChangeFeed_FeedRange_FromV0Token()
{
ContainerResponse largerContainer = await this.database.CreateContainerAsync(
new ContainerProperties(id : Guid.NewGuid().ToString(), partitionKeyPath : "/pk"),
throughput : 20000,
cancellationToken : this.cancellationToken);
ContainerInternal container = (ContainerInlineCore)largerContainer;
int expected = 100;
int count = 0;
await this.CreateRandomItems((ContainerCore)container, expected, randomPartitionKey : true);
IReadOnlyList <FeedRange> feedRanges = await container.GetFeedRangesAsync();
List <string> continuations = new List <string>();
// First do one request to construct the old model information based on Etag
foreach (FeedRange feedRange in feedRanges)
{
IEnumerable <string> pkRangeIds = await container.GetPartitionKeyRangesAsync(feedRange);
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
PageSizeHint = 1
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(
changeFeedStartFrom: ChangeFeedStartFrom.Beginning(feedRange),
changeFeedMode: ChangeFeedMode.Incremental,
changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
// Construct the continuation's range, using PKRangeId + ETag
List <dynamic> ct = new List <dynamic>()
{
new
{
FeedRange = new
{
type = "Physical Partition Key Range Id",
value = pkRangeIds.First()
},
State = new
{
type = "continuation",
value = JObject.Parse(firstResponse.ContinuationToken)["Continuation"][0]["State"]["value"].ToString()
}
}
};
if (firstResponse.Content != null)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(firstResponse.Content).Data;
count += response.Count;
}
// Extract Etag and manually construct the continuation
dynamic oldContinuation = new
{
V = 2,
Rid = await container.GetCachedRIDAsync(cancellationToken : this.cancellationToken),
Continuation = ct
};
continuations.Add(JsonConvert.SerializeObject(oldContinuation));
}
// Now start the new iterators with the constructed continuations from migration
foreach (string continuation in continuations)
{
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
PageSizeHint = 100,
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(
changeFeedStartFrom: ChangeFeedStartFrom.ContinuationToken(continuation),
changeFeedMode: ChangeFeedMode.Incremental,
changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
if (firstResponse.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(firstResponse.Content).Data;
count += response.Count;
string migratedContinuation = firstResponse.ContinuationToken;
TryCatch <CosmosElement> monadicParsedToken = CosmosElement.Monadic.Parse(migratedContinuation);
Assert.IsFalse(monadicParsedToken.Failed);
TryCatch <VersionedAndRidCheckedCompositeToken> monadicVersionedToken = VersionedAndRidCheckedCompositeToken
.MonadicCreateFromCosmosElement(monadicParsedToken.Result);
Assert.IsFalse(monadicVersionedToken.Failed);
VersionedAndRidCheckedCompositeToken versionedAndRidCheckedCompositeToken = monadicVersionedToken.Result;
Assert.AreEqual(VersionedAndRidCheckedCompositeToken.Version.V2, versionedAndRidCheckedCompositeToken.VersionNumber);
}
}
Assert.AreEqual(expected, count);
}
public async Task ShouldContinueUntilResponseOk()
{
// Setting 3 ranges, first one returns a 304, second returns Ok
MultiRangeMockDocumentClient documentClient = new MultiRangeMockDocumentClient();
using CosmosClient client = MockCosmosUtil.CreateMockCosmosClient();
Mock <CosmosClientContext> mockContext = new Mock <CosmosClientContext>();
mockContext.Setup(x => x.ClientOptions).Returns(MockCosmosUtil.GetDefaultConfiguration());
mockContext.Setup(x => x.DocumentClient).Returns(documentClient);
mockContext.Setup(x => x.SerializerCore).Returns(MockCosmosUtil.Serializer);
mockContext.Setup(x => x.Client).Returns(client);
mockContext.Setup(x => x.CreateLink(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())).Returns("/dbs/test/colls/test");
ResponseMessage firstResponse = new ResponseMessage(HttpStatusCode.NotModified);
firstResponse.Headers.ETag = "FirstContinuation";
ResponseMessage secondResponse = new ResponseMessage(HttpStatusCode.OK);
secondResponse.Headers.ETag = "SecondContinuation";
mockContext.SetupSequence(x => x.ProcessResourceOperationAsync <ResponseMessage>(
It.IsAny <string>(),
It.IsAny <Documents.ResourceType>(),
It.IsAny <Documents.OperationType>(),
It.IsAny <RequestOptions>(),
It.IsAny <ContainerInternal>(),
It.IsAny <PartitionKey?>(),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(firstResponse))
.Returns(Task.FromResult(secondResponse));
DatabaseInternal databaseCore = new DatabaseInlineCore(mockContext.Object, "mydb");
StandByFeedIteratorCore iterator = new StandByFeedIteratorCore(
mockContext.Object,
new ContainerInlineCore(mockContext.Object, databaseCore, "myColl"),
ChangeFeedStartFrom.Beginning(),
new ChangeFeedRequestOptions()
{
PageSizeHint = 10,
});
ResponseMessage firstRequest = await iterator.ReadNextAsync();
Assert.IsTrue(firstRequest.Headers.ContinuationToken.Contains(firstResponse.Headers.ETag), "Response should contain the first continuation");
Assert.IsTrue(firstRequest.Headers.ContinuationToken.Contains(secondResponse.Headers.ETag), "Response should contain the second continuation");
Assert.AreEqual(HttpStatusCode.OK, firstRequest.StatusCode);
mockContext.Verify(x => x.ProcessResourceOperationAsync <ResponseMessage>(
It.IsAny <string>(),
It.IsAny <Documents.ResourceType>(),
It.IsAny <Documents.OperationType>(),
It.IsAny <RequestOptions>(),
It.IsAny <ContainerInternal>(),
It.IsAny <PartitionKey?>(),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <CancellationToken>()), Times.Exactly(2));
}
public async Task ChangeFeedIteratorCore_OfT_ReadAll()
{
int totalCount = 0;
int firstRunTotal = 25;
int batchSize = 25;
ContainerInternal itemsCore = await this.InitializeContainerAsync();
await this.CreateRandomItems(itemsCore, batchSize, randomPartitionKey : true);
FeedIterator <ToDoActivity> feedIterator = itemsCore.GetChangeFeedIterator <ToDoActivity>(ChangeFeedStartFrom.Beginning(), ChangeFeedMode.Incremental);
string continuation = null;
while (feedIterator.HasMoreResults)
{
try
{
FeedResponse <ToDoActivity> feedResponse = await feedIterator.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
continuation = feedResponse.ContinuationToken;
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == HttpStatusCode.NotModified)
{
continuation = cosmosException.Headers.ContinuationToken;
break;
}
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
// Insert another batch of 25 and use the last FeedToken from the first cycle
await this.CreateRandomItems(itemsCore, batchSize, randomPartitionKey : true);
FeedIterator <ToDoActivity> setIteratorNew = itemsCore.GetChangeFeedIterator <ToDoActivity>(ChangeFeedStartFrom.ContinuationToken(continuation), ChangeFeedMode.Incremental);
while (setIteratorNew.HasMoreResults)
{
try
{
FeedResponse <ToDoActivity> feedResponse = await feedIterator.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == HttpStatusCode.NotModified)
{
break;
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
public async Task ChangeFeedIteratorCore_PartitionKey_ReadAll()
{
int totalCount = 0;
int firstRunTotal = 25;
int batchSize = 25;
string pkToRead = "pkToRead";
string otherPK = "otherPK";
ContainerInternal itemsCore = await this.InitializeContainerAsync();
for (int i = 0; i < batchSize; i++)
{
await itemsCore.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: pkToRead));
}
for (int i = 0; i < batchSize; i++)
{
await itemsCore.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: otherPK));
}
ChangeFeedIteratorCore feedIterator = itemsCore.GetChangeFeedStreamIterator(
ChangeFeedStartFrom.Beginning(
FeedRange.FromPartitionKey(
new PartitionKey(pkToRead))),
ChangeFeedMode.Incremental,
new ChangeFeedRequestOptions()
{
PageSizeHint = 1,
}) as ChangeFeedIteratorCore;
string continuation = null;
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
if (!responseMessage.IsSuccessStatusCode)
{
continuation = responseMessage.ContinuationToken;
break;
}
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
foreach (ToDoActivity toDoActivity in response)
{
Assert.AreEqual(pkToRead, toDoActivity.pk);
}
}
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
// Insert another batch of 25 and use the last FeedToken from the first cycle
for (int i = 0; i < batchSize; i++)
{
await itemsCore.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: pkToRead));
}
ChangeFeedIteratorCore setIteratorNew = itemsCore.GetChangeFeedStreamIterator(
ChangeFeedStartFrom.ContinuationToken(continuation),
ChangeFeedMode.Incremental) as ChangeFeedIteratorCore;
while (setIteratorNew.HasMoreResults)
{
using (ResponseMessage responseMessage =
await setIteratorNew.ReadNextAsync(this.cancellationToken))
{
if (!responseMessage.IsSuccessStatusCode)
{
break;
}
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
foreach (ToDoActivity toDoActivity in response)
{
Assert.AreEqual(pkToRead, toDoActivity.pk);
}
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
public async Task ChangeFeedIteratorCore_OfT_ReadAll()
{
int totalCount = 0;
int firstRunTotal = 25;
int batchSize = 25;
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey : true);
ContainerInternal itemsCore = this.Container;
FeedIterator <ToDoActivity> feedIterator = itemsCore.GetChangeFeedIterator <ToDoActivity>(ChangeFeedStartFrom.Beginning());
string continuation = null;
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await feedIterator.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
continuation = feedResponse.ContinuationToken;
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
// Insert another batch of 25 and use the last FeedToken from the first cycle
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey : true);
FeedIterator <ToDoActivity> setIteratorNew = itemsCore.GetChangeFeedIterator <ToDoActivity>(ChangeFeedStartFrom.ContinuationToken(continuation));
while (setIteratorNew.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await setIteratorNew.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
public async Task ChangeFeed_FeedRange_FromV0Token()
{
ContainerResponse largerContainer = await this.database.CreateContainerAsync(
new ContainerProperties(id : Guid.NewGuid().ToString(), partitionKeyPath : "/status"),
throughput : 20000,
cancellationToken : this.cancellationToken);
ContainerInternal container = (ContainerInlineCore)largerContainer;
int expected = 100;
int count = 0;
await this.CreateRandomItems((ContainerCore)container, expected, randomPartitionKey : true);
IReadOnlyList <FeedRange> feedRanges = await container.GetFeedRangesAsync();
List <string> continuations = new List <string>();
// First do one request to construct the old model information based on Etag
foreach (FeedRange feedRange in feedRanges)
{
IEnumerable <string> pkRangeIds = await container.GetPartitionKeyRangesAsync(feedRange);
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
PageSizeHint = 1
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(
changeFeedStartFrom: ChangeFeedStartFrom.Beginning(feedRange),
changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
FeedRangeEpk FeedRangeEpk = feedRange as FeedRangeEpk;
// Construct the continuation's range, using PKRangeId + ETag
List <dynamic> ct = new List <dynamic>()
{
new
{
min = FeedRangeEpk.Range.Min,
max = FeedRangeEpk.Range.Max,
token = (string)null
}
};
// Extract Etag and manually construct the continuation
dynamic oldContinuation = new
{
V = 0,
Rid = await container.GetCachedRIDAsync(cancellationToken : this.cancellationToken),
Continuation = ct
};
continuations.Add(JsonConvert.SerializeObject(oldContinuation));
}
// Now start the new iterators with the constructed continuations from migration
foreach (string continuation in continuations)
{
ChangeFeedRequestOptions requestOptions = new ChangeFeedRequestOptions()
{
PageSizeHint = 100,
EmitOldContinuationToken = true,
};
ChangeFeedIteratorCore feedIterator = container.GetChangeFeedStreamIterator(
changeFeedStartFrom: ChangeFeedStartFrom.ContinuationToken(continuation),
changeFeedRequestOptions: requestOptions) as ChangeFeedIteratorCore;
ResponseMessage firstResponse = await feedIterator.ReadNextAsync();
if (firstResponse.IsSuccessStatusCode)
{
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(firstResponse.Content).Data;
count += response.Count;
string migratedContinuation = firstResponse.ContinuationToken;
Assert.IsTrue(FeedRangeContinuation.TryParse(migratedContinuation, out FeedRangeContinuation feedRangeContinuation));
Assert.IsTrue(feedRangeContinuation.FeedRange is FeedRangeEpk);
}
}
Assert.AreEqual(expected, count);
}
public async Task RunAsync(
CTLConfig config,
CosmosClient cosmosClient,
ILogger logger,
IMetrics metrics,
string loggingContextIdentifier,
CancellationToken cancellationToken)
{
Stopwatch stopWatch = Stopwatch.StartNew();
GaugeOptions documentGauge = new GaugeOptions {
Name = "#Documents received", Context = loggingContextIdentifier
};
Container container = cosmosClient.GetContainer(config.Database, config.Collection);
try
{
while (stopWatch.Elapsed <= config.RunningTimeDurationAsTimespan)
{
long documentTotal = 0;
string continuation = null;
using FeedIterator <Dictionary <string, string> > changeFeedPull
= container.GetChangeFeedIterator <Dictionary <string, string> >(ChangeFeedStartFrom.Beginning(), ChangeFeedMode.Incremental);
try
{
while (changeFeedPull.HasMoreResults)
{
FeedResponse <Dictionary <string, string> > response = await changeFeedPull.ReadNextAsync();
documentTotal += response.Count;
continuation = response.ContinuationToken;
if (response.StatusCode == HttpStatusCode.NotModified)
{
break;
}
}
metrics.Measure.Gauge.SetValue(documentGauge, documentTotal);
if (config.PreCreatedDocuments > 0)
{
if (this.initializationResult.InsertedDocuments == documentTotal)
{
logger.LogInformation($"Success: The number of new documents match the number of pre-created documents: {this.initializationResult.InsertedDocuments}");
}
else
{
logger.LogError($"The prepopulated documents and the change feed documents don't match. Preconfigured Docs = {this.initializationResult.InsertedDocuments}, Change feed Documents = {documentTotal}.{Environment.NewLine}{continuation}");
}
}
}
catch (CosmosException ce) when(ce.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
{
//Logging 429s is not relevant
}
catch (Exception ex)
{
metrics.Measure.Gauge.SetValue(documentGauge, documentTotal);
logger.LogError(ex, "Failure while looping through change feed documents");
}
}
}
catch (Exception ex)
{
logger.LogError(ex, "Failure during Change Feed Pull scenario");
}
finally
{
stopWatch.Stop();
if (this.initializationResult.CreatedContainer)
{
await cosmosClient.GetContainer(config.Database, config.Collection).DeleteContainerStreamAsync();
}
if (this.initializationResult.CreatedDatabase)
{
await cosmosClient.GetDatabase(config.Database).DeleteStreamAsync();
}
}
}
public async Task StandByFeedIterator()
{
int totalCount = 0;
string lastcontinuation = string.Empty;
int firstRunTotal = 25;
int batchSize = 25;
int pkRangesCount = (await this.Container.ClientContext.DocumentClient.ReadPartitionKeyRangeFeedAsync(this.Container.LinkUri)).Count;
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey : true);
ContainerCore itemsCore = (ContainerCore)this.Container;
FeedIterator feedIterator = itemsCore.GetStandByFeedIterator(
ChangeFeedStartFrom.Beginning());
while (feedIterator.HasMoreResults)
{
using (ResponseMessage responseMessage =
await feedIterator.ReadNextAsync(this.cancellationToken))
{
lastcontinuation = responseMessage.Headers.ContinuationToken;
Assert.AreEqual(responseMessage.ContinuationToken, responseMessage.Headers.ContinuationToken);
List <CompositeContinuationToken> deserializedToken = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(lastcontinuation);
Assert.AreEqual(pkRangesCount, deserializedToken.Count);
if (!responseMessage.IsSuccessStatusCode)
{
break;
}
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
// Insert another batch of 25 and use the last continuation token from the first cycle
await this.CreateRandomItems(this.Container, batchSize, randomPartitionKey : true);
FeedIterator setIteratorNew =
itemsCore.GetStandByFeedIterator(
ChangeFeedStartFrom.ContinuationToken(lastcontinuation));
while (setIteratorNew.HasMoreResults)
{
using (ResponseMessage responseMessage =
await setIteratorNew.ReadNextAsync(this.cancellationToken))
{
lastcontinuation = responseMessage.Headers.ContinuationToken;
Assert.AreEqual(responseMessage.ContinuationToken, responseMessage.Headers.ContinuationToken);
if (!responseMessage.IsSuccessStatusCode)
{
break;
}
Collection <ToDoActivity> response = TestCommon.SerializerCore.FromStream <CosmosFeedResponseUtil <ToDoActivity> >(responseMessage.Content).Data;
totalCount += response.Count;
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
public async Task RunAsync(
CTLConfig config,
CosmosClient cosmosClient,
ILogger logger,
IMetrics metrics,
string loggingContextIdentifier,
CancellationToken cancellationToken)
{
Stopwatch stopWatch = Stopwatch.StartNew();
GaugeOptions documentGauge = new GaugeOptions {
Name = "#Documents received", Context = loggingContextIdentifier
};
TimerOptions readLatencyTimer = new TimerOptions
{
Name = "Latency",
MeasurementUnit = Unit.Requests,
DurationUnit = TimeUnit.Milliseconds,
RateUnit = TimeUnit.Seconds,
Context = loggingContextIdentifier,
Reservoir = () => new App.Metrics.ReservoirSampling.Uniform.DefaultAlgorithmRReservoir()
};
Container container = cosmosClient.GetContainer(config.Database, config.Collection);
try
{
while (stopWatch.Elapsed <= config.RunningTimeDurationAsTimespan)
{
long documentTotal = 0;
string continuation = null;
using FeedIterator <Dictionary <string, string> > changeFeedPull
= container.GetChangeFeedIterator <Dictionary <string, string> >(ChangeFeedStartFrom.Beginning(), ChangeFeedMode.Incremental);
try
{
while (changeFeedPull.HasMoreResults)
{
FeedResponse <Dictionary <string, string> > response;
using (TimerContext timerContext = metrics.Measure.Timer.Time(readLatencyTimer))
{
response = await changeFeedPull.ReadNextAsync();
Utils.LogDiagnostics(
logger: logger,
operationName: nameof(ChangeFeedPullScenario),
timerContextLatency: timerContext.Elapsed,
config: config,
cosmosDiagnostics: response.Diagnostics);
}
documentTotal += response.Count;
continuation = response.ContinuationToken;
if (response.StatusCode == HttpStatusCode.NotModified)
{
break;
}
}
metrics.Measure.Gauge.SetValue(documentGauge, documentTotal);
if (config.PreCreatedDocuments > 0)
{
if (this.initializationResult.InsertedDocuments != documentTotal)
{
Utils.LogError(logger, loggingContextIdentifier, $"The prepopulated documents and the change feed documents don't match. Preconfigured Docs = {this.initializationResult.InsertedDocuments}, Change feed Documents = {documentTotal}.{Environment.NewLine}{continuation}");
}
}
}
catch (Exception ex)
{
metrics.Measure.Gauge.SetValue(documentGauge, documentTotal);
Utils.LogError(logger, loggingContextIdentifier, ex);
}
}
}
catch (Exception ex)
{
Utils.LogError(logger, loggingContextIdentifier, ex);
}
finally
{
stopWatch.Stop();
if (this.initializationResult.CreatedContainer)
{
await cosmosClient.GetContainer(config.Database, config.Collection).DeleteContainerStreamAsync();
}
if (this.initializationResult.CreatedDatabase)
{
await cosmosClient.GetDatabase(config.Database).DeleteStreamAsync();
}
}
}
public async Task ChangeFeedIteratorCore_PartitionKey_OfT_ReadAll()
{
int totalCount = 0;
int firstRunTotal = 25;
int batchSize = 25;
string pkToRead = "pkToRead";
string otherPK = "otherPK";
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead));
}
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(otherPK));
}
ContainerInternal itemsCore = this.Container;
FeedIterator <ToDoActivity> feedIterator = itemsCore.GetChangeFeedIterator <ToDoActivity>(
ChangeFeedStartFrom.Beginning(
new FeedRangePartitionKey(
new PartitionKey(pkToRead))),
new ChangeFeedRequestOptions()
{
PageSizeHint = 1,
});
string continuation = null;
while (feedIterator.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await feedIterator.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.status);
}
continuation = feedResponse.ContinuationToken;
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
// Insert another batch of 25 and use the last FeedToken from the first cycle
for (int i = 0; i < batchSize; i++)
{
await this.Container.CreateItemAsync(this.CreateRandomToDoActivity(pkToRead));
}
FeedIterator <ToDoActivity> setIteratorNew = itemsCore.GetChangeFeedIterator <ToDoActivity>(
ChangeFeedStartFrom.ContinuationToken(continuation));
while (setIteratorNew.HasMoreResults)
{
FeedResponse <ToDoActivity> feedResponse = await setIteratorNew.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.status);
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
public override FeedIterator <T> GetChangeFeedIterator <T>(ChangeFeedStartFrom changeFeedStartFrom, ChangeFeedMode changeFeedMode,
ChangeFeedRequestOptions changeFeedRequestOptions = null) =>
throw new NotImplementedException();
public ChangeFeedIteratorCore(
IDocumentContainer documentContainer,
ChangeFeedRequestOptions changeFeedRequestOptions,
ChangeFeedStartFrom changeFeedStartFrom)
{
if (changeFeedStartFrom == null)
{
throw new ArgumentNullException(nameof(changeFeedStartFrom));
}
this.documentContainer = documentContainer ?? throw new ArgumentNullException(nameof(documentContainer));
this.changeFeedRequestOptions = changeFeedRequestOptions ?? new ChangeFeedRequestOptions();
this.lazyMonadicEnumerator = new AsyncLazy <TryCatch <CrossPartitionChangeFeedAsyncEnumerator> >(
valueFactory: async(cancellationToken) =>
{
if (changeFeedStartFrom is ChangeFeedStartFromContinuation startFromContinuation)
{
TryCatch <CosmosElement> monadicParsedToken = CosmosElement.Monadic.Parse(startFromContinuation.Continuation);
if (monadicParsedToken.Failed)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse continuation token: {startFromContinuation.Continuation}.",
innerException: monadicParsedToken.Exception)));
}
TryCatch <VersionedAndRidCheckedCompositeToken> monadicVersionedToken = VersionedAndRidCheckedCompositeToken
.MonadicCreateFromCosmosElement(monadicParsedToken.Result);
if (monadicVersionedToken.Failed)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse continuation token: {startFromContinuation.Continuation}.",
innerException: monadicVersionedToken.Exception)));
}
VersionedAndRidCheckedCompositeToken versionedAndRidCheckedCompositeToken = monadicVersionedToken.Result;
if (versionedAndRidCheckedCompositeToken.VersionNumber == VersionedAndRidCheckedCompositeToken.Version.V1)
{
// Need to migrate continuation token
if (!(versionedAndRidCheckedCompositeToken.ContinuationToken is CosmosArray cosmosArray))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse get array continuation token: {startFromContinuation.Continuation}.")));
}
List <CosmosElement> changeFeedTokensV2 = new List <CosmosElement>();
foreach (CosmosElement arrayItem in cosmosArray)
{
if (!(arrayItem is CosmosObject cosmosObject))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse get object in composite continuation: {startFromContinuation.Continuation}.")));
}
if (!cosmosObject.TryGetValue("min", out CosmosString min))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse start of range: {cosmosObject}.")));
}
if (!cosmosObject.TryGetValue("max", out CosmosString max))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse end of range: {cosmosObject}.")));
}
if (!cosmosObject.TryGetValue("token", out CosmosElement token))
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Failed to parse token: {cosmosObject}.")));
}
FeedRangeEpk feedRangeEpk = new FeedRangeEpk(new Documents.Routing.Range <string>(
min: min.Value,
max: max.Value,
isMinInclusive: true,
isMaxInclusive: false));
ChangeFeedState state = token is CosmosNull ? ChangeFeedState.Beginning() : ChangeFeedStateContinuation.Continuation(token);
FeedRangeState <ChangeFeedState> feedRangeState = new FeedRangeState <ChangeFeedState>(feedRangeEpk, state);
changeFeedTokensV2.Add(ChangeFeedFeedRangeStateSerializer.ToCosmosElement(feedRangeState));
}
CosmosArray changeFeedTokensArrayV2 = CosmosArray.Create(changeFeedTokensV2);
versionedAndRidCheckedCompositeToken = new VersionedAndRidCheckedCompositeToken(
VersionedAndRidCheckedCompositeToken.Version.V2,
changeFeedTokensArrayV2,
versionedAndRidCheckedCompositeToken.Rid);
}
if (versionedAndRidCheckedCompositeToken.VersionNumber != VersionedAndRidCheckedCompositeToken.Version.V2)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Wrong version number: {versionedAndRidCheckedCompositeToken.VersionNumber}.")));
}
string collectionRid = await documentContainer.GetResourceIdentifierAsync(cancellationToken);
if (versionedAndRidCheckedCompositeToken.Rid != collectionRid)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"rids mismatched. Expected: {collectionRid} but got {versionedAndRidCheckedCompositeToken.Rid}.")));
}
changeFeedStartFrom = ChangeFeedStartFrom.ContinuationToken(versionedAndRidCheckedCompositeToken.ContinuationToken.ToString());
}
TryCatch <ChangeFeedCrossFeedRangeState> monadicChangeFeedCrossFeedRangeState = changeFeedStartFrom.Accept(ChangeFeedStateFromToChangeFeedCrossFeedRangeState.Singleton);
if (monadicChangeFeedCrossFeedRangeState.Failed)
{
return(TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromException(
new MalformedChangeFeedContinuationTokenException(
message: $"Could not convert to {nameof(ChangeFeedCrossFeedRangeState)}.",
innerException: monadicChangeFeedCrossFeedRangeState.Exception)));
}
CrossPartitionChangeFeedAsyncEnumerator enumerator = CrossPartitionChangeFeedAsyncEnumerator.Create(
documentContainer,
changeFeedRequestOptions,
new CrossFeedRangeState <ChangeFeedState>(monadicChangeFeedCrossFeedRangeState.Result.FeedRangeStates),
cancellationToken: default);
TryCatch <CrossPartitionChangeFeedAsyncEnumerator> monadicEnumerator = TryCatch <CrossPartitionChangeFeedAsyncEnumerator> .FromResult(enumerator);
return(monadicEnumerator);
});
this.hasMoreResults = true;
}
internal async Task <(string, ResponseMessage)> ReadNextInternalAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (this.compositeContinuationToken == null)
{
PartitionKeyRangeCache pkRangeCache = await this.clientContext.DocumentClient.GetPartitionKeyRangeCacheAsync();
this.containerRid = await this.container.GetRIDAsync(cancellationToken);
if (this.changeFeedStartFrom is ChangeFeedStartFromContinuation startFromContinuation)
{
this.compositeContinuationToken = await StandByFeedContinuationToken.CreateAsync(
this.containerRid,
startFromContinuation.Continuation,
pkRangeCache.TryGetOverlappingRangesAsync);
(CompositeContinuationToken token, string id) = await this.compositeContinuationToken.GetCurrentTokenAsync();
if (token.Token != null)
{
this.changeFeedStartFrom = ChangeFeedStartFrom.ContinuationToken(token.Token);
}
else
{
this.changeFeedStartFrom = ChangeFeedStartFrom.Beginning();
}
}
else
{
this.compositeContinuationToken = await StandByFeedContinuationToken.CreateAsync(
this.containerRid,
initialStandByFeedContinuationToken : null,
pkRangeCache.TryGetOverlappingRangesAsync);
}
}
(CompositeContinuationToken currentRangeToken, string rangeId) = await this.compositeContinuationToken.GetCurrentTokenAsync();
FeedRange feedRange = new FeedRangePartitionKeyRange(rangeId);
if (currentRangeToken.Token != null)
{
this.changeFeedStartFrom = new ChangeFeedStartFromContinuationAndFeedRange(currentRangeToken.Token, (FeedRangeInternal)feedRange);
}
else
{
this.changeFeedStartFrom = ChangeFeedStartFrom.Beginning(feedRange);
}
ResponseMessage response = await this.NextResultSetDelegateAsync(this.changeFeedOptions, cancellationToken);
if (await this.ShouldRetryFailureAsync(response, cancellationToken))
{
return(await this.ReadNextInternalAsync(cancellationToken));
}
if (response.IsSuccessStatusCode ||
response.StatusCode == HttpStatusCode.NotModified)
{
// Change Feed read uses Etag for continuation
currentRangeToken.Token = response.Headers.ETag;
}
return(rangeId, response);
}
public async Task ChangeFeedIteratorCore_PartitionKey_OfT_ReadAll()
{
int totalCount = 0;
int firstRunTotal = 25;
int batchSize = 25;
string pkToRead = "pkToRead";
string otherPK = "otherPK";
ContainerInternal itemsCore = await this.InitializeContainerAsync();
for (int i = 0; i < batchSize; i++)
{
await itemsCore.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: pkToRead));
}
for (int i = 0; i < batchSize; i++)
{
await itemsCore.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: otherPK));
}
FeedIterator <ToDoActivity> feedIterator = itemsCore.GetChangeFeedIterator <ToDoActivity>(
ChangeFeedStartFrom.Beginning(
new FeedRangePartitionKey(
new PartitionKey(pkToRead))),
ChangeFeedMode.Incremental,
new ChangeFeedRequestOptions()
{
PageSizeHint = 1,
});
string continuation = null;
while (feedIterator.HasMoreResults)
{
try
{
FeedResponse <ToDoActivity> feedResponse = await feedIterator.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.pk);
}
continuation = feedResponse.ContinuationToken;
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == HttpStatusCode.NotModified)
{
continuation = cosmosException.Headers.ContinuationToken;
break;
}
}
Assert.AreEqual(firstRunTotal, totalCount);
int expectedFinalCount = 50;
// Insert another batch of 25 and use the last FeedToken from the first cycle
for (int i = 0; i < batchSize; i++)
{
await itemsCore.CreateItemAsync(ToDoActivity.CreateRandomToDoActivity(pk: pkToRead));
}
FeedIterator <ToDoActivity> setIteratorNew = itemsCore.GetChangeFeedIterator <ToDoActivity>(
ChangeFeedStartFrom.ContinuationToken(continuation),
ChangeFeedMode.Incremental);
while (setIteratorNew.HasMoreResults)
{
try
{
FeedResponse <ToDoActivity> feedResponse = await setIteratorNew.ReadNextAsync(this.cancellationToken);
totalCount += feedResponse.Count;
foreach (ToDoActivity toDoActivity in feedResponse)
{
Assert.AreEqual(pkToRead, toDoActivity.pk);
}
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == HttpStatusCode.NotModified)
{
break;
}
}
Assert.AreEqual(expectedFinalCount, totalCount);
}
public async Task ContinuationTokenIsNotUpdatedOnFails()
{
MultiRangeMockDocumentClient documentClient = new MultiRangeMockDocumentClient();
CosmosClient client = MockCosmosUtil.CreateMockCosmosClient();
Mock <CosmosClientContext> mockContext = new Mock <CosmosClientContext>();
mockContext.Setup(x => x.ClientOptions).Returns(MockCosmosUtil.GetDefaultConfiguration());
mockContext.Setup(x => x.DocumentClient).Returns(documentClient);
mockContext.Setup(x => x.SerializerCore).Returns(MockCosmosUtil.Serializer);
mockContext.Setup(x => x.Client).Returns(client);
mockContext.Setup(x => x.CreateLink(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())).Returns("/dbs/test/colls/test");
ResponseMessage firstResponse = new ResponseMessage(HttpStatusCode.NotModified);
firstResponse.Headers.ETag = "FirstContinuation";
ResponseMessage secondResponse = new ResponseMessage(
statusCode: HttpStatusCode.NotFound,
requestMessage: null,
headers: new Headers()
{
ETag = "ShouldNotContainThis"
},
cosmosException: CosmosExceptionFactory.CreateNotFoundException("something"),
diagnostics: new CosmosDiagnosticsContextCore());
mockContext.SetupSequence(x => x.ProcessResourceOperationAsync <ResponseMessage>(
It.IsAny <string>(),
It.IsAny <Documents.ResourceType>(),
It.IsAny <Documents.OperationType>(),
It.IsAny <RequestOptions>(),
It.IsAny <ContainerInternal>(),
It.IsAny <PartitionKey?>(),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(firstResponse))
.Returns(Task.FromResult(secondResponse));
DatabaseInternal databaseCore = new DatabaseInlineCore(mockContext.Object, "mydb");
StandByFeedIteratorCore iterator = new StandByFeedIteratorCore(
mockContext.Object,
new ContainerInlineCore(mockContext.Object, databaseCore, "myColl"),
ChangeFeedStartFrom.Beginning(),
new ChangeFeedRequestOptions()
{
PageSizeHint = 10,
});
ResponseMessage firstRequest = await iterator.ReadNextAsync();
Assert.IsTrue(firstRequest.Headers.ContinuationToken.Contains(firstResponse.Headers.ETag), "Response should contain the first continuation");
Assert.IsTrue(!firstRequest.Headers.ContinuationToken.Contains(secondResponse.Headers.ETag), "Response should not contain the second continuation");
Assert.AreEqual(HttpStatusCode.NotFound, firstRequest.StatusCode);
mockContext.Verify(x => x.ProcessResourceOperationAsync <ResponseMessage>(
It.IsAny <string>(),
It.IsAny <Documents.ResourceType>(),
It.IsAny <Documents.OperationType>(),
It.IsAny <RequestOptions>(),
It.IsAny <ContainerInternal>(),
It.IsAny <PartitionKey?>(),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <CancellationToken>()), Times.Exactly(2));
}
public async Task ChangeFeedIteratorCore_WithFullFidelity()
{
ContainerProperties properties = new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: ChangeFeedIteratorCoreTests.PartitionKey);
properties.ChangeFeedPolicy.FullFidelityRetention = TimeSpan.FromMinutes(5);
ContainerResponse response = await this.database.CreateContainerAsync(
properties,
cancellationToken : this.cancellationToken);
ContainerInternal container = (ContainerInternal)response;
// FF does not work with StartFromBeginning currently, so we capture an initial continuation.
FeedIterator <ToDoActivityWithMetadata> fullFidelityIterator = container.GetChangeFeedIterator <ToDoActivityWithMetadata>(
ChangeFeedStartFrom.Now(),
ChangeFeedMode.FullFidelity);
string initialContinuation = null;
while (fullFidelityIterator.HasMoreResults)
{
try
{
FeedResponse <ToDoActivityWithMetadata> feedResponse = await fullFidelityIterator.ReadNextAsync(this.cancellationToken);
initialContinuation = feedResponse.ContinuationToken;
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == HttpStatusCode.NotModified)
{
initialContinuation = cosmosException.Headers.ContinuationToken;
break;
}
}
// Insert documents and then delete them
int totalDocuments = 50;
IList <ToDoActivity> createdItems = await this.CreateRandomItems(container, totalDocuments, randomPartitionKey : true);
foreach (ToDoActivity item in createdItems)
{
await container.DeleteItemAsync <ToDoActivity>(item.id, new PartitionKey(item.pk));
}
// Resume Change Feed and verify we pickup all the events
fullFidelityIterator = container.GetChangeFeedIterator <ToDoActivityWithMetadata>(
ChangeFeedStartFrom.ContinuationToken(initialContinuation),
ChangeFeedMode.FullFidelity);
int detectedEvents = 0;
bool hasInserts = false;
bool hasDeletes = false;
while (fullFidelityIterator.HasMoreResults)
{
try
{
FeedResponse <ToDoActivityWithMetadata> feedResponse = await fullFidelityIterator.ReadNextAsync(this.cancellationToken);
foreach (ToDoActivityWithMetadata item in feedResponse)
{
Assert.IsNotNull(item.metadata, "Metadata not present");
Assert.IsNotNull(item.metadata.operationType, "Metadata has no operationType");
hasInserts |= item.metadata.operationType == "create";
hasDeletes |= item.metadata.operationType == "delete";
}
detectedEvents += feedResponse.Count;
}
catch (CosmosException cosmosException) when(cosmosException.StatusCode == HttpStatusCode.NotModified)
{
break;
}
}
Assert.AreEqual(2 * totalDocuments, detectedEvents, "Full Fidelity should include inserts and delete events.");
Assert.IsTrue(hasInserts, "No metadata for create operationType found");
Assert.IsTrue(hasDeletes, "No metadata for delete operationType found");
}
public async Task TestCancellationTokenAsync()
{
CancellationTokenRequestHandler cancellationTokenHandler = new CancellationTokenRequestHandler();
ContainerInternal itemsCore = await this.InitializeContainerAsync();
await this.CreateRandomItems(itemsCore, 100, randomPartitionKey : true);
// Inject validating handler
RequestHandler currentInnerHandler = this.cosmosClient.RequestHandler.InnerHandler;
this.cosmosClient.RequestHandler.InnerHandler = cancellationTokenHandler;
cancellationTokenHandler.InnerHandler = currentInnerHandler;
{
// Test to see if the token flows to the pipeline
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
ChangeFeedIteratorCore feedIterator = itemsCore.GetChangeFeedStreamIterator(
ChangeFeedStartFrom.Beginning(),
ChangeFeedMode.Incremental) as ChangeFeedIteratorCore;
await feedIterator.ReadNextAsync(cancellationTokenSource.Token);
Assert.AreEqual(cancellationTokenSource.Token, cancellationTokenHandler.LastUsedToken, "The token passed did not reach the pipeline");
}
// See if cancellation token is honored for first request
try
{
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.Cancel();
ChangeFeedIteratorCore feedIterator = itemsCore.GetChangeFeedStreamIterator(
ChangeFeedStartFrom.Beginning(),
ChangeFeedMode.Incremental) as ChangeFeedIteratorCore;
await feedIterator.ReadNextAsync(cancellationTokenSource.Token);
Assert.Fail("Expected exception.");
}
catch (OperationCanceledException)
{
}
// See if cancellation token is honored for second request
try
{
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.Cancel();
ChangeFeedIteratorCore feedIterator = itemsCore.GetChangeFeedStreamIterator(
ChangeFeedStartFrom.Beginning(),
ChangeFeedMode.Incremental) as ChangeFeedIteratorCore;
await feedIterator.ReadNextAsync();
await feedIterator.ReadNextAsync(cancellationTokenSource.Token);
Assert.Fail("Expected exception.");
}
catch (OperationCanceledException)
{
}
// See if cancellation token is honored mid draining
try
{
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
ChangeFeedIteratorCore feedIterator = itemsCore.GetChangeFeedStreamIterator(
ChangeFeedStartFrom.Beginning(),
ChangeFeedMode.Incremental) as ChangeFeedIteratorCore;
await feedIterator.ReadNextAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
await feedIterator.ReadNextAsync(cancellationTokenSource.Token);
Assert.Fail("Expected exception.");
}
catch (OperationCanceledException)
{
}
}
