public async Task ContinuationTokenIsNotUpdatedOnFails()
{
MultiRangeMockDocumentClient documentClient = new MultiRangeMockDocumentClient();
CosmosClient client = MockCosmosUtil.CreateMockCosmosClient();
Mock <CosmosClientContext> mockContext = new Mock <CosmosClientContext>();
mockContext.Setup(x => x.ClientOptions).Returns(MockCosmosUtil.GetDefaultConfiguration());
mockContext.Setup(x => x.DocumentClient).Returns(documentClient);
mockContext.Setup(x => x.SerializerCore).Returns(MockCosmosUtil.Serializer);
mockContext.Setup(x => x.Client).Returns(client);
mockContext.Setup(x => x.CreateLink(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())).Returns(new Uri("/dbs/test/colls/test", UriKind.Relative));
ResponseMessage firstResponse = new ResponseMessage(HttpStatusCode.NotModified);
firstResponse.Headers.ETag = "FirstContinuation";
ResponseMessage secondResponse = new ResponseMessage(
statusCode: HttpStatusCode.NotFound,
requestMessage: null,
headers: new Headers()
{
ETag = "ShouldNotContainThis"
},
cosmosException: CosmosExceptionFactory.CreateNotFoundException("something"),
diagnostics: CosmosDiagnosticsContext.Create());
mockContext.SetupSequence(x => x.ProcessResourceOperationAsync <ResponseMessage>(
It.IsAny <Uri>(),
It.IsAny <Documents.ResourceType>(),
It.IsAny <Documents.OperationType>(),
It.IsAny <RequestOptions>(),
It.IsAny <ContainerCore>(),
It.IsAny <PartitionKey?>(),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(firstResponse))
.Returns(Task.FromResult(secondResponse));
DatabaseCore databaseCore = new DatabaseCore(mockContext.Object, "mydb");
StandByFeedIteratorCore iterator = new StandByFeedIteratorCore(
mockContext.Object, new ContainerCore(mockContext.Object, databaseCore, "myColl"), null, 10, new ChangeFeedRequestOptions());
ResponseMessage firstRequest = await iterator.ReadNextAsync();
Assert.IsTrue(firstRequest.Headers.ContinuationToken.Contains(firstResponse.Headers.ETag), "Response should contain the first continuation");
Assert.IsTrue(!firstRequest.Headers.ContinuationToken.Contains(secondResponse.Headers.ETag), "Response should not contain the second continuation");
Assert.AreEqual(HttpStatusCode.NotFound, firstRequest.StatusCode);
mockContext.Verify(x => x.ProcessResourceOperationAsync <ResponseMessage>(
It.IsAny <Uri>(),
It.IsAny <Documents.ResourceType>(),
It.IsAny <Documents.OperationType>(),
It.IsAny <RequestOptions>(),
It.IsAny <ContainerCore>(),
It.IsAny <PartitionKey?>(),
It.IsAny <Stream>(),
It.IsAny <Action <RequestMessage> >(),
It.IsAny <Func <ResponseMessage, ResponseMessage> >(),
It.IsAny <CosmosDiagnosticsContext>(),
It.IsAny <CancellationToken>()), Times.Exactly(2));
}
public static QueryIterator Create(
CosmosQueryClient client,
CosmosClientContext clientContext,
SqlQuerySpec sqlQuerySpec,
string continuationToken,
FeedRangeInternal feedRangeInternal,
QueryRequestOptions queryRequestOptions,
string resourceLink,
bool isContinuationExpected,
bool allowNonValueAggregateQuery,
bool forcePassthrough,
PartitionedQueryExecutionInfo partitionedQueryExecutionInfo)
{
if (queryRequestOptions == null)
{
queryRequestOptions = new QueryRequestOptions();
}
CosmosDiagnosticsContext queryPipelineCreationDiagnostics = CosmosDiagnosticsContext.Create(queryRequestOptions);
CosmosQueryContextCore cosmosQueryContext = new CosmosQueryContextCore(
client: client,
queryRequestOptions: queryRequestOptions,
resourceTypeEnum: Documents.ResourceType.Document,
operationType: Documents.OperationType.Query,
resourceType: typeof(QueryResponseCore),
resourceLink: resourceLink,
isContinuationExpected: isContinuationExpected,
allowNonValueAggregateQuery: allowNonValueAggregateQuery,
diagnosticsContext: queryPipelineCreationDiagnostics,
correlatedActivityId: Guid.NewGuid());
CosmosElement requestContinuationToken;
switch (queryRequestOptions.ExecutionEnvironment.GetValueOrDefault(ExecutionEnvironment.Client))
{
case ExecutionEnvironment.Client:
if (continuationToken != null)
{
TryCatch <CosmosElement> tryParse = CosmosElement.Monadic.Parse(continuationToken);
if (tryParse.Failed)
{
return(new QueryIterator(
cosmosQueryContext,
new QueryExecutionContextWithException(
new MalformedContinuationTokenException(
message: $"Malformed Continuation Token: {continuationToken}",
innerException: tryParse.Exception)),
queryRequestOptions.CosmosSerializationFormatOptions,
queryRequestOptions,
clientContext));
}
requestContinuationToken = tryParse.Result;
}
else
{
requestContinuationToken = null;
}
break;
case ExecutionEnvironment.Compute:
requestContinuationToken = queryRequestOptions.CosmosElementContinuationToken;
break;
default:
throw new ArgumentOutOfRangeException($"Unknown {nameof(ExecutionEnvironment)}: {queryRequestOptions.ExecutionEnvironment.Value}.");
}
CosmosQueryExecutionContextFactory.InputParameters inputParameters = new CosmosQueryExecutionContextFactory.InputParameters(
sqlQuerySpec: sqlQuerySpec,
initialUserContinuationToken: requestContinuationToken,
initialFeedRange: feedRangeInternal,
maxConcurrency: queryRequestOptions.MaxConcurrency,
maxItemCount: queryRequestOptions.MaxItemCount,
maxBufferedItemCount: queryRequestOptions.MaxBufferedItemCount,
partitionKey: queryRequestOptions.PartitionKey,
properties: queryRequestOptions.Properties,
partitionedQueryExecutionInfo: partitionedQueryExecutionInfo,
executionEnvironment: queryRequestOptions.ExecutionEnvironment,
returnResultsInDeterministicOrder: queryRequestOptions.ReturnResultsInDeterministicOrder,
forcePassthrough: forcePassthrough,
testInjections: queryRequestOptions.TestSettings);
return(new QueryIterator(
cosmosQueryContext,
CosmosQueryExecutionContextFactory.Create(cosmosQueryContext, inputParameters),
queryRequestOptions.CosmosSerializationFormatOptions,
queryRequestOptions,
clientContext));
}
public override async Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default)
{
CosmosDiagnosticsContext diagnostics = CosmosDiagnosticsContext.Create(this.requestOptions);
using (diagnostics.GetOverallScope())
{
QueryResponseCore responseCore;
try
{
// This catches exception thrown by the pipeline and converts it to QueryResponse
responseCore = await this.cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken);
}
catch (OperationCanceledException ex) when(!(ex is CosmosOperationCanceledException))
{
throw new CosmosOperationCanceledException(ex, diagnostics);
}
finally
{
// This swaps the diagnostics in the context. This shows all the page reads between the previous ReadNextAsync and the current ReadNextAsync
diagnostics.AddDiagnosticsInternal(this.cosmosQueryContext.GetAndResetDiagnostics());
}
if (responseCore.IsSuccess)
{
return(QueryResponse.CreateSuccess(
result: responseCore.CosmosElements,
count: responseCore.CosmosElements.Count,
responseLengthBytes: responseCore.ResponseLengthBytes,
diagnostics: diagnostics,
serializationOptions: this.cosmosSerializationFormatOptions,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
this.cosmosQueryContext.ResourceTypeEnum,
this.cosmosQueryContext.ContainerResourceId)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId,
SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown
}));
}
if (responseCore.CosmosException != null)
{
return(responseCore.CosmosException.ToCosmosResponseMessage(null));
}
return(QueryResponse.CreateFailure(
statusCode: responseCore.StatusCode,
cosmosException: responseCore.CosmosException,
requestMessage: null,
diagnostics: diagnostics,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
this.cosmosQueryContext.ResourceTypeEnum,
this.cosmosQueryContext.ContainerResourceId)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId,
SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown,
}));
}
}
public async Task TestItemProducerTreeWithFailure()
{
int callBackCount = 0;
Mock <CosmosQueryContext> mockQueryContext = new Mock <CosmosQueryContext>();
SqlQuerySpec sqlQuerySpec = new SqlQuerySpec("Select * from t");
PartitionKeyRange partitionKeyRange = new PartitionKeyRange {
Id = "0", MinInclusive = "A", MaxExclusive = "B"
};
void produceAsyncCompleteCallback(
ItemProducerTree producer,
int itemsBuffered,
double resourceUnitUsage,
IReadOnlyCollection <QueryPageDiagnostics> queryPageDiagnostics,
long responseLengthBytes,
CancellationToken token)
{
callBackCount++;
}
Mock <IComparer <ItemProducerTree> > comparer = new Mock <IComparer <ItemProducerTree> >();
Mock <IEqualityComparer <CosmosElement> > cosmosElementComparer = new Mock <IEqualityComparer <CosmosElement> >();
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
IReadOnlyList <CosmosElement> cosmosElements = new List <CosmosElement>()
{
new Mock <CosmosElement>(CosmosElementType.Object).Object
};
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create();
diagnosticsContext.AddDiagnosticsInternal(new PointOperationStatistics(
Guid.NewGuid().ToString(),
System.Net.HttpStatusCode.OK,
subStatusCode: SubStatusCodes.Unknown,
requestCharge: 42,
errorMessage: null,
method: HttpMethod.Post,
requestUri: new Uri("http://localhost.com"),
requestSessionToken: null,
responseSessionToken: null,
clientSideRequestStatistics: null));
QueryPageDiagnostics diagnostics = new QueryPageDiagnostics(
partitionKeyRangeId: "0",
queryMetricText: "SomeRandomQueryMetricText",
indexUtilizationText: null,
diagnosticsContext: diagnosticsContext,
schedulingStopwatch: new SchedulingStopwatch());
IReadOnlyCollection <QueryPageDiagnostics> pageDiagnostics = new List <QueryPageDiagnostics>()
{
diagnostics
};
mockQueryContext.Setup(x => x.ContainerResourceId).Returns("MockCollectionRid");
mockQueryContext.Setup(x => x.ExecuteQueryAsync(
sqlQuerySpec,
It.IsAny <string>(),
It.IsAny <PartitionKeyRangeIdentity>(),
It.IsAny <bool>(),
It.IsAny <int>(),
It.IsAny <SchedulingStopwatch>(),
cancellationTokenSource.Token)).Returns(
Task.FromResult(QueryResponseCore.CreateSuccess(
result: cosmosElements,
requestCharge: 42,
activityId: "AA470D71-6DEF-4D61-9A08-272D8C9ABCFE",
diagnostics: pageDiagnostics,
responseLengthBytes: 500,
disallowContinuationTokenMessage: null,
continuationToken: "TestToken")));
ItemProducerTree itemProducerTree = new ItemProducerTree(
queryContext: mockQueryContext.Object,
querySpecForInit: sqlQuerySpec,
partitionKeyRange: partitionKeyRange,
produceAsyncCompleteCallback: produceAsyncCompleteCallback,
itemProducerTreeComparer: comparer.Object,
equalityComparer: cosmosElementComparer.Object,
testSettings: new TestInjections(simulate429s: false, simulateEmptyPages: false),
deferFirstPage: false,
collectionRid: "collectionRid",
initialContinuationToken: null,
initialPageSize: 50);
// Buffer to success responses
await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token);
await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token);
CosmosDiagnosticsContext diagnosticsContextInternalServerError = CosmosDiagnosticsContext.Create();
diagnosticsContextInternalServerError.AddDiagnosticsInternal(new PointOperationStatistics(
Guid.NewGuid().ToString(),
System.Net.HttpStatusCode.InternalServerError,
subStatusCode: SubStatusCodes.Unknown,
requestCharge: 10.2,
errorMessage: "Error message",
method: HttpMethod.Post,
requestUri: new Uri("http://localhost.com"),
requestSessionToken: null,
responseSessionToken: null,
clientSideRequestStatistics: null));
diagnostics = new QueryPageDiagnostics(
partitionKeyRangeId: "0",
queryMetricText: null,
indexUtilizationText: null,
diagnosticsContext: diagnosticsContextInternalServerError,
schedulingStopwatch: new SchedulingStopwatch());
pageDiagnostics = new List <QueryPageDiagnostics>()
{
diagnostics
};
// Buffer a failure
mockQueryContext.Setup(x => x.ExecuteQueryAsync(
sqlQuerySpec,
It.IsAny <string>(),
It.IsAny <PartitionKeyRangeIdentity>(),
It.IsAny <bool>(),
It.IsAny <int>(),
It.IsAny <SchedulingStopwatch>(),
cancellationTokenSource.Token)).Returns(
Task.FromResult(QueryResponseCore.CreateFailure(
statusCode: HttpStatusCode.InternalServerError,
subStatusCodes: null,
cosmosException: CosmosExceptionFactory.CreateInternalServerErrorException(
"Error message"),
requestCharge: 10.2,
activityId: Guid.NewGuid().ToString(),
diagnostics: pageDiagnostics)));
await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token);
// First item should be a success
{
(bool movedToNextPage, QueryResponseCore? failureResponse) = await itemProducerTree.TryMoveNextPageAsync(cancellationTokenSource.Token);
Assert.IsTrue(movedToNextPage);
Assert.IsNull(failureResponse);
Assert.IsTrue(itemProducerTree.TryMoveNextDocumentWithinPage());
Assert.IsFalse(itemProducerTree.TryMoveNextDocumentWithinPage());
Assert.IsTrue(itemProducerTree.HasMoreResults);
}
// Second item should be a success
{
(bool movedToNextPage, QueryResponseCore? failureResponse) = await itemProducerTree.TryMoveNextPageAsync(cancellationTokenSource.Token);
Assert.IsTrue(movedToNextPage);
Assert.IsNull(failureResponse);
Assert.IsTrue(itemProducerTree.TryMoveNextDocumentWithinPage());
Assert.IsFalse(itemProducerTree.TryMoveNextDocumentWithinPage());
Assert.IsTrue(itemProducerTree.HasMoreResults);
}
// Third item should be a failure
{
(bool movedToNextPage, QueryResponseCore? failureResponse) = await itemProducerTree.TryMoveNextPageAsync(cancellationTokenSource.Token);
Assert.IsFalse(movedToNextPage);
Assert.IsNotNull(failureResponse);
Assert.IsFalse(itemProducerTree.HasMoreResults);
}
// Try to buffer after failure. It should return the previous cached failure and not try to buffer again.
mockQueryContext.Setup(x => x.ExecuteQueryAsync(
sqlQuerySpec,
It.IsAny <string>(),
It.IsAny <PartitionKeyRangeIdentity>(),
It.IsAny <bool>(),
It.IsAny <int>(),
It.IsAny <SchedulingStopwatch>(),
cancellationTokenSource.Token)).
Throws(new Exception("Previous buffer failed. Operation should return original failure and not try again"));
await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token);
Assert.IsFalse(itemProducerTree.HasMoreResults);
}
public void ValidateDiagnosticsContext()
{
CosmosDiagnosticsContext cosmosDiagnostics = CosmosDiagnosticsContext.Create();
string diagnostics = cosmosDiagnostics.ToString();
//Test the default user agent string
JObject jObject = JObject.Parse(diagnostics);
JToken summary = jObject["Summary"];
Assert.IsTrue(summary["UserAgent"].ToString().Contains("cosmos-netstandard-sdk"), "Diagnostics should have user agent string");
Assert.AreEqual("[]", jObject["Context"].ToString());
// Test all the different operations on diagnostics context
using (cosmosDiagnostics.CreateOverallScope("OverallScope"))
{
Thread.Sleep(TimeSpan.FromSeconds(1));
using (cosmosDiagnostics.CreateScope("ValidateScope"))
{
Thread.Sleep(TimeSpan.FromSeconds(1));
cosmosDiagnostics.AddDiagnosticsInternal(new PointOperationStatistics(
new Guid("692ab2f2-41ba-486b-aad7-8c7c6c52379f").ToString(),
(HttpStatusCode)429,
Documents.SubStatusCodes.Unknown,
42,
null,
HttpMethod.Get,
new Uri("http://MockUri.com"),
null,
null,
null));
}
using (cosmosDiagnostics.CreateScope("SuccessScope"))
{
cosmosDiagnostics.AddDiagnosticsInternal(new PointOperationStatistics(
new Guid("de09baab-71a4-4897-a163-470711c93ed3").ToString(),
HttpStatusCode.OK,
Documents.SubStatusCodes.Unknown,
42,
null,
HttpMethod.Get,
new Uri("http://MockUri.com"),
null,
null,
null));
}
}
cosmosDiagnostics.SetSdkUserAgent("MyCustomUserAgentString");
string result = cosmosDiagnostics.ToString();
string regex = @"\{""Summary"":\{""StartUtc"":"".+Z"",""ElapsedTime"":""00:00:.+"",""UserAgent"":""MyCustomUserAgentString"",""TotalRequestCount"":2,""FailedRequestCount"":1\},""Context"":\[\{""Id"":""OverallScope"",""ElapsedTime"":""00:00:0.+""\},\{""Id"":""ValidateScope"",""ElapsedTime"":""00:00:0.+""\},\{""Id"":""PointOperationStatistics"",""ActivityId"":""692ab2f2-41ba-486b-aad7-8c7c6c52379f"",""StatusCode"":429,""SubStatusCode"":0,""RequestCharge"":42.0,""RequestUri"":""http://MockUri.com"",""RequestSessionToken"":null,""ResponseSessionToken"":null\},\{""Id"":""SuccessScope"",""ElapsedTime"":""00:00:.+""\},\{""Id"":""PointOperationStatistics"",""ActivityId"":""de09baab-71a4-4897-a163-470711c93ed3"",""StatusCode"":200,""SubStatusCode"":0,""RequestCharge"":42.0,""RequestUri"":""http://MockUri.com"",""RequestSessionToken"":null,""ResponseSessionToken"":null\}\]\}";
Assert.IsTrue(Regex.IsMatch(result, regex), result);
JToken jToken = JToken.Parse(result);
TimeSpan total = jToken["Summary"]["ElapsedTime"].ToObject <TimeSpan>();
Assert.IsTrue(total > TimeSpan.FromSeconds(2));
TimeSpan overalScope = jToken["Context"][0]["ElapsedTime"].ToObject <TimeSpan>();
Assert.IsTrue(total == overalScope);
TimeSpan innerScope = jToken["Context"][1]["ElapsedTime"].ToObject <TimeSpan>();
Assert.IsTrue(innerScope > TimeSpan.FromSeconds(1));
}
internal void GetDeserializedObjectsFromQueryResponseTest(JsonSerializationFormat jsonSerializationFormat)
{
// Constants to use for vertex document property key/values
const string vertex1Id = "v_0";
const string vertex2Id = "v_1";
const string vertex1Label = "l_0";
const string vertex2Label = "l_1";
const string vertex1PkValue = "pk_0";
const string vertex2PkValue = "pk_1";
const string property1Name = "p_0";
const string vertex1Property1Value = "v_0_p_0_v_0";
const string vertex2Property1Value = "v_1_p_0_v_0";
const string property2Name = "p_1";
const double vertex1Property2Value = 12.34;
const long vertex2Property2Value = 5678;
// Compose two initial vertex documents using eager CosmosElements
CosmosObject initialVertex1EagerObject = this.CreateVertexDocument(
vertex1Id,
vertex1Label,
GremlinScenarioTests.PartitionKeyPropertyName,
vertex1PkValue,
new Tuple <string, IEnumerable <object> >[]
{
Tuple.Create <string, IEnumerable <object> >(property1Name, new object[] { vertex1Property1Value }),
Tuple.Create <string, IEnumerable <object> >(property2Name, new object[] { vertex1Property2Value }),
});
CosmosObject initialVertex2EagerObject = this.CreateVertexDocument(
vertex2Id,
vertex2Label,
GremlinScenarioTests.PartitionKeyPropertyName,
vertex2PkValue,
new Tuple <string, IEnumerable <object> >[]
{
Tuple.Create <string, IEnumerable <object> >(property1Name, new object[] { vertex2Property1Value }),
Tuple.Create <string, IEnumerable <object> >(property2Name, new object[] { vertex2Property2Value }),
});
// Serialize the initial vertex object into a document using the specified serialization format
IJsonWriter jsonWriter = JsonWriter.Create(jsonSerializationFormat);
initialVertex1EagerObject.WriteTo(jsonWriter);
ReadOnlyMemory <byte> vertex1JsonWriterResult = jsonWriter.GetResult();
Assert.IsTrue(vertex1JsonWriterResult.Length > 0, "IJsonWriter result data is empty.");
jsonWriter = JsonWriter.Create(jsonSerializationFormat);
initialVertex2EagerObject.WriteTo(jsonWriter);
ReadOnlyMemory <byte> vertex2JsonWriterResult = jsonWriter.GetResult();
Assert.IsTrue(vertex2JsonWriterResult.Length > 0, "IJsonWriter result data is empty.");
// Navigate into the serialized vertex documents using lazy CosmosElements
CosmosElement vertex1LazyObject = CosmosElement.CreateFromBuffer(vertex1JsonWriterResult);
CosmosElement vertex2LazyObject = CosmosElement.CreateFromBuffer(vertex2JsonWriterResult);
// Create a dynamically-typed QueryResponse backed by the vertex document CosmosElements
CosmosArray vertexArray = CosmosArray.Create(
new CosmosElement[]
{
vertex1LazyObject,
vertex2LazyObject,
});
QueryResponse queryResponse = QueryResponse.CreateSuccess(
vertexArray,
count: 2,
responseLengthBytes: vertex1JsonWriterResult.Length + vertex2JsonWriterResult.Length,
serializationOptions: null,
responseHeaders: CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders(
sourceHeaders: null,
resourceType: ResourceType.Document,
containerRid: GremlinScenarioTests.CreateRandomString(10)),
diagnostics: CosmosDiagnosticsContext.Create());
QueryResponse <dynamic> cosmosElementQueryResponse =
QueryResponse <dynamic> .CreateResponse <dynamic>(
queryResponse,
MockCosmosUtil.Serializer);
// Assert that other objects (anything besides the lazy CosmosElements that we created earlier) are deserialized
// from the backing CosmosElement contents rather than being directly returned as CosmosElements
List <dynamic> responseCosmosElements = new List <dynamic>(cosmosElementQueryResponse.Resource);
Assert.AreEqual(vertexArray.Count, responseCosmosElements.Count);
Assert.AreNotSame(vertex1LazyObject, responseCosmosElements[0]);
Assert.AreNotSame(vertex2LazyObject, responseCosmosElements[1]);
}
