public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken token)
{
token.ThrowIfCancellationRequested();
QueryResponse sourcePage = await base.DrainAsync(maxElements, token);
if (!sourcePage.IsSuccessStatusCode)
{
return(sourcePage);
}
// skip the documents but keep all the other headers
List <CosmosElement> documentsAfterSkip = sourcePage.CosmosElements.Skip(this.skipCount).ToList();
int numberOfDocumentsSkipped = sourcePage.Count - documentsAfterSkip.Count;
this.skipCount -= numberOfDocumentsSkipped;
string updatedContinuationToken = null;
if (sourcePage.QueryHeaders.DisallowContinuationTokenMessage == null)
{
if (!this.IsDone)
{
updatedContinuationToken = new OffsetContinuationToken(
this.skipCount,
sourcePage.Headers.ContinuationToken).ToString();
}
}
return(QueryResponse.CreateSuccess(
result: documentsAfterSkip,
count: documentsAfterSkip.Count(),
responseHeaders: sourcePage.QueryHeaders.CloneKnownProperties(updatedContinuationToken, sourcePage.QueryHeaders.DisallowContinuationTokenMessage),
responseLengthBytes: sourcePage.ResponseLengthBytes));
}
/// <summary>
/// Gets the next page of results from this context.
/// </summary>
/// <param name="token">The cancellation token.</param>
/// <returns>A task to await on that in turn returns a DoucmentFeedResponse of results.</returns>
public override async Task <QueryResponse> ExecuteNextAsync(CancellationToken token)
{
try
{
QueryResponse queryResponse = await this.component.DrainAsync(this.actualPageSize, token);
if (!queryResponse.IsSuccessStatusCode)
{
this.component.Stop();
return(queryResponse);
}
List <CosmosElement> dynamics = new List <CosmosElement>();
foreach (CosmosElement element in queryResponse.CosmosElements)
{
dynamics.Add(element);
}
return(QueryResponse.CreateSuccess(
dynamics,
queryResponse.Count,
queryResponse.ResponseLengthBytes,
queryResponse.QueryHeaders.CloneKnownProperties()));
}
catch (Exception)
{
this.component.Stop();
throw;
}
}
public override async Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default)
{
CosmosDiagnosticsContext diagnostics = new CosmosDiagnosticsContext();
using (diagnostics.CreateScope("QueryReadNextAsync"))
{
// This catches exception thrown by the pipeline and converts it to QueryResponse
QueryResponseCore responseCore = await this.cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken);
CosmosQueryContext cosmosQueryContext = this.cosmosQueryContext;
foreach (QueryPageDiagnostics queryPage in responseCore.Diagnostics)
{
diagnostics.Summary.Append(queryPage.DiagnosticsContext.Summary);
diagnostics.AddDiagnosticsInternal(queryPage);
}
QueryResponse queryResponse;
if (responseCore.IsSuccess)
{
queryResponse = QueryResponse.CreateSuccess(
result: responseCore.CosmosElements,
count: responseCore.CosmosElements.Count,
responseLengthBytes: responseCore.ResponseLengthBytes,
diagnostics: diagnostics,
serializationOptions: this.cosmosSerializationFormatOptions,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
cosmosQueryContext.ResourceTypeEnum,
cosmosQueryContext.ContainerResourceId)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId,
SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown
});
}
else
{
queryResponse = QueryResponse.CreateFailure(
statusCode: responseCore.StatusCode,
error: null,
errorMessage: responseCore.ErrorMessage,
requestMessage: null,
diagnostics: diagnostics,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
cosmosQueryContext.ResourceTypeEnum,
cosmosQueryContext.ContainerResourceId)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId,
SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown
});
}
return(queryResponse);
}
}
public static (QueryResponse queryResponse, IList <ToDoItem> items) Create(
string itemIdPrefix,
string continuationToken,
string collectionRid,
int itemCount = 50)
{
// Use -1 to represent a split response
if (itemCount == QueryResponseMessageFactory.SPLIT)
{
return(CreateSplitResponse(collectionRid), new List <ToDoItem>().AsReadOnly());
}
IList <ToDoItem> items = ToDoItem.CreateItems(itemCount, itemIdPrefix);
MemoryStream memoryStream = (MemoryStream)cosmosSerializer.ToStream <IList <ToDoItem> >(items);
long responseLengthBytes = memoryStream.Length;
IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray());
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode);
Headers headers = new Headers();
headers.ContinuationToken = continuationToken;
headers.ActivityId = Guid.NewGuid().ToString();
QueryResponse message = QueryResponse.CreateSuccess(
result: cosmosArray,
count: itemCount,
responseHeaders: CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders(headers, ResourceType.Document, collectionRid),
responseLengthBytes: responseLengthBytes);
return(message, items);
}
public void VerifyCosmosQueryResponseStream()
{
string contianerRid = "mockContainerRid";
(QueryResponseCore response, IList <ToDoItem> items)factoryResponse = QueryResponseMessageFactory.Create(
itemIdPrefix: $"TestPage",
continuationToken: "SomeContinuationToken",
collectionRid: contianerRid,
itemCount: 100);
QueryResponseCore responseCore = factoryResponse.response;
QueryResponse queryResponse = QueryResponse.CreateSuccess(
result: responseCore.CosmosElements,
count: responseCore.CosmosElements.Count,
responseLengthBytes: responseCore.ResponseLengthBytes,
queryMetrics: responseCore.QueryMetrics,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
ResourceType.Document,
contianerRid)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId
});
using (Stream stream = queryResponse.Content)
{
using (Stream innerStream = queryResponse.Content)
{
Assert.IsTrue(object.ReferenceEquals(stream, innerStream), "Content should return the same stream");
}
}
}
//Retorna uma lista de genérica com todos os dados de uma dada tabela
public QueryResponse <List <T> > GetAllData <T>(DbCommand command)
{
try
{
using (DbCurrentConnection connection = new DbCurrentConnection(command))
{
connection.OpenConnection();
DbDataReader reader = command.ExecuteReader();
DataTable table = new DataTable();
table.Load(reader);
if (table.Rows.Count == 0)
{
List <T> datan = new List <T>();
return(QueryResponse <List <T> > .CreateSuccess(datan, "Nenhum registro cadastrado."));
}
List <T> data = table.ConvertDataTable <T>();
return(QueryResponse <List <T> > .CreateSuccess(data, "Dados retornados com sucesso."));
}
}
catch (Exception ex)
{
return(QueryResponse <List <T> > .CreateFailureException("Ocorreu um erro, contate o administrador.", ex));
}
}
/// <summary>
/// Drains documents from this component. This has the common drain logic for each implementation.
/// </summary>
/// <param name="maxElements">The maximum number of documents to drain.</param>
/// <param name="cancellationToken">The cancellation token to cancel tasks.</param>
/// <returns>A task that when awaited on returns a feed response.</returns>
public async override Task <QueryResponse> DrainAsync(int maxElements, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
// The initialization or previous Drain Async failed. Just return the failure.
if (this.FailureResponse != null)
{
this.Stop();
return(this.FailureResponse);
}
// Drain the results. If there is no results and a failure then return the failure.
IList <CosmosElement> results = await this.InternalDrainAsync(maxElements, cancellationToken);
if ((results == null || results.Count == 0) && this.FailureResponse != null)
{
this.Stop();
return(this.FailureResponse);
}
return(QueryResponse.CreateSuccess(
result: results,
count: results.Count,
responseHeaders: this.GetResponseHeaders(),
responseLengthBytes: this.GetAndResetResponseLengthBytes()));
}
public override async Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default)
{
CosmosDiagnosticsContext diagnostics = CosmosDiagnosticsContext.Create(this.requestOptions);
using (diagnostics.GetOverallScope())
{
// This catches exception thrown by the pipeline and converts it to QueryResponse
QueryResponseCore responseCore = await this.cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken);
// This swaps the diagnostics in the context. This shows all the page reads between the previous ReadNextAsync and the current ReadNextAsync
diagnostics.AddDiagnosticsInternal(this.cosmosQueryContext.GetAndResetDiagnostics());
if (responseCore.IsSuccess)
{
List <CosmosElement> decryptedCosmosElements = null;
if (this.clientContext.ClientOptions.Encryptor != null)
{
decryptedCosmosElements = await this.GetDecryptedElementResponseAsync(responseCore.CosmosElements, diagnostics, cancellationToken);
}
return(QueryResponse.CreateSuccess(
result: decryptedCosmosElements ?? responseCore.CosmosElements,
count: responseCore.CosmosElements.Count,
responseLengthBytes: responseCore.ResponseLengthBytes,
diagnostics: diagnostics,
serializationOptions: this.cosmosSerializationFormatOptions,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
this.cosmosQueryContext.ResourceTypeEnum,
this.cosmosQueryContext.ContainerResourceId)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId,
SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown
}));
}
if (responseCore.CosmosException != null)
{
return(responseCore.CosmosException.ToCosmosResponseMessage(null));
}
return(QueryResponse.CreateFailure(
statusCode: responseCore.StatusCode,
cosmosException: responseCore.CosmosException,
requestMessage: null,
diagnostics: diagnostics,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
this.cosmosQueryContext.ResourceTypeEnum,
this.cosmosQueryContext.ContainerResourceId)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId,
SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown,
}));
}
}
public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken token)
{
token.ThrowIfCancellationRequested();
QueryResponse results = await base.DrainAsync(maxElements, token);
if (!results.IsSuccessStatusCode)
{
return(results);
}
List <CosmosElement> takedDocuments = results.CosmosElements.Take(this.takeCount).ToList();
this.takeCount -= takedDocuments.Count;
string updatedContinuationToken = null;
if (results.QueryHeaders.DisallowContinuationTokenMessage == null)
{
if (!this.IsDone)
{
string sourceContinuation = results.Headers.ContinuationToken;
TakeContinuationToken takeContinuationToken;
switch (this.takeEnum)
{
case TakeEnum.Limit:
takeContinuationToken = new LimitContinuationToken(
this.takeCount,
sourceContinuation);
break;
case TakeEnum.Top:
takeContinuationToken = new TopContinuationToken(
this.takeCount,
sourceContinuation);
break;
default:
throw new ArgumentException($"Unknown {nameof(TakeEnum)}: {this.takeEnum}");
}
updatedContinuationToken = takeContinuationToken.ToString();
}
}
return(QueryResponse.CreateSuccess(
takedDocuments,
takedDocuments.Count,
results.ResponseLengthBytes,
results.QueryHeaders.CloneKnownProperties(updatedContinuationToken, results.QueryHeaders.DisallowContinuationTokenMessage),
results.queryMetrics));
}
/// <summary>
/// Drains a page of results returning only distinct elements.
/// </summary>
/// <param name="maxElements">The maximum number of items to drain.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>A page of distinct results.</returns>
public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken cancellationToken)
{
List <CosmosElement> distinctResults = new List <CosmosElement>();
QueryResponse cosmosQueryResponse = await base.DrainAsync(maxElements, cancellationToken);
if (!cosmosQueryResponse.IsSuccessStatusCode)
{
return(cosmosQueryResponse);
}
foreach (CosmosElement document in cosmosQueryResponse.CosmosElements)
{
if (this.distinctMap.Add(document, out this.lastHash))
{
distinctResults.Add(document);
}
}
string updatedContinuationToken;
if (!this.IsDone)
{
updatedContinuationToken = new DistinctContinuationToken(
this.lastHash,
cosmosQueryResponse.Headers.ContinuationToken).ToString();
}
else
{
this.Source.Stop();
updatedContinuationToken = null;
}
string disallowContinuationTokenMessage = this.distinctQueryType == DistinctQueryType.Ordered ? null : RMResources.UnorderedDistinctQueryContinuationToken;
return(QueryResponse.CreateSuccess(
distinctResults,
distinctResults.Count,
cosmosQueryResponse.ResponseLengthBytes,
cosmosQueryResponse.QueryHeaders.CloneKnownProperties(updatedContinuationToken, disallowContinuationTokenMessage),
cosmosQueryResponse.queryMetrics));
}
public static QueryResponse CreateQueryResponse(
IList <ToDoItem> items,
bool isOrderByQuery,
string continuationToken,
string collectionRid)
{
MemoryStream memoryStream;
string json;
if (isOrderByQuery)
{
memoryStream = SerializeForOrderByQuery(items);
using (StreamReader sr = new StreamReader(SerializeForOrderByQuery(items)))
{
json = sr.ReadToEnd();
}
}
else
{
memoryStream = (MemoryStream)cosmosSerializer.ToStream <IList <ToDoItem> >(items);
}
long responseLengthBytes = memoryStream.Length;
IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray());
IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode();
CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode);
Headers headers = new Headers();
headers.ContinuationToken = continuationToken;
headers.ActivityId = Guid.NewGuid().ToString();
QueryResponse message = QueryResponse.CreateSuccess(
result: cosmosArray,
count: items.Count,
responseHeaders: CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders(headers, ResourceType.Document, collectionRid),
responseLengthBytes: responseLengthBytes);
return(message);
}
public void VerifyItemQueryResponseResult()
{
string contianerRid = "mockContainerRid";
(QueryResponseCore response, IList <ToDoItem> items)factoryResponse = QueryResponseMessageFactory.Create(
itemIdPrefix: $"TestPage",
continuationToken: "SomeContinuationToken",
collectionRid: contianerRid,
itemCount: 100);
QueryResponseCore responseCore = factoryResponse.response;
List <CosmosElement> cosmosElements = new List <CosmosElement>(responseCore.CosmosElements);
QueryResponse queryResponse = QueryResponse.CreateSuccess(
result: cosmosElements,
count: cosmosElements.Count,
responseLengthBytes: responseCore.ResponseLengthBytes,
serializationOptions: null,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
ResourceType.Document,
contianerRid)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId
},
diagnostics: new CosmosDiagnosticsContextCore());
QueryResponse <ToDoItem> itemQueryResponse = QueryResponseMessageFactory.CreateQueryResponse <ToDoItem>(queryResponse);
List <ToDoItem> resultItems = new List <ToDoItem>(itemQueryResponse.Resource);
ToDoItemComparer comparer = new ToDoItemComparer();
Assert.AreEqual(factoryResponse.items.Count, resultItems.Count);
for (int i = 0; i < factoryResponse.items.Count; i++)
{
Assert.AreNotSame(factoryResponse.items[i], resultItems[i]);
Assert.AreEqual(0, comparer.Compare(factoryResponse.items[i], resultItems[i]));
}
}
private ResponseMessage CreateQueryResponse()
{
List <CosmosElement> cosmosElements = new List <CosmosElement>();
string serializedItem = this.GetSerializedToDoActivity();
CosmosObject cosmosObject = CosmosObject.Parse(serializedItem);
cosmosElements.Add(cosmosObject);
ResponseMessage cosmosResponse = QueryResponse.CreateSuccess(
cosmosElements,
1,
Encoding.UTF8.GetByteCount(serializedItem),
new CosmosQueryResponseMessageHeaders(
continauationToken: null,
disallowContinuationTokenMessage: null,
resourceType: Documents.ResourceType.Document,
"+o4fAPfXPzw="),
new CosmosDiagnosticsContextCore(),
null);
return(cosmosResponse);
}
/// <summary>
/// Drains documents from this execution context.
/// </summary>
/// <param name="maxElements">The maximum number of documents to drains.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>A task that when awaited on returns a DoucmentFeedResponse of results.</returns>
public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken token)
{
// In order to maintain the continuation token for the user we must drain with a few constraints
// 1) We fully drain from the left most partition before moving on to the next partition
// 2) We drain only full pages from the document producer so we aren't left with a partial page
// otherwise we would need to add to the continuation token how many items to skip over on that page.
// Only drain from the leftmost (current) document producer tree
DocumentProducerTree currentDocumentProducerTree = this.PopCurrentDocumentProducerTree();
// This might be the first time we have seen this document producer tree so we need to buffer documents
if (currentDocumentProducerTree.Current == null)
{
await currentDocumentProducerTree.MoveNextAsync(token);
}
int itemsLeftInCurrentPage = currentDocumentProducerTree.ItemsLeftInCurrentPage;
// Only drain full pages or less if this is a top query.
List <CosmosElement> results = new List <CosmosElement>();
for (int i = 0; i < Math.Min(itemsLeftInCurrentPage, maxElements); i++)
{
results.Add(currentDocumentProducerTree.Current);
await currentDocumentProducerTree.MoveNextAsync(token);
}
this.PushCurrentDocumentProducerTree(currentDocumentProducerTree);
// At this point the document producer tree should have internally called MoveNextPage, since we fully drained a page.
return(QueryResponse.CreateSuccess(
result: results,
count: results.Count,
responseLengthBytes: this.GetAndResetResponseLengthBytes(),
responseHeaders: this.GetResponseHeaders()));
}
//Retorna os dados de uma linha específica na tabela de banco de dados
public QueryResponse <T> GetSingleData <T>(DbCommand command)
{
try
{
using (DbCurrentConnection connection = new DbCurrentConnection(command))
{
connection.OpenConnection();
DbDataReader reader = command.ExecuteReader();
DataTable table = new DataTable();
table.Load(reader);
if (table.Rows.Count == 0)
{
return(QueryResponse <T> .CreateFailure("Registro não encontrado."));
}
T data = table.ConvertDataTable <T>().FirstOrDefault();
return(QueryResponse <T> .CreateSuccess(data, "Dados retornados com sucesso."));
}
}
catch (Exception ex)
{
return(QueryResponse <T> .CreateFailureException("Ocorreu um erro, contate o administrador.", ex));
}
}
internal void GetDeserializedObjectsFromQueryResponseTest(JsonSerializationFormat jsonSerializationFormat)
{
// Constants to use for vertex document property key/values
const string vertex1Id = "v_0";
const string vertex2Id = "v_1";
const string vertex1Label = "l_0";
const string vertex2Label = "l_1";
const string vertex1PkValue = "pk_0";
const string vertex2PkValue = "pk_1";
const string property1Name = "p_0";
const string vertex1Property1Value = "v_0_p_0_v_0";
const string vertex2Property1Value = "v_1_p_0_v_0";
const string property2Name = "p_1";
const double vertex1Property2Value = 12.34;
const long vertex2Property2Value = 5678;
// Compose two initial vertex documents using eager CosmosElements
CosmosObject initialVertex1EagerObject = this.CreateVertexDocument(
vertex1Id,
vertex1Label,
GremlinScenarioTests.PartitionKeyPropertyName,
vertex1PkValue,
new Tuple <string, IEnumerable <object> >[]
{
Tuple.Create <string, IEnumerable <object> >(property1Name, new object[] { vertex1Property1Value }),
Tuple.Create <string, IEnumerable <object> >(property2Name, new object[] { vertex1Property2Value }),
});
CosmosObject initialVertex2EagerObject = this.CreateVertexDocument(
vertex2Id,
vertex2Label,
GremlinScenarioTests.PartitionKeyPropertyName,
vertex2PkValue,
new Tuple <string, IEnumerable <object> >[]
{
Tuple.Create <string, IEnumerable <object> >(property1Name, new object[] { vertex2Property1Value }),
Tuple.Create <string, IEnumerable <object> >(property2Name, new object[] { vertex2Property2Value }),
});
// Serialize the initial vertex object into a document using the specified serialization format
IJsonWriter jsonWriter = JsonWriter.Create(jsonSerializationFormat);
initialVertex1EagerObject.WriteTo(jsonWriter);
ReadOnlyMemory <byte> vertex1JsonWriterResult = jsonWriter.GetResult();
Assert.IsTrue(vertex1JsonWriterResult.Length > 0, "IJsonWriter result data is empty.");
jsonWriter = JsonWriter.Create(jsonSerializationFormat);
initialVertex2EagerObject.WriteTo(jsonWriter);
ReadOnlyMemory <byte> vertex2JsonWriterResult = jsonWriter.GetResult();
Assert.IsTrue(vertex2JsonWriterResult.Length > 0, "IJsonWriter result data is empty.");
// Navigate into the serialized vertex documents using lazy CosmosElements
CosmosElement vertex1LazyObject = CosmosElement.CreateFromBuffer(vertex1JsonWriterResult);
CosmosElement vertex2LazyObject = CosmosElement.CreateFromBuffer(vertex2JsonWriterResult);
// Create a dynamically-typed QueryResponse backed by the vertex document CosmosElements
CosmosArray vertexArray = CosmosArray.Create(
new CosmosElement[]
{
vertex1LazyObject,
vertex2LazyObject,
});
QueryResponse queryResponse = QueryResponse.CreateSuccess(
vertexArray,
count: 2,
responseLengthBytes: vertex1JsonWriterResult.Length + vertex2JsonWriterResult.Length,
serializationOptions: null,
responseHeaders: CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders(
sourceHeaders: null,
resourceType: ResourceType.Document,
containerRid: GremlinScenarioTests.CreateRandomString(10)),
diagnostics: null);
QueryResponse <dynamic> cosmosElementQueryResponse =
QueryResponse <dynamic> .CreateResponse <dynamic>(
queryResponse,
MockCosmosUtil.Serializer);
// Assert that other objects (anything besides the lazy CosmosElements that we created earlier) are deserialized
// from the backing CosmosElement contents rather than being directly returned as CosmosElements
List <dynamic> responseCosmosElements = new List <dynamic>(cosmosElementQueryResponse.Resource);
Assert.AreEqual(vertexArray.Count, responseCosmosElements.Count);
Assert.AreNotSame(vertex1LazyObject, responseCosmosElements[0]);
Assert.AreNotSame(vertex2LazyObject, responseCosmosElements[1]);
}
public override async Task <QueryResponse> DrainAsync(
int maxElements,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
// Draining GROUP BY is broken down into two stages:
QueryResponse response;
if (!this.Source.IsDone)
{
// Stage 1:
// Drain the groupings fully from all continuation and all partitions
QueryResponse sourceResponse = await base.DrainAsync(int.MaxValue, cancellationToken);
if (!sourceResponse.IsSuccessStatusCode)
{
return(sourceResponse);
}
this.containerRid = sourceResponse.QueryHeaders.ContainerRid;
foreach (CosmosElement result in sourceResponse.CosmosElements)
{
// Aggregate the values for all groupings across all continuations.
RewrittenGroupByProjection groupByItem = new RewrittenGroupByProjection(result);
this.distinctMap.Add(groupByItem.GroupByItems, out UInt192? groupByKeysHash);
if (!groupByKeysHash.HasValue)
{
throw new InvalidOperationException("hash invariant was broken");
}
if (!this.groupingTable.TryGetValue(groupByKeysHash.Value, out SingleGroupAggregator singleGroupAggregator))
{
singleGroupAggregator = SingleGroupAggregator.Create(
EmptyAggregateOperators,
this.groupByAliasToAggregateType,
this.hasSelectValue);
this.groupingTable[groupByKeysHash.Value] = singleGroupAggregator;
}
CosmosElement payload = groupByItem.Payload;
singleGroupAggregator.AddValues(payload);
}
string updatedContinuationToken = null;
// We need to give empty pages until the results are fully drained.
response = QueryResponse.CreateSuccess(
EmptyResults,
EmptyResults.Count,
sourceResponse.ResponseLengthBytes,
sourceResponse.QueryHeaders.CloneKnownProperties(updatedContinuationToken, GroupByDocumentQueryExecutionComponent.ContinuationTokenNotSupportedWithGroupBy));
this.isDone = false;
}
else
{
// Stage 2:
// Emit the results from the grouping table page by page
IEnumerable <SingleGroupAggregator> groupByValuesList = this.groupingTable
.OrderBy(kvp => kvp.Key)
.Skip(this.numPagesDrainedFromGroupingTable * maxElements)
.Take(maxElements)
.Select(kvp => kvp.Value);
List <CosmosElement> results = new List <CosmosElement>();
foreach (SingleGroupAggregator groupByValues in groupByValuesList)
{
results.Add(groupByValues.GetResult());
}
response = QueryResponse.CreateSuccess(
results,
results.Count,
responseLengthBytes: 0,
responseHeaders: new CosmosQueryResponseMessageHeaders(
continauationToken: null,
disallowContinuationTokenMessage: GroupByDocumentQueryExecutionComponent.ContinuationTokenNotSupportedWithGroupBy,
resourceType: Documents.ResourceType.Document,
containerRid: this.containerRid));
this.numPagesDrainedFromGroupingTable++;
if (this.numPagesDrainedFromGroupingTable * maxElements >= this.groupingTable.Count)
{
this.isDone = true;
}
}
return(response);
}
/// <summary>
/// Drains at most 'maxElements' documents from the <see cref="AggregateDocumentQueryExecutionComponent"/> .
/// </summary>
/// <param name="maxElements">This value is ignored, since the aggregates are aggregated for you.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The aggregate result after all the continuations have been followed.</returns>
/// <remarks>
/// Note that this functions follows all continuations meaning that it won't return until all continuations are drained.
/// This means that if you have a long running query this function will take a very long time to return.
/// </remarks>
public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken token)
{
token.ThrowIfCancellationRequested();
// Note-2016-10-25-felixfan: Given what we support now, we should expect to return only 1 document.
double requestCharge = 0;
long responseLengthBytes = 0;
List <Uri> replicaUris = new List <Uri>();
ClientSideRequestStatistics requestStatistics = new ClientSideRequestStatistics();
PartitionedQueryMetrics partitionedQueryMetrics = new PartitionedQueryMetrics();
while (!this.IsDone)
{
QueryResponse result = await base.DrainAsync(int.MaxValue, token);
if (!result.IsSuccessStatusCode)
{
return(result);
}
requestCharge += result.Headers.RequestCharge;
responseLengthBytes += result.ResponseLengthBytes;
//partitionedQueryMetrics += new PartitionedQueryMetrics(result.QueryMetrics);
if (result.RequestStatistics != null)
{
replicaUris.AddRange(result.RequestStatistics.ContactedReplicas);
}
foreach (CosmosElement item in result.CosmosElements)
{
if (!(item is CosmosArray comosArray))
{
throw new InvalidOperationException("Expected an array of aggregate results from the execution context.");
}
List <AggregateItem> aggregateItems = new List <AggregateItem>();
foreach (CosmosElement arrayItem in comosArray)
{
aggregateItems.Add(new AggregateItem(arrayItem));
}
Debug.Assert(
aggregateItems.Count == this.aggregators.Length,
$"Expected {this.aggregators.Length} values, but received {aggregateItems.Count}.");
for (int i = 0; i < this.aggregators.Length; ++i)
{
this.aggregators[i].Aggregate(aggregateItems[i].Item);
}
}
}
List <CosmosElement> finalResult = this.BindAggregateResults(
this.aggregators.Select(aggregator => aggregator.GetResult()));
// The replicaUris may have duplicates.
requestStatistics.ContactedReplicas.AddRange(replicaUris);
return(QueryResponse.CreateSuccess(
result: finalResult,
count: finalResult.Count,
responseLengthBytes: responseLengthBytes,
responseHeaders: new CosmosQueryResponseMessageHeaders(continauationToken: null, disallowContinuationTokenMessage: null)
{
RequestCharge = requestCharge
}));
}
/// <summary>
/// Drains at most 'maxElements' documents from the AggregateDocumentQueryExecutionComponent.
/// </summary>
/// <param name="maxElements">This value is ignored, since the aggregates are aggregated for you.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The aggregate result after all the continuations have been followed.</returns>
/// <remarks>
/// Note that this functions follows all continuations meaning that it won't return until all continuations are drained.
/// This means that if you have a long running query this function will take a very long time to return.
/// </remarks>
public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken token)
{
// Note-2016-10-25-felixfan: Given what we support now, we should expect to return only 1 document.
// Note-2019-07-11-brchon: We can return empty pages until all the documents are drained,
// but then we will have to design a continuation token.
double requestCharge = 0;
long responseLengthBytes = 0;
List <Uri> replicaUris = new List <Uri>();
ClientSideRequestStatistics requestStatistics = new ClientSideRequestStatistics();
PartitionedQueryMetrics partitionedQueryMetrics = new PartitionedQueryMetrics();
ResourceType resourceType = ResourceType.Document;
string containerRid = null;
while (!this.IsDone)
{
QueryResponse result = await base.DrainAsync(int.MaxValue, token);
if (!result.IsSuccessStatusCode)
{
return(result);
}
containerRid = result.QueryHeaders.ContainerRid;
resourceType = result.QueryHeaders.ResourceType;
requestCharge += result.Headers.RequestCharge;
responseLengthBytes += result.ResponseLengthBytes;
// DEVNOTE: Add when query metrics is supported
// partitionedQueryMetrics += new PartitionedQueryMetrics(results.QueryMetrics);
if (result.RequestStatistics != null)
{
replicaUris.AddRange(result.RequestStatistics.ContactedReplicas);
}
foreach (CosmosElement element in result.CosmosElements)
{
RewrittenAggregateProjections rewrittenAggregateProjections = new RewrittenAggregateProjections(
this.isValueAggregateQuery,
element);
this.singleGroupAggregator.AddValues(rewrittenAggregateProjections.Payload);
}
}
List <CosmosElement> finalResult = new List <CosmosElement>();
CosmosElement aggregationResult = this.singleGroupAggregator.GetResult();
if (aggregationResult != null)
{
finalResult.Add(aggregationResult);
}
// The replicaUris may have duplicates.
requestStatistics.ContactedReplicas.AddRange(replicaUris);
return(QueryResponse.CreateSuccess(
result: finalResult,
count: finalResult.Count,
responseLengthBytes: responseLengthBytes,
responseHeaders: new CosmosQueryResponseMessageHeaders(
continauationToken: null,
disallowContinuationTokenMessage: null,
resourceType: resourceType,
containerRid: containerRid)
{
RequestCharge = requestCharge
},
queryMetrics: this.GetQueryMetrics()));
}
public override async Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default(CancellationToken))
{
// This catches exception thrown by the pipeline and converts it to QueryResponse
ResponseMessage response;
try
{
QueryResponseCore responseCore = await this.cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken);
CosmosQueryContext cosmosQueryContext = this.cosmosQueryExecutionContext.CosmosQueryContext;
QueryResponse queryResponse;
if (responseCore.IsSuccess)
{
queryResponse = QueryResponse.CreateSuccess(
result: responseCore.CosmosElements,
count: responseCore.CosmosElements.Count,
responseLengthBytes: responseCore.ResponseLengthBytes,
queryMetrics: responseCore.QueryMetrics,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
cosmosQueryContext.ResourceTypeEnum,
cosmosQueryContext.ContainerResourceId)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId
});
}
else
{
queryResponse = QueryResponse.CreateFailure(
statusCode: responseCore.StatusCode,
error: null,
errorMessage: responseCore.ErrorMessage,
requestMessage: null,
responseHeaders: new CosmosQueryResponseMessageHeaders(
responseCore.ContinuationToken,
responseCore.DisallowContinuationTokenMessage,
cosmosQueryContext.ResourceTypeEnum,
cosmosQueryContext.ContainerResourceId)
{
RequestCharge = responseCore.RequestCharge,
ActivityId = responseCore.ActivityId
});
}
if (responseCore.QueryMetrics != null && responseCore.QueryMetrics.Count > 0)
{
queryResponse.Diagnostics = new QueryOperationStatistics(responseCore.QueryMetrics);
}
queryResponse.CosmosSerializationOptions = cosmosQueryContext.QueryRequestOptions.CosmosSerializationFormatOptions;
response = queryResponse;
}
catch (Documents.DocumentClientException exception)
{
response = exception.ToCosmosResponseMessage(request: null);
}
catch (CosmosException exception)
{
response = exception.ToCosmosResponseMessage(request: null);
}
catch (AggregateException ae)
{
response = TransportHandler.AggregateExceptionConverter(ae, null);
if (response == null)
{
throw;
}
}
return(response);
}
public async Task TestItemProducerTreeWithFailure()
{
int callBackCount = 0;
Mock <CosmosQueryContext> mockQueryContext = new Mock <CosmosQueryContext>();
SqlQuerySpec sqlQuerySpec = new SqlQuerySpec("Select * from t");
PartitionKeyRange partitionKeyRange = new PartitionKeyRange {
Id = "0", MinInclusive = "A", MaxExclusive = "B"
};
Action <ItemProducerTree, int, double, QueryMetrics, long, CancellationToken> produceAsyncCompleteCallback = (
ItemProducerTree producer,
int itemsBuffered,
double resourceUnitUsage,
QueryMetrics queryMetrics,
long responseLengthBytes,
CancellationToken token) => { callBackCount++; };
Mock <IComparer <ItemProducerTree> > comparer = new Mock <IComparer <ItemProducerTree> >();
Mock <IEqualityComparer <CosmosElement> > cosmosElementComparer = new Mock <IEqualityComparer <CosmosElement> >();
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
IEnumerable <CosmosElement> cosmosElements = new List <CosmosElement>()
{
new Mock <CosmosElement>(CosmosElementType.Object).Object
};
CosmosQueryResponseMessageHeaders headers = new CosmosQueryResponseMessageHeaders("TestToken", null)
{
ActivityId = "AA470D71-6DEF-4D61-9A08-272D8C9ABCFE",
RequestCharge = 42
};
mockQueryContext.Setup(x => x.ExecuteQueryAsync(sqlQuerySpec, cancellationTokenSource.Token, It.IsAny <Action <CosmosRequestMessage> >())).Returns(
Task.FromResult(QueryResponse.CreateSuccess(cosmosElements, 1, 500, headers)));
ItemProducerTree itemProducerTree = new ItemProducerTree(
queryContext: mockQueryContext.Object,
querySpecForInit: sqlQuerySpec,
partitionKeyRange: partitionKeyRange,
produceAsyncCompleteCallback: produceAsyncCompleteCallback,
itemProducerTreeComparer: comparer.Object,
equalityComparer: cosmosElementComparer.Object,
deferFirstPage: false,
collectionRid: "collectionRid",
initialContinuationToken: null,
initialPageSize: 50);
// Buffer to success responses
await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token);
await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token);
// Buffer a failure
mockQueryContext.Setup(x => x.ExecuteQueryAsync(sqlQuerySpec, cancellationTokenSource.Token, It.IsAny <Action <CosmosRequestMessage> >())).Returns(
Task.FromResult(QueryResponse.CreateFailure(headers, HttpStatusCode.InternalServerError, null, "Error message", null)));
await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token);
// First item should be a success
var result = await itemProducerTree.MoveNextAsync(cancellationTokenSource.Token);
Assert.IsTrue(result.successfullyMovedNext);
Assert.IsNull(result.failureResponse);
Assert.IsTrue(itemProducerTree.HasMoreResults);
// Second item should be a success
result = await itemProducerTree.MoveNextAsync(cancellationTokenSource.Token);
Assert.IsTrue(result.successfullyMovedNext);
Assert.IsNull(result.failureResponse);
Assert.IsTrue(itemProducerTree.HasMoreResults);
// Third item should be a failure
result = await itemProducerTree.MoveNextAsync(cancellationTokenSource.Token);
Assert.IsFalse(result.successfullyMovedNext);
Assert.IsNotNull(result.failureResponse);
Assert.IsFalse(itemProducerTree.HasMoreResults);
// Try to buffer after failure. It should return the previous cached failure and not try to buffer again.
mockQueryContext.Setup(x => x.ExecuteQueryAsync(sqlQuerySpec, cancellationTokenSource.Token, It.IsAny <Action <CosmosRequestMessage> >())).
Throws(new Exception("Previous buffer failed. Operation should return original failure and not try again"));
await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token);
Assert.IsFalse(result.successfullyMovedNext);
Assert.IsNotNull(result.failureResponse);
Assert.IsFalse(itemProducerTree.HasMoreResults);
}
/// <summary>
/// Drains a page of documents from this context.
/// </summary>
/// <param name="maxElements">The maximum number of elements.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>A task that when awaited on return a page of documents.</returns>
public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken cancellationToken)
{
//// In order to maintain the continuation toke for the user we must drain with a few constraints
//// 1) We always drain from the partition, which has the highest priority item first
//// 2) If multiple partitions have the same priority item then we drain from the left most first
//// otherwise we would need to keep track of how many of each item we drained from each partition
//// (just like parallel queries).
//// Visually that look the following case where we have three partitions that are numbered and store letters.
//// For teaching purposes I have made each item a tuple of the following form:
//// <item stored in partition, partition number>
//// So that duplicates across partitions are distinct, but duplicates within partitions are indistinguishable.
//// |-------| |-------| |-------|
//// | <a,1> | | <a,2> | | <a,3> |
//// | <a,1> | | <b,2> | | <c,3> |
//// | <a,1> | | <b,2> | | <c,3> |
//// | <d,1> | | <c,2> | | <c,3> |
//// | <d,1> | | <e,2> | | <f,3> |
//// | <e,1> | | <h,2> | | <j,3> |
//// | <f,1> | | <i,2> | | <k,3> |
//// |-------| |-------| |-------|
//// Now the correct drain order in this case is:
//// <a,1>,<a,1>,<a,1>,<a,2>,<a,3>,<b,2>,<b,2>,<c,2>,<c,3>,<c,3>,<c,3>,
//// <d,1>,<d,1>,<e,1>,<e,2>,<f,1>,<f,3>,<h,2>,<i,2>,<j,3>,<k,3>
//// In more mathematical terms
//// 1) <x, y> always comes before <z, y> where x < z
//// 2) <i, j> always come before <i, k> where j < k
List <CosmosElement> results = new List <CosmosElement>();
while (!this.IsDone && results.Count < maxElements)
{
// Only drain from the highest priority document producer
// We need to pop and push back the document producer tree, since the priority changes according to the sort order.
DocumentProducerTree currentDocumentProducerTree = this.PopCurrentDocumentProducerTree();
OrderByQueryResult orderByQueryResult = new OrderByQueryResult(currentDocumentProducerTree.Current);
// Only add the payload, since other stuff is garbage from the caller's perspective.
results.Add(orderByQueryResult.Payload);
// If we are at the begining of the page and seeing an rid from the previous page we should increment the skip count
// due to the fact that JOINs can make a document appear multiple times and across continuations, so we don't want to
// surface this more than needed. More information can be found in the continuation token docs.
if (this.ShouldIncrementSkipCount(currentDocumentProducerTree.CurrentDocumentProducerTree.Root))
{
++this.skipCount;
}
else
{
this.skipCount = 0;
}
this.previousRid = orderByQueryResult.Rid;
await currentDocumentProducerTree.MoveNextAsync(cancellationToken);
this.PushCurrentDocumentProducerTree(currentDocumentProducerTree);
}
return(QueryResponse.CreateSuccess(
result: results,
count: results.Count,
responseHeaders: this.GetResponseHeaders(),
responseLengthBytes: this.GetAndResetResponseLengthBytes()));
}
