public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken token) { token.ThrowIfCancellationRequested(); QueryResponse sourcePage = await base.DrainAsync(maxElements, token); if (!sourcePage.IsSuccessStatusCode) { return(sourcePage); } // skip the documents but keep all the other headers List <CosmosElement> documentsAfterSkip = sourcePage.CosmosElements.Skip(this.skipCount).ToList(); int numberOfDocumentsSkipped = sourcePage.Count - documentsAfterSkip.Count; this.skipCount -= numberOfDocumentsSkipped; string updatedContinuationToken = null; if (sourcePage.QueryHeaders.DisallowContinuationTokenMessage == null) { if (!this.IsDone) { updatedContinuationToken = new OffsetContinuationToken( this.skipCount, sourcePage.Headers.ContinuationToken).ToString(); } } return(QueryResponse.CreateSuccess( result: documentsAfterSkip, count: documentsAfterSkip.Count(), responseHeaders: sourcePage.QueryHeaders.CloneKnownProperties(updatedContinuationToken, sourcePage.QueryHeaders.DisallowContinuationTokenMessage), responseLengthBytes: sourcePage.ResponseLengthBytes)); }
/// <summary> /// Gets the next page of results from this context. /// </summary> /// <param name="token">The cancellation token.</param> /// <returns>A task to await on that in turn returns a DoucmentFeedResponse of results.</returns> public override async Task <QueryResponse> ExecuteNextAsync(CancellationToken token) { try { QueryResponse queryResponse = await this.component.DrainAsync(this.actualPageSize, token); if (!queryResponse.IsSuccessStatusCode) { this.component.Stop(); return(queryResponse); } List <CosmosElement> dynamics = new List <CosmosElement>(); foreach (CosmosElement element in queryResponse.CosmosElements) { dynamics.Add(element); } return(QueryResponse.CreateSuccess( dynamics, queryResponse.Count, queryResponse.ResponseLengthBytes, queryResponse.QueryHeaders.CloneKnownProperties())); } catch (Exception) { this.component.Stop(); throw; } }
public override async Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default) { CosmosDiagnosticsContext diagnostics = new CosmosDiagnosticsContext(); using (diagnostics.CreateScope("QueryReadNextAsync")) { // This catches exception thrown by the pipeline and converts it to QueryResponse QueryResponseCore responseCore = await this.cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken); CosmosQueryContext cosmosQueryContext = this.cosmosQueryContext; foreach (QueryPageDiagnostics queryPage in responseCore.Diagnostics) { diagnostics.Summary.Append(queryPage.DiagnosticsContext.Summary); diagnostics.AddDiagnosticsInternal(queryPage); } QueryResponse queryResponse; if (responseCore.IsSuccess) { queryResponse = QueryResponse.CreateSuccess( result: responseCore.CosmosElements, count: responseCore.CosmosElements.Count, responseLengthBytes: responseCore.ResponseLengthBytes, diagnostics: diagnostics, serializationOptions: this.cosmosSerializationFormatOptions, responseHeaders: new CosmosQueryResponseMessageHeaders( responseCore.ContinuationToken, responseCore.DisallowContinuationTokenMessage, cosmosQueryContext.ResourceTypeEnum, cosmosQueryContext.ContainerResourceId) { RequestCharge = responseCore.RequestCharge, ActivityId = responseCore.ActivityId, SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown }); } else { queryResponse = QueryResponse.CreateFailure( statusCode: responseCore.StatusCode, error: null, errorMessage: responseCore.ErrorMessage, requestMessage: null, diagnostics: diagnostics, responseHeaders: new CosmosQueryResponseMessageHeaders( responseCore.ContinuationToken, responseCore.DisallowContinuationTokenMessage, cosmosQueryContext.ResourceTypeEnum, cosmosQueryContext.ContainerResourceId) { RequestCharge = responseCore.RequestCharge, ActivityId = responseCore.ActivityId, SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown }); } return(queryResponse); } }
public static (QueryResponse queryResponse, IList <ToDoItem> items) Create( string itemIdPrefix, string continuationToken, string collectionRid, int itemCount = 50) { // Use -1 to represent a split response if (itemCount == QueryResponseMessageFactory.SPLIT) { return(CreateSplitResponse(collectionRid), new List <ToDoItem>().AsReadOnly()); } IList <ToDoItem> items = ToDoItem.CreateItems(itemCount, itemIdPrefix); MemoryStream memoryStream = (MemoryStream)cosmosSerializer.ToStream <IList <ToDoItem> >(items); long responseLengthBytes = memoryStream.Length; IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray()); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode); Headers headers = new Headers(); headers.ContinuationToken = continuationToken; headers.ActivityId = Guid.NewGuid().ToString(); QueryResponse message = QueryResponse.CreateSuccess( result: cosmosArray, count: itemCount, responseHeaders: CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders(headers, ResourceType.Document, collectionRid), responseLengthBytes: responseLengthBytes); return(message, items); }
public void VerifyCosmosQueryResponseStream() { string contianerRid = "mockContainerRid"; (QueryResponseCore response, IList <ToDoItem> items)factoryResponse = QueryResponseMessageFactory.Create( itemIdPrefix: $"TestPage", continuationToken: "SomeContinuationToken", collectionRid: contianerRid, itemCount: 100); QueryResponseCore responseCore = factoryResponse.response; QueryResponse queryResponse = QueryResponse.CreateSuccess( result: responseCore.CosmosElements, count: responseCore.CosmosElements.Count, responseLengthBytes: responseCore.ResponseLengthBytes, queryMetrics: responseCore.QueryMetrics, responseHeaders: new CosmosQueryResponseMessageHeaders( responseCore.ContinuationToken, responseCore.DisallowContinuationTokenMessage, ResourceType.Document, contianerRid) { RequestCharge = responseCore.RequestCharge, ActivityId = responseCore.ActivityId }); using (Stream stream = queryResponse.Content) { using (Stream innerStream = queryResponse.Content) { Assert.IsTrue(object.ReferenceEquals(stream, innerStream), "Content should return the same stream"); } } }
//Retorna uma lista de genérica com todos os dados de uma dada tabela public QueryResponse <List <T> > GetAllData <T>(DbCommand command) { try { using (DbCurrentConnection connection = new DbCurrentConnection(command)) { connection.OpenConnection(); DbDataReader reader = command.ExecuteReader(); DataTable table = new DataTable(); table.Load(reader); if (table.Rows.Count == 0) { List <T> datan = new List <T>(); return(QueryResponse <List <T> > .CreateSuccess(datan, "Nenhum registro cadastrado.")); } List <T> data = table.ConvertDataTable <T>(); return(QueryResponse <List <T> > .CreateSuccess(data, "Dados retornados com sucesso.")); } } catch (Exception ex) { return(QueryResponse <List <T> > .CreateFailureException("Ocorreu um erro, contate o administrador.", ex)); } }
/// <summary> /// Drains documents from this component. This has the common drain logic for each implementation. /// </summary> /// <param name="maxElements">The maximum number of documents to drain.</param> /// <param name="cancellationToken">The cancellation token to cancel tasks.</param> /// <returns>A task that when awaited on returns a feed response.</returns> public async override Task <QueryResponse> DrainAsync(int maxElements, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); // The initialization or previous Drain Async failed. Just return the failure. if (this.FailureResponse != null) { this.Stop(); return(this.FailureResponse); } // Drain the results. If there is no results and a failure then return the failure. IList <CosmosElement> results = await this.InternalDrainAsync(maxElements, cancellationToken); if ((results == null || results.Count == 0) && this.FailureResponse != null) { this.Stop(); return(this.FailureResponse); } return(QueryResponse.CreateSuccess( result: results, count: results.Count, responseHeaders: this.GetResponseHeaders(), responseLengthBytes: this.GetAndResetResponseLengthBytes())); }
public override async Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default) { CosmosDiagnosticsContext diagnostics = CosmosDiagnosticsContext.Create(this.requestOptions); using (diagnostics.GetOverallScope()) { // This catches exception thrown by the pipeline and converts it to QueryResponse QueryResponseCore responseCore = await this.cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken); // This swaps the diagnostics in the context. This shows all the page reads between the previous ReadNextAsync and the current ReadNextAsync diagnostics.AddDiagnosticsInternal(this.cosmosQueryContext.GetAndResetDiagnostics()); if (responseCore.IsSuccess) { List <CosmosElement> decryptedCosmosElements = null; if (this.clientContext.ClientOptions.Encryptor != null) { decryptedCosmosElements = await this.GetDecryptedElementResponseAsync(responseCore.CosmosElements, diagnostics, cancellationToken); } return(QueryResponse.CreateSuccess( result: decryptedCosmosElements ?? responseCore.CosmosElements, count: responseCore.CosmosElements.Count, responseLengthBytes: responseCore.ResponseLengthBytes, diagnostics: diagnostics, serializationOptions: this.cosmosSerializationFormatOptions, responseHeaders: new CosmosQueryResponseMessageHeaders( responseCore.ContinuationToken, responseCore.DisallowContinuationTokenMessage, this.cosmosQueryContext.ResourceTypeEnum, this.cosmosQueryContext.ContainerResourceId) { RequestCharge = responseCore.RequestCharge, ActivityId = responseCore.ActivityId, SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown })); } if (responseCore.CosmosException != null) { return(responseCore.CosmosException.ToCosmosResponseMessage(null)); } return(QueryResponse.CreateFailure( statusCode: responseCore.StatusCode, cosmosException: responseCore.CosmosException, requestMessage: null, diagnostics: diagnostics, responseHeaders: new CosmosQueryResponseMessageHeaders( responseCore.ContinuationToken, responseCore.DisallowContinuationTokenMessage, this.cosmosQueryContext.ResourceTypeEnum, this.cosmosQueryContext.ContainerResourceId) { RequestCharge = responseCore.RequestCharge, ActivityId = responseCore.ActivityId, SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown, })); } }
public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken token) { token.ThrowIfCancellationRequested(); QueryResponse results = await base.DrainAsync(maxElements, token); if (!results.IsSuccessStatusCode) { return(results); } List <CosmosElement> takedDocuments = results.CosmosElements.Take(this.takeCount).ToList(); this.takeCount -= takedDocuments.Count; string updatedContinuationToken = null; if (results.QueryHeaders.DisallowContinuationTokenMessage == null) { if (!this.IsDone) { string sourceContinuation = results.Headers.ContinuationToken; TakeContinuationToken takeContinuationToken; switch (this.takeEnum) { case TakeEnum.Limit: takeContinuationToken = new LimitContinuationToken( this.takeCount, sourceContinuation); break; case TakeEnum.Top: takeContinuationToken = new TopContinuationToken( this.takeCount, sourceContinuation); break; default: throw new ArgumentException($"Unknown {nameof(TakeEnum)}: {this.takeEnum}"); } updatedContinuationToken = takeContinuationToken.ToString(); } } return(QueryResponse.CreateSuccess( takedDocuments, takedDocuments.Count, results.ResponseLengthBytes, results.QueryHeaders.CloneKnownProperties(updatedContinuationToken, results.QueryHeaders.DisallowContinuationTokenMessage), results.queryMetrics)); }
/// <summary> /// Drains a page of results returning only distinct elements. /// </summary> /// <param name="maxElements">The maximum number of items to drain.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns>A page of distinct results.</returns> public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken cancellationToken) { List <CosmosElement> distinctResults = new List <CosmosElement>(); QueryResponse cosmosQueryResponse = await base.DrainAsync(maxElements, cancellationToken); if (!cosmosQueryResponse.IsSuccessStatusCode) { return(cosmosQueryResponse); } foreach (CosmosElement document in cosmosQueryResponse.CosmosElements) { if (this.distinctMap.Add(document, out this.lastHash)) { distinctResults.Add(document); } } string updatedContinuationToken; if (!this.IsDone) { updatedContinuationToken = new DistinctContinuationToken( this.lastHash, cosmosQueryResponse.Headers.ContinuationToken).ToString(); } else { this.Source.Stop(); updatedContinuationToken = null; } string disallowContinuationTokenMessage = this.distinctQueryType == DistinctQueryType.Ordered ? null : RMResources.UnorderedDistinctQueryContinuationToken; return(QueryResponse.CreateSuccess( distinctResults, distinctResults.Count, cosmosQueryResponse.ResponseLengthBytes, cosmosQueryResponse.QueryHeaders.CloneKnownProperties(updatedContinuationToken, disallowContinuationTokenMessage), cosmosQueryResponse.queryMetrics)); }
public static QueryResponse CreateQueryResponse( IList <ToDoItem> items, bool isOrderByQuery, string continuationToken, string collectionRid) { MemoryStream memoryStream; string json; if (isOrderByQuery) { memoryStream = SerializeForOrderByQuery(items); using (StreamReader sr = new StreamReader(SerializeForOrderByQuery(items))) { json = sr.ReadToEnd(); } } else { memoryStream = (MemoryStream)cosmosSerializer.ToStream <IList <ToDoItem> >(items); } long responseLengthBytes = memoryStream.Length; IJsonNavigator jsonNavigator = JsonNavigator.Create(memoryStream.ToArray()); IJsonNavigatorNode jsonNavigatorNode = jsonNavigator.GetRootNode(); CosmosArray cosmosArray = CosmosArray.Create(jsonNavigator, jsonNavigatorNode); Headers headers = new Headers(); headers.ContinuationToken = continuationToken; headers.ActivityId = Guid.NewGuid().ToString(); QueryResponse message = QueryResponse.CreateSuccess( result: cosmosArray, count: items.Count, responseHeaders: CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders(headers, ResourceType.Document, collectionRid), responseLengthBytes: responseLengthBytes); return(message); }
public void VerifyItemQueryResponseResult() { string contianerRid = "mockContainerRid"; (QueryResponseCore response, IList <ToDoItem> items)factoryResponse = QueryResponseMessageFactory.Create( itemIdPrefix: $"TestPage", continuationToken: "SomeContinuationToken", collectionRid: contianerRid, itemCount: 100); QueryResponseCore responseCore = factoryResponse.response; List <CosmosElement> cosmosElements = new List <CosmosElement>(responseCore.CosmosElements); QueryResponse queryResponse = QueryResponse.CreateSuccess( result: cosmosElements, count: cosmosElements.Count, responseLengthBytes: responseCore.ResponseLengthBytes, serializationOptions: null, responseHeaders: new CosmosQueryResponseMessageHeaders( responseCore.ContinuationToken, responseCore.DisallowContinuationTokenMessage, ResourceType.Document, contianerRid) { RequestCharge = responseCore.RequestCharge, ActivityId = responseCore.ActivityId }, diagnostics: new CosmosDiagnosticsContextCore()); QueryResponse <ToDoItem> itemQueryResponse = QueryResponseMessageFactory.CreateQueryResponse <ToDoItem>(queryResponse); List <ToDoItem> resultItems = new List <ToDoItem>(itemQueryResponse.Resource); ToDoItemComparer comparer = new ToDoItemComparer(); Assert.AreEqual(factoryResponse.items.Count, resultItems.Count); for (int i = 0; i < factoryResponse.items.Count; i++) { Assert.AreNotSame(factoryResponse.items[i], resultItems[i]); Assert.AreEqual(0, comparer.Compare(factoryResponse.items[i], resultItems[i])); } }
private ResponseMessage CreateQueryResponse() { List <CosmosElement> cosmosElements = new List <CosmosElement>(); string serializedItem = this.GetSerializedToDoActivity(); CosmosObject cosmosObject = CosmosObject.Parse(serializedItem); cosmosElements.Add(cosmosObject); ResponseMessage cosmosResponse = QueryResponse.CreateSuccess( cosmosElements, 1, Encoding.UTF8.GetByteCount(serializedItem), new CosmosQueryResponseMessageHeaders( continauationToken: null, disallowContinuationTokenMessage: null, resourceType: Documents.ResourceType.Document, "+o4fAPfXPzw="), new CosmosDiagnosticsContextCore(), null); return(cosmosResponse); }
/// <summary> /// Drains documents from this execution context. /// </summary> /// <param name="maxElements">The maximum number of documents to drains.</param> /// <param name="token">The cancellation token.</param> /// <returns>A task that when awaited on returns a DoucmentFeedResponse of results.</returns> public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken token) { // In order to maintain the continuation token for the user we must drain with a few constraints // 1) We fully drain from the left most partition before moving on to the next partition // 2) We drain only full pages from the document producer so we aren't left with a partial page // otherwise we would need to add to the continuation token how many items to skip over on that page. // Only drain from the leftmost (current) document producer tree DocumentProducerTree currentDocumentProducerTree = this.PopCurrentDocumentProducerTree(); // This might be the first time we have seen this document producer tree so we need to buffer documents if (currentDocumentProducerTree.Current == null) { await currentDocumentProducerTree.MoveNextAsync(token); } int itemsLeftInCurrentPage = currentDocumentProducerTree.ItemsLeftInCurrentPage; // Only drain full pages or less if this is a top query. List <CosmosElement> results = new List <CosmosElement>(); for (int i = 0; i < Math.Min(itemsLeftInCurrentPage, maxElements); i++) { results.Add(currentDocumentProducerTree.Current); await currentDocumentProducerTree.MoveNextAsync(token); } this.PushCurrentDocumentProducerTree(currentDocumentProducerTree); // At this point the document producer tree should have internally called MoveNextPage, since we fully drained a page. return(QueryResponse.CreateSuccess( result: results, count: results.Count, responseLengthBytes: this.GetAndResetResponseLengthBytes(), responseHeaders: this.GetResponseHeaders())); }
//Retorna os dados de uma linha específica na tabela de banco de dados public QueryResponse <T> GetSingleData <T>(DbCommand command) { try { using (DbCurrentConnection connection = new DbCurrentConnection(command)) { connection.OpenConnection(); DbDataReader reader = command.ExecuteReader(); DataTable table = new DataTable(); table.Load(reader); if (table.Rows.Count == 0) { return(QueryResponse <T> .CreateFailure("Registro não encontrado.")); } T data = table.ConvertDataTable <T>().FirstOrDefault(); return(QueryResponse <T> .CreateSuccess(data, "Dados retornados com sucesso.")); } } catch (Exception ex) { return(QueryResponse <T> .CreateFailureException("Ocorreu um erro, contate o administrador.", ex)); } }
internal void GetDeserializedObjectsFromQueryResponseTest(JsonSerializationFormat jsonSerializationFormat) { // Constants to use for vertex document property key/values const string vertex1Id = "v_0"; const string vertex2Id = "v_1"; const string vertex1Label = "l_0"; const string vertex2Label = "l_1"; const string vertex1PkValue = "pk_0"; const string vertex2PkValue = "pk_1"; const string property1Name = "p_0"; const string vertex1Property1Value = "v_0_p_0_v_0"; const string vertex2Property1Value = "v_1_p_0_v_0"; const string property2Name = "p_1"; const double vertex1Property2Value = 12.34; const long vertex2Property2Value = 5678; // Compose two initial vertex documents using eager CosmosElements CosmosObject initialVertex1EagerObject = this.CreateVertexDocument( vertex1Id, vertex1Label, GremlinScenarioTests.PartitionKeyPropertyName, vertex1PkValue, new Tuple <string, IEnumerable <object> >[] { Tuple.Create <string, IEnumerable <object> >(property1Name, new object[] { vertex1Property1Value }), Tuple.Create <string, IEnumerable <object> >(property2Name, new object[] { vertex1Property2Value }), }); CosmosObject initialVertex2EagerObject = this.CreateVertexDocument( vertex2Id, vertex2Label, GremlinScenarioTests.PartitionKeyPropertyName, vertex2PkValue, new Tuple <string, IEnumerable <object> >[] { Tuple.Create <string, IEnumerable <object> >(property1Name, new object[] { vertex2Property1Value }), Tuple.Create <string, IEnumerable <object> >(property2Name, new object[] { vertex2Property2Value }), }); // Serialize the initial vertex object into a document using the specified serialization format IJsonWriter jsonWriter = JsonWriter.Create(jsonSerializationFormat); initialVertex1EagerObject.WriteTo(jsonWriter); ReadOnlyMemory <byte> vertex1JsonWriterResult = jsonWriter.GetResult(); Assert.IsTrue(vertex1JsonWriterResult.Length > 0, "IJsonWriter result data is empty."); jsonWriter = JsonWriter.Create(jsonSerializationFormat); initialVertex2EagerObject.WriteTo(jsonWriter); ReadOnlyMemory <byte> vertex2JsonWriterResult = jsonWriter.GetResult(); Assert.IsTrue(vertex2JsonWriterResult.Length > 0, "IJsonWriter result data is empty."); // Navigate into the serialized vertex documents using lazy CosmosElements CosmosElement vertex1LazyObject = CosmosElement.CreateFromBuffer(vertex1JsonWriterResult); CosmosElement vertex2LazyObject = CosmosElement.CreateFromBuffer(vertex2JsonWriterResult); // Create a dynamically-typed QueryResponse backed by the vertex document CosmosElements CosmosArray vertexArray = CosmosArray.Create( new CosmosElement[] { vertex1LazyObject, vertex2LazyObject, }); QueryResponse queryResponse = QueryResponse.CreateSuccess( vertexArray, count: 2, responseLengthBytes: vertex1JsonWriterResult.Length + vertex2JsonWriterResult.Length, serializationOptions: null, responseHeaders: CosmosQueryResponseMessageHeaders.ConvertToQueryHeaders( sourceHeaders: null, resourceType: ResourceType.Document, containerRid: GremlinScenarioTests.CreateRandomString(10)), diagnostics: null); QueryResponse <dynamic> cosmosElementQueryResponse = QueryResponse <dynamic> .CreateResponse <dynamic>( queryResponse, MockCosmosUtil.Serializer); // Assert that other objects (anything besides the lazy CosmosElements that we created earlier) are deserialized // from the backing CosmosElement contents rather than being directly returned as CosmosElements List <dynamic> responseCosmosElements = new List <dynamic>(cosmosElementQueryResponse.Resource); Assert.AreEqual(vertexArray.Count, responseCosmosElements.Count); Assert.AreNotSame(vertex1LazyObject, responseCosmosElements[0]); Assert.AreNotSame(vertex2LazyObject, responseCosmosElements[1]); }
public override async Task <QueryResponse> DrainAsync( int maxElements, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); // Draining GROUP BY is broken down into two stages: QueryResponse response; if (!this.Source.IsDone) { // Stage 1: // Drain the groupings fully from all continuation and all partitions QueryResponse sourceResponse = await base.DrainAsync(int.MaxValue, cancellationToken); if (!sourceResponse.IsSuccessStatusCode) { return(sourceResponse); } this.containerRid = sourceResponse.QueryHeaders.ContainerRid; foreach (CosmosElement result in sourceResponse.CosmosElements) { // Aggregate the values for all groupings across all continuations. RewrittenGroupByProjection groupByItem = new RewrittenGroupByProjection(result); this.distinctMap.Add(groupByItem.GroupByItems, out UInt192? groupByKeysHash); if (!groupByKeysHash.HasValue) { throw new InvalidOperationException("hash invariant was broken"); } if (!this.groupingTable.TryGetValue(groupByKeysHash.Value, out SingleGroupAggregator singleGroupAggregator)) { singleGroupAggregator = SingleGroupAggregator.Create( EmptyAggregateOperators, this.groupByAliasToAggregateType, this.hasSelectValue); this.groupingTable[groupByKeysHash.Value] = singleGroupAggregator; } CosmosElement payload = groupByItem.Payload; singleGroupAggregator.AddValues(payload); } string updatedContinuationToken = null; // We need to give empty pages until the results are fully drained. response = QueryResponse.CreateSuccess( EmptyResults, EmptyResults.Count, sourceResponse.ResponseLengthBytes, sourceResponse.QueryHeaders.CloneKnownProperties(updatedContinuationToken, GroupByDocumentQueryExecutionComponent.ContinuationTokenNotSupportedWithGroupBy)); this.isDone = false; } else { // Stage 2: // Emit the results from the grouping table page by page IEnumerable <SingleGroupAggregator> groupByValuesList = this.groupingTable .OrderBy(kvp => kvp.Key) .Skip(this.numPagesDrainedFromGroupingTable * maxElements) .Take(maxElements) .Select(kvp => kvp.Value); List <CosmosElement> results = new List <CosmosElement>(); foreach (SingleGroupAggregator groupByValues in groupByValuesList) { results.Add(groupByValues.GetResult()); } response = QueryResponse.CreateSuccess( results, results.Count, responseLengthBytes: 0, responseHeaders: new CosmosQueryResponseMessageHeaders( continauationToken: null, disallowContinuationTokenMessage: GroupByDocumentQueryExecutionComponent.ContinuationTokenNotSupportedWithGroupBy, resourceType: Documents.ResourceType.Document, containerRid: this.containerRid)); this.numPagesDrainedFromGroupingTable++; if (this.numPagesDrainedFromGroupingTable * maxElements >= this.groupingTable.Count) { this.isDone = true; } } return(response); }
/// <summary> /// Drains at most 'maxElements' documents from the <see cref="AggregateDocumentQueryExecutionComponent"/> . /// </summary> /// <param name="maxElements">This value is ignored, since the aggregates are aggregated for you.</param> /// <param name="token">The cancellation token.</param> /// <returns>The aggregate result after all the continuations have been followed.</returns> /// <remarks> /// Note that this functions follows all continuations meaning that it won't return until all continuations are drained. /// This means that if you have a long running query this function will take a very long time to return. /// </remarks> public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken token) { token.ThrowIfCancellationRequested(); // Note-2016-10-25-felixfan: Given what we support now, we should expect to return only 1 document. double requestCharge = 0; long responseLengthBytes = 0; List <Uri> replicaUris = new List <Uri>(); ClientSideRequestStatistics requestStatistics = new ClientSideRequestStatistics(); PartitionedQueryMetrics partitionedQueryMetrics = new PartitionedQueryMetrics(); while (!this.IsDone) { QueryResponse result = await base.DrainAsync(int.MaxValue, token); if (!result.IsSuccessStatusCode) { return(result); } requestCharge += result.Headers.RequestCharge; responseLengthBytes += result.ResponseLengthBytes; //partitionedQueryMetrics += new PartitionedQueryMetrics(result.QueryMetrics); if (result.RequestStatistics != null) { replicaUris.AddRange(result.RequestStatistics.ContactedReplicas); } foreach (CosmosElement item in result.CosmosElements) { if (!(item is CosmosArray comosArray)) { throw new InvalidOperationException("Expected an array of aggregate results from the execution context."); } List <AggregateItem> aggregateItems = new List <AggregateItem>(); foreach (CosmosElement arrayItem in comosArray) { aggregateItems.Add(new AggregateItem(arrayItem)); } Debug.Assert( aggregateItems.Count == this.aggregators.Length, $"Expected {this.aggregators.Length} values, but received {aggregateItems.Count}."); for (int i = 0; i < this.aggregators.Length; ++i) { this.aggregators[i].Aggregate(aggregateItems[i].Item); } } } List <CosmosElement> finalResult = this.BindAggregateResults( this.aggregators.Select(aggregator => aggregator.GetResult())); // The replicaUris may have duplicates. requestStatistics.ContactedReplicas.AddRange(replicaUris); return(QueryResponse.CreateSuccess( result: finalResult, count: finalResult.Count, responseLengthBytes: responseLengthBytes, responseHeaders: new CosmosQueryResponseMessageHeaders(continauationToken: null, disallowContinuationTokenMessage: null) { RequestCharge = requestCharge })); }
/// <summary> /// Drains at most 'maxElements' documents from the AggregateDocumentQueryExecutionComponent. /// </summary> /// <param name="maxElements">This value is ignored, since the aggregates are aggregated for you.</param> /// <param name="token">The cancellation token.</param> /// <returns>The aggregate result after all the continuations have been followed.</returns> /// <remarks> /// Note that this functions follows all continuations meaning that it won't return until all continuations are drained. /// This means that if you have a long running query this function will take a very long time to return. /// </remarks> public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken token) { // Note-2016-10-25-felixfan: Given what we support now, we should expect to return only 1 document. // Note-2019-07-11-brchon: We can return empty pages until all the documents are drained, // but then we will have to design a continuation token. double requestCharge = 0; long responseLengthBytes = 0; List <Uri> replicaUris = new List <Uri>(); ClientSideRequestStatistics requestStatistics = new ClientSideRequestStatistics(); PartitionedQueryMetrics partitionedQueryMetrics = new PartitionedQueryMetrics(); ResourceType resourceType = ResourceType.Document; string containerRid = null; while (!this.IsDone) { QueryResponse result = await base.DrainAsync(int.MaxValue, token); if (!result.IsSuccessStatusCode) { return(result); } containerRid = result.QueryHeaders.ContainerRid; resourceType = result.QueryHeaders.ResourceType; requestCharge += result.Headers.RequestCharge; responseLengthBytes += result.ResponseLengthBytes; // DEVNOTE: Add when query metrics is supported // partitionedQueryMetrics += new PartitionedQueryMetrics(results.QueryMetrics); if (result.RequestStatistics != null) { replicaUris.AddRange(result.RequestStatistics.ContactedReplicas); } foreach (CosmosElement element in result.CosmosElements) { RewrittenAggregateProjections rewrittenAggregateProjections = new RewrittenAggregateProjections( this.isValueAggregateQuery, element); this.singleGroupAggregator.AddValues(rewrittenAggregateProjections.Payload); } } List <CosmosElement> finalResult = new List <CosmosElement>(); CosmosElement aggregationResult = this.singleGroupAggregator.GetResult(); if (aggregationResult != null) { finalResult.Add(aggregationResult); } // The replicaUris may have duplicates. requestStatistics.ContactedReplicas.AddRange(replicaUris); return(QueryResponse.CreateSuccess( result: finalResult, count: finalResult.Count, responseLengthBytes: responseLengthBytes, responseHeaders: new CosmosQueryResponseMessageHeaders( continauationToken: null, disallowContinuationTokenMessage: null, resourceType: resourceType, containerRid: containerRid) { RequestCharge = requestCharge }, queryMetrics: this.GetQueryMetrics())); }
public override async Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default(CancellationToken)) { // This catches exception thrown by the pipeline and converts it to QueryResponse ResponseMessage response; try { QueryResponseCore responseCore = await this.cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken); CosmosQueryContext cosmosQueryContext = this.cosmosQueryExecutionContext.CosmosQueryContext; QueryResponse queryResponse; if (responseCore.IsSuccess) { queryResponse = QueryResponse.CreateSuccess( result: responseCore.CosmosElements, count: responseCore.CosmosElements.Count, responseLengthBytes: responseCore.ResponseLengthBytes, queryMetrics: responseCore.QueryMetrics, responseHeaders: new CosmosQueryResponseMessageHeaders( responseCore.ContinuationToken, responseCore.DisallowContinuationTokenMessage, cosmosQueryContext.ResourceTypeEnum, cosmosQueryContext.ContainerResourceId) { RequestCharge = responseCore.RequestCharge, ActivityId = responseCore.ActivityId }); } else { queryResponse = QueryResponse.CreateFailure( statusCode: responseCore.StatusCode, error: null, errorMessage: responseCore.ErrorMessage, requestMessage: null, responseHeaders: new CosmosQueryResponseMessageHeaders( responseCore.ContinuationToken, responseCore.DisallowContinuationTokenMessage, cosmosQueryContext.ResourceTypeEnum, cosmosQueryContext.ContainerResourceId) { RequestCharge = responseCore.RequestCharge, ActivityId = responseCore.ActivityId }); } if (responseCore.QueryMetrics != null && responseCore.QueryMetrics.Count > 0) { queryResponse.Diagnostics = new QueryOperationStatistics(responseCore.QueryMetrics); } queryResponse.CosmosSerializationOptions = cosmosQueryContext.QueryRequestOptions.CosmosSerializationFormatOptions; response = queryResponse; } catch (Documents.DocumentClientException exception) { response = exception.ToCosmosResponseMessage(request: null); } catch (CosmosException exception) { response = exception.ToCosmosResponseMessage(request: null); } catch (AggregateException ae) { response = TransportHandler.AggregateExceptionConverter(ae, null); if (response == null) { throw; } } return(response); }
public async Task TestItemProducerTreeWithFailure() { int callBackCount = 0; Mock <CosmosQueryContext> mockQueryContext = new Mock <CosmosQueryContext>(); SqlQuerySpec sqlQuerySpec = new SqlQuerySpec("Select * from t"); PartitionKeyRange partitionKeyRange = new PartitionKeyRange { Id = "0", MinInclusive = "A", MaxExclusive = "B" }; Action <ItemProducerTree, int, double, QueryMetrics, long, CancellationToken> produceAsyncCompleteCallback = ( ItemProducerTree producer, int itemsBuffered, double resourceUnitUsage, QueryMetrics queryMetrics, long responseLengthBytes, CancellationToken token) => { callBackCount++; }; Mock <IComparer <ItemProducerTree> > comparer = new Mock <IComparer <ItemProducerTree> >(); Mock <IEqualityComparer <CosmosElement> > cosmosElementComparer = new Mock <IEqualityComparer <CosmosElement> >(); CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); IEnumerable <CosmosElement> cosmosElements = new List <CosmosElement>() { new Mock <CosmosElement>(CosmosElementType.Object).Object }; CosmosQueryResponseMessageHeaders headers = new CosmosQueryResponseMessageHeaders("TestToken", null) { ActivityId = "AA470D71-6DEF-4D61-9A08-272D8C9ABCFE", RequestCharge = 42 }; mockQueryContext.Setup(x => x.ExecuteQueryAsync(sqlQuerySpec, cancellationTokenSource.Token, It.IsAny <Action <CosmosRequestMessage> >())).Returns( Task.FromResult(QueryResponse.CreateSuccess(cosmosElements, 1, 500, headers))); ItemProducerTree itemProducerTree = new ItemProducerTree( queryContext: mockQueryContext.Object, querySpecForInit: sqlQuerySpec, partitionKeyRange: partitionKeyRange, produceAsyncCompleteCallback: produceAsyncCompleteCallback, itemProducerTreeComparer: comparer.Object, equalityComparer: cosmosElementComparer.Object, deferFirstPage: false, collectionRid: "collectionRid", initialContinuationToken: null, initialPageSize: 50); // Buffer to success responses await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token); await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token); // Buffer a failure mockQueryContext.Setup(x => x.ExecuteQueryAsync(sqlQuerySpec, cancellationTokenSource.Token, It.IsAny <Action <CosmosRequestMessage> >())).Returns( Task.FromResult(QueryResponse.CreateFailure(headers, HttpStatusCode.InternalServerError, null, "Error message", null))); await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token); // First item should be a success var result = await itemProducerTree.MoveNextAsync(cancellationTokenSource.Token); Assert.IsTrue(result.successfullyMovedNext); Assert.IsNull(result.failureResponse); Assert.IsTrue(itemProducerTree.HasMoreResults); // Second item should be a success result = await itemProducerTree.MoveNextAsync(cancellationTokenSource.Token); Assert.IsTrue(result.successfullyMovedNext); Assert.IsNull(result.failureResponse); Assert.IsTrue(itemProducerTree.HasMoreResults); // Third item should be a failure result = await itemProducerTree.MoveNextAsync(cancellationTokenSource.Token); Assert.IsFalse(result.successfullyMovedNext); Assert.IsNotNull(result.failureResponse); Assert.IsFalse(itemProducerTree.HasMoreResults); // Try to buffer after failure. It should return the previous cached failure and not try to buffer again. mockQueryContext.Setup(x => x.ExecuteQueryAsync(sqlQuerySpec, cancellationTokenSource.Token, It.IsAny <Action <CosmosRequestMessage> >())). Throws(new Exception("Previous buffer failed. Operation should return original failure and not try again")); await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token); Assert.IsFalse(result.successfullyMovedNext); Assert.IsNotNull(result.failureResponse); Assert.IsFalse(itemProducerTree.HasMoreResults); }
/// <summary> /// Drains a page of documents from this context. /// </summary> /// <param name="maxElements">The maximum number of elements.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns>A task that when awaited on return a page of documents.</returns> public override async Task <QueryResponse> DrainAsync(int maxElements, CancellationToken cancellationToken) { //// In order to maintain the continuation toke for the user we must drain with a few constraints //// 1) We always drain from the partition, which has the highest priority item first //// 2) If multiple partitions have the same priority item then we drain from the left most first //// otherwise we would need to keep track of how many of each item we drained from each partition //// (just like parallel queries). //// Visually that look the following case where we have three partitions that are numbered and store letters. //// For teaching purposes I have made each item a tuple of the following form: //// <item stored in partition, partition number> //// So that duplicates across partitions are distinct, but duplicates within partitions are indistinguishable. //// |-------| |-------| |-------| //// | <a,1> | | <a,2> | | <a,3> | //// | <a,1> | | <b,2> | | <c,3> | //// | <a,1> | | <b,2> | | <c,3> | //// | <d,1> | | <c,2> | | <c,3> | //// | <d,1> | | <e,2> | | <f,3> | //// | <e,1> | | <h,2> | | <j,3> | //// | <f,1> | | <i,2> | | <k,3> | //// |-------| |-------| |-------| //// Now the correct drain order in this case is: //// <a,1>,<a,1>,<a,1>,<a,2>,<a,3>,<b,2>,<b,2>,<c,2>,<c,3>,<c,3>,<c,3>, //// <d,1>,<d,1>,<e,1>,<e,2>,<f,1>,<f,3>,<h,2>,<i,2>,<j,3>,<k,3> //// In more mathematical terms //// 1) <x, y> always comes before <z, y> where x < z //// 2) <i, j> always come before <i, k> where j < k List <CosmosElement> results = new List <CosmosElement>(); while (!this.IsDone && results.Count < maxElements) { // Only drain from the highest priority document producer // We need to pop and push back the document producer tree, since the priority changes according to the sort order. DocumentProducerTree currentDocumentProducerTree = this.PopCurrentDocumentProducerTree(); OrderByQueryResult orderByQueryResult = new OrderByQueryResult(currentDocumentProducerTree.Current); // Only add the payload, since other stuff is garbage from the caller's perspective. results.Add(orderByQueryResult.Payload); // If we are at the begining of the page and seeing an rid from the previous page we should increment the skip count // due to the fact that JOINs can make a document appear multiple times and across continuations, so we don't want to // surface this more than needed. More information can be found in the continuation token docs. if (this.ShouldIncrementSkipCount(currentDocumentProducerTree.CurrentDocumentProducerTree.Root)) { ++this.skipCount; } else { this.skipCount = 0; } this.previousRid = orderByQueryResult.Rid; await currentDocumentProducerTree.MoveNextAsync(cancellationToken); this.PushCurrentDocumentProducerTree(currentDocumentProducerTree); } return(QueryResponse.CreateSuccess( result: results, count: results.Count, responseHeaders: this.GetResponseHeaders(), responseLengthBytes: this.GetAndResetResponseLengthBytes())); }