public static void VerifyQueryDiagnostics(CosmosDiagnostics diagnostics, bool isFirstPage) { string info = diagnostics.ToString(); Assert.IsNotNull(info); JObject jObject = JObject.Parse(info); JToken summary = jObject["Summary"]; Assert.IsNotNull(summary["UserAgent"].ToString()); Assert.IsNotNull(summary["StartUtc"].ToString()); JArray contextList = jObject["Context"].ToObject <JArray>(); Assert.IsTrue(contextList.Count > 0); // Find the PointOperationStatistics object JObject page = GetJObjectInContextList( contextList, "0", "PKRangeId"); // First page will have a request // Query might use cache pages which don't have the following info. It was returned in the previous call. if (isFirstPage || page != null) { string queryMetrics = page["QueryMetric"].ToString(); Assert.IsNotNull(queryMetrics); Assert.IsNotNull(page["IndexUtilization"].ToString()); Assert.IsNotNull(page["PKRangeId"].ToString()); JArray requestDiagnostics = page["Context"].ToObject <JArray>(); Assert.IsNotNull(requestDiagnostics); } }
public static void VerifyBulkPointDiagnostics(CosmosDiagnostics diagnostics) { string info = diagnostics.ToString(); Assert.IsNotNull(info); JObject jObject = JObject.Parse(info); JToken summary = jObject["Summary"]; Assert.IsNotNull(summary["UserAgent"].ToString()); Assert.IsNotNull(summary["StartUtc"].ToString()); Assert.IsNotNull(jObject["Context"].ToString()); JArray contextList = jObject["Context"].ToObject <JArray>(); Assert.IsTrue(contextList.Count > 2); // Find the PointOperationStatistics object JObject pointStatistics = GetJObjectInContextList( contextList, "PointOperationStatistics"); if (pointStatistics != null) { ValidatePointOperation(pointStatistics); } else { JObject storeResponseStatistics = GetJObjectInContextList( contextList, "StoreResponseStatistics"); ValidateStoreResponseStatistics(storeResponseStatistics); } }
public static void VerifyPointDiagnostics(CosmosDiagnostics diagnostics) { string info = diagnostics.ToString(); Assert.IsNotNull(info); JObject jObject = JObject.Parse(info); Assert.IsNotNull(jObject["ActivityId"].ToString()); Assert.IsNotNull(jObject["StatusCode"].ToString()); Assert.IsNotNull(jObject["RequestCharge"].ToString()); Assert.IsNotNull(jObject["RequestUri"].ToString()); Assert.IsNotNull(jObject["requestStartTime"].ToString()); Assert.IsNotNull(jObject["requestEndTime"].ToString()); Assert.IsNotNull(jObject["responseStatisticsList"].ToString()); Assert.IsNotNull(jObject["supplementalResponseStatisticsList"].ToString()); Assert.IsNotNull(jObject["addressResolutionStatistics"].ToString()); Assert.IsNotNull(jObject["contactedReplicas"].ToString()); Assert.IsNotNull(jObject["failedReplicas"].ToString()); Assert.IsNotNull(jObject["regionsContacted"].ToString()); Assert.IsNotNull(jObject["requestLatency"].ToString()); int statusCode = jObject["StatusCode"].ToObject <int>(); // Session token only expected on success if (statusCode >= 200 && statusCode < 300) { Assert.IsNotNull(jObject["ResponseSessionToken"].ToString()); } }
public static void Log(CosmosDiagnostics cosmosDiagnostics) { TimeSpan elapsedTime = cosmosDiagnostics.GetClientElapsedTime(); // Require the diagnostics to be at least 10 seconds apart to avoid getting the // diagnostics from the exact same time frame to avoid the same issue multiple times if (stopwatch.Elapsed > CosmosDiagnosticsLogger.minimumDelayBetweenDiagnostics && elapsedTime > CosmosDiagnosticsLogger.maxTimeSpan) { // This can be called concurrently by multiple tasks. Take a lock and // validate the check again and update the times. lock (CosmosDiagnosticsLogger.UpdateLock) { if (stopwatch.Elapsed <= CosmosDiagnosticsLogger.minimumDelayBetweenDiagnostics && elapsedTime <= CosmosDiagnosticsLogger.maxTimeSpan) { return; } stopwatch.Restart(); maxTimeSpan = elapsedTime; } CosmosDiagnosticsLogger.CosmosDiagnosticsToLog.Enqueue(cosmosDiagnostics); if (CosmosDiagnosticsToLog.Count > MaxSize) { CosmosDiagnosticsToLog.TryDequeue(out _); } } }
public static void VerifyPointDiagnostics(CosmosDiagnostics diagnostics, bool disableDiagnostics) { string info = diagnostics.ToString(); if (disableDiagnostics) { Assert.AreEqual(string.Empty, info); return; } Assert.IsNotNull(info); JObject jObject = JObject.Parse(info); JToken summary = jObject["Summary"]; Assert.IsNotNull(summary["UserAgent"].ToString()); Assert.IsNotNull(summary["StartUtc"].ToString()); Assert.IsNotNull(summary["ElapsedTime"].ToString()); Assert.IsNotNull(jObject["Context"].ToString()); JArray contextList = jObject["Context"].ToObject <JArray>(); Assert.IsTrue(contextList.Count > 3); // Find the PointOperationStatistics object JObject pointStatistics = GetJObjectInContextList( contextList, "PointOperationStatistics"); ValidatePointOperation(pointStatistics); }
public void VerifyNegativeCosmosQueryResponseStream() { string contianerRid = "mockContainerRid"; string errorMessage = "TestErrorMessage"; string activityId = "TestActivityId"; double requestCharge = 42.42; Mock <CosmosDiagnostics> mockDiagnostics = new Mock <CosmosDiagnostics>(); CosmosDiagnostics diagnostics = mockDiagnostics.Object; QueryResponse queryResponse = QueryResponse.CreateFailure( statusCode: HttpStatusCode.NotFound, errorMessage: errorMessage, requestMessage: null, error: null, responseHeaders: new CosmosQueryResponseMessageHeaders( null, null, ResourceType.Document, contianerRid) { RequestCharge = requestCharge, ActivityId = activityId }, diagnostics: diagnostics); Assert.AreEqual(HttpStatusCode.NotFound, queryResponse.StatusCode); Assert.AreEqual(errorMessage, queryResponse.ErrorMessage); Assert.AreEqual(requestCharge, queryResponse.Headers.RequestCharge); Assert.AreEqual(activityId, queryResponse.Headers.ActivityId); Assert.AreEqual(diagnostics, queryResponse.Diagnostics); Assert.IsNull(queryResponse.Content); }
internal void CollectRegionContacted(CosmosDiagnostics cosmosDiagnostics) { if (cosmosDiagnostics == null) { throw new ArgumentNullException(nameof(cosmosDiagnostics)); } ClientTelemetryHelper.GetContactedRegions(cosmosDiagnostics); }
public ChangeFeedEstimatorFeedResponse( CosmosDiagnostics cosmosDiagnostics, ReadOnlyCollection <ChangeFeedProcessorState> remainingLeaseWorks, double ruCost) { this.cosmosDiagnostics = cosmosDiagnostics ?? throw new ArgumentNullException(nameof(cosmosDiagnostics)); this.remainingLeaseWorks = remainingLeaseWorks ?? throw new ArgumentNullException(nameof(remainingLeaseWorks)); this.headers = new Headers(); this.headers.RequestCharge = ruCost; }
public EncryptionCosmosException( string message, HttpStatusCode statusCode, int subStatusCode, string activityId, double requestCharge, CosmosDiagnostics encryptionCosmosDiagnostics) : base(message, statusCode, subStatusCode, activityId, requestCharge) { this.encryptionCosmosDiagnostics = encryptionCosmosDiagnostics ?? throw new ArgumentNullException(nameof(encryptionCosmosDiagnostics)); }
/// <summary> /// A private constructor to ensure the factory is used to create the object. /// This will prevent memory leaks when handling the HttpResponseMessage /// </summary> internal StoredProcedureExecuteResponse( HttpStatusCode httpStatusCode, Headers headers, T response, CosmosDiagnostics diagnostics) { this.StatusCode = httpStatusCode; this.Headers = headers; this.Resource = response; this.Diagnostics = diagnostics; }
/// <summary> /// A private constructor to ensure the factory is used to create the object. /// This will prevent memory leaks when handling the HttpResponseMessage /// </summary> internal UserDefinedFunctionResponse( HttpStatusCode httpStatusCode, Headers headers, UserDefinedFunctionProperties userDefinedFunctionProperties, CosmosDiagnostics diagnostics) { this.StatusCode = httpStatusCode; this.Headers = headers; this.Resource = userDefinedFunctionProperties; this.Diagnostics = diagnostics; }
/// <summary> /// A private constructor to ensure the factory is used to create the object. /// This will prevent memory leaks when handling the HttpResponseMessage /// </summary> internal TriggerResponse( HttpStatusCode httpStatusCode, Headers headers, TriggerProperties triggerProperties, CosmosDiagnostics diagnostics) { this.StatusCode = httpStatusCode; this.Headers = headers; this.Resource = triggerProperties; this.Diagnostics = diagnostics; }
/// <summary> /// A private constructor to ensure the factory is used to create the object. /// This will prevent memory leaks when handling the HttpResponseMessage /// </summary> internal StoredProcedureResponse( HttpStatusCode httpStatusCode, Headers headers, StoredProcedureProperties storedProcedureProperties, CosmosDiagnostics diagnostics) { this.StatusCode = httpStatusCode; this.Headers = headers; this.Resource = storedProcedureProperties; this.Diagnostics = diagnostics; }
public EncryptionTransactionalBatchResponse( IReadOnlyList <TransactionalBatchOperationResult> results, TransactionalBatchResponse response, CosmosSerializer cosmosSerializer, CosmosDiagnostics diagnostics) { this.results = results; this.response = response; this.cosmosSerializer = cosmosSerializer; this.diagnostics = diagnostics; }
public static void VerifyPointDiagnostics(CosmosDiagnostics diagnostics, bool disableDiagnostics) { string info = diagnostics.ToString(); if (disableDiagnostics) { Assert.AreEqual(string.Empty, info); return; } Assert.IsNotNull(info); JObject jObject = JObject.Parse(info); JToken summary = jObject["Summary"]; Assert.IsNotNull(summary["UserAgent"].ToString()); Assert.IsNotNull(summary["StartUtc"].ToString()); Assert.IsNotNull(summary["ElapsedTime"].ToString()); Assert.IsNotNull(summary["TotalRequestCount"].ToString()); Assert.IsNotNull(summary["FailedRequestCount"].ToString()); Assert.IsNotNull(jObject["Context"].ToString()); JArray contextList = jObject["Context"].ToObject <JArray>(); Assert.IsTrue(contextList.Count > 3); JObject addressResolutionStatistics = GetJObjectInContextList( contextList, "AddressResolutionStatistics"); // Address resolution doesn't happen on every request. if (addressResolutionStatistics != null) { ValidateAddressResolutionStatistics(addressResolutionStatistics); } // Find the PointOperationStatistics object JObject pointStatistics = GetJObjectInContextList( contextList, "PointOperationStatistics"); if (pointStatistics != null) { ValidatePointOperation(pointStatistics); } else { JObject storeResponseStatistics = GetJObjectInContextList( contextList, "StoreResponseStatistics"); ValidateStoreResponseStatistics(storeResponseStatistics); } }
private async Task <OperationResult> ExecuteOnceAsyncWithPagination() { string continuationToken = null; double totalCharge = 0; CosmosDiagnostics lastDiagnostics = null; do { FeedIterator <Dictionary <string, object> > feedIterator = this.container.GetItemQueryIterator <Dictionary <string, object> >( queryDefinition: this.QueryDefinition, continuationToken: continuationToken, requestOptions: this.QueryRequestOptions); FeedResponse <Dictionary <string, object> > feedResponse = await feedIterator.ReadNextAsync(); if (feedResponse == null || feedResponse.StatusCode != HttpStatusCode.OK) { throw new Exception($"QueryTV3BenchmarkOperation failed with {feedResponse?.StatusCode} " + $"where pagination : {this.IsPaginationEnabled} and cross partition : {this.IsCrossPartitioned}"); } foreach (Dictionary <string, object> item in feedResponse) { // No-op check that forces any lazy logic to be executed if (item == null) { throw new Exception("Null item was returned"); } } totalCharge += feedResponse.Headers.RequestCharge; lastDiagnostics = feedResponse.Diagnostics; continuationToken = feedResponse.ContinuationToken; if (!feedIterator.HasMoreResults) { break; } } while (true); return(new OperationResult() { DatabseName = databaseName, ContainerName = containerName, RuCharges = totalCharge, CosmosDiagnostics = lastDiagnostics, LazyDiagnostics = () => lastDiagnostics?.ToString(), }); }
public QueryPageDiagnostics( string partitionKeyRangeId, string queryMetricText, string indexUtilizationText, CosmosDiagnostics requestDiagnostics, SchedulingStopwatch schedulingStopwatch) { this.PartitionKeyRangeId = partitionKeyRangeId ?? throw new ArgumentNullException(nameof(partitionKeyRangeId)); this.QueryMetricText = queryMetricText ?? string.Empty; this.IndexUtilizationText = indexUtilizationText ?? string.Empty; this.RequestDiagnostics = requestDiagnostics; this.SchedulingTimeSpan = schedulingStopwatch.Elapsed; }
public void IncludeDiagnosticToStringHelper( CosmosDiagnostics cosmosDiagnostics) { if (!this.IncludeDiagnosticsToString) { return; } string diagnostics = cosmosDiagnostics.ToString(); if (string.IsNullOrEmpty(diagnostics)) { throw new Exception(); } }
public EncryptionCosmosDiagnostics( CosmosDiagnostics coreDiagnostics, JObject encryptContent, JObject decryptContent) { this.coreDiagnostics = coreDiagnostics ?? throw new ArgumentNullException(nameof(coreDiagnostics)); if (encryptContent?.Count > 0) { this.encryptContent = encryptContent; } if (decryptContent?.Count > 0) { this.decryptContent = decryptContent; } }
public static void VerifyChangeFeedDiagnostics( CosmosDiagnostics diagnostics, bool disableDiagnostics) { string info = diagnostics.ToString(); if (disableDiagnostics) { Assert.AreEqual(string.Empty, info); return; } CosmosDiagnosticsContext diagnosticsContext = (diagnostics as CosmosDiagnosticsCore).Context; DiagnosticValidator.ValidateChangeFeedOperationDiagnostics(diagnosticsContext); }
public static void VerifyQueryDiagnostics(CosmosDiagnostics diagnostics) { string info = diagnostics.ToString(); Assert.IsNotNull(info); JArray jArray = JArray.Parse(info); foreach (JToken jObject in jArray) { string queryMetrics = jObject["QueryMetricText"].ToString(); Assert.IsNotNull(queryMetrics); Assert.IsNotNull(jObject["IndexUtilizationText"].ToString()); Assert.IsNotNull(jObject["PartitionKeyRangeId"].ToString()); JObject requestDiagnostics = jObject["RequestDiagnostics"].Value<JObject>(); Assert.IsNotNull(requestDiagnostics); Assert.IsNotNull(requestDiagnostics["ActivityId"].ToString()); } }
public async Task <OperationResult> ExecuteOnceAsync() { FeedIterator <Dictionary <string, object> > feedIterator = this.container.GetItemQueryIterator <Dictionary <string, object> >( queryDefinition: new QueryDefinition("select * from T where T.id = @id").WithParameter("@id", this.executionItemId), continuationToken: null, requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey(this.executionItemPartitionKey) }); double totalCharge = 0; CosmosDiagnostics lastDiagnostics = null; while (feedIterator.HasMoreResults) { FeedResponse <Dictionary <string, object> > feedResponse = await feedIterator.ReadNextAsync(); totalCharge += feedResponse.Headers.RequestCharge; lastDiagnostics = feedResponse.Diagnostics; if (feedResponse.StatusCode != HttpStatusCode.OK) { throw new Exception($"QuerySinglePkStreamV3BenchmarkOperation failed with {feedResponse.StatusCode}"); } foreach (Dictionary <string, object> item in feedResponse) { // No-op check that forces any lazy logic to be executed if (item == null) { throw new Exception("Null item was returned"); } } } return(new OperationResult() { DatabseName = databsaeName, ContainerName = containerName, RuCharges = totalCharge, CosmosDiagnostics = lastDiagnostics, LazyDiagnostics = () => lastDiagnostics?.ToString(), }); }
private async Task <OperationResult> ExecuteOnceAsyncWithStreamsAndPagination() { string continuationToken = null; double totalCharge = 0; CosmosDiagnostics lastDiagnostics = null; do { FeedIterator feedIterator = this.container.GetItemQueryStreamIterator( queryDefinition: this.QueryDefinition, continuationToken: continuationToken, requestOptions: this.QueryRequestOptions); ResponseMessage feedResponse = await feedIterator.ReadNextAsync(); if (feedResponse == null || feedResponse.StatusCode != HttpStatusCode.OK) { throw new Exception($"QueryTV3BenchmarkOperation failed with {feedResponse?.StatusCode} " + $"where pagination : {this.IsPaginationEnabled} and cross partition : {this.IsCrossPartitioned}"); } totalCharge += feedResponse.Headers.RequestCharge; lastDiagnostics = feedResponse.Diagnostics; continuationToken = feedResponse.ContinuationToken; if (!feedIterator.HasMoreResults) { break; } } while (true); return(new OperationResult() { DatabseName = databaseName, ContainerName = containerName, RuCharges = totalCharge, CosmosDiagnostics = lastDiagnostics, LazyDiagnostics = () => lastDiagnostics?.ToString(), }); }
public EncryptionCosmosDiagnostics( CosmosDiagnostics coreDiagnostics, JObject encryptContent, JObject decryptContent, TimeSpan processingDuration) { this.coreDiagnostics = coreDiagnostics ?? throw new ArgumentNullException(nameof(coreDiagnostics)); if (encryptContent?.Count > 0) { this.encryptContent = encryptContent; } if (decryptContent?.Count > 0) { this.decryptContent = decryptContent; } this.processingDuration = processingDuration; }
public static void LogDiagnostics( ILogger logger, string operationName, TimeSpan timerContextLatency, CTLConfig config, CosmosDiagnostics cosmosDiagnostics) { if (timerContextLatency > config.DiagnosticsThresholdDurationAsTimespan) { logger.LogInformation($"{operationName}; LatencyInMs:{timerContextLatency.TotalMilliseconds}; request took more than latency threshold {config.DiagnosticsThresholdDuration}, diagnostics: {cosmosDiagnostics}"); } CosmosTraceDiagnostics traceDiagnostics = (CosmosTraceDiagnostics)cosmosDiagnostics; if (traceDiagnostics.IsGoneExceptionHit()) { logger.LogInformation($"{operationName}; LatencyInMs:{timerContextLatency.TotalMilliseconds}; request contains 410(GoneExceptions), diagnostics:{cosmosDiagnostics}"); return; } }
private async Task <OperationResult> ExecuteOnceAsyncWithStreams() { if (this.IsPaginationEnabled) { return(await this.ExecuteOnceAsyncWithStreamsAndPagination()); } FeedIterator feedIterator = this.container.GetItemQueryStreamIterator( queryDefinition: this.QueryDefinition, continuationToken: null, requestOptions: this.QueryRequestOptions); double totalCharge = 0; CosmosDiagnostics lastDiagnostics = null; while (feedIterator.HasMoreResults) { using (ResponseMessage feedResponse = await feedIterator.ReadNextAsync()) { totalCharge += feedResponse.Headers.RequestCharge; lastDiagnostics = feedResponse.Diagnostics; if (feedResponse.StatusCode != HttpStatusCode.OK) { throw new Exception($"QueryTV3BenchmarkOperation failed with {feedResponse.StatusCode}"); } // Access the stream to catch any lazy logic using Stream stream = feedResponse.Content; } } return(new OperationResult() { DatabseName = databaseName, ContainerName = containerName, RuCharges = totalCharge, CosmosDiagnostics = lastDiagnostics, LazyDiagnostics = () => lastDiagnostics?.ToString(), }); }
public async Task <OperationResult> ExecuteOnceAsync() { FeedIterator feedIterator = this.container.GetItemQueryStreamIterator( queryDefinition: new QueryDefinition("select * from T where T.id = @id").WithParameter("@id", this.executionItemId), continuationToken: null, requestOptions: new QueryRequestOptions() { PartitionKey = new PartitionKey(this.executionItemPartitionKey) }); double totalCharge = 0; CosmosDiagnostics lastDiagnostics = null; while (feedIterator.HasMoreResults) { using (ResponseMessage feedResponse = await feedIterator.ReadNextAsync()) { totalCharge += feedResponse.Headers.RequestCharge; lastDiagnostics = feedResponse.Diagnostics; if (feedResponse.StatusCode != HttpStatusCode.OK) { throw new Exception($"QuerySinglePkStreamV3BenchmarkOperation failed with {feedResponse.StatusCode}"); } // Access the stream to catch any lazy logic using Stream stream = feedResponse.Content; } } return(new OperationResult() { DatabseName = databsaeName, ContainerName = containerName, RuCharges = totalCharge, CosmosDiagnostics = lastDiagnostics, LazyDiagnostics = () => lastDiagnostics?.ToString(), }); }
public static void VerifyQueryDiagnostics( CosmosDiagnostics diagnostics, bool isFirstPage, bool disableDiagnostics) { string info = diagnostics.ToString(); if (disableDiagnostics) { Assert.AreEqual(string.Empty, info); return; } CosmosDiagnosticsContext diagnosticsContext = (diagnostics as CosmosDiagnosticsCore).Context; // If all the pages are buffered then several of the normal summary validation will fail. if (diagnosticsContext.GetTotalResponseCount() > 0) { DiagnosticValidator.ValidateCosmosDiagnosticsContext(diagnosticsContext); } DiagnosticValidator.ValidateQueryDiagnostics(diagnosticsContext, isFirstPage); }
public ChangeFeedEstimatorEmptyFeedResponse(CosmosDiagnostics cosmosDiagnostics) { this.cosmosDiagnostics = cosmosDiagnostics ?? throw new ArgumentNullException(nameof(cosmosDiagnostics)); this.headers = new Headers(); }
public CosmosDiagnosticsSerializerBaselineInput(string description, CosmosDiagnostics cosmosDiagnostics) : base(description) { this.CosmosDiagnostics = cosmosDiagnostics ?? throw new ArgumentNullException(nameof(cosmosDiagnostics)); }