private async Task <PartitionKeyRangeBatchExecutionResult> ExecuteAsync(
PartitionKeyRangeServerBatchRequest serverRequest,
CancellationToken cancellationToken)
{
CosmosDiagnosticsContext diagnosticsContext = new CosmosDiagnosticsContext();
CosmosDiagnosticScope limiterScope = diagnosticsContext.CreateScope("BatchAsyncContainerExecutor.Limiter");
SemaphoreSlim limiter = this.GetOrAddLimiterForPartitionKeyRange(serverRequest.PartitionKeyRangeId);
using (await limiter.UsingWaitAsync(cancellationToken))
{
limiterScope.Dispose();
using (Stream serverRequestPayload = serverRequest.TransferBodyStream())
{
Debug.Assert(serverRequestPayload != null, "Server request payload expected to be non-null");
ResponseMessage responseMessage = await this.cosmosClientContext.ProcessResourceOperationStreamAsync(
this.cosmosContainer.LinkUri,
ResourceType.Document,
OperationType.Batch,
new RequestOptions(),
cosmosContainerCore : this.cosmosContainer,
partitionKey : null,
streamPayload : serverRequestPayload,
requestEnricher : requestMessage => BatchAsyncContainerExecutor.AddHeadersToRequestMessage(requestMessage, serverRequest.PartitionKeyRangeId),
diagnosticsScope : diagnosticsContext,
cancellationToken : cancellationToken).ConfigureAwait(false);
using (diagnosticsContext.CreateScope("BatchAsyncContainerExecutor.ToResponse"))
{
TransactionalBatchResponse serverResponse = await TransactionalBatchResponse.FromResponseMessageAsync(responseMessage, serverRequest, this.cosmosClientContext.SerializerCore).ConfigureAwait(false);
return(new PartitionKeyRangeBatchExecutionResult(serverRequest.PartitionKeyRangeId, serverRequest.Operations, serverResponse));
}
}
}
}
/// <summary>
/// Get the next set of results from the cosmos service
/// </summary>
/// <param name="cancellationToken">(Optional) <see cref="CancellationToken"/> representing request cancellation.</param>
/// <returns>A query response from cosmos service</returns>
public override async Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default)
{
CosmosDiagnosticsContext diagnostics = CosmosDiagnosticsContext.Create(this.changeFeedOptions);
using (diagnostics.GetOverallScope())
{
diagnostics.AddDiagnosticsInternal(new FeedRangeStatistics(this.changeFeedOptions.FeedRange));
if (!this.lazyContainerRid.ValueInitialized)
{
using (diagnostics.CreateScope("InitializeContainerResourceId"))
{
TryCatch <string> tryInitializeContainerRId = await this.lazyContainerRid.GetValueAsync(cancellationToken);
if (!tryInitializeContainerRId.Succeeded)
{
if (!(tryInitializeContainerRId.Exception.InnerException is CosmosException cosmosException))
{
throw new InvalidOperationException("Failed to convert to CosmosException.");
}
return(cosmosException.ToCosmosResponseMessage(
new RequestMessage(
method: null,
requestUri: null,
diagnosticsContext: diagnostics)));
}
}
if (this.FeedRangeContinuation == null)
{
using (diagnostics.CreateScope("InitializeContinuation"))
{
await this.InitializeFeedContinuationAsync(cancellationToken);
}
}
TryCatch validateContainer = this.FeedRangeContinuation.ValidateContainer(this.lazyContainerRid.Result.Result);
if (!validateContainer.Succeeded)
{
return(CosmosExceptionFactory
.CreateBadRequestException(
message: validateContainer.Exception.InnerException.Message,
innerException: validateContainer.Exception.InnerException,
diagnosticsContext: diagnostics)
.ToCosmosResponseMessage(
new RequestMessage(
method: null,
requestUri: null,
diagnosticsContext: diagnostics)));
}
}
return(await this.ReadNextInternalAsync(diagnostics, cancellationToken));
}
}
internal async Task <InMemoryRawDek> UnwrapAsync(
DataEncryptionKeyProperties dekProperties,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
EncryptionKeyWrapProvider encryptionKeyWrapProvider = this.ClientContext.ClientOptions.EncryptionKeyWrapProvider;
if (encryptionKeyWrapProvider == null)
{
throw new ArgumentException(ClientResources.EncryptionKeyWrapProviderNotConfigured);
}
EncryptionKeyUnwrapResult unwrapResult = null;
using (diagnosticsContext.CreateScope("UnwrapDataEncryptionKey"))
{
unwrapResult = await encryptionKeyWrapProvider.UnwrapKeyAsync(
dekProperties.WrappedDataEncryptionKey,
dekProperties.EncryptionKeyWrapMetadata,
cancellationToken);
}
EncryptionAlgorithm encryptionAlgorithm = this.GetEncryptionAlgorithm(unwrapResult.DataEncryptionKey, dekProperties.EncryptionAlgorithmId);
return(new InMemoryRawDek(unwrapResult.DataEncryptionKey, encryptionAlgorithm, unwrapResult.ClientCacheTimeToLive));
}
internal override async Task <ContainerProperties> GetCachedContainerPropertiesAsync(
string containerUri,
ITrace trace,
CancellationToken cancellationToken)
{
using (ITrace childTrace = trace.StartChild("Get Container Properties", TraceComponent.Transport, Tracing.TraceLevel.Info))
{
this.ThrowIfDisposed();
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContextCore.Create(requestOptions: null);
using (diagnosticsContext.GetOverallScope())
{
ClientCollectionCache collectionCache = await this.DocumentClient.GetCollectionCacheAsync();
try
{
using (diagnosticsContext.CreateScope("ContainerCache.ResolveByNameAsync"))
{
return(await collectionCache.ResolveByNameAsync(
HttpConstants.Versions.CurrentVersion,
containerUri,
forceRefesh : false,
cancellationToken));
}
}
catch (DocumentClientException ex)
{
throw CosmosExceptionFactory.Create(ex, diagnosticsContext);
}
}
}
}
internal override async Task <ContainerProperties> GetCachedContainerPropertiesAsync(
string containerUri,
CancellationToken cancellationToken)
{
this.ThrowIfDisposed();
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContextCore.Create(requestOptions: null);
using (diagnosticsContext.GetOverallScope())
{
ClientCollectionCache collectionCache = await this.DocumentClient.GetCollectionCacheAsync();
try
{
using (diagnosticsContext.CreateScope("ContainerCache.ResolveByNameAsync"))
{
return(await collectionCache.ResolveByNameAsync(
HttpConstants.Versions.CurrentVersion,
containerUri,
cancellationToken));
}
}
catch (DocumentClientException ex)
{
throw CosmosExceptionFactory.Create(ex, diagnosticsContext);
}
}
}
internal override async Task <Stream> EncryptItemAsync(
Stream input,
EncryptionOptions encryptionOptions,
DatabaseCore database,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
if (input == null)
{
throw new ArgumentException(ClientResources.InvalidRequestWithEncryptionOptions);
}
Debug.Assert(encryptionOptions != null);
Debug.Assert(database != null);
Debug.Assert(diagnosticsContext != null);
using (diagnosticsContext.CreateScope("Encrypt"))
{
return(await this.EncryptionProcessor.EncryptAsync(
input,
encryptionOptions,
database,
this.ClientOptions.EncryptionKeyWrapProvider,
diagnosticsContext,
cancellationToken));
}
}
internal async Task <(byte[], EncryptionKeyWrapMetadata, InMemoryRawDek)> WrapAsync(
byte[] key,
CosmosEncryptionAlgorithm encryptionAlgorithmId,
EncryptionKeyWrapMetadata metadata,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
EncryptionKeyWrapProvider encryptionKeyWrapProvider = this.ClientContext.ClientOptions.EncryptionKeyWrapProvider;
if (encryptionKeyWrapProvider == null)
{
throw new ArgumentException(ClientResources.EncryptionKeyWrapProviderNotConfigured);
}
EncryptionKeyWrapResult keyWrapResponse;
using (diagnosticsContext.CreateScope("WrapDataEncryptionKey"))
{
keyWrapResponse = await encryptionKeyWrapProvider.WrapKeyAsync(key, metadata, cancellationToken);
}
// Verify
DataEncryptionKeyProperties tempDekProperties = new DataEncryptionKeyProperties(this.Id, encryptionAlgorithmId, keyWrapResponse.WrappedDataEncryptionKey, keyWrapResponse.EncryptionKeyWrapMetadata);
InMemoryRawDek roundTripResponse = await this.UnwrapAsync(tempDekProperties, diagnosticsContext, cancellationToken);
if (!roundTripResponse.RawDek.SequenceEqual(key))
{
throw CosmosExceptionFactory.CreateBadRequestException(ClientResources.KeyWrappingDidNotRoundtrip,
diagnosticsContext: diagnosticsContext);
}
return(keyWrapResponse.WrappedDataEncryptionKey, keyWrapResponse.EncryptionKeyWrapMetadata, roundTripResponse);
}
/// <summary>
/// Get the next set of results from the cosmos service
/// </summary>
/// <param name="cancellationToken">(Optional) <see cref="CancellationToken"/> representing request cancellation.</param>
/// <returns>A query response from cosmos service</returns>
public override Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default(CancellationToken))
{
CosmosDiagnosticsContext diagnostics = CosmosDiagnosticsContext.Create(this.changeFeedOptions);
using (diagnostics.CreateScope("ChangeFeedReadNextAsync"))
{
return(this.ReadNextInternalAsync(diagnostics, cancellationToken));
}
}
private async ValueTask RefreshCachedTokenWithRetryHelperAsync(
CosmosDiagnosticsContext diagnosticsContext)
{
// A different thread is already updating the access token
bool skipRefreshBecause = this.backgroundRefreshLock.CurrentCount == 1;
await this.backgroundRefreshLock.WaitAsync();
try
{
// Token was already refreshed successfully from another thread.
if (skipRefreshBecause && this.cachedAccessToken.ExpiresOn > DateTime.UtcNow)
{
return;
}
Exception lastException = null;
const int totalRetryCount = 3;
for (int retry = 0; retry < totalRetryCount; retry++)
{
if (this.cancellationToken.IsCancellationRequested)
{
DefaultTrace.TraceInformation(
"Stop RefreshTokenWithIndefiniteRetries because cancellation is requested");
break;
}
try
{
using (diagnosticsContext.CreateScope(nameof(this.RefreshCachedTokenWithRetryHelperAsync)))
{
this.cachedAccessToken = await this.tokenCredential.GetTokenAsync(
this.tokenRequestContext,
this.cancellationToken);
if (!this.userDefinedBackgroundTokenCredentialRefreshInterval.HasValue)
{
double totalSecondUntilExpire = (this.cachedAccessToken.ExpiresOn - DateTimeOffset.UtcNow).TotalSeconds * DefaultBackgroundTokenCredentialRefreshIntervalPercentage;
this.systemBackgroundTokenCredentialRefreshInterval = TimeSpan.FromSeconds(totalSecondUntilExpire);
}
return;
}
}
catch (RequestFailedException requestFailedException)
{
lastException = requestFailedException;
diagnosticsContext.AddDiagnosticsInternal(
new PointOperationStatistics(
activityId: Trace.CorrelationManager.ActivityId.ToString(),
statusCode: (HttpStatusCode)requestFailedException.Status,
subStatusCode: SubStatusCodes.Unknown,
responseTimeUtc: DateTime.UtcNow,
requestCharge: default,
/// <summary>
/// Get the next set of results from the cosmos service
/// </summary>
/// <param name="cancellationToken">(Optional) <see cref="CancellationToken"/> representing request cancellation.</param>
/// <returns>A query response from cosmos service</returns>
public override async Task <ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default)
{
CosmosDiagnosticsContext diagnostics = CosmosDiagnosticsContext.Create(this.queryRequestOptions);
using (diagnostics.GetOverallScope())
{
if (!this.lazyContainerRid.ValueInitialized)
{
using (diagnostics.CreateScope("InitializeContainerResourceId"))
{
TryCatch <string> tryInitializeContainerRId = await this.lazyContainerRid.GetValueAsync(cancellationToken);
if (!tryInitializeContainerRId.Succeeded)
{
CosmosException cosmosException = tryInitializeContainerRId.Exception.InnerException as CosmosException;
return(cosmosException.ToCosmosResponseMessage(new RequestMessage(method: null, requestUriString: null, diagnosticsContext: diagnostics)));
}
}
using (diagnostics.CreateScope("InitializeContinuation"))
{
if (this.FeedRangeContinuation != null)
{
TryCatch validateContainer = this.FeedRangeContinuation.ValidateContainer(this.lazyContainerRid.Result.Result);
if (!validateContainer.Succeeded)
{
return(CosmosExceptionFactory.CreateBadRequestException(
message: validateContainer.Exception.InnerException.Message,
innerException: validateContainer.Exception.InnerException,
diagnosticsContext: diagnostics).ToCosmosResponseMessage(new RequestMessage(method: null, requestUriString: null, diagnosticsContext: diagnostics)));
}
}
else
{
await this.InitializeFeedContinuationAsync(cancellationToken);
}
}
}
return(await this.ReadNextInternalAsync(diagnostics, cancellationToken));
}
}
private async Task <DataEncryptionKeyProperties> ReadResourceAsync(
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
using (diagnosticsContext.CreateScope("ReadDataEncryptionKey"))
{
return(await this.ReadInternalAsync(
requestOptions : null,
diagnosticsContext : diagnosticsContext,
cancellationToken : cancellationToken));
}
}
public static async Task <IDisposable> UsingWaitAsync(
this SemaphoreSlim semaphoreSlim,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
using (diagnosticsContext?.CreateScope(nameof(UsingWaitAsync)))
{
await semaphoreSlim.WaitAsync(cancellationToken).ConfigureAwait(false);
return(new UsableSemaphoreWrapper(semaphoreSlim));
}
}
public Task <ResponseMessage> PatchItemStreamAsync(
CosmosDiagnosticsContext diagnosticsContext,
string id,
PartitionKey partitionKey,
IReadOnlyList <PatchOperation> patchOperations,
ItemRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
if (diagnosticsContext == null)
{
throw new ArgumentNullException(nameof(diagnosticsContext));
}
if (string.IsNullOrWhiteSpace(id))
{
throw new ArgumentNullException(nameof(id));
}
if (partitionKey == null)
{
throw new ArgumentNullException(nameof(partitionKey));
}
if (patchOperations == null ||
!patchOperations.Any())
{
throw new ArgumentNullException(nameof(patchOperations));
}
Stream patchOperationsStream;
using (diagnosticsContext.CreateScope("PatchOperationsSerialize"))
{
patchOperationsStream = this.ClientContext.SerializerCore.ToStream(patchOperations);
}
return(this.ClientContext.ProcessResourceOperationStreamAsync(
resourceUri: this.GetResourceUri(
requestOptions,
OperationType.Patch,
id),
resourceType: ResourceType.Document,
operationType: OperationType.Patch,
requestOptions: requestOptions,
cosmosContainerCore: this,
partitionKey: partitionKey,
itemId: id,
streamPayload: patchOperationsStream,
requestEnricher: null,
diagnosticsContext: diagnosticsContext,
cancellationToken: cancellationToken));
}
private Task <TResult> RunWithSynchronizationContextAndDiagnosticsHelperAsync <TResult>(
CosmosDiagnosticsContext diagnosticsContext,
Func <CosmosDiagnosticsContext, Task <TResult> > task)
{
Debug.Assert(SynchronizationContext.Current != null, "This should only be used when a SynchronizationContext is specified");
// Used on NETFX applications with SynchronizationContext when doing locking calls
IDisposable synchronizationContextScope = diagnosticsContext.CreateScope("SynchronizationContext");
return(Task.Run(() =>
{
synchronizationContextScope.Dispose();
return this.RunWithDiagnosticsHelperAsync <TResult>(
diagnosticsContext,
task);
}));
}
private Task <TResult> RunWithSynchronizationContextAndDiagnosticsHelperAsync <TResult>(
string operationName,
RequestOptions requestOptions,
Func <CosmosDiagnosticsContext, Task <TResult> > task)
{
Debug.Assert(SynchronizationContext.Current != null, "This should only be used when a SynchronizationContext is specified");
CosmosDiagnosticsContext diagnosticsContext = this.CreateDiagnosticContext(
operationName,
requestOptions);
// Used on NETFX applications with SynchronizationContext when doing locking calls
return(Task.Run(async() =>
{
using (diagnosticsContext.GetOverallScope())
using (diagnosticsContext.CreateScope("SynchronizationContext"))
{
return await task(diagnosticsContext);
}
}));
}
private Task <TResult> RunWithSynchronizationContextAndDiagnosticsHelperAsync <TResult>(
CosmosDiagnosticsContext diagnosticsContext,
Func <CosmosDiagnosticsContext, Task <TResult> > task)
{
Debug.Assert(SynchronizationContext.Current != null, "This should only be used when a SynchronizationContext is specified");
// Used on NETFX applications with SynchronizationContext when doing locking calls
IDisposable synchronizationContextScope = diagnosticsContext.CreateScope("SynchronizationContext");
return(Task.Run(() =>
{
using (new ActivityScope(Guid.NewGuid()))
{
// The goal of synchronizationContextScope is to log how much latency the Task.Run added to the latency.
// Dispose of it here so it only measures the latency added by the Task.Run.
synchronizationContextScope.Dispose();
return this.RunWithDiagnosticsHelperAsync <TResult>(
diagnosticsContext,
task);
}
}));
}
internal override async Task <Stream> DecryptItemAsync(
Stream input,
DatabaseInternal database,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
if (input == null || this.ClientOptions.Encryptor == null)
{
return(input);
}
Debug.Assert(database != null);
Debug.Assert(diagnosticsContext != null);
using (diagnosticsContext.CreateScope("Decrypt"))
{
return(await this.EncryptionProcessor.DecryptAsync(
input,
this.ClientOptions.Encryptor,
diagnosticsContext,
cancellationToken));
}
}
private async Task <HttpResponseMessage> SendHttpHelperAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
CosmosDiagnosticsContext diagnosticsContext,
HttpTimeoutPolicy timeoutPolicy,
CancellationToken cancellationToken)
{
bool isDefaultCancellationToken = cancellationToken == default;
DateTime startDateTimeUtc = DateTime.UtcNow;
IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.TimeoutEnumerator;
timeoutEnumerator.MoveNext();
while (true)
{
(TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
{
// If the default cancellation token is passed then use the timeout policy
CancellationTokenSource cancellationTokenSource = null;
if (isDefaultCancellationToken)
{
cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(requestTimeout);
cancellationToken = cancellationTokenSource.Token;
}
cancellationToken.ThrowIfCancellationRequested();
try
{
using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpHelperAsync)))
{
return(await this.ExecuteHttpHelperAsync(
requestMessage,
resourceType,
cancellationToken));
}
}
catch (Exception e)
{
// Log the error message
diagnosticsContext.AddDiagnosticsInternal(
new PointOperationStatistics(
activityId: Trace.CorrelationManager.ActivityId.ToString(),
statusCode: HttpStatusCode.ServiceUnavailable,
subStatusCode: SubStatusCodes.Unknown,
responseTimeUtc: DateTime.UtcNow,
requestCharge: 0,
errorMessage: e.ToString(),
method: requestMessage.Method,
requestUri: requestMessage.RequestUri.OriginalString,
requestSessionToken: null,
responseSessionToken: null));
bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
!timeoutEnumerator.MoveNext(); // No more retries are configured
switch (e)
{
case OperationCanceledException operationCanceledException:
// Throw if the user passed in cancellation was requested
if (!isDefaultCancellationToken && cancellationToken.IsCancellationRequested)
{
throw;
}
// Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
// the request timed out and was not user canceled operation.
if (isOutOfRetries || requestMessage.Method != HttpMethod.Get)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
string message =
$"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {Trace.CorrelationManager.ActivityId};";
throw CosmosExceptionFactory.CreateRequestTimeoutException(
message,
innerException: operationCanceledException,
diagnosticsContext: diagnosticsContext);
}
break;
case WebException webException:
if (isOutOfRetries || (requestMessage.Method != HttpMethod.Get && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
{
throw;
}
break;
case HttpRequestException httpRequestException:
if (isOutOfRetries || requestMessage.Method != HttpMethod.Get)
{
throw;
}
break;
default:
throw;
}
}
}
if (delayForNextRequest != TimeSpan.Zero)
{
using (diagnosticsContext.CreateScope($"HttpRetryDelay; Delay:{delayForNextRequest} seconds; Current request timeout {requestTimeout}; TimeoutPolicy: {timeoutPolicy.TimeoutPolicyName}"))
{
await Task.Delay(delayForNextRequest);
}
}
}
}
// Extracted partition key might be invalid as CollectionCache might be stale.
// Stale collection cache is refreshed through PartitionKeyMismatchRetryPolicy
// and partition-key is extracted again.
private async Task <ResponseMessage> ExtractPartitionKeyAndProcessItemStreamAsync <T>(
PartitionKey?partitionKey,
string itemId,
T item,
OperationType operationType,
ItemRequestOptions requestOptions,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
if (diagnosticsContext == null)
{
throw new ArgumentNullException(nameof(diagnosticsContext));
}
Stream itemStream;
using (diagnosticsContext.CreateScope("ItemSerialize"))
{
itemStream = this.ClientContext.SerializerCore.ToStream <T>(item);
}
// User specified PK value, no need to extract it
if (partitionKey.HasValue)
{
PartitionKeyDefinition pKeyDefinition = await this.GetPartitionKeyDefinitionAsync();
if (partitionKey.HasValue && partitionKey.Value != PartitionKey.None && partitionKey.Value.InternalKey.Components.Count != pKeyDefinition.Paths.Count)
{
throw new ArgumentException(RMResources.MissingPartitionKeyValue);
}
return(await this.ProcessItemStreamAsync(
partitionKey,
itemId,
itemStream,
operationType,
requestOptions,
diagnosticsContext : diagnosticsContext,
cancellationToken : cancellationToken));
}
PartitionKeyMismatchRetryPolicy requestRetryPolicy = null;
while (true)
{
using (diagnosticsContext.CreateScope("ExtractPkValue"))
{
partitionKey = await this.GetPartitionKeyValueFromStreamAsync(itemStream, cancellationToken);
}
ResponseMessage responseMessage = await this.ProcessItemStreamAsync(
partitionKey,
itemId,
itemStream,
operationType,
requestOptions,
diagnosticsContext : diagnosticsContext,
cancellationToken : cancellationToken);
if (responseMessage.IsSuccessStatusCode)
{
return(responseMessage);
}
if (requestRetryPolicy == null)
{
requestRetryPolicy = new PartitionKeyMismatchRetryPolicy(await this.ClientContext.DocumentClient.GetCollectionCacheAsync(), null);
}
ShouldRetryResult retryResult = await requestRetryPolicy.ShouldRetryAsync(responseMessage, cancellationToken);
if (!retryResult.ShouldRetry)
{
return(responseMessage);
}
}
}
public async Task <IReadOnlyList <FeedRange> > GetFeedRangesAsync(
CosmosDiagnosticsContext diagnosticsContext,
ITrace trace,
CancellationToken cancellationToken = default)
{
PartitionKeyRangeCache partitionKeyRangeCache = await this.ClientContext.DocumentClient.GetPartitionKeyRangeCacheAsync();
string containerRId;
using (diagnosticsContext.CreateScope(nameof(GetCachedRIDAsync)))
{
containerRId = await this.GetCachedRIDAsync(
forceRefresh : false,
cancellationToken);
}
IReadOnlyList <PartitionKeyRange> partitionKeyRanges;
using (diagnosticsContext.CreateScope(nameof(partitionKeyRangeCache.TryGetOverlappingRangesAsync)))
{
partitionKeyRanges = await partitionKeyRangeCache.TryGetOverlappingRangesAsync(
containerRId,
ContainerCore.allRanges,
forceRefresh : true);
}
if (partitionKeyRanges == null)
{
string refreshedContainerRId;
using (diagnosticsContext.CreateScope("GetRIDAsyncForceRefresh"))
{
refreshedContainerRId = await this.GetCachedRIDAsync(
forceRefresh : true,
cancellationToken);
}
if (string.Equals(containerRId, refreshedContainerRId))
{
throw CosmosExceptionFactory.CreateInternalServerErrorException(
$"Container rid {containerRId} did not have a partition key range after refresh",
diagnosticsContext: diagnosticsContext);
}
using (diagnosticsContext.CreateScope(nameof(partitionKeyRangeCache.TryGetOverlappingRangesAsync)))
{
partitionKeyRanges = await partitionKeyRangeCache.TryGetOverlappingRangesAsync(
containerRId,
ContainerCore.allRanges,
forceRefresh : true);
}
if (partitionKeyRanges == null)
{
throw CosmosExceptionFactory.CreateInternalServerErrorException(
$"Container rid {containerRId} returned partitionKeyRanges null after Container RID refresh",
diagnosticsContext: diagnosticsContext);
}
}
List <FeedRange> feedTokens = new List <FeedRange>(partitionKeyRanges.Count);
foreach (PartitionKeyRange partitionKeyRange in partitionKeyRanges)
{
feedTokens.Add(new FeedRangeEpk(partitionKeyRange.ToRange()));
}
return(feedTokens);
}
public override Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
diagnosticsContext ??= new CosmosDiagnosticsContextCore();
HttpRequestMessage requestMessage = null;
Func <Task <HttpResponseMessage> > funcDelegate = async() =>
{
using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpAsync)))
{
using (requestMessage = await createRequestMessageAsync())
{
DateTime sendTimeUtc = DateTime.UtcNow;
Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces
Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
this.eventSource.Request(
requestedActivityId,
localGuid,
requestMessage.RequestUri.ToString(),
resourceType.ToResourceTypeString(),
requestMessage.Headers);
// Only read the header initially. The content gets copied into a memory stream later
// if we read the content http client will buffer the message and then it will get buffered
// again when it is copied to the memory stream.
HttpResponseMessage responseMessage = await this.httpClient.SendAsync(
requestMessage,
HttpCompletionOption.ResponseHeadersRead,
cancellationToken);
// WebAssembly HttpClient does not set the RequestMessage property on SendAsync
if (responseMessage.RequestMessage == null)
{
responseMessage.RequestMessage = requestMessage;
}
DateTime receivedTimeUtc = DateTime.UtcNow;
TimeSpan durationTimeSpan = receivedTimeUtc - sendTimeUtc;
Guid activityId = Guid.Empty;
if (responseMessage.Headers.TryGetValues(
HttpConstants.HttpHeaders.ActivityId,
out IEnumerable <string> headerValues) && headerValues.Any())
{
activityId = new Guid(headerValues.First());
}
this.eventSource.Response(
activityId,
localGuid,
(short)responseMessage.StatusCode,
durationTimeSpan.TotalMilliseconds,
responseMessage.Headers);
return(responseMessage);
}
}
};
HttpRequestMessage GetHttpRequestMessage()
{
return(requestMessage);
}
return(BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(
callbackMethod : funcDelegate,
retryPolicy : new TransientHttpClientRetryPolicy(
getHttpRequestMessage: GetHttpRequestMessage,
gatewayRequestTimeout: this.httpClient.Timeout,
diagnosticsContext: diagnosticsContext),
cancellationToken : cancellationToken));
}
internal async Task <ResponseMessage> ProcessItemStreamAsync(
PartitionKey?partitionKey,
string itemId,
Stream streamPayload,
OperationType operationType,
ItemRequestOptions requestOptions,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
if (requestOptions != null && requestOptions.IsEffectivePartitionKeyRouting)
{
partitionKey = null;
}
ContainerCore.ValidatePartitionKey(partitionKey, requestOptions);
Uri resourceUri = this.GetResourceUri(requestOptions, operationType, itemId);
if (diagnosticsContext == null)
{
diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
}
using (diagnosticsContext.CreateOverallScope("ProcessItemStream"))
{
if (requestOptions != null && requestOptions.EncryptionOptions != null)
{
if (streamPayload == null)
{
throw new ArgumentException(ClientResources.InvalidRequestWithEncryptionOptions);
}
using (diagnosticsContext.CreateScope("Encrypt"))
{
streamPayload = await this.ClientContext.EncryptionProcessor.EncryptAsync(
streamPayload,
requestOptions.EncryptionOptions,
(DatabaseCore)this.Database,
this.ClientContext.ClientOptions.EncryptionKeyWrapProvider,
diagnosticsContext,
cancellationToken);
}
}
ResponseMessage responseMessage = await this.ClientContext.ProcessResourceOperationStreamAsync(
resourceUri : resourceUri,
resourceType : ResourceType.Document,
operationType : operationType,
requestOptions : requestOptions,
cosmosContainerCore : this,
partitionKey : partitionKey,
itemId : itemId,
streamPayload : streamPayload,
requestEnricher : null,
diagnosticsScope : diagnosticsContext,
cancellationToken : cancellationToken);
if (responseMessage.Content != null && this.ClientContext.ClientOptions.EncryptionKeyWrapProvider != null)
{
using (diagnosticsContext.CreateScope("Decrypt"))
{
responseMessage.Content = await this.ClientContext.EncryptionProcessor.DecryptAsync(
responseMessage.Content,
(DatabaseCore)this.Database,
this.ClientContext.ClientOptions.EncryptionKeyWrapProvider,
diagnosticsContext,
cancellationToken);
}
}
return(responseMessage);
}
}
private async Task <HttpResponseMessage> InvokeClientAsync(
DocumentServiceRequest request,
ResourceType resourceType,
Uri physicalAddress,
CancellationToken cancellationToken)
{
Func <Task <HttpResponseMessage> > funcDelegate = async() =>
{
using (HttpRequestMessage requestMessage = await this.PrepareRequestMessageAsync(request, physicalAddress))
{
DateTime sendTimeUtc = DateTime.UtcNow;
Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces
// DEVNOTE: This is temporary until IClientSideRequestStats is modified to support gateway calls.
CosmosDiagnosticsContext diagnosticsContext = null;
if (request.RequestContext.ClientRequestStatistics is CosmosClientSideRequestStatistics clientSideRequestStatistics)
{
diagnosticsContext = clientSideRequestStatistics.DiagnosticsContext;
}
else
{
// ClientRequestStatistics are not passed in use Empty to avoid null checks.
// Caches do not pass in the ClientRequestStatistics
diagnosticsContext = EmptyCosmosDiagnosticsContext.Singleton;
}
Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
this.eventSource.Request(
requestedActivityId,
localGuid,
requestMessage.RequestUri.ToString(),
resourceType.ToResourceTypeString(),
requestMessage.Headers);
TimeSpan durationTimeSpan;
try
{
HttpResponseMessage responseMessage;
using (diagnosticsContext.CreateScope("GatewayRequestTime"))
{
responseMessage = await this.httpClient.SendAsync(requestMessage, cancellationToken);
}
DateTime receivedTimeUtc = DateTime.UtcNow;
durationTimeSpan = receivedTimeUtc - sendTimeUtc;
IEnumerable <string> headerValues;
Guid activityId = Guid.Empty;
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.ActivityId, out headerValues) &&
headerValues.Count() != 0)
{
activityId = new Guid(headerValues.First());
}
this.eventSource.Response(
activityId,
localGuid,
(short)responseMessage.StatusCode,
durationTimeSpan.TotalMilliseconds,
responseMessage.Headers);
return(responseMessage);
}
catch (TaskCanceledException ex)
{
if (!cancellationToken.IsCancellationRequested)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
}
else
{
throw;
}
}
catch (OperationCanceledException ex)
{
if (!cancellationToken.IsCancellationRequested)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
}
else
{
throw;
}
}
}
};
return(await BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(funcDelegate, new WebExceptionRetryPolicy(), cancellationToken));
}
private Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
HttpCompletionOption httpCompletionOption,
ResourceType resourceType,
CosmosDiagnosticsContext diagnosticsContext,
CancellationToken cancellationToken)
{
diagnosticsContext ??= new CosmosDiagnosticsContextCore();
HttpRequestMessage requestMessage = null;
Func <Task <HttpResponseMessage> > funcDelegate = async() =>
{
using (diagnosticsContext.CreateScope(nameof(CosmosHttpClientCore.SendHttpAsync)))
{
using (requestMessage = await createRequestMessageAsync())
{
DateTime sendTimeUtc = DateTime.UtcNow;
Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces
Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
this.eventSource.Request(
requestedActivityId,
localGuid,
requestMessage.RequestUri.ToString(),
resourceType.ToResourceTypeString(),
requestMessage.Headers);
HttpResponseMessage responseMessage = await this.httpClient.SendAsync(
requestMessage,
httpCompletionOption,
cancellationToken);
DateTime receivedTimeUtc = DateTime.UtcNow;
TimeSpan durationTimeSpan = receivedTimeUtc - sendTimeUtc;
Guid activityId = Guid.Empty;
if (responseMessage.Headers.TryGetValues(
HttpConstants.HttpHeaders.ActivityId,
out IEnumerable <string> headerValues) && headerValues.Any())
{
activityId = new Guid(headerValues.First());
}
this.eventSource.Response(
activityId,
localGuid,
(short)responseMessage.StatusCode,
durationTimeSpan.TotalMilliseconds,
responseMessage.Headers);
return(responseMessage);
}
}
};
HttpRequestMessage GetHttpRequestMessage() => requestMessage;
return(BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(
callbackMethod : funcDelegate,
retryPolicy : new TransientHttpClientRetryPolicy(
getHttpRequestMessage: GetHttpRequestMessage,
gatewayRequestTimeout: this.httpClient.Timeout,
diagnosticsContext: diagnosticsContext),
cancellationToken : cancellationToken));
}
// Extracted partition key might be invalid as CollectionCache might be stale.
// Stale collection cache is refreshed through PartitionKeyMismatchRetryPolicy
// and partition-key is extracted again.
internal async Task <ResponseMessage> ExtractPartitionKeyAndProcessItemStreamAsync <T>(
PartitionKey?partitionKey,
string itemId,
T item,
OperationType operationType,
RequestOptions requestOptions,
CancellationToken cancellationToken)
{
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions);
using (diagnosticsContext.CreateOverallScope("ItemStream"))
{
Stream itemStream;
using (diagnosticsContext.CreateScope("ItemSerialize"))
{
itemStream = this.ClientContext.SerializerCore.ToStream <T>(item);
}
// User specified PK value, no need to extract it
if (partitionKey.HasValue)
{
return(await this.ProcessItemStreamAsync(
partitionKey,
itemId,
itemStream,
operationType,
requestOptions,
diagnosticsScope : diagnosticsContext,
cancellationToken : cancellationToken));
}
PartitionKeyMismatchRetryPolicy requestRetryPolicy = null;
while (true)
{
using (diagnosticsContext.CreateScope("ExtractPkValue"))
{
partitionKey = await this.GetPartitionKeyValueFromStreamAsync(itemStream, cancellationToken);
}
ResponseMessage responseMessage = await this.ProcessItemStreamAsync(
partitionKey,
itemId,
itemStream,
operationType,
requestOptions,
diagnosticsScope : diagnosticsContext,
cancellationToken : cancellationToken);
if (responseMessage.IsSuccessStatusCode)
{
return(responseMessage);
}
if (requestRetryPolicy == null)
{
requestRetryPolicy = new PartitionKeyMismatchRetryPolicy(await this.ClientContext.DocumentClient.GetCollectionCacheAsync(), null);
}
ShouldRetryResult retryResult = await requestRetryPolicy.ShouldRetryAsync(responseMessage, cancellationToken);
if (!retryResult.ShouldRetry)
{
return(responseMessage);
}
}
}
}