public abstract Task <HttpResponseMessage> GetAsync(
Uri uri,
INameValueCollection additionalHeaders,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken);
internal virtual async Task <ContainerProperties> ResolveByNameAsync(
string apiVersion,
string resourceAddress,
bool forceRefesh,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
string resourceFullName = PathsHelper.GetCollectionPath(resourceAddress);
InternalCache cache = this.GetCache(apiVersion);
if (forceRefesh)
{
cache.collectionInfoByName.TryRemoveIfCompleted(resourceFullName);
}
return(await cache.collectionInfoByName.GetAsync(
resourceFullName,
null,
async() =>
{
DateTime currentTime = DateTime.UtcNow;
ContainerProperties collection = await this.GetByNameAsync(apiVersion, resourceFullName, trace, clientSideRequestStatistics, cancellationToken);
cache.collectionInfoById.Set(collection.ResourceId, collection);
cache.collectionInfoByNameLastRefreshTime.AddOrUpdate(resourceFullName, currentTime,
(string currentKey, DateTime currentValue) => currentTime);
cache.collectionInfoByIdLastRefreshTime.AddOrUpdate(collection.ResourceId, currentTime,
(string currentKey, DateTime currentValue) => currentTime);
return collection;
},
cancellationToken));
}
/// <summary>
/// Method that is called before a request is sent to allow the retry policy implementation
/// to modify the state of the request.
/// </summary>
/// <param name="request">The request being sent to the service.</param>
public void OnBeforeSendRequest(DocumentServiceRequest request)
{
this.isReadRequest = request.IsReadOnlyRequest;
this.canUseMultipleWriteLocations = this.globalEndpointManager.CanUseMultipleWriteLocations(request);
if (request.RequestContext.ClientRequestStatistics == null)
{
if (this.sharedStatistics == null)
{
this.sharedStatistics = new CosmosClientSideRequestStatistics();
}
request.RequestContext.ClientRequestStatistics = this.sharedStatistics;
}
else
{
this.sharedStatistics = request.RequestContext.ClientRequestStatistics;
}
// clear previous location-based routing directive
request.RequestContext.ClearRouteToLocation();
if (this.retryContext != null)
{
// set location-based routing directive based on request retry context
request.RequestContext.RouteToLocation(this.retryContext.RetryLocationIndex, this.retryContext.RetryRequestOnPreferredLocations);
}
// Resolve the endpoint for the request and pin the resolution to the resolved endpoint
// This enables marking the endpoint unavailability on endpoint failover/unreachability
this.locationEndpoint = this.globalEndpointManager.ResolveServiceEndpoint(request);
request.RequestContext.RouteToLocation(this.locationEndpoint);
}
/// <summary>
/// Should the caller retry the operation.
/// </summary>
/// <param name="exception">Exception that occurred when the operation was tried</param>
/// <param name="cancellationToken"></param>
/// <returns>True indicates caller should retry, False otherwise</returns>
public async Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
this.retryContext = null;
// Received Connection error (HttpRequestException), initiate the endpoint rediscovery
if (exception is HttpRequestException _)
{
DefaultTrace.TraceWarning("Endpoint not reachable. Refresh cache and retry");
return(await this.ShouldRetryOnEndpointFailureAsync(
isReadRequest : this.isReadRequest,
forceRefresh : false,
retryOnPreferredLocations : true));
}
DocumentClientException clientException = exception as DocumentClientException;
if (clientException?.RequestStatistics != null)
{
this.sharedStatistics = clientException.RequestStatistics;
}
ShouldRetryResult shouldRetryResult = await this.ShouldRetryInternalAsync(
clientException?.StatusCode,
clientException?.GetSubStatus());
if (shouldRetryResult != null)
{
return(shouldRetryResult);
}
return(await this.throttlingRetry.ShouldRetryAsync(exception, cancellationToken));
}
public async Task <PartitionKeyRange> TryGetRangeByPartitionKeyRangeIdAsync(string collectionRid,
string partitionKeyRangeId,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics)
{
try
{
CollectionRoutingMap routingMap = await this.routingMapCache.GetAsync(
key : collectionRid,
singleValueInitFunc : (_) => this.GetRoutingMapForCollectionAsync(
collectionRid: collectionRid,
previousRoutingMap: null,
trace: trace,
clientSideRequestStatistics: clientSideRequestStatistics),
forceRefresh : (_) => false);
return(routingMap.TryGetRangeByPartitionKeyRangeId(partitionKeyRangeId));
}
catch (DocumentClientException ex)
{
if (ex.StatusCode == HttpStatusCode.NotFound)
{
return(null);
}
throw;
}
}
public async Task <PartitionKeyRange> TryGetRangeByPartitionKeyRangeIdAsync(string collectionRid,
string partitionKeyRangeId,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics)
{
try
{
CollectionRoutingMap routingMap = await this.routingMapCache.GetAsync(
collectionRid,
null,
() => this.GetRoutingMapForCollectionAsync(collectionRid, null, trace, clientSideRequestStatistics, CancellationToken.None),
CancellationToken.None);
return(routingMap.TryGetRangeByPartitionKeyRangeId(partitionKeyRangeId));
}
catch (DocumentClientException ex)
{
if (ex.StatusCode == HttpStatusCode.NotFound)
{
return(null);
}
throw;
}
}
private Task <ContainerProperties> ResolveByRidAsync(
string apiVersion,
string resourceId,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
ResourceId resourceIdParsed = ResourceId.Parse(resourceId);
string collectionResourceId = resourceIdParsed.DocumentCollectionId.ToString();
InternalCache cache = this.GetCache(apiVersion);
return(cache.collectionInfoById.GetAsync(
collectionResourceId,
null,
async() =>
{
DateTime currentTime = DateTime.UtcNow;
ContainerProperties collection = await this.GetByRidAsync(apiVersion, collectionResourceId, trace, clientSideRequestStatistics, cancellationToken);
cache.collectionInfoByIdLastRefreshTime.AddOrUpdate(collectionResourceId, currentTime,
(string currentKey, DateTime currentValue) => currentTime);
return collection;
},
cancellationToken));
}
private async Task <ContainerProperties> ReadCollectionAsync(
string collectionLink,
IDocumentClientRetryPolicy retryPolicyInstance,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
using (ITrace childTrace = trace.StartChild("Read Collection", TraceComponent.Transport, TraceLevel.Info))
{
cancellationToken.ThrowIfCancellationRequested();
RequestNameValueCollection headers = new RequestNameValueCollection();
using (DocumentServiceRequest request = DocumentServiceRequest.Create(
OperationType.Read,
ResourceType.Collection,
collectionLink,
AuthorizationTokenType.PrimaryMasterKey,
headers))
{
headers.XDate = Rfc1123DateTimeCache.UtcNow();
request.RequestContext.ClientRequestStatistics = clientSideRequestStatistics ?? new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow, trace.Summary);
if (clientSideRequestStatistics == null)
{
childTrace.AddDatum(
"Client Side Request Stats",
request.RequestContext.ClientRequestStatistics);
}
string authorizationToken = await this.tokenProvider.GetUserAuthorizationTokenAsync(
request.ResourceAddress,
PathsHelper.GetResourcePath(request.ResourceType),
HttpConstants.HttpMethods.Get,
request.Headers,
AuthorizationTokenType.PrimaryMasterKey,
childTrace);
headers.Authorization = authorizationToken;
using (new ActivityScope(Guid.NewGuid()))
{
retryPolicyInstance?.OnBeforeSendRequest(request);
try
{
using (DocumentServiceResponse response =
await this.storeModel.ProcessMessageAsync(request))
{
return(CosmosResource.FromStream <ContainerProperties>(response));
}
}
catch (DocumentClientException ex)
{
childTrace.AddDatum("Exception Message", ex.Message);
throw;
}
}
}
}
}
internal DocumentFeedResponse(
IEnumerable <T> result,
int count,
INameValueCollection responseHeaders,
IClientSideRequestStatistics requestStats,
long responseLengthBytes)
: this(result, count, responseHeaders, false, null, requestStats, responseLengthBytes : responseLengthBytes)
{
}
private static void LogPartitionCacheRefresh(
IClientSideRequestStatistics clientSideRequestStatistics,
PartitionAddressInformation old,
PartitionAddressInformation updated)
{
if (clientSideRequestStatistics is ClientSideRequestStatisticsTraceDatum traceDatum)
{
traceDatum.RecordAddressCachRefreshContent(old, updated);
}
}
internal Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > requestMessage,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken = default)
{
return(this.httpClient.SendHttpAsync(
createRequestMessageAsync: requestMessage,
resourceType: resourceType,
timeoutPolicy: timeoutPolicy,
clientSideRequestStatistics: clientSideRequestStatistics,
cancellationToken: cancellationToken));
}
protected override Task <ContainerProperties> GetByNameAsync(string apiVersion,
string resourceAddress,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
IDocumentClientRetryPolicy retryPolicyInstance = new ClearingSessionContainerClientRetryPolicy(this.sessionContainer, this.retryPolicy.GetRequestPolicy());
return(TaskHelper.InlineIfPossible(
() => this.ReadCollectionAsync(resourceAddress, retryPolicyInstance, trace, clientSideRequestStatistics, cancellationToken),
retryPolicyInstance,
cancellationToken));
}
private static void ThrowTransportExceptionOnItemOperation(
Uri physicalAddress,
ResourceOperation resourceOperation,
DocumentServiceRequest request)
{
if (request.ResourceType == ResourceType.Document)
{
TransportException transportException = new TransportException(
errorCode: TransportErrorCode.ConnectionBroken,
innerException: null,
activityId: Guid.NewGuid(),
requestUri: physicalAddress,
sourceDescription: "SourceDescription",
userPayload: true,
payloadSent: false);
DocumentClientException documentClientException = new DocumentClientException(
message: "Exception",
innerException: transportException,
statusCode: System.Net.HttpStatusCode.Gone);
IClientSideRequestStatistics requestStatistics = request.RequestContext.ClientRequestStatistics;
requestStatistics.RecordResponse(
request,
new StoreResult(
storeResponse: null,
exception: documentClientException,
partitionKeyRangeId: "PkRange",
lsn: 42,
quorumAckedLsn: 4242,
requestCharge: 9000.42,
currentReplicaSetSize: 3,
currentWriteQuorum: 4,
isValid: true,
storePhysicalAddress: physicalAddress,
globalCommittedLSN: 2,
numberOfReadRegions: 1,
itemLSN: 5,
sessionToken: null,
usingLocalLSN: true,
activityId: Guid.NewGuid().ToString(),
backendRequestDurationInMs: "0",
retryAfterInMs: "42",
transportRequestStats: new TransportRequestStats()),
DateTime.MinValue,
DateTime.MaxValue);
throw Documents.Rntbd.TransportExceptions.GetServiceUnavailableException(physicalAddress, Guid.NewGuid(),
transportException);
}
}
internal override Task <ContainerProperties> ResolveByNameAsync(
string apiVersion,
string resourceAddress,
bool forceRefesh,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
if (forceRefesh && this.sessionContainer != null)
{
return(TaskHelper.InlineIfPossible(
async() =>
{
string oldRid = (await base.ResolveByNameAsync(
apiVersion,
resourceAddress,
forceRefesh: false,
trace,
clientSideRequestStatistics,
cancellationToken))?.ResourceId;
ContainerProperties propertiesAfterRefresh = await base.ResolveByNameAsync(
apiVersion,
resourceAddress,
forceRefesh,
trace,
clientSideRequestStatistics,
cancellationToken);
if (oldRid != null && oldRid != propertiesAfterRefresh?.ResourceId)
{
string resourceFullName = PathsHelper.GetCollectionPath(resourceAddress);
this.sessionContainer.ClearTokenByCollectionFullname(resourceFullName);
}
return propertiesAfterRefresh;
},
retryPolicy: null,
cancellationToken));
}
return(TaskHelper.InlineIfPossible(
() => base.ResolveByNameAsync(
apiVersion, resourceAddress, forceRefesh, trace, clientSideRequestStatistics, cancellationToken),
retryPolicy: null,
cancellationToken));
}
public ClientRetryPolicy(
GlobalEndpointManager globalEndpointManager,
bool enableEndpointDiscovery,
RetryOptions retryOptions)
{
this.throttlingRetry = new ResourceThrottleRetryPolicy(
retryOptions.MaxRetryAttemptsOnThrottledRequests,
retryOptions.MaxRetryWaitTimeInSeconds);
this.globalEndpointManager = globalEndpointManager;
this.failoverRetryCount = 0;
this.enableEndpointDiscovery = enableEndpointDiscovery;
this.sessionTokenRetryCount = 0;
this.canUseMultipleWriteLocations = false;
this.sharedStatistics = new CosmosClientSideRequestStatistics();
}
public override Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
if (createRequestMessageAsync == null)
{
throw new ArgumentNullException(nameof(createRequestMessageAsync));
}
return(this.SendHttpHelperAsync(
createRequestMessageAsync,
resourceType,
timeoutPolicy,
clientSideRequestStatistics,
cancellationToken));
}
internal DocumentFeedResponse(
IEnumerable <T> result,
int count,
INameValueCollection responseHeaders,
bool useETagAsContinuation = false,
IReadOnlyDictionary <string, QueryMetrics> queryMetrics = null,
IClientSideRequestStatistics requestStats = null,
string disallowContinuationTokenMessage = null,
long responseLengthBytes = 0)
: this(result)
{
this.Count = count;
this.responseHeaders = (INameValueCollection)responseHeaders.Clone();
this.usageHeaders = new Dictionary <string, long>();
this.quotaHeaders = new Dictionary <string, long>();
this.useETagAsContinuation = useETagAsContinuation;
this.queryMetrics = queryMetrics;
this.disallowContinuationTokenMessage = disallowContinuationTokenMessage;
this.ResponseLengthBytes = responseLengthBytes;
}
public override Task <HttpResponseMessage> GetAsync(
Uri uri,
INameValueCollection additionalHeaders,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
if (uri == null)
{
throw new ArgumentNullException(nameof(uri));
}
// GetAsync doesn't let clients to pass in additional headers. So, we are
// internally using SendAsync and add the additional headers to requestMessage.
ValueTask <HttpRequestMessage> CreateRequestMessage()
{
HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Get, uri);
if (additionalHeaders != null)
{
foreach (string header in additionalHeaders)
{
if (GatewayStoreClient.IsAllowedRequestHeader(header))
{
requestMessage.Headers.TryAddWithoutValidation(header, additionalHeaders[header]);
}
}
}
return(new ValueTask <HttpRequestMessage>(requestMessage));
}
return(this.SendHttpAsync(
CreateRequestMessage,
resourceType,
timeoutPolicy,
clientSideRequestStatistics,
cancellationToken));
}
private async Task RefreshAsync(DocumentServiceRequest request,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
System.Diagnostics.Debug.Assert(request.IsNameBased);
InternalCache cache = this.GetCache(request.Headers[HttpConstants.HttpHeaders.Version]);
string resourceFullName = PathsHelper.GetCollectionPath(request.ResourceAddress);
if (request.RequestContext.ResolvedCollectionRid != null)
{
// Here we will issue backend call only if cache wasn't already refreshed (if whatever is there corresponds to presiously resolved collection rid).
await cache.collectionInfoByName.GetAsync(
resourceFullName,
ContainerProperties.CreateWithResourceId(request.RequestContext.ResolvedCollectionRid),
async() =>
{
DateTime currentTime = DateTime.UtcNow;
ContainerProperties collection = await this.GetByNameAsync(request.Headers[HttpConstants.HttpHeaders.Version], resourceFullName, trace, clientSideRequestStatistics, cancellationToken);
cache.collectionInfoById.Set(collection.ResourceId, collection);
cache.collectionInfoByNameLastRefreshTime.AddOrUpdate(resourceFullName, currentTime,
(string currentKey, DateTime currentValue) => currentTime);
cache.collectionInfoByIdLastRefreshTime.AddOrUpdate(collection.ResourceId, currentTime,
(string currentKey, DateTime currentValue) => currentTime);
return(collection);
},
cancellationToken);
}
else
{
// In case of ForceRefresh directive coming from client, there will be no ResolvedCollectionRid, so we
// need to refresh unconditionally.
this.Refresh(request.ResourceAddress, request.Headers[HttpConstants.HttpHeaders.Version]);
}
request.RequestContext.ResolvedCollectionRid = null;
}
internal static async Task <DocumentServiceResponse> ParseResponseAsync(HttpResponseMessage responseMessage, JsonSerializerSettings serializerSettings = null, DocumentServiceRequest request = null)
{
using (responseMessage)
{
IClientSideRequestStatistics requestStatistics = request?.RequestContext?.ClientRequestStatistics;
if ((int)responseMessage.StatusCode < 400)
{
INameValueCollection headers = GatewayStoreClient.ExtractResponseHeaders(responseMessage);
Stream contentStream = await GatewayStoreClient.BufferContentIfAvailableAsync(responseMessage);
return(new DocumentServiceResponse(
body: contentStream,
headers: headers,
statusCode: responseMessage.StatusCode,
clientSideRequestStatistics: requestStatistics,
serializerSettings: serializerSettings));
}
else if (request != null &&
request.IsValidStatusCodeForExceptionlessRetry((int)responseMessage.StatusCode))
{
INameValueCollection headers = GatewayStoreClient.ExtractResponseHeaders(responseMessage);
Stream contentStream = await GatewayStoreClient.BufferContentIfAvailableAsync(responseMessage);
return(new DocumentServiceResponse(
body: contentStream,
headers: headers,
statusCode: responseMessage.StatusCode,
clientSideRequestStatistics: requestStatistics,
serializerSettings: serializerSettings));
}
else
{
throw await GatewayStoreClient.CreateDocumentClientExceptionAsync(responseMessage, requestStatistics);
}
}
}
public virtual async Task <AccountProperties> GetDatabaseAccountAsync(Func <ValueTask <HttpRequestMessage> > requestMessage,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken = default)
{
AccountProperties databaseAccount = null;
// Get the ServiceDocumentResource from the gateway.
using (HttpResponseMessage responseMessage = await this.gatewayStoreClient.SendHttpAsync(
requestMessage,
ResourceType.DatabaseAccount,
HttpTimeoutPolicyControlPlaneRead.Instance,
clientSideRequestStatistics,
cancellationToken))
{
using (DocumentServiceResponse documentServiceResponse = await ClientExtensions.ParseResponseAsync(responseMessage))
{
databaseAccount = CosmosResource.FromStream <AccountProperties>(documentServiceResponse);
}
long longValue;
IEnumerable <string> headerValues;
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.MaxMediaStorageUsageInMB, out headerValues) &&
(headerValues.Count() != 0))
{
if (long.TryParse(headerValues.First(), out longValue))
{
databaseAccount.MaxMediaStorageUsageInMB = longValue;
}
}
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.CurrentMediaStorageUsageInMB, out headerValues) &&
(headerValues.Count() != 0))
{
if (long.TryParse(headerValues.First(), out longValue))
{
databaseAccount.MediaStorageUsageInMB = longValue;
}
}
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.DatabaseAccountConsumedDocumentStorageInMB, out headerValues) &&
(headerValues.Count() != 0))
{
if (long.TryParse(headerValues.First(), out longValue))
{
databaseAccount.ConsumedDocumentStorageInMB = longValue;
}
}
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.DatabaseAccountProvisionedDocumentStorageInMB, out headerValues) &&
(headerValues.Count() != 0))
{
if (long.TryParse(headerValues.First(), out longValue))
{
databaseAccount.ProvisionedDocumentStorageInMB = longValue;
}
}
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.DatabaseAccountReservedDocumentStorageInMB, out headerValues) &&
(headerValues.Count() != 0))
{
if (long.TryParse(headerValues.First(), out longValue))
{
databaseAccount.ReservedDocumentStorageInMB = longValue;
}
}
}
return(databaseAccount);
}
internal static async Task <DocumentClientException> CreateDocumentClientExceptionAsync(
HttpResponseMessage responseMessage,
IClientSideRequestStatistics requestStatistics)
{
bool isNameBased = false;
bool isFeed = false;
string resourceTypeString;
string resourceIdOrFullName;
string resourceLink = responseMessage.RequestMessage.RequestUri.LocalPath;
if (!PathsHelper.TryParsePathSegments(resourceLink, out isFeed, out resourceTypeString, out resourceIdOrFullName, out isNameBased))
{
// if resourceLink is invalid - we will not set resourceAddress in exception.
}
// If service rejects the initial payload like header is to large it will return an HTML error instead of JSON.
if (string.Equals(responseMessage.Content?.Headers?.ContentType?.MediaType, "application/json", StringComparison.OrdinalIgnoreCase))
{
Stream readStream = await responseMessage.Content.ReadAsStreamAsync();
Error error = Documents.Resource.LoadFrom <Error>(readStream);
return(new DocumentClientException(
error,
responseMessage.Headers,
responseMessage.StatusCode)
{
StatusDescription = responseMessage.ReasonPhrase,
ResourceAddress = resourceIdOrFullName,
RequestStatistics = requestStatistics
});
}
else
{
StringBuilder context = new StringBuilder();
context.AppendLine(await responseMessage.Content.ReadAsStringAsync());
HttpRequestMessage requestMessage = responseMessage.RequestMessage;
if (requestMessage != null)
{
context.AppendLine($"RequestUri: {requestMessage.RequestUri.ToString()};");
context.AppendLine($"RequestMethod: {requestMessage.Method.Method};");
if (requestMessage.Headers != null)
{
foreach (KeyValuePair <string, IEnumerable <string> > header in requestMessage.Headers)
{
context.AppendLine($"Header: {header.Key} Length: {string.Join(",", header.Value).Length};");
}
}
}
String message = await responseMessage.Content.ReadAsStringAsync();
return(new DocumentClientException(
message: context.ToString(),
innerException: null,
responseHeaders: responseMessage.Headers,
statusCode: responseMessage.StatusCode,
requestUri: responseMessage.RequestMessage.RequestUri)
{
StatusDescription = responseMessage.ReasonPhrase,
ResourceAddress = resourceIdOrFullName,
RequestStatistics = requestStatistics
});
}
}
private async Task <HttpResponseMessage> SendHttpHelperAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
DateTime startDateTimeUtc = DateTime.UtcNow;
IEnumerator <(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> timeoutEnumerator = timeoutPolicy.GetTimeoutEnumerator();
timeoutEnumerator.MoveNext();
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
(TimeSpan requestTimeout, TimeSpan delayForNextRequest) = timeoutEnumerator.Current;
using (HttpRequestMessage requestMessage = await createRequestMessageAsync())
{
// If the default cancellation token is passed then use the timeout policy
using CancellationTokenSource cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cancellationTokenSource.CancelAfter(requestTimeout);
try
{
HttpResponseMessage responseMessage = await this.ExecuteHttpHelperAsync(
requestMessage,
resourceType,
cancellationTokenSource.Token);
if (clientSideRequestStatistics is ClientSideRequestStatisticsTraceDatum datum)
{
datum.RecordHttpResponse(requestMessage, responseMessage, resourceType, DateTime.UtcNow);
}
return(responseMessage);
}
catch (Exception e)
{
if (clientSideRequestStatistics is ClientSideRequestStatisticsTraceDatum datum)
{
datum.RecordHttpException(requestMessage, e, resourceType, DateTime.UtcNow);
}
bool isOutOfRetries = (DateTime.UtcNow - startDateTimeUtc) > timeoutPolicy.MaximumRetryTimeLimit || // Maximum of time for all retries
!timeoutEnumerator.MoveNext(); // No more retries are configured
switch (e)
{
case OperationCanceledException operationCanceledException:
// Throw if the user passed in cancellation was requested
if (cancellationToken.IsCancellationRequested)
{
throw;
}
// Convert OperationCanceledException to 408 when the HTTP client throws it. This makes it clear that the
// the request timed out and was not user canceled operation.
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
// throw current exception (caught in transport handler)
string message =
$"GatewayStoreClient Request Timeout. Start Time UTC:{startDateTimeUtc}; Total Duration:{(DateTime.UtcNow - startDateTimeUtc).TotalMilliseconds} Ms; Request Timeout {requestTimeout.TotalMilliseconds} Ms; Http Client Timeout:{this.httpClient.Timeout.TotalMilliseconds} Ms; Activity id: {System.Diagnostics.Trace.CorrelationManager.ActivityId};";
e.Data.Add("Message", message);
throw;
}
break;
case WebException webException:
if (isOutOfRetries || (!timeoutPolicy.IsSafeToRetry(requestMessage.Method) && !WebExceptionUtility.IsWebExceptionRetriable(webException)))
{
throw;
}
break;
case HttpRequestException httpRequestException:
if (isOutOfRetries || !timeoutPolicy.IsSafeToRetry(requestMessage.Method))
{
throw;
}
break;
default:
throw;
}
}
}
if (delayForNextRequest != TimeSpan.Zero)
{
await Task.Delay(delayForNextRequest);
}
}
}
private async Task <CollectionRoutingMap> GetRoutingMapForCollectionAsync(
string collectionRid,
CollectionRoutingMap previousRoutingMap,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
List <PartitionKeyRange> ranges = new List <PartitionKeyRange>();
string changeFeedNextIfNoneMatch = previousRoutingMap == null ? null : previousRoutingMap.ChangeFeedNextIfNoneMatch;
HttpStatusCode lastStatusCode = HttpStatusCode.OK;
do
{
INameValueCollection headers = new StoreRequestNameValueCollection();
headers.Set(HttpConstants.HttpHeaders.PageSize, PageSizeString);
headers.Set(HttpConstants.HttpHeaders.A_IM, HttpConstants.A_IMHeaderValues.IncrementalFeed);
if (changeFeedNextIfNoneMatch != null)
{
headers.Set(HttpConstants.HttpHeaders.IfNoneMatch, changeFeedNextIfNoneMatch);
}
RetryOptions retryOptions = new RetryOptions();
using (DocumentServiceResponse response = await BackoffRetryUtility <DocumentServiceResponse> .ExecuteAsync(
() => this.ExecutePartitionKeyRangeReadChangeFeedAsync(collectionRid, headers, trace, clientSideRequestStatistics),
new ResourceThrottleRetryPolicy(retryOptions.MaxRetryAttemptsOnThrottledRequests, retryOptions.MaxRetryWaitTimeInSeconds),
cancellationToken))
{
lastStatusCode = response.StatusCode;
changeFeedNextIfNoneMatch = response.Headers[HttpConstants.HttpHeaders.ETag];
FeedResource <PartitionKeyRange> feedResource = response.GetResource <FeedResource <PartitionKeyRange> >();
if (feedResource != null)
{
ranges.AddRange(feedResource);
}
}
}while (lastStatusCode != HttpStatusCode.NotModified);
IEnumerable <Tuple <PartitionKeyRange, ServiceIdentity> > tuples = ranges.Select(range => Tuple.Create(range, (ServiceIdentity)null));
CollectionRoutingMap routingMap;
if (previousRoutingMap == null)
{
// Splits could have happened during change feed query and we might have a mix of gone and new ranges.
HashSet <string> goneRanges = new HashSet <string>(ranges.SelectMany(range => range.Parents ?? Enumerable.Empty <string>()));
routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap(
tuples.Where(tuple => !goneRanges.Contains(tuple.Item1.Id)),
string.Empty,
changeFeedNextIfNoneMatch);
}
else
{
routingMap = previousRoutingMap.TryCombine(tuples, changeFeedNextIfNoneMatch);
}
if (routingMap == null)
{
// Range information either doesn't exist or is not complete.
throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: GetRoutingMapForCollectionAsync(collectionRid: {collectionRid}), Range information either doesn't exist or is not complete.");
}
return(routingMap);
}
protected abstract Task <ContainerProperties> GetByNameAsync(string apiVersion,
string resourceAddress,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken);
protected abstract Task <ContainerProperties> GetByRidAsync(string apiVersion,
string collectionRid,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken);
private async Task <HttpResponseMessage> InvokeClientAsync(
DocumentServiceRequest request,
ResourceType resourceType,
Uri physicalAddress,
CancellationToken cancellationToken)
{
Func <Task <HttpResponseMessage> > funcDelegate = async() =>
{
using (HttpRequestMessage requestMessage = await this.PrepareRequestMessageAsync(request, physicalAddress))
{
DateTime sendTimeUtc = DateTime.UtcNow;
Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces
IClientSideRequestStatistics clientSideRequestStatistics = request.RequestContext.ClientRequestStatistics;
if (clientSideRequestStatistics == null)
{
clientSideRequestStatistics = new CosmosClientSideRequestStatistics();
request.RequestContext.ClientRequestStatistics = clientSideRequestStatistics;
}
Guid requestedActivityId = Trace.CorrelationManager.ActivityId;
this.eventSource.Request(
requestedActivityId,
localGuid,
requestMessage.RequestUri.ToString(),
resourceType.ToResourceTypeString(),
requestMessage.Headers);
TimeSpan durationTimeSpan;
string recordAddressResolutionId = clientSideRequestStatistics.RecordAddressResolutionStart(requestMessage.RequestUri);
try
{
HttpResponseMessage responseMessage = await this.httpClient.SendAsync(requestMessage, cancellationToken);
DateTime receivedTimeUtc = DateTime.UtcNow;
durationTimeSpan = receivedTimeUtc - sendTimeUtc;
IEnumerable <string> headerValues;
Guid activityId = Guid.Empty;
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.ActivityId, out headerValues) &&
headerValues.Count() != 0)
{
activityId = new Guid(headerValues.First());
}
this.eventSource.Response(
activityId,
localGuid,
(short)responseMessage.StatusCode,
durationTimeSpan.TotalMilliseconds,
responseMessage.Headers);
return(responseMessage);
}
catch (TaskCanceledException ex)
{
if (!cancellationToken.IsCancellationRequested)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
}
else
{
throw;
}
}
catch (OperationCanceledException ex)
{
if (!cancellationToken.IsCancellationRequested)
{
// throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out)
durationTimeSpan = DateTime.UtcNow - sendTimeUtc;
string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};";
throw new RequestTimeoutException(message, ex, requestMessage.RequestUri);
}
else
{
throw;
}
}
finally
{
clientSideRequestStatistics.RecordAddressResolutionEnd(recordAddressResolutionId);
}
}
};
return(await BackoffRetryUtility <HttpResponseMessage> .ExecuteAsync(funcDelegate, new WebExceptionRetryPolicy(), cancellationToken));
}
public abstract Task <HttpResponseMessage> SendHttpAsync(
Func <ValueTask <HttpRequestMessage> > createRequestMessageAsync,
ResourceType resourceType,
HttpTimeoutPolicy timeoutPolicy,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken);
private async Task <DocumentServiceResponse> ExecutePartitionKeyRangeReadChangeFeedAsync(string collectionRid,
INameValueCollection headers,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics)
{
using (ITrace childTrace = trace.StartChild("Read PartitionKeyRange Change Feed", TraceComponent.Transport, Tracing.TraceLevel.Info))
{
using (DocumentServiceRequest request = DocumentServiceRequest.Create(
OperationType.ReadFeed,
collectionRid,
ResourceType.PartitionKeyRange,
AuthorizationTokenType.PrimaryMasterKey,
headers))
{
string authorizationToken = null;
try
{
authorizationToken = await this.authorizationTokenProvider.GetUserAuthorizationTokenAsync(
request.ResourceAddress,
PathsHelper.GetResourcePath(request.ResourceType),
HttpConstants.HttpMethods.Get,
request.Headers,
AuthorizationTokenType.PrimaryMasterKey,
childTrace);
}
catch (UnauthorizedException)
{
}
if (authorizationToken == null)
{
// User doesn't have rid based resource token. Maybe he has name based.
throw new NotSupportedException("Resource tokens are not supported");
////CosmosContainerSettings collection = await this.collectionCache.ResolveCollectionAsync(request, CancellationToken.None);
////authorizationToken =
//// this.authorizationTokenProvider.GetUserAuthorizationTokenAsync(
//// collection.AltLink,
//// PathsHelper.GetResourcePath(request.ResourceType),
//// HttpConstants.HttpMethods.Get,
//// request.Headers,
//// AuthorizationTokenType.PrimaryMasterKey);
}
request.Headers[HttpConstants.HttpHeaders.Authorization] = authorizationToken;
request.RequestContext.ClientRequestStatistics = clientSideRequestStatistics ?? new ClientSideRequestStatisticsTraceDatum(DateTime.UtcNow);
if (clientSideRequestStatistics == null)
{
childTrace.AddDatum("Client Side Request Stats", request.RequestContext.ClientRequestStatistics);
}
using (new ActivityScope(Guid.NewGuid()))
{
try
{
return(await this.storeModel.ProcessMessageAsync(request));
}
catch (DocumentClientException ex)
{
childTrace.AddDatum("Exception Message", ex.Message);
throw;
}
}
}
}
}
private async Task <ContainerProperties> ResolveByPartitionKeyRangeIdentityAsync(string apiVersion,
PartitionKeyRangeIdentity partitionKeyRangeIdentity,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
// if request is targeted at specific partition using x-ms-documentd-partitionkeyrangeid header,
// which contains value "<collectionrid>,<partitionkeyrangeid>", then resolve to collection rid in this header.
if (partitionKeyRangeIdentity != null && partitionKeyRangeIdentity.CollectionRid != null)
{
try
{
return(await this.ResolveByRidAsync(apiVersion, partitionKeyRangeIdentity.CollectionRid, trace, clientSideRequestStatistics, cancellationToken));
}
catch (NotFoundException)
{
// This is signal to the upper logic either in Gateway or client SDK to refresh
// collection cache and retry.
throw new InvalidPartitionException(RMResources.InvalidDocumentCollection);
}
}
return(null);
}