public static StoreResponse ReturnThrottledStoreResponseOnItemOperation(
Uri physicalAddress,
ResourceOperation resourceOperation,
DocumentServiceRequest request,
Guid activityId,
string errorMessage)
{
if (request.ResourceType == ResourceType.Document)
{
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers.Add(HttpConstants.HttpHeaders.ActivityId, activityId.ToString());
headers.Add(WFConstants.BackendHeaders.SubStatus, ((int)SubStatusCodes.WriteForbidden).ToString(CultureInfo.InvariantCulture));
headers.Add(HttpConstants.HttpHeaders.RetryAfterInMilliseconds, TimeSpan.FromMilliseconds(100).TotalMilliseconds.ToString(CultureInfo.InvariantCulture));
headers.Add(HttpConstants.HttpHeaders.RequestCharge, ((double)9001).ToString(CultureInfo.InvariantCulture));
StoreResponse storeResponse = new StoreResponse()
{
Status = 429,
Headers = headers,
ResponseBody = new MemoryStream(Encoding.UTF8.GetBytes(errorMessage))
};
return(storeResponse);
}
return(null);
}
public async Task TestGatewayModelSession()
{
ContainerProperties containerProperties = await this.Container.GetCachedContainerPropertiesAsync(
false,
Trace.GetRootTrace("Test"),
CancellationToken.None);
ISessionContainer sessionContainer = this.cosmosClient.DocumentClient.sessionContainer;
string docLink = "dbs/" + this.database.Id + "/colls/" + containerProperties.Id + "/docs/3";
Documents.Collections.INameValueCollection headers = new StoreRequestNameValueCollection();
headers.Set(HttpConstants.HttpHeaders.PartitionKey, "[\"Status3\"]");
DocumentServiceRequest request = DocumentServiceRequest.Create(OperationType.Read, ResourceType.Document, docLink, AuthorizationTokenType.PrimaryMasterKey, headers);
string globalSessionToken = sessionContainer.ResolveGlobalSessionToken(request);
Assert.IsTrue(globalSessionToken.Split(',').Length > 1);
await GatewayStoreModel.ApplySessionTokenAsync(request,
Cosmos.ConsistencyLevel.Session,
sessionContainer,
await this.cosmosClient.DocumentClient.GetPartitionKeyRangeCacheAsync(NoOpTrace.Singleton),
await this.cosmosClient.DocumentClient.GetCollectionCacheAsync(NoOpTrace.Singleton));
string sessionToken = request.Headers[HttpConstants.HttpHeaders.SessionToken];
Assert.IsTrue(!string.IsNullOrEmpty(sessionToken) && sessionToken.Split(',').Length == 1);
}
public async Task AddPartitionKeyRangeToContinuationTokenOnNotNullBackendContinuation()
{
ResolvedRangeInfo currentPartitionKeyRange = new ResolvedRangeInfo(new PartitionKeyRange {
Id = "1", MinInclusive = "B", MaxExclusive = "C"
}, null);
Mock <IRoutingMapProvider> routingMapProvider = new Mock <IRoutingMapProvider>();
routingMapProvider.Setup(m => m.TryGetOverlappingRangesAsync(
It.IsAny <string>(),
It.IsAny <Range <string> >(),
It.IsAny <bool>()
)).Returns(Task.FromResult <IReadOnlyList <PartitionKeyRange> >(null)).Verifiable();
PartitionRoutingHelper partitionRoutingHelper = new PartitionRoutingHelper();
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers.Add(HttpConstants.HttpHeaders.Continuation, "something");
bool result = await partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
headers,
null,
routingMapProvider.Object,
CollectionId,
currentPartitionKeyRange,
RntdbEnumerationDirection.Reverse
);
Assert.IsTrue(true);
routingMapProvider.Verify(m => m.TryGetOverlappingRangesAsync(
It.IsAny <string>(),
It.IsAny <Range <string> >(),
It.IsAny <bool>()
), Times.Never);
}
private async Task <DocumentServiceResponse> GetFeedResponseAsync(string resourceLink, ResourceType resourceType, IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken)
{
INameValueCollection headers = new StoreRequestNameValueCollection();
if (this.feedOptions.MaxItemCount.HasValue)
{
headers.Set(HttpConstants.HttpHeaders.PageSize, this.feedOptions.MaxItemCount.ToString());
}
if (this.feedOptions.SessionToken != null)
{
headers.Set(HttpConstants.HttpHeaders.SessionToken, this.feedOptions.SessionToken);
}
if (resourceType.IsPartitioned() && this.feedOptions.PartitionKeyRangeId == null && this.feedOptions.PartitionKey == null)
{
throw new ForbiddenException(RMResources.PartitionKeyRangeIdOrPartitionKeyMustBeSpecified);
}
// On REST level, change feed is using IfNoneMatch/ETag instead of continuation.
if (this.nextIfNoneMatch != null)
{
headers.Set(HttpConstants.HttpHeaders.IfNoneMatch, this.nextIfNoneMatch);
}
if (this.ifModifiedSince != null)
{
headers.Set(HttpConstants.HttpHeaders.IfModifiedSince, this.ifModifiedSince);
}
headers.Set(HttpConstants.HttpHeaders.A_IM, HttpConstants.A_IMHeaderValues.IncrementalFeed);
if (this.feedOptions.PartitionKey != null)
{
PartitionKeyInternal partitionKey = this.feedOptions.PartitionKey.InternalKey;
headers.Set(HttpConstants.HttpHeaders.PartitionKey, partitionKey.ToJsonString());
}
if (this.feedOptions.IncludeTentativeWrites)
{
headers.Set(HttpConstants.HttpHeaders.IncludeTentativeWrites, bool.TrueString);
}
using (DocumentServiceRequest request = this.client.CreateDocumentServiceRequest(
OperationType.ReadFeed,
resourceLink,
resourceType,
headers))
{
if (resourceType.IsPartitioned() && this.feedOptions.PartitionKeyRangeId != null)
{
request.RouteTo(new PartitionKeyRangeIdentity(this.feedOptions.PartitionKeyRangeId));
}
return(await this.client.ReadFeedAsync(request, retryPolicyInstance, cancellationToken));
}
}
private async Task <DocumentServiceResponse> GetMasterAddressesViaGatewayAsync(
DocumentServiceRequest request,
ResourceType resourceType,
string resourceAddress,
string entryUrl,
bool forceRefresh,
bool useMasterCollectionResolver)
{
INameValueCollection addressQuery = new StoreRequestNameValueCollection
{
{ HttpConstants.QueryStrings.Url, HttpUtility.UrlEncode(entryUrl) }
};
INameValueCollection headers = new StoreRequestNameValueCollection();
if (forceRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceRefresh, bool.TrueString);
}
if (useMasterCollectionResolver)
{
headers.Set(HttpConstants.HttpHeaders.UseMasterCollectionResolver, bool.TrueString);
}
if (request.ForceCollectionRoutingMapRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceCollectionRoutingMapRefresh, bool.TrueString);
}
addressQuery.Add(HttpConstants.QueryStrings.Filter, this.protocolFilter);
string resourceTypeToSign = PathsHelper.GetResourcePath(resourceType);
headers.Set(HttpConstants.HttpHeaders.XDate, DateTime.UtcNow.ToString("r", CultureInfo.InvariantCulture));
using (ITrace trace = Trace.GetRootTrace(nameof(GetMasterAddressesViaGatewayAsync), TraceComponent.Authorization, TraceLevel.Info))
{
string token = await this.tokenProvider.GetUserAuthorizationTokenAsync(
resourceAddress,
resourceTypeToSign,
HttpConstants.HttpMethods.Get,
headers,
AuthorizationTokenType.PrimaryMasterKey,
trace);
headers.Set(HttpConstants.HttpHeaders.Authorization, token);
Uri targetEndpoint = UrlUtility.SetQuery(this.addressEndpoint, UrlUtility.CreateQuery(addressQuery));
string identifier = GatewayAddressCache.LogAddressResolutionStart(request, targetEndpoint);
using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync(
uri: targetEndpoint,
additionalHeaders: headers,
resourceType: resourceType,
timeoutPolicy: HttpTimeoutPolicyControlPlaneRetriableHotPath.Instance,
clientSideRequestStatistics: request.RequestContext?.ClientRequestStatistics,
cancellationToken: default))
public async Task ValidateRetryOnWriteForbiddenExceptionAsync()
{
this.Initialize(
useMultipleWriteLocations: false,
enableEndpointDiscovery: true,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, true, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false))
{
int retryCount = 0;
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryCount++;
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 1)
{
this.mockedClient.ResetCalls();
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.WriteForbidden).ToString();
DocumentClientException forbiddenException = new ForbiddenException(RMResources.Forbidden, headers);
throw forbiddenException;
}
else if (retryCount == 2)
{
this.mockedClient.Verify(client => client.GetDatabaseAccountInternalAsync(It.IsAny <Uri>(), It.IsAny <CancellationToken>()), Times.Once);
// Next request must go to next preferred endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
return(Task.FromResult(true));
}
else
{
Assert.Fail();
}
return(Task.FromResult(true));
},
retryPolicy);
}
}
private async Task <FeedResource <Address> > GetMasterAddressesViaGatewayAsync(
DocumentServiceRequest request,
ResourceType resourceType,
string resourceAddress,
string entryUrl,
bool forceRefresh,
bool useMasterCollectionResolver)
{
INameValueCollection addressQuery = new StoreRequestNameValueCollection
{
{ HttpConstants.QueryStrings.Url, HttpUtility.UrlEncode(entryUrl) }
};
INameValueCollection headers = new StoreRequestNameValueCollection();
if (forceRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceRefresh, bool.TrueString);
}
if (useMasterCollectionResolver)
{
headers.Set(HttpConstants.HttpHeaders.UseMasterCollectionResolver, bool.TrueString);
}
if (request.ForceCollectionRoutingMapRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceCollectionRoutingMapRefresh, bool.TrueString);
}
addressQuery.Add(HttpConstants.QueryStrings.Filter, this.protocolFilter);
string resourceTypeToSign = PathsHelper.GetResourcePath(resourceType);
headers.Set(HttpConstants.HttpHeaders.XDate, DateTime.UtcNow.ToString("r", CultureInfo.InvariantCulture));
(string token, string _) = await this.tokenProvider.GetUserAuthorizationAsync(
resourceAddress,
resourceTypeToSign,
HttpConstants.HttpMethods.Get,
headers,
AuthorizationTokenType.PrimaryMasterKey);
headers.Set(HttpConstants.HttpHeaders.Authorization, token);
Uri targetEndpoint = UrlUtility.SetQuery(this.addressEndpoint, UrlUtility.CreateQuery(addressQuery));
string identifier = GatewayAddressCache.LogAddressResolutionStart(request, targetEndpoint);
using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync(
uri: targetEndpoint,
additionalHeaders: headers,
resourceType: resourceType,
diagnosticsContext: null,
cancellationToken: default))
private void ValidateLazyHeadersAreNotCreated(CosmosMessageHeadersInternal internalHeaders)
{
StoreRequestNameValueCollection storeRequestHeaders = (StoreRequestNameValueCollection)internalHeaders;
FieldInfo lazyHeaders = typeof(StoreRequestNameValueCollection).GetField("lazyNotCommonHeaders", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic);
Lazy <Dictionary <string, string> > lazyNotCommonHeaders = (Lazy <Dictionary <string, string> >)lazyHeaders.GetValue(storeRequestHeaders);
// Use the if instead of Assert.IsFalse to avoid creating the dictionary in the error message
if (lazyNotCommonHeaders.IsValueCreated)
{
Assert.Fail($"The lazy dictionary should not be created. Please add the following headers to the {nameof(StoreRequestNameValueCollection)}: {JsonConvert.SerializeObject(lazyNotCommonHeaders.Value)}");
}
}
public async Task AddPartitionKeyRangeToContinuationTokenOnSplit()
{
const string BackendToken = "backendToken";
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
List <CompositeContinuationToken> compositeContinuationTokensFromSplit = new List <CompositeContinuationToken>
{
new CompositeContinuationToken {
Token = "someToken", Range = new Range <string>("A", "B", true, false)
},
new CompositeContinuationToken {
Token = "anotherToken", Range = new Range <string>("B", "C", true, false)
}
};
PartitionRoutingHelper partitionRoutingHelper = new PartitionRoutingHelper();
//With backend header
headers.Add(HttpConstants.HttpHeaders.Continuation, BackendToken);
ResolvedRangeInfo resolvedRangeInfo = new ResolvedRangeInfo(new PartitionKeyRange(), new List <CompositeContinuationToken>(compositeContinuationTokensFromSplit));
bool result = await partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
headers,
null,
null,
null,
resolvedRangeInfo,
NoOpTrace.Singleton,
RntdbEnumerationDirection.Reverse);
List <CompositeContinuationToken> compositeContinuationTokens = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(headers.Get(HttpConstants.HttpHeaders.Continuation));
Assert.IsTrue(result);
Assert.AreEqual(compositeContinuationTokensFromSplit.Count, compositeContinuationTokens.Count);
Assert.AreEqual(BackendToken, compositeContinuationTokens.First().Token);
Assert.AreNotEqual(BackendToken, compositeContinuationTokens.Last().Token);
//Without backend header
headers.Remove(HttpConstants.HttpHeaders.Continuation);
resolvedRangeInfo = new ResolvedRangeInfo(new PartitionKeyRange(), new List <CompositeContinuationToken>(compositeContinuationTokensFromSplit));
result = await partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
headers,
null,
null,
null,
resolvedRangeInfo,
NoOpTrace.Singleton,
RntdbEnumerationDirection.Reverse);
compositeContinuationTokens = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(headers.Get(HttpConstants.HttpHeaders.Continuation));
Assert.IsTrue(result);
Assert.IsTrue(compositeContinuationTokens.Count == compositeContinuationTokensFromSplit.Count - 1);
Assert.AreEqual(compositeContinuationTokensFromSplit.Last().Token, compositeContinuationTokens.First().Token);
}
public void StoreRequestHeadersPointRead()
{
_ = new StoreRequestNameValueCollection
{
{ HttpConstants.HttpHeaders.Authorization, AuthValue },
{ HttpConstants.HttpHeaders.XDate, Date },
{ HttpConstants.HttpHeaders.ClientRetryAttemptCount, Retry },
{ HttpConstants.HttpHeaders.PartitionKey, Partitionkey },
{ HttpConstants.HttpHeaders.RemainingTimeInMsOnClientRequest, Remaining },
{ HttpConstants.HttpHeaders.TransportRequestID, Transport },
{ WFConstants.BackendHeaders.CollectionRid, Rid }
};
}
public async Task GatewaySameSessionTokenTest()
{
string createSessionToken = null;
GatewaySessionTokenTests.HttpClientHandlerHelper httpClientHandler = new HttpClientHandlerHelper
{
ResponseCallBack = (result) =>
{
HttpResponseMessage response = result.Result;
if (response.StatusCode != HttpStatusCode.Created)
{
return(response);
}
response.Headers.TryGetValues("x-ms-session-token", out IEnumerable <string> sessionTokens);
foreach (string singleToken in sessionTokens)
{
createSessionToken = singleToken;
break;
}
return(response);
}
};
using (CosmosClient client = TestCommon.CreateCosmosClient(builder => builder
.WithConnectionModeGateway()
.WithConsistencyLevel(Cosmos.ConsistencyLevel.Session)
.WithHttpClientFactory(() => new HttpClient(httpClientHandler))))
{
Container container = client.GetContainer(this.database.Id, this.Container.Id);
ToDoActivity item = ToDoActivity.CreateRandomToDoActivity("Status1001", "1001");
ItemResponse <ToDoActivity> itemResponse = await container.CreateItemAsync(item);
// Read back the created Item and check if the session token is identical.
string docLink = "dbs/" + this.database.Id + "/colls/" + this.Container.Id + "/docs/1001";
Documents.Collections.INameValueCollection headers = new StoreRequestNameValueCollection();
headers.Set(HttpConstants.HttpHeaders.PartitionKey, "[\"Status1001\"]");
DocumentServiceRequest request = DocumentServiceRequest.Create(OperationType.Read, ResourceType.Document, docLink, AuthorizationTokenType.PrimaryMasterKey, headers);
await GatewayStoreModel.ApplySessionTokenAsync(request,
Cosmos.ConsistencyLevel.Session,
client.DocumentClient.sessionContainer,
await client.DocumentClient.GetPartitionKeyRangeCacheAsync(NoOpTrace.Singleton),
await client.DocumentClient.GetCollectionCacheAsync(NoOpTrace.Singleton));
string readSessionToken = request.Headers[HttpConstants.HttpHeaders.SessionToken];
Assert.AreEqual(readSessionToken, createSessionToken);
}
}
private async Task ValidateRetryOnSessionNotAvailabeWithEndpointDiscoveryDisabled(bool isPreferredLocationsListEmpty, bool useMultipleWriteLocations, bool isReadRequest)
{
const bool enableEndpointDiscovery = false;
this.Initialize(
useMultipleWriteLocations: useMultipleWriteLocations,
enableEndpointDiscovery: enableEndpointDiscovery,
isPreferredLocationsListEmpty: isPreferredLocationsListEmpty);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 0)
{
Assert.AreEqual(request.RequestContext.LocationEndpointToRoute, this.endpointManager.ReadEndpoints[0]);
}
else
{
Assert.Fail();
}
retryCount++;
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString();
DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers);
throw notFoundException;
},
retryPolicy);
Assert.Fail();
}
catch (NotFoundException)
{
DefaultTrace.TraceInformation("Received expected notFoundException");
Assert.AreEqual(1, retryCount);
}
}
}
[Ignore] /* TODO: There is a TODO in PartitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken that it's refering to some pending deployment */
public void TestExtractPartitionKeyRangeFromHeaders()
{
Func <string, INameValueCollection> getHeadersWithContinuation = (string continuationToken) =>
{
INameValueCollection headers = new StoreRequestNameValueCollection();
headers[HttpConstants.HttpHeaders.Continuation] = continuationToken;
return(headers);
};
using (Stream stream = new MemoryStream(Properties.Resources.BaselineTest_PartitionRoutingHelper_ExtractPartitionKeyRangeFromHeaders))
{
using (StreamReader reader = new StreamReader(stream))
{
TestSet <ExtractPartitionKeyRangeFromHeadersTestData> testSet = JsonConvert.DeserializeObject <TestSet <ExtractPartitionKeyRangeFromHeadersTestData> >(reader.ReadToEnd());
foreach (ExtractPartitionKeyRangeFromHeadersTestData testData in testSet.Postive)
{
INameValueCollection headers = getHeadersWithContinuation(testData.CompositeContinuationToken);
List <CompositeContinuationToken> suppliedTokens;
Range <string> range = this.partitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken(headers, out suppliedTokens);
if (suppliedTokens != null)
{
Assert.AreEqual(testData.ContinuationToken, headers[HttpConstants.HttpHeaders.Continuation]);
Assert.AreEqual(JsonConvert.SerializeObject(testData.PartitionKeyRange), JsonConvert.SerializeObject(range));
}
else
{
Assert.IsTrue(testData.ContinuationToken == headers[HttpConstants.HttpHeaders.Continuation] || testData.ContinuationToken == null);
}
}
foreach (ExtractPartitionKeyRangeFromHeadersTestData testData in testSet.Negative)
{
INameValueCollection headers = getHeadersWithContinuation(testData.CompositeContinuationToken);
try
{
List <CompositeContinuationToken> suppliedTokens;
Range <string> rangeOrId = this.partitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken(headers, out suppliedTokens);
Assert.Fail("Expect BadRequestException");
}
catch (BadRequestException)
{
}
}
}
}
}
public void AuthorizationBaselineTests()
{
string key = "VGhpcyBpcyBhIHNhbXBsZSBzdHJpbmc=";
for (int i = 0; i < this.AuthorizationBaseline.Length; i++)
{
string[] baseline = this.AuthorizationBaseline[i];
string[] baselineResults = this.AuthorizationBaselineResults[i];
StoreRequestNameValueCollection nvc = new StoreRequestNameValueCollection();
nvc.Add(HttpConstants.HttpHeaders.XDate, baseline[4]);
Uri uri = new Uri(baseline[0]);
string authorization = AuthorizationHelper.GenerateKeyAuthorizationSignature(
verb: baseline[2],
uri: uri,
headers: nvc,
stringHMACSHA256Helper: new StringHMACSHA256Hash(key));
string authorization2 = AuthorizationHelper.GenerateKeyAuthorizationSignature(
verb: baseline[2],
resourceId: baseline[1],
resourceType: baseline[3],
headers: nvc,
new StringHMACSHA256Hash(key),
out string payload2);
string authorization3 = AuthorizationHelper.GenerateKeyAuthorizationSignature(
verb: baseline[2],
resourceId: baseline[1],
resourceType: baseline[3],
headers: nvc,
key);
Assert.AreEqual(authorization, baselineResults[0]);
Assert.AreEqual(authorization2, baselineResults[0]);
Assert.AreEqual(authorization3, baselineResults[0]);
Assert.AreEqual(payload2.Replace("\n", "[n]"), baselineResults[1]);
AuthorizationHelper.ParseAuthorizationToken(authorization, out ReadOnlyMemory <char> typeOutput1, out ReadOnlyMemory <char> versionoutput1, out ReadOnlyMemory <char> tokenOutput1);
Assert.AreEqual("master", typeOutput1.ToString());
AuthorizationHelper.ParseAuthorizationToken(authorization2, out ReadOnlyMemory <char> typeOutput2, out ReadOnlyMemory <char> versionoutput2, out ReadOnlyMemory <char> tokenOutput2);
Assert.AreEqual("master", typeOutput2.ToString());
AuthorizationHelper.ParseAuthorizationToken(authorization3, out ReadOnlyMemory <char> typeOutput3, out ReadOnlyMemory <char> versionoutput3, out ReadOnlyMemory <char> tokenOutput3);
Assert.AreEqual("master", typeOutput3.ToString());
Assert.IsTrue(AuthorizationHelper.CheckPayloadUsingKey(tokenOutput1, baseline[2], baseline[1], baseline[3], nvc, key));
Assert.IsTrue(AuthorizationHelper.CheckPayloadUsingKey(tokenOutput2, baseline[2], baseline[1], baseline[3], nvc, key));
Assert.IsTrue(AuthorizationHelper.CheckPayloadUsingKey(tokenOutput3, baseline[2], baseline[1], baseline[3], nvc, key));
}
}
public static async Task <ICollection <T> > ListAllAsync <T>(this HttpClient client,
Uri collectionUri,
INameValueCollection headers = null) where T : Resource, new()
{
Collection <T> responseCollection = new Collection <T>();
string responseContinuation = null;
if (headers == null)
{
headers = new StoreRequestNameValueCollection();
}
do
{
if (responseContinuation != null)
{
headers[HttpConstants.HttpHeaders.Continuation] = responseContinuation;
}
HttpResponseMessage responseMessage;
foreach (string header in headers.AllKeys())
{
client.DefaultRequestHeaders.Add(header, headers[header]);
}
responseMessage = await client.GetAsync(collectionUri, HttpCompletionOption.ResponseHeadersRead);
FeedResource <T> feedResource = await responseMessage.ToResourceAsync <FeedResource <T> >();
foreach (T resource in feedResource)
{
responseCollection.Add(resource);
}
if (responseMessage.Headers.TryGetValues(HttpConstants.HttpHeaders.Continuation,
out IEnumerable <string> continuationToken))
{
responseContinuation = continuationToken.SingleOrDefault();
}
else
{
responseContinuation = null;
}
} while (!string.IsNullOrEmpty(responseContinuation));
return(responseCollection);
}
public void CompositeContinuationTokenIsNotPassedToBackend()
{
Range <string> expectedRange = new Range <string>("A", "B", true, false);
string expectedToken = "someToken";
CompositeContinuationToken compositeContinuationToken = new CompositeContinuationToken {
Range = expectedRange, Token = expectedToken
};
string continuation = JsonConvert.SerializeObject(compositeContinuationToken);
PartitionRoutingHelper partitionRoutingHelper = new PartitionRoutingHelper();
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers.Add(HttpConstants.HttpHeaders.Continuation, continuation);
Range <string> range = partitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken(headers, out List <CompositeContinuationToken> compositeContinuationTokens);
Assert.IsTrue(expectedRange.Equals(range));
Assert.AreEqual(expectedToken, headers.Get(HttpConstants.HttpHeaders.Continuation)); //not a composite token
}
public void AuthorizationGenerateAndCheckKeyAuthSignature()
{
Random r = new Random();
byte[] hashKey = new byte[16];
r.NextBytes(hashKey);
string key = Convert.ToBase64String(hashKey);
foreach (string method in this.HttpMethods)
{
foreach (string resourceType in this.ResourceTypesArray)
{
foreach (string resourceName in this.ResourceNameValues)
{
StoreRequestNameValueCollection nvc = new StoreRequestNameValueCollection();
nvc.Add(HttpConstants.HttpHeaders.XDate, new DateTime(2020, 02, 01, 10, 00, 00).ToString("r"));
string authorizationKey = AuthorizationHelper.GenerateKeyAuthorizationSignature(
method,
resourceName,
resourceType,
nvc,
new StringHMACSHA256Hash(key),
out string payload);
AuthorizationHelper.ParseAuthorizationToken(authorizationKey,
out ReadOnlyMemory <char> typeOutput,
out ReadOnlyMemory <char> versionOutput,
out ReadOnlyMemory <char> tokenOutput);
Assert.AreEqual("master", typeOutput.ToString());
Assert.IsTrue(AuthorizationHelper.CheckPayloadUsingKey(
tokenOutput,
method,
resourceName,
resourceType,
nvc,
key));
}
}
}
}
public static void ThrowForbiddendExceptionOnItemOperation(
Uri physicalAddress,
DocumentServiceRequest request,
string activityId,
string errorMessage)
{
if (request.ResourceType == ResourceType.Document)
{
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers.Add(HttpConstants.HttpHeaders.ActivityId, activityId.ToString());
headers.Add(WFConstants.BackendHeaders.SubStatus, ((int)SubStatusCodes.WriteForbidden).ToString(CultureInfo.InvariantCulture));
headers.Add(HttpConstants.HttpHeaders.RequestCharge, ((double)9001).ToString(CultureInfo.InvariantCulture));
ForbiddenException forbiddenException = new ForbiddenException(
errorMessage,
headers,
physicalAddress);
throw forbiddenException;
}
}
public void VerifyUnKnownHeader()
{
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
Assert.AreEqual(0, headers.Keys().Count());
string key = Guid.NewGuid().ToString();
string value = Guid.NewGuid().ToString();
headers[key] = value;
Assert.AreEqual(value, headers[key]);
Assert.AreEqual(value, headers[key.ToLower()]);
Assert.AreEqual(value, headers[key.ToUpper()]);
Assert.AreEqual(value, headers.Get(key));
Assert.AreEqual(value, headers.Get(key.ToLower()));
Assert.AreEqual(value, headers.Get(key.ToUpper()));
Assert.AreEqual(key, headers.Keys().First());
headers.Remove(key);
Assert.AreEqual(0, headers.Keys().Count());
Assert.IsNull(headers[key]);
}
public async Task TestGetPartitionRoutingInfo()
{
using (Stream stream = new MemoryStream(Properties.Resources.BaselineTest_PartitionRoutingHelper_GetPartitionRoutingInfo))
{
using (StreamReader reader = new StreamReader(stream))
{
GetPartitionRoutingInfoTestData testData = JsonConvert.DeserializeObject <GetPartitionRoutingInfoTestData>(reader.ReadToEnd());
CollectionRoutingMap routingMap =
CollectionRoutingMap.TryCreateCompleteRoutingMap(
testData.RoutingMap.Select(range => Tuple.Create(range, (ServiceIdentity)null)), string.Empty);
foreach (GetPartitionRoutingInfoTestCase testCase in testData.TestCases)
{
List <string> actualPartitionKeyRangeIds = new List <string>();
Range <string> startRange = Range <string> .GetEmptyRange(PartitionKeyInternal.MinimumInclusiveEffectivePartitionKey);
for (Range <string> currentRange = startRange; currentRange != null;)
{
RoutingMapProvider routingMapProvider = new RoutingMapProvider(routingMap);
PartitionRoutingHelper.ResolvedRangeInfo resolvedRangeInfo = await this.partitionRoutingHelper.TryGetTargetRangeFromContinuationTokenRangeAsync(testCase.ProvidedRanges, routingMapProvider, string.Empty, currentRange, null, NoOpTrace.Singleton);
actualPartitionKeyRangeIds.Add(resolvedRangeInfo.ResolvedRange.Id);
INameValueCollection headers = new StoreRequestNameValueCollection();
await this.partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(headers, testCase.ProvidedRanges, routingMapProvider, string.Empty, resolvedRangeInfo, NoOpTrace.Singleton);
List <CompositeContinuationToken> suppliedTokens;
Range <string> nextRange = this.partitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken(headers, out suppliedTokens);
currentRange = nextRange.IsEmpty ? null : nextRange;
}
Assert.AreEqual(string.Join(", ", testCase.RoutingRangeIds), string.Join(", ", actualPartitionKeyRangeIds));
}
}
}
}
private void AddFormattedContinuationHeaderHelper(AddFormattedContinuationToHeaderTestUnit positiveTestData, out INameValueCollection headers, out List <PartitionKeyRange> resolvedRanges, out List <CompositeContinuationToken> resolvedContinuationTokens)
{
Func <string, INameValueCollection> getHeadersWithContinuation = (string continuationToken) =>
{
INameValueCollection localHeaders = new StoreRequestNameValueCollection();
if (continuationToken != null)
{
localHeaders[HttpConstants.HttpHeaders.Continuation] = continuationToken;
}
return(localHeaders);
};
resolvedRanges = positiveTestData.ResolvedRanges.Select(x => new PartitionKeyRange()
{
MinInclusive = x.Min, MaxExclusive = x.Max
}).ToList();
resolvedContinuationTokens = new List <CompositeContinuationToken>();
CompositeContinuationToken[] initialContinuationTokens = null;
if (!string.IsNullOrEmpty(positiveTestData.InputCompositeContinuationToken))
{
if (positiveTestData.InputCompositeContinuationToken.Trim().StartsWith("[", StringComparison.Ordinal))
{
initialContinuationTokens = JsonConvert.DeserializeObject <CompositeContinuationToken[]>(positiveTestData.InputCompositeContinuationToken);
}
else
{
initialContinuationTokens = new CompositeContinuationToken[] { JsonConvert.DeserializeObject <CompositeContinuationToken>(positiveTestData.InputCompositeContinuationToken) };
}
}
if (resolvedRanges.Count > 1)
{
CompositeContinuationToken continuationToBeCopied;
if (initialContinuationTokens != null && initialContinuationTokens.Length > 0)
{
continuationToBeCopied = (CompositeContinuationToken)initialContinuationTokens[0].ShallowCopy();
}
else
{
continuationToBeCopied = new CompositeContinuationToken();
continuationToBeCopied.Token = string.Empty;
}
headers = getHeadersWithContinuation(continuationToBeCopied.Token);
foreach (PartitionKeyRange pkrange in resolvedRanges)
{
CompositeContinuationToken token = (CompositeContinuationToken)continuationToBeCopied.ShallowCopy();
token.Range = pkrange.ToRange();
resolvedContinuationTokens.Add(token);
}
if (initialContinuationTokens != null)
{
resolvedContinuationTokens.AddRange(initialContinuationTokens.Skip(1));
}
}
else
{
headers = getHeadersWithContinuation(null);
}
}
internal static void ValidateQuery <T>(DocumentClient client, string collectionLink, string queryProperty, string queryPropertyValue, int expectedCount, INameValueCollection headers = null)
where T : Resource, new()
{
INameValueCollection inputHeaders = headers;
headers = new StoreRequestNameValueCollection();
if (inputHeaders != null)
{
headers.Add(inputHeaders); // dont mess with the input headers
}
int maxTries = 5;
const int minIndexInterval = 5000; // 5 seconds
while (maxTries-- > 0)
{
DocumentFeedResponse <dynamic> resourceFeed = null;
IDocumentQuery <dynamic> queryService = null;
string queryString = @"select * from root r where r." + queryProperty + @"=""" + queryPropertyValue + @"""";
if (typeof(T) == typeof(Database))
{
queryService = client.CreateDatabaseQuery(queryString).AsDocumentQuery();
}
else if (typeof(T) == typeof(DocumentCollection))
{
queryService = client.CreateDocumentCollectionQuery(collectionLink, queryString).AsDocumentQuery();
}
else if (typeof(T) == typeof(Document))
{
queryService = client.CreateDocumentQuery(collectionLink, queryString).AsDocumentQuery();
}
else
{
Assert.Fail("Unexpected type");
}
while (queryService.HasMoreResults)
{
resourceFeed = queryService.ExecuteNextAsync().Result;
if (resourceFeed.Count > 0)
{
Assert.IsNotNull(resourceFeed, "Query result is null");
Assert.AreNotEqual(0, resourceFeed.Count, "Query result is invalid");
foreach (T resource in resourceFeed)
{
if (queryProperty.Equals("name", StringComparison.CurrentCultureIgnoreCase))
{
Assert.AreEqual(resource.Id, queryPropertyValue, "Result contain invalid result");
}
}
return;
}
}
Task.Delay(minIndexInterval);
}
Assert.Fail("Query did not return result after max tries");
}
private async Task ValidateRetryOnDatabaseAccountNotFoundAsync(bool enableMultipleWriteLocations, bool isReadRequest)
{
this.Initialize(
useMultipleWriteLocations: enableMultipleWriteLocations,
enableEndpointDiscovery: true,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, true, new RetryOptions());
int expectedRetryCount = isReadRequest || enableMultipleWriteLocations ? 2 : 1;
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: isReadRequest, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryCount++;
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 1)
{
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.DatabaseAccountNotFound).ToString();
DocumentClientException forbiddenException = new ForbiddenException(RMResources.NotFound, headers);
throw forbiddenException;
}
else if (retryCount == 2)
{
// Next request must go to next preferred endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
return(Task.FromResult(true));
}
else
{
Assert.Fail();
}
return(Task.FromResult(true));
},
retryPolicy);
}
catch (ForbiddenException)
{
if (expectedRetryCount == 1)
{
DefaultTrace.TraceInformation("Received expected ForbiddenException");
}
else
{
Assert.Fail();
}
}
Assert.AreEqual(expectedRetryCount, retryCount);
}
}
private async Task ValidateRetryOnWriteSessionNotAvailabeWithEnableMultipleWriteLocationsAsync()
{
const bool useMultipleWriteLocations = true;
bool enableEndpointDiscovery = true;
this.Initialize(
useMultipleWriteLocations: useMultipleWriteLocations,
enableEndpointDiscovery: enableEndpointDiscovery,
isPreferredLocationsListEmpty: false);
await this.endpointManager.RefreshLocationAsync(this.databaseAccount);
ClientRetryPolicy retryPolicy = new ClientRetryPolicy(this.endpointManager, enableEndpointDiscovery, new RetryOptions());
using (DocumentServiceRequest request = this.CreateRequest(isReadRequest: false, isMasterResourceType: false))
{
int retryCount = 0;
try
{
await BackoffRetryUtility <bool> .ExecuteAsync(
() =>
{
retryPolicy.OnBeforeSendRequest(request);
if (retryCount == 0)
{
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[0]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 1)
{
// Second request must go to first write endpoint
Uri expectedEndpoint = new Uri(this.databaseAccount.WriteLocationsInternal[0].Endpoint);
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 2)
{
// Second request must go to first write endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[1]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else if (retryCount == 3)
{
// Second request must go to first write endpoint
Uri expectedEndpoint = LocationCacheTests.EndpointByLocation[this.preferredLocations[2]];
Assert.AreEqual(expectedEndpoint, request.RequestContext.LocationEndpointToRoute);
}
else
{
Assert.Fail();
}
retryCount++;
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers[WFConstants.BackendHeaders.SubStatus] = ((int)SubStatusCodes.ReadSessionNotAvailable).ToString();
DocumentClientException notFoundException = new NotFoundException(RMResources.NotFound, headers);
throw notFoundException;
},
retryPolicy);
Assert.Fail();
}
catch (NotFoundException)
{
DefaultTrace.TraceInformation("Received expected notFoundException");
Assert.AreEqual(4, retryCount);
}
}
}
public async Task <INameValueCollection> CreateCommonHeadersAsync(FeedOptions feedOptions)
{
INameValueCollection requestHeaders = new StoreRequestNameValueCollection();
Cosmos.ConsistencyLevel defaultConsistencyLevel = (Cosmos.ConsistencyLevel) await this.Client.GetDefaultConsistencyLevelAsync();
Cosmos.ConsistencyLevel?desiredConsistencyLevel = (Cosmos.ConsistencyLevel?) await this.Client.GetDesiredConsistencyLevelAsync();
if (!string.IsNullOrEmpty(feedOptions.SessionToken) && !ReplicatedResourceClient.IsReadingFromMaster(this.ResourceTypeEnum, OperationType.ReadFeed))
{
if (defaultConsistencyLevel == Cosmos.ConsistencyLevel.Session || (desiredConsistencyLevel.HasValue && desiredConsistencyLevel.Value == Cosmos.ConsistencyLevel.Session))
{
// Query across partitions is not supported today. Master resources (for e.g., database)
// can span across partitions, whereas server resources (viz: collection, document and attachment)
// don't span across partitions. Hence, session token returned by one partition should not be used
// when quering resources from another partition.
// Since master resources can span across partitions, don't send session token to the backend.
// As master resources are sync replicated, we should always get consistent query result for master resources,
// irrespective of the chosen replica.
// For server resources, which don't span partitions, specify the session token
// for correct replica to be chosen for servicing the query result.
requestHeaders[HttpConstants.HttpHeaders.SessionToken] = feedOptions.SessionToken;
}
}
requestHeaders[HttpConstants.HttpHeaders.Continuation] = feedOptions.RequestContinuationToken;
requestHeaders[HttpConstants.HttpHeaders.IsQuery] = bool.TrueString;
// Flow the pageSize only when we are not doing client eval
if (feedOptions.MaxItemCount.HasValue)
{
requestHeaders[HttpConstants.HttpHeaders.PageSize] = feedOptions.MaxItemCount.ToString();
}
requestHeaders[HttpConstants.HttpHeaders.EnableCrossPartitionQuery] = feedOptions.EnableCrossPartitionQuery.ToString();
if (feedOptions.MaxDegreeOfParallelism != 0)
{
requestHeaders[HttpConstants.HttpHeaders.ParallelizeCrossPartitionQuery] = bool.TrueString;
}
if (this.feedOptions.EnableScanInQuery != null)
{
requestHeaders[HttpConstants.HttpHeaders.EnableScanInQuery] = this.feedOptions.EnableScanInQuery.ToString();
}
if (this.feedOptions.EmitVerboseTracesInQuery != null)
{
requestHeaders[HttpConstants.HttpHeaders.EmitVerboseTracesInQuery] = this.feedOptions.EmitVerboseTracesInQuery.ToString();
}
if (this.feedOptions.EnableLowPrecisionOrderBy != null)
{
requestHeaders[HttpConstants.HttpHeaders.EnableLowPrecisionOrderBy] = this.feedOptions.EnableLowPrecisionOrderBy.ToString();
}
if (!string.IsNullOrEmpty(this.feedOptions.FilterBySchemaResourceId))
{
requestHeaders[HttpConstants.HttpHeaders.FilterBySchemaResourceId] = this.feedOptions.FilterBySchemaResourceId;
}
if (this.feedOptions.ResponseContinuationTokenLimitInKb != null)
{
requestHeaders[HttpConstants.HttpHeaders.ResponseContinuationTokenLimitInKB] = this.feedOptions.ResponseContinuationTokenLimitInKb.ToString();
}
if (this.feedOptions.ConsistencyLevel.HasValue)
{
await this.Client.EnsureValidOverwriteAsync((Documents.ConsistencyLevel) feedOptions.ConsistencyLevel.Value);
requestHeaders.Set(HttpConstants.HttpHeaders.ConsistencyLevel, this.feedOptions.ConsistencyLevel.Value.ToString());
}
else if (desiredConsistencyLevel.HasValue)
{
requestHeaders.Set(HttpConstants.HttpHeaders.ConsistencyLevel, desiredConsistencyLevel.Value.ToString());
}
if (this.feedOptions.EnumerationDirection.HasValue)
{
requestHeaders.Set(HttpConstants.HttpHeaders.EnumerationDirection, this.feedOptions.EnumerationDirection.Value.ToString());
}
if (this.feedOptions.ReadFeedKeyType.HasValue)
{
requestHeaders.Set(HttpConstants.HttpHeaders.ReadFeedKeyType, this.feedOptions.ReadFeedKeyType.Value.ToString());
}
if (this.feedOptions.StartId != null)
{
requestHeaders.Set(HttpConstants.HttpHeaders.StartId, this.feedOptions.StartId);
}
if (this.feedOptions.EndId != null)
{
requestHeaders.Set(HttpConstants.HttpHeaders.EndId, this.feedOptions.EndId);
}
if (this.feedOptions.StartEpk != null)
{
requestHeaders.Set(HttpConstants.HttpHeaders.StartEpk, this.feedOptions.StartEpk);
}
if (this.feedOptions.EndEpk != null)
{
requestHeaders.Set(HttpConstants.HttpHeaders.EndEpk, this.feedOptions.EndEpk);
}
if (this.feedOptions.PopulateQueryMetrics)
{
requestHeaders[HttpConstants.HttpHeaders.PopulateQueryMetrics] = bool.TrueString;
}
if (this.feedOptions.ForceQueryScan)
{
requestHeaders[HttpConstants.HttpHeaders.ForceQueryScan] = bool.TrueString;
}
if (this.feedOptions.MergeStaticId != null)
{
requestHeaders.Set(HttpConstants.HttpHeaders.MergeStaticId, this.feedOptions.MergeStaticId);
}
if (this.feedOptions.CosmosSerializationFormatOptions != null)
{
requestHeaders[HttpConstants.HttpHeaders.ContentSerializationFormat] = this.feedOptions.CosmosSerializationFormatOptions.ContentSerializationFormat;
}
else if (this.feedOptions.ContentSerializationFormat.HasValue)
{
requestHeaders[HttpConstants.HttpHeaders.ContentSerializationFormat] = this.feedOptions.ContentSerializationFormat.Value.ToString();
}
return(requestHeaders);
}
public async Task AddPartitionKeyRangeToContinuationTokenOnBoundry()
{
List <Range <string> > providedRanges = new List <Range <string> > {
new Range <string>(
"A",
"D",
isMinInclusive: true,
isMaxInclusive: false)
};
//Reverse
ResolvedRangeInfo currentPartitionKeyRange = new ResolvedRangeInfo(new PartitionKeyRange {
Id = "0", MinInclusive = "A", MaxExclusive = "B"
}, null);
IReadOnlyList <PartitionKeyRange> overlappingRanges = new List <PartitionKeyRange> {
new PartitionKeyRange {
Id = "0", MinInclusive = "A", MaxExclusive = "B"
},
}.AsReadOnly();
Mock <IRoutingMapProvider> routingMapProvider = new Mock <IRoutingMapProvider>();
routingMapProvider.Setup(m => m.TryGetOverlappingRangesAsync(
It.IsAny <string>(),
It.Is <Range <string> >(x => x.Min == providedRanges.Single().Min&& x.Max == providedRanges.Single().Max),
It.IsAny <ITrace>(),
It.Is <bool>(x => x == false)
)).Returns(Task.FromResult(overlappingRanges)).Verifiable();
PartitionRoutingHelper partitionRoutingHelper = new PartitionRoutingHelper();
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
bool result = await partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
headers,
providedRanges,
routingMapProvider.Object,
CollectionId,
currentPartitionKeyRange,
NoOpTrace.Singleton,
RntdbEnumerationDirection.Reverse
);
Assert.IsTrue(result);
routingMapProvider.Verify();
string expectedContinuationToken = JsonConvert.SerializeObject(new CompositeContinuationToken
{
Token = null,
Range = overlappingRanges.First().ToRange(),
});
Assert.IsNull(headers.Get(HttpConstants.HttpHeaders.Continuation));
//Forward
currentPartitionKeyRange = new ResolvedRangeInfo(new PartitionKeyRange {
Id = "0", MinInclusive = "A", MaxExclusive = "D"
}, null);
overlappingRanges = new List <PartitionKeyRange> {
new PartitionKeyRange {
Id = "0", MinInclusive = "A", MaxExclusive = "D"
},
}.AsReadOnly();
routingMapProvider.Setup(m => m.TryGetOverlappingRangesAsync(
It.IsAny <string>(),
It.IsAny <Range <string> >(),
It.IsAny <ITrace>(),
It.IsAny <bool>()
)).Returns(Task.FromResult(overlappingRanges));
headers = new StoreRequestNameValueCollection();
result = await partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
headers,
providedRanges,
routingMapProvider.Object,
CollectionId,
currentPartitionKeyRange,
NoOpTrace.Singleton,
RntdbEnumerationDirection.Forward
);
Assert.IsTrue(result);
routingMapProvider.Verify(m => m.TryGetOverlappingRangesAsync(
It.IsAny <string>(),
It.Is <Range <string> >(e => e.IsMaxInclusive),
It.IsAny <ITrace>(),
It.IsAny <bool>()
), Times.Never);
expectedContinuationToken = JsonConvert.SerializeObject(new CompositeContinuationToken
{
Token = null,
Range = overlappingRanges.Last().ToRange(),
});
Assert.IsNull(headers.Get(HttpConstants.HttpHeaders.Continuation));
}
public async Task QueryRequestRateTest(bool directMode)
{
string firstItemIdAndPk = "BasicQueryItem" + Guid.NewGuid();
// Prevent the test from changing the static client
{
CosmosClient client = directMode ? DirectCosmosClient : GatewayCosmosClient;
Container container = client.GetContainer(DatabaseId, ContainerId);
List <string> createdIds = new List <string>()
{
firstItemIdAndPk,
"BasicQueryItem2" + Guid.NewGuid(),
"BasicQueryItem3" + Guid.NewGuid()
};
foreach (string id in createdIds)
{
dynamic item = new
{
id = id,
pk = id,
};
await container.CreateItemAsync <dynamic>(item : item);
}
}
CosmosClient clientWithThrottle;
if (directMode)
{
clientWithThrottle = TestCommon.CreateCosmosClient();
}
else
{
clientWithThrottle = TestCommon.CreateCosmosClient((builder) => builder.WithConnectionModeGateway());
}
Container containerWithThrottle = clientWithThrottle.GetContainer(DatabaseId, ContainerId);
// Do a read to warm up all the caches to prevent them from getting the throttle errors
using (await containerWithThrottle.ReadItemStreamAsync(firstItemIdAndPk, new PartitionKey(firstItemIdAndPk))) { }
Documents.IStoreModel storeModel = clientWithThrottle.ClientContext.DocumentClient.StoreModel;
Mock <Documents.IStoreModel> mockStore = new Mock <Documents.IStoreModel>();
clientWithThrottle.ClientContext.DocumentClient.StoreModel = mockStore.Object;
// Cause 429 after the first call
int callCount = 0;
string activityId = null;
string errorMessage = "QueryRequestRateTest Resource Not Found";
mockStore.Setup(x => x.ProcessMessageAsync(It.IsAny <Documents.DocumentServiceRequest>(), It.IsAny <CancellationToken>()))
.Returns <Documents.DocumentServiceRequest, CancellationToken>((dsr, token) =>
{
callCount++;
if (callCount > 1)
{
INameValueCollection headers = new StoreRequestNameValueCollection();
headers.Add(Documents.HttpConstants.HttpHeaders.RetryAfterInMilliseconds, "42");
activityId = Guid.NewGuid().ToString();
headers.Add(Documents.HttpConstants.HttpHeaders.ActivityId, activityId);
Documents.DocumentServiceResponse response = new Documents.DocumentServiceResponse(
body: TestCommon.GenerateStreamFromString(@"{""Errors"":[""" + errorMessage + @"""]}"),
headers: headers,
statusCode: (HttpStatusCode)429,
clientSideRequestStatistics: dsr.RequestContext.ClientRequestStatistics);
return(Task.FromResult(response));
}
return(storeModel.ProcessMessageAsync(dsr, token));
});
List <dynamic> results = new List <dynamic>();
try
{
using (FeedIterator <dynamic> feedIterator = containerWithThrottle.GetItemQueryIterator <dynamic>(
"select * from T where STARTSWITH(T.id, \"BasicQueryItem\")",
requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1,
MaxConcurrency = 1
}))
{
while (feedIterator.HasMoreResults)
{
FeedResponse <dynamic> response = await feedIterator.ReadNextAsync();
Assert.IsTrue(response.Count <= 1);
Assert.IsTrue(response.Resource.Count() <= 1);
results.AddRange(response);
}
}
Assert.Fail("Should throw 429 exception after the first page.");
}
catch (CosmosException ce)
{
Assert.IsTrue(ce.RetryAfter.HasValue);
Assert.AreEqual(42, ce.RetryAfter.Value.TotalMilliseconds);
Assert.AreEqual(activityId, ce.ActivityId);
Assert.IsNotNull(ce.DiagnosticsContext);
Assert.IsTrue(ce.Message.Contains(errorMessage));
}
callCount = 0;
FeedIterator streamIterator = containerWithThrottle.GetItemQueryStreamIterator(
"select * from T where STARTSWITH(T.id, \"BasicQueryItem\")",
requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1,
MaxConcurrency = 1
});
// First request should be a success
using (ResponseMessage response = await streamIterator.ReadNextAsync())
{
response.EnsureSuccessStatusCode();
Assert.IsNotNull(response.Content);
}
// Second page should be a failure
using (ResponseMessage response = await streamIterator.ReadNextAsync())
{
Assert.AreEqual(429, (int)response.StatusCode);
Assert.AreEqual("42", response.Headers.RetryAfterLiteral);
Assert.AreEqual(activityId, response.Headers.ActivityId);
Assert.IsNotNull(response.DiagnosticsContext);
Assert.IsTrue(response.ErrorMessage.Contains(errorMessage));
}
}
private async Task <CollectionRoutingMap> GetRoutingMapForCollectionAsync(
string collectionRid,
CollectionRoutingMap previousRoutingMap,
ITrace trace,
IClientSideRequestStatistics clientSideRequestStatistics,
CancellationToken cancellationToken)
{
List <PartitionKeyRange> ranges = new List <PartitionKeyRange>();
string changeFeedNextIfNoneMatch = previousRoutingMap == null ? null : previousRoutingMap.ChangeFeedNextIfNoneMatch;
HttpStatusCode lastStatusCode = HttpStatusCode.OK;
do
{
INameValueCollection headers = new StoreRequestNameValueCollection();
headers.Set(HttpConstants.HttpHeaders.PageSize, PageSizeString);
headers.Set(HttpConstants.HttpHeaders.A_IM, HttpConstants.A_IMHeaderValues.IncrementalFeed);
if (changeFeedNextIfNoneMatch != null)
{
headers.Set(HttpConstants.HttpHeaders.IfNoneMatch, changeFeedNextIfNoneMatch);
}
RetryOptions retryOptions = new RetryOptions();
using (DocumentServiceResponse response = await BackoffRetryUtility <DocumentServiceResponse> .ExecuteAsync(
() => this.ExecutePartitionKeyRangeReadChangeFeedAsync(collectionRid, headers, trace, clientSideRequestStatistics),
new ResourceThrottleRetryPolicy(retryOptions.MaxRetryAttemptsOnThrottledRequests, retryOptions.MaxRetryWaitTimeInSeconds),
cancellationToken))
{
lastStatusCode = response.StatusCode;
changeFeedNextIfNoneMatch = response.Headers[HttpConstants.HttpHeaders.ETag];
FeedResource <PartitionKeyRange> feedResource = response.GetResource <FeedResource <PartitionKeyRange> >();
if (feedResource != null)
{
ranges.AddRange(feedResource);
}
}
}while (lastStatusCode != HttpStatusCode.NotModified);
IEnumerable <Tuple <PartitionKeyRange, ServiceIdentity> > tuples = ranges.Select(range => Tuple.Create(range, (ServiceIdentity)null));
CollectionRoutingMap routingMap;
if (previousRoutingMap == null)
{
// Splits could have happened during change feed query and we might have a mix of gone and new ranges.
HashSet <string> goneRanges = new HashSet <string>(ranges.SelectMany(range => range.Parents ?? Enumerable.Empty <string>()));
routingMap = CollectionRoutingMap.TryCreateCompleteRoutingMap(
tuples.Where(tuple => !goneRanges.Contains(tuple.Item1.Id)),
string.Empty,
changeFeedNextIfNoneMatch);
}
else
{
routingMap = previousRoutingMap.TryCombine(tuples, changeFeedNextIfNoneMatch);
}
if (routingMap == null)
{
// Range information either doesn't exist or is not complete.
throw new NotFoundException($"{DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture)}: GetRoutingMapForCollectionAsync(collectionRid: {collectionRid}), Range information either doesn't exist or is not complete.");
}
return(routingMap);
}
public void VerifyAllKnownProperties()
{
Dictionary <string, string> httpHeadersMap = typeof(HttpConstants.HttpHeaders).GetFields(BindingFlags.Public | BindingFlags.Static)
.ToDictionary(x => x.Name, x => (string)x.GetValue(null));
Dictionary <string, string> backendHeadersMap = typeof(WFConstants.BackendHeaders).GetFields(BindingFlags.Public | BindingFlags.Static)
.ToDictionary(x => x.Name, x => (string)x.GetValue(null));
PropertyInfo[] optimizedResponseHeaders = typeof(StoreRequestNameValueCollection).GetProperties(BindingFlags.Public | BindingFlags.Instance)
.Where(x => !string.Equals("Item", x.Name)).ToArray();
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
foreach (PropertyInfo propertyInfo in optimizedResponseHeaders)
{
Assert.AreEqual(0, headers.Count());
Assert.AreEqual(0, headers.Keys().Count());
// Test property first
string value = Guid.NewGuid().ToString();
propertyInfo.SetValue(headers, value);
Assert.AreEqual(value, propertyInfo.GetValue(headers));
if (!httpHeadersMap.TryGetValue(propertyInfo.Name, out string key))
{
if (!backendHeadersMap.TryGetValue(propertyInfo.Name, out key))
{
Assert.Fail($"The property name {propertyInfo.Name} should match a header constant name");
}
}
Assert.AreEqual(1, headers.Count());
Assert.AreEqual(1, headers.Keys().Count());
Assert.AreEqual(key, headers.Keys().First());
Assert.AreEqual(value, headers.Get(key));
Assert.AreEqual(value, headers.Get(key.ToUpper()));
Assert.AreEqual(value, headers.Get(key.ToLower()));
// Reset the value back to null
propertyInfo.SetValue(headers, null);
Assert.AreEqual(0, headers.Count());
Assert.AreEqual(0, headers.Keys().Count());
// Check adding via the interface sets the property correctly
headers.Add(key, value);
Assert.AreEqual(value, propertyInfo.GetValue(headers));
Assert.AreEqual(value, headers.Get(key));
Assert.AreEqual(1, headers.Count());
Assert.AreEqual(1, headers.Keys().Count());
Assert.AreEqual(key, headers.Keys().First());
Assert.AreEqual(value, headers.Get(key));
// Check setting via the interface sets the property correctly
value = Guid.NewGuid().ToString();
headers.Set(key, value);
Assert.AreEqual(value, propertyInfo.GetValue(headers));
Assert.AreEqual(value, headers.Get(key));
Assert.AreEqual(1, headers.Count());
Assert.AreEqual(1, headers.Keys().Count());
Assert.AreEqual(key, headers.Keys().First());
Assert.AreEqual(value, headers.Get(key));
// Check setting via the interface sets the property correctly
headers.Remove(key);
Assert.AreEqual(null, propertyInfo.GetValue(headers));
Assert.AreEqual(null, headers.Get(key));
Assert.AreEqual(0, headers.Count());
Assert.AreEqual(0, headers.Keys().Count());
}
}