public static StoreResponse ReturnThrottledStoreResponseOnItemOperation(
Uri physicalAddress,
ResourceOperation resourceOperation,
DocumentServiceRequest request,
Guid activityId,
string errorMessage)
{
if (request.ResourceType == ResourceType.Document)
{
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers.Add(HttpConstants.HttpHeaders.ActivityId, activityId.ToString());
headers.Add(WFConstants.BackendHeaders.SubStatus, ((int)SubStatusCodes.WriteForbidden).ToString(CultureInfo.InvariantCulture));
headers.Add(HttpConstants.HttpHeaders.RetryAfterInMilliseconds, TimeSpan.FromMilliseconds(100).TotalMilliseconds.ToString(CultureInfo.InvariantCulture));
headers.Add(HttpConstants.HttpHeaders.RequestCharge, ((double)9001).ToString(CultureInfo.InvariantCulture));
StoreResponse storeResponse = new StoreResponse()
{
Status = 429,
Headers = headers,
ResponseBody = new MemoryStream(Encoding.UTF8.GetBytes(errorMessage))
};
return(storeResponse);
}
return(null);
}
public async Task AddPartitionKeyRangeToContinuationTokenOnNotNullBackendContinuation()
{
ResolvedRangeInfo currentPartitionKeyRange = new ResolvedRangeInfo(new PartitionKeyRange {
Id = "1", MinInclusive = "B", MaxExclusive = "C"
}, null);
Mock <IRoutingMapProvider> routingMapProvider = new Mock <IRoutingMapProvider>();
routingMapProvider.Setup(m => m.TryGetOverlappingRangesAsync(
It.IsAny <string>(),
It.IsAny <Range <string> >(),
It.IsAny <bool>()
)).Returns(Task.FromResult <IReadOnlyList <PartitionKeyRange> >(null)).Verifiable();
PartitionRoutingHelper partitionRoutingHelper = new PartitionRoutingHelper();
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers.Add(HttpConstants.HttpHeaders.Continuation, "something");
bool result = await partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
headers,
null,
routingMapProvider.Object,
CollectionId,
currentPartitionKeyRange,
RntdbEnumerationDirection.Reverse
);
Assert.IsTrue(true);
routingMapProvider.Verify(m => m.TryGetOverlappingRangesAsync(
It.IsAny <string>(),
It.IsAny <Range <string> >(),
It.IsAny <bool>()
), Times.Never);
}
private async Task <DocumentServiceResponse> GetMasterAddressesViaGatewayAsync(
DocumentServiceRequest request,
ResourceType resourceType,
string resourceAddress,
string entryUrl,
bool forceRefresh,
bool useMasterCollectionResolver)
{
INameValueCollection addressQuery = new StoreRequestNameValueCollection
{
{ HttpConstants.QueryStrings.Url, HttpUtility.UrlEncode(entryUrl) }
};
INameValueCollection headers = new StoreRequestNameValueCollection();
if (forceRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceRefresh, bool.TrueString);
}
if (useMasterCollectionResolver)
{
headers.Set(HttpConstants.HttpHeaders.UseMasterCollectionResolver, bool.TrueString);
}
if (request.ForceCollectionRoutingMapRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceCollectionRoutingMapRefresh, bool.TrueString);
}
addressQuery.Add(HttpConstants.QueryStrings.Filter, this.protocolFilter);
string resourceTypeToSign = PathsHelper.GetResourcePath(resourceType);
headers.Set(HttpConstants.HttpHeaders.XDate, DateTime.UtcNow.ToString("r", CultureInfo.InvariantCulture));
using (ITrace trace = Trace.GetRootTrace(nameof(GetMasterAddressesViaGatewayAsync), TraceComponent.Authorization, TraceLevel.Info))
{
string token = await this.tokenProvider.GetUserAuthorizationTokenAsync(
resourceAddress,
resourceTypeToSign,
HttpConstants.HttpMethods.Get,
headers,
AuthorizationTokenType.PrimaryMasterKey,
trace);
headers.Set(HttpConstants.HttpHeaders.Authorization, token);
Uri targetEndpoint = UrlUtility.SetQuery(this.addressEndpoint, UrlUtility.CreateQuery(addressQuery));
string identifier = GatewayAddressCache.LogAddressResolutionStart(request, targetEndpoint);
using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync(
uri: targetEndpoint,
additionalHeaders: headers,
resourceType: resourceType,
timeoutPolicy: HttpTimeoutPolicyControlPlaneRetriableHotPath.Instance,
clientSideRequestStatistics: request.RequestContext?.ClientRequestStatistics,
cancellationToken: default))
private async Task <FeedResource <Address> > GetMasterAddressesViaGatewayAsync(
DocumentServiceRequest request,
ResourceType resourceType,
string resourceAddress,
string entryUrl,
bool forceRefresh,
bool useMasterCollectionResolver)
{
INameValueCollection addressQuery = new StoreRequestNameValueCollection
{
{ HttpConstants.QueryStrings.Url, HttpUtility.UrlEncode(entryUrl) }
};
INameValueCollection headers = new StoreRequestNameValueCollection();
if (forceRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceRefresh, bool.TrueString);
}
if (useMasterCollectionResolver)
{
headers.Set(HttpConstants.HttpHeaders.UseMasterCollectionResolver, bool.TrueString);
}
if (request.ForceCollectionRoutingMapRefresh)
{
headers.Set(HttpConstants.HttpHeaders.ForceCollectionRoutingMapRefresh, bool.TrueString);
}
addressQuery.Add(HttpConstants.QueryStrings.Filter, this.protocolFilter);
string resourceTypeToSign = PathsHelper.GetResourcePath(resourceType);
headers.Set(HttpConstants.HttpHeaders.XDate, DateTime.UtcNow.ToString("r", CultureInfo.InvariantCulture));
(string token, string _) = await this.tokenProvider.GetUserAuthorizationAsync(
resourceAddress,
resourceTypeToSign,
HttpConstants.HttpMethods.Get,
headers,
AuthorizationTokenType.PrimaryMasterKey);
headers.Set(HttpConstants.HttpHeaders.Authorization, token);
Uri targetEndpoint = UrlUtility.SetQuery(this.addressEndpoint, UrlUtility.CreateQuery(addressQuery));
string identifier = GatewayAddressCache.LogAddressResolutionStart(request, targetEndpoint);
using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync(
uri: targetEndpoint,
additionalHeaders: headers,
resourceType: resourceType,
diagnosticsContext: null,
cancellationToken: default))
public static void ThrowForbiddendExceptionOnItemOperation(
Uri physicalAddress,
DocumentServiceRequest request,
string activityId,
string errorMessage)
{
if (request.ResourceType == ResourceType.Document)
{
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers.Add(HttpConstants.HttpHeaders.ActivityId, activityId.ToString());
headers.Add(WFConstants.BackendHeaders.SubStatus, ((int)SubStatusCodes.WriteForbidden).ToString(CultureInfo.InvariantCulture));
headers.Add(HttpConstants.HttpHeaders.RequestCharge, ((double)9001).ToString(CultureInfo.InvariantCulture));
ForbiddenException forbiddenException = new ForbiddenException(
errorMessage,
headers,
physicalAddress);
throw forbiddenException;
}
}
public async Task AddPartitionKeyRangeToContinuationTokenOnSplit()
{
const string BackendToken = "backendToken";
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
List <CompositeContinuationToken> compositeContinuationTokensFromSplit = new List <CompositeContinuationToken>
{
new CompositeContinuationToken {
Token = "someToken", Range = new Range <string>("A", "B", true, false)
},
new CompositeContinuationToken {
Token = "anotherToken", Range = new Range <string>("B", "C", true, false)
}
};
PartitionRoutingHelper partitionRoutingHelper = new PartitionRoutingHelper();
//With backend header
headers.Add(HttpConstants.HttpHeaders.Continuation, BackendToken);
ResolvedRangeInfo resolvedRangeInfo = new ResolvedRangeInfo(new PartitionKeyRange(), new List <CompositeContinuationToken>(compositeContinuationTokensFromSplit));
bool result = await partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
headers,
null,
null,
null,
resolvedRangeInfo,
NoOpTrace.Singleton,
RntdbEnumerationDirection.Reverse);
List <CompositeContinuationToken> compositeContinuationTokens = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(headers.Get(HttpConstants.HttpHeaders.Continuation));
Assert.IsTrue(result);
Assert.AreEqual(compositeContinuationTokensFromSplit.Count, compositeContinuationTokens.Count);
Assert.AreEqual(BackendToken, compositeContinuationTokens.First().Token);
Assert.AreNotEqual(BackendToken, compositeContinuationTokens.Last().Token);
//Without backend header
headers.Remove(HttpConstants.HttpHeaders.Continuation);
resolvedRangeInfo = new ResolvedRangeInfo(new PartitionKeyRange(), new List <CompositeContinuationToken>(compositeContinuationTokensFromSplit));
result = await partitionRoutingHelper.TryAddPartitionKeyRangeToContinuationTokenAsync(
headers,
null,
null,
null,
resolvedRangeInfo,
NoOpTrace.Singleton,
RntdbEnumerationDirection.Reverse);
compositeContinuationTokens = JsonConvert.DeserializeObject <List <CompositeContinuationToken> >(headers.Get(HttpConstants.HttpHeaders.Continuation));
Assert.IsTrue(result);
Assert.IsTrue(compositeContinuationTokens.Count == compositeContinuationTokensFromSplit.Count - 1);
Assert.AreEqual(compositeContinuationTokensFromSplit.Last().Token, compositeContinuationTokens.First().Token);
}
public void AuthorizationBaselineTests()
{
string key = "VGhpcyBpcyBhIHNhbXBsZSBzdHJpbmc=";
for (int i = 0; i < this.AuthorizationBaseline.Length; i++)
{
string[] baseline = this.AuthorizationBaseline[i];
string[] baselineResults = this.AuthorizationBaselineResults[i];
StoreRequestNameValueCollection nvc = new StoreRequestNameValueCollection();
nvc.Add(HttpConstants.HttpHeaders.XDate, baseline[4]);
Uri uri = new Uri(baseline[0]);
string authorization = AuthorizationHelper.GenerateKeyAuthorizationSignature(
verb: baseline[2],
uri: uri,
headers: nvc,
stringHMACSHA256Helper: new StringHMACSHA256Hash(key));
string authorization2 = AuthorizationHelper.GenerateKeyAuthorizationSignature(
verb: baseline[2],
resourceId: baseline[1],
resourceType: baseline[3],
headers: nvc,
new StringHMACSHA256Hash(key),
out string payload2);
string authorization3 = AuthorizationHelper.GenerateKeyAuthorizationSignature(
verb: baseline[2],
resourceId: baseline[1],
resourceType: baseline[3],
headers: nvc,
key);
Assert.AreEqual(authorization, baselineResults[0]);
Assert.AreEqual(authorization2, baselineResults[0]);
Assert.AreEqual(authorization3, baselineResults[0]);
Assert.AreEqual(payload2.Replace("\n", "[n]"), baselineResults[1]);
AuthorizationHelper.ParseAuthorizationToken(authorization, out ReadOnlyMemory <char> typeOutput1, out ReadOnlyMemory <char> versionoutput1, out ReadOnlyMemory <char> tokenOutput1);
Assert.AreEqual("master", typeOutput1.ToString());
AuthorizationHelper.ParseAuthorizationToken(authorization2, out ReadOnlyMemory <char> typeOutput2, out ReadOnlyMemory <char> versionoutput2, out ReadOnlyMemory <char> tokenOutput2);
Assert.AreEqual("master", typeOutput2.ToString());
AuthorizationHelper.ParseAuthorizationToken(authorization3, out ReadOnlyMemory <char> typeOutput3, out ReadOnlyMemory <char> versionoutput3, out ReadOnlyMemory <char> tokenOutput3);
Assert.AreEqual("master", typeOutput3.ToString());
Assert.IsTrue(AuthorizationHelper.CheckPayloadUsingKey(tokenOutput1, baseline[2], baseline[1], baseline[3], nvc, key));
Assert.IsTrue(AuthorizationHelper.CheckPayloadUsingKey(tokenOutput2, baseline[2], baseline[1], baseline[3], nvc, key));
Assert.IsTrue(AuthorizationHelper.CheckPayloadUsingKey(tokenOutput3, baseline[2], baseline[1], baseline[3], nvc, key));
}
}
public void CompositeContinuationTokenIsNotPassedToBackend()
{
Range <string> expectedRange = new Range <string>("A", "B", true, false);
string expectedToken = "someToken";
CompositeContinuationToken compositeContinuationToken = new CompositeContinuationToken {
Range = expectedRange, Token = expectedToken
};
string continuation = JsonConvert.SerializeObject(compositeContinuationToken);
PartitionRoutingHelper partitionRoutingHelper = new PartitionRoutingHelper();
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
headers.Add(HttpConstants.HttpHeaders.Continuation, continuation);
Range <string> range = partitionRoutingHelper.ExtractPartitionKeyRangeFromContinuationToken(headers, out List <CompositeContinuationToken> compositeContinuationTokens);
Assert.IsTrue(expectedRange.Equals(range));
Assert.AreEqual(expectedToken, headers.Get(HttpConstants.HttpHeaders.Continuation)); //not a composite token
}
public void AuthorizationGenerateAndCheckKeyAuthSignature()
{
Random r = new Random();
byte[] hashKey = new byte[16];
r.NextBytes(hashKey);
string key = Convert.ToBase64String(hashKey);
foreach (string method in this.HttpMethods)
{
foreach (string resourceType in this.ResourceTypesArray)
{
foreach (string resourceName in this.ResourceNameValues)
{
StoreRequestNameValueCollection nvc = new StoreRequestNameValueCollection();
nvc.Add(HttpConstants.HttpHeaders.XDate, new DateTime(2020, 02, 01, 10, 00, 00).ToString("r"));
string authorizationKey = AuthorizationHelper.GenerateKeyAuthorizationSignature(
method,
resourceName,
resourceType,
nvc,
new StringHMACSHA256Hash(key),
out string payload);
AuthorizationHelper.ParseAuthorizationToken(authorizationKey,
out ReadOnlyMemory <char> typeOutput,
out ReadOnlyMemory <char> versionOutput,
out ReadOnlyMemory <char> tokenOutput);
Assert.AreEqual("master", typeOutput.ToString());
Assert.IsTrue(AuthorizationHelper.CheckPayloadUsingKey(
tokenOutput,
method,
resourceName,
resourceType,
nvc,
key));
}
}
}
}
public async Task QueryRequestRateTest(bool directMode)
{
string firstItemIdAndPk = "BasicQueryItem" + Guid.NewGuid();
// Prevent the test from changing the static client
{
CosmosClient client = directMode ? DirectCosmosClient : GatewayCosmosClient;
Container container = client.GetContainer(DatabaseId, ContainerId);
List <string> createdIds = new List <string>()
{
firstItemIdAndPk,
"BasicQueryItem2" + Guid.NewGuid(),
"BasicQueryItem3" + Guid.NewGuid()
};
foreach (string id in createdIds)
{
dynamic item = new
{
id = id,
pk = id,
};
await container.CreateItemAsync <dynamic>(item : item);
}
}
CosmosClient clientWithThrottle;
if (directMode)
{
clientWithThrottle = TestCommon.CreateCosmosClient();
}
else
{
clientWithThrottle = TestCommon.CreateCosmosClient((builder) => builder.WithConnectionModeGateway());
}
Container containerWithThrottle = clientWithThrottle.GetContainer(DatabaseId, ContainerId);
// Do a read to warm up all the caches to prevent them from getting the throttle errors
using (await containerWithThrottle.ReadItemStreamAsync(firstItemIdAndPk, new PartitionKey(firstItemIdAndPk))) { }
Documents.IStoreModel storeModel = clientWithThrottle.ClientContext.DocumentClient.StoreModel;
Mock <Documents.IStoreModel> mockStore = new Mock <Documents.IStoreModel>();
clientWithThrottle.ClientContext.DocumentClient.StoreModel = mockStore.Object;
// Cause 429 after the first call
int callCount = 0;
string activityId = null;
string errorMessage = "QueryRequestRateTest Resource Not Found";
mockStore.Setup(x => x.ProcessMessageAsync(It.IsAny <Documents.DocumentServiceRequest>(), It.IsAny <CancellationToken>()))
.Returns <Documents.DocumentServiceRequest, CancellationToken>((dsr, token) =>
{
callCount++;
if (callCount > 1)
{
INameValueCollection headers = new StoreRequestNameValueCollection();
headers.Add(Documents.HttpConstants.HttpHeaders.RetryAfterInMilliseconds, "42");
activityId = Guid.NewGuid().ToString();
headers.Add(Documents.HttpConstants.HttpHeaders.ActivityId, activityId);
Documents.DocumentServiceResponse response = new Documents.DocumentServiceResponse(
body: TestCommon.GenerateStreamFromString(@"{""Errors"":[""" + errorMessage + @"""]}"),
headers: headers,
statusCode: (HttpStatusCode)429,
clientSideRequestStatistics: dsr.RequestContext.ClientRequestStatistics);
return(Task.FromResult(response));
}
return(storeModel.ProcessMessageAsync(dsr, token));
});
List <dynamic> results = new List <dynamic>();
try
{
using (FeedIterator <dynamic> feedIterator = containerWithThrottle.GetItemQueryIterator <dynamic>(
"select * from T where STARTSWITH(T.id, \"BasicQueryItem\")",
requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1,
MaxConcurrency = 1
}))
{
while (feedIterator.HasMoreResults)
{
FeedResponse <dynamic> response = await feedIterator.ReadNextAsync();
Assert.IsTrue(response.Count <= 1);
Assert.IsTrue(response.Resource.Count() <= 1);
results.AddRange(response);
}
}
Assert.Fail("Should throw 429 exception after the first page.");
}
catch (CosmosException ce)
{
Assert.IsTrue(ce.RetryAfter.HasValue);
Assert.AreEqual(42, ce.RetryAfter.Value.TotalMilliseconds);
Assert.AreEqual(activityId, ce.ActivityId);
Assert.IsNotNull(ce.DiagnosticsContext);
Assert.IsTrue(ce.Message.Contains(errorMessage));
}
callCount = 0;
FeedIterator streamIterator = containerWithThrottle.GetItemQueryStreamIterator(
"select * from T where STARTSWITH(T.id, \"BasicQueryItem\")",
requestOptions: new QueryRequestOptions()
{
MaxItemCount = 1,
MaxConcurrency = 1
});
// First request should be a success
using (ResponseMessage response = await streamIterator.ReadNextAsync())
{
response.EnsureSuccessStatusCode();
Assert.IsNotNull(response.Content);
}
// Second page should be a failure
using (ResponseMessage response = await streamIterator.ReadNextAsync())
{
Assert.AreEqual(429, (int)response.StatusCode);
Assert.AreEqual("42", response.Headers.RetryAfterLiteral);
Assert.AreEqual(activityId, response.Headers.ActivityId);
Assert.IsNotNull(response.DiagnosticsContext);
Assert.IsTrue(response.ErrorMessage.Contains(errorMessage));
}
}
public void VerifyAllKnownProperties()
{
Dictionary <string, string> httpHeadersMap = typeof(HttpConstants.HttpHeaders).GetFields(BindingFlags.Public | BindingFlags.Static)
.ToDictionary(x => x.Name, x => (string)x.GetValue(null));
Dictionary <string, string> backendHeadersMap = typeof(WFConstants.BackendHeaders).GetFields(BindingFlags.Public | BindingFlags.Static)
.ToDictionary(x => x.Name, x => (string)x.GetValue(null));
PropertyInfo[] optimizedResponseHeaders = typeof(StoreRequestNameValueCollection).GetProperties(BindingFlags.Public | BindingFlags.Instance)
.Where(x => !string.Equals("Item", x.Name)).ToArray();
StoreRequestNameValueCollection headers = new StoreRequestNameValueCollection();
foreach (PropertyInfo propertyInfo in optimizedResponseHeaders)
{
Assert.AreEqual(0, headers.Count());
Assert.AreEqual(0, headers.Keys().Count());
// Test property first
string value = Guid.NewGuid().ToString();
propertyInfo.SetValue(headers, value);
Assert.AreEqual(value, propertyInfo.GetValue(headers));
if (!httpHeadersMap.TryGetValue(propertyInfo.Name, out string key))
{
if (!backendHeadersMap.TryGetValue(propertyInfo.Name, out key))
{
Assert.Fail($"The property name {propertyInfo.Name} should match a header constant name");
}
}
Assert.AreEqual(1, headers.Count());
Assert.AreEqual(1, headers.Keys().Count());
Assert.AreEqual(key, headers.Keys().First());
Assert.AreEqual(value, headers.Get(key));
Assert.AreEqual(value, headers.Get(key.ToUpper()));
Assert.AreEqual(value, headers.Get(key.ToLower()));
// Reset the value back to null
propertyInfo.SetValue(headers, null);
Assert.AreEqual(0, headers.Count());
Assert.AreEqual(0, headers.Keys().Count());
// Check adding via the interface sets the property correctly
headers.Add(key, value);
Assert.AreEqual(value, propertyInfo.GetValue(headers));
Assert.AreEqual(value, headers.Get(key));
Assert.AreEqual(1, headers.Count());
Assert.AreEqual(1, headers.Keys().Count());
Assert.AreEqual(key, headers.Keys().First());
Assert.AreEqual(value, headers.Get(key));
// Check setting via the interface sets the property correctly
value = Guid.NewGuid().ToString();
headers.Set(key, value);
Assert.AreEqual(value, propertyInfo.GetValue(headers));
Assert.AreEqual(value, headers.Get(key));
Assert.AreEqual(1, headers.Count());
Assert.AreEqual(1, headers.Keys().Count());
Assert.AreEqual(key, headers.Keys().First());
Assert.AreEqual(value, headers.Get(key));
// Check setting via the interface sets the property correctly
headers.Remove(key);
Assert.AreEqual(null, propertyInfo.GetValue(headers));
Assert.AreEqual(null, headers.Get(key));
Assert.AreEqual(0, headers.Count());
Assert.AreEqual(0, headers.Keys().Count());
}
}
internal static void ValidateQuery <T>(DocumentClient client, string collectionLink, string queryProperty, string queryPropertyValue, int expectedCount, INameValueCollection headers = null)
where T : Resource, new()
{
INameValueCollection inputHeaders = headers;
headers = new StoreRequestNameValueCollection();
if (inputHeaders != null)
{
headers.Add(inputHeaders); // dont mess with the input headers
}
int maxTries = 5;
const int minIndexInterval = 5000; // 5 seconds
while (maxTries-- > 0)
{
DocumentFeedResponse <dynamic> resourceFeed = null;
IDocumentQuery <dynamic> queryService = null;
string queryString = @"select * from root r where r." + queryProperty + @"=""" + queryPropertyValue + @"""";
if (typeof(T) == typeof(Database))
{
queryService = client.CreateDatabaseQuery(queryString).AsDocumentQuery();
}
else if (typeof(T) == typeof(DocumentCollection))
{
queryService = client.CreateDocumentCollectionQuery(collectionLink, queryString).AsDocumentQuery();
}
else if (typeof(T) == typeof(Document))
{
queryService = client.CreateDocumentQuery(collectionLink, queryString).AsDocumentQuery();
}
else
{
Assert.Fail("Unexpected type");
}
while (queryService.HasMoreResults)
{
resourceFeed = queryService.ExecuteNextAsync().Result;
if (resourceFeed.Count > 0)
{
Assert.IsNotNull(resourceFeed, "Query result is null");
Assert.AreNotEqual(0, resourceFeed.Count, "Query result is invalid");
foreach (T resource in resourceFeed)
{
if (queryProperty.Equals("name", StringComparison.CurrentCultureIgnoreCase))
{
Assert.AreEqual(resource.Id, queryPropertyValue, "Result contain invalid result");
}
}
return;
}
}
Task.Delay(minIndexInterval);
}
Assert.Fail("Query did not return result after max tries");
}