private static async Task <TResult> ExecuteWithRetries <TResult>(Func <Task <TResult> > clientFunc)
{
// From: https://blogs.msdn.microsoft.com/bigdatasupport/2015/09/02/dealing-with-requestratetoolarge-errors-in-azure-documentdb-and-testing-performance/
TimeSpan sleepTime = TimeSpan.Zero;
while (true)
{
try
{
return(await clientFunc());
}
catch (DocumentClientException dce)
{
if ((int)dce.StatusCode != 429)
{
throw;
}
}
catch (AggregateException ae)
{
if (!(ae.InnerException is DocumentClientException))
{
throw;
}
DocumentClientException dce = (DocumentClientException)ae.InnerException;
if ((int)dce.StatusCode != 429)
{
throw;
}
sleepTime = dce.RetryAfter;
await Task.Delay(sleepTime);
}
}
}
/// <summary>
/// Execute the function with retries on throttle.
/// </summary>
/// <typeparam name="V">The type of return value from the execution.</typeparam>
/// <param name="client">The DocumentDB client instance.</param>
/// <param name="function">The function to execute.</param>
/// <returns>The response from the execution.</returns>
public static async Task <V> ExecuteWithRetries <V>(IReliableReadWriteDocumentClient client, Func <Task <V> > function)
{
TimeSpan sleepTime = TimeSpan.Zero;
while (true)
{
try
{
return(await function());
}
catch (DocumentClientException de)
{
if ((int)de.StatusCode != 429)
{
throw;
}
sleepTime = de.RetryAfter;
}
catch (AggregateException ae)
{
if (!(ae.InnerException is DocumentClientException))
{
throw;
}
DocumentClientException de = (DocumentClientException)ae.InnerException;
if ((int)de.StatusCode != 429)
{
throw;
}
sleepTime = de.RetryAfter;
}
await Task.Delay(sleepTime);
}
}
private static string ParseDocumentClientExceptionMessage(DocumentClientException ex)
{
var errorMessage = string.Empty;
try
{
var message = ex.Message;
var openBrace = message.IndexOf('{');
var closeBrace = message.IndexOf('}', openBrace);
var errorsJson = message.Substring(openBrace, closeBrace - openBrace + 1);
dynamic errorsObj = JObject.Parse(errorsJson);
foreach (var error in errorsObj.Errors)
{
errorMessage += error;
}
}
catch
{
errorMessage = ex.Message;
}
return(errorMessage);
}
public Task <ShouldRetryResult> ShouldRetryAsync(
Exception exception,
CancellationToken cancellationToken)
{
DocumentClientException clientException = exception as DocumentClientException;
ShouldRetryResult shouldRetryResult = this.ShouldRetryInternal(
clientException?.StatusCode,
clientException?.GetSubStatus(),
clientException?.ResourceAddress);
if (shouldRetryResult != null)
{
return(Task.FromResult(shouldRetryResult));
}
if (this.nextRetryPolicy == null)
{
return(Task.FromResult(ShouldRetryResult.NoRetry()));
}
return(this.nextRetryPolicy.ShouldRetryAsync(exception, cancellationToken));
}
public void ValidateTransportHandlerLogging()
{
DocumentClientException dce = new DocumentClientException(
"test",
null,
new StoreResponseNameValueCollection(),
HttpStatusCode.Gone,
SubStatusCodes.PartitionKeyRangeGone,
new Uri("htts://localhost.com"));
CosmosDiagnosticsContext diagnosticsContext = new CosmosDiagnosticsContextCore();
RequestMessage requestMessage = new RequestMessage(
HttpMethod.Get,
"/dbs/test/colls/abc/docs/123",
diagnosticsContext,
Microsoft.Azure.Cosmos.Tracing.NoOpTrace.Singleton);
ResponseMessage response = dce.ToCosmosResponseMessage(requestMessage);
Assert.AreEqual(HttpStatusCode.Gone, response.StatusCode);
Assert.AreEqual(SubStatusCodes.PartitionKeyRangeGone, response.Headers.SubStatusCode);
bool visited = false;
foreach (CosmosDiagnosticsInternal cosmosDiagnosticsInternal in diagnosticsContext)
{
if (cosmosDiagnosticsInternal is PointOperationStatistics operationStatistics)
{
visited = true;
Assert.AreEqual(operationStatistics.StatusCode, HttpStatusCode.Gone);
Assert.AreEqual(operationStatistics.SubStatusCode, SubStatusCodes.PartitionKeyRangeGone);
}
}
Assert.IsTrue(visited, "PointOperationStatistics was not found in the diagnostics.");
}
public async Task Execute(Experiment experiment, DocumentClient client, DocumentCollection collection, Uri documentCollectionUri, int taskId, ConfigurationOptions options)
{
try
{
ResourceResponse <DocumentCollection> response = await client.ReadDocumentCollectionAsync(collection.SelfLink, new RequestOptions()
{
});
experiment.IncrementOperationCount();
experiment.UpdateRequestUnits(taskId, response.RequestCharge);
}
catch (Exception e)
{
Trace.TraceError("Failed to read {0}. Exception was {1}", collection.SelfLink, e);
if (e is DocumentClientException)
{
DocumentClientException de = (DocumentClientException)e;
if (de.StatusCode == HttpStatusCode.Forbidden)
{
experiment.IncrementOperationCount();
}
}
}
}
private async Task <ResourceResponse <Document> > WriteDocumentWithRetries(string collectionLink, dynamic document)
{
TimeSpan sleepTime = TimeSpan.Zero;
while (true)
{
try
{
return(await _client.CreateDocumentAsync(collectionLink, document));
}
catch (DocumentClientException de)
{
if ((int)de.StatusCode != 429)
{
throw;
}
sleepTime = de.RetryAfter;
}
catch (AggregateException ae)
{
if (!(ae.InnerException is DocumentClientException))
{
throw;
}
DocumentClientException de = (DocumentClientException)ae.InnerException;
if ((int)de.StatusCode != 429)
{
throw;
}
sleepTime = de.RetryAfter;
}
await Task.Delay(sleepTime);
}
}
internal async Task <ThroughputResponse> ReplaceThroughputIfExistsAsync(
string targetRID,
int throughput,
RequestOptions requestOptions,
CancellationToken cancellationToken = default(CancellationToken))
{
try
{
OfferV2 offerV2 = await this.GetOfferV2Async(targetRID, failIfNotConfigured : true, cancellationToken : cancellationToken);
OfferV2 newOffer = new OfferV2(offerV2, throughput);
return(await this.GetThroughputResponseAsync(
streamPayload : this.ClientContext.PropertiesSerializer.ToStream(newOffer),
operationType : OperationType.Replace,
linkUri : new Uri(offerV2.SelfLink, UriKind.Relative),
resourceType : ResourceType.Offer,
requestOptions : requestOptions,
cancellationToken : cancellationToken));
}
catch (DocumentClientException dce)
{
return(new ThroughputResponse(
dce.StatusCode ?? HttpStatusCode.InternalServerError,
headers: null,
throughputProperties: null));
}
catch (AggregateException ex)
{
DocumentClientException dce = ex.Flatten().InnerExceptions.FirstOrDefault(innerEx => innerEx is DocumentClientException) as DocumentClientException;
return(new ThroughputResponse(
dce?.StatusCode ?? HttpStatusCode.InternalServerError,
headers: null,
throughputProperties: null));
}
}
static void Main(string[] args)
{
string serviceEndpoint = ConfigurationManager.AppSettings["serviceEndpoint"];
string authKey = ConfigurationManager.AppSettings["authKey"];
if (string.IsNullOrWhiteSpace(serviceEndpoint) || string.IsNullOrWhiteSpace(authKey) ||
String.Equals(serviceEndpoint, "TODO - [YOUR ENDPOINT]", StringComparison.OrdinalIgnoreCase) ||
String.Equals(authKey, "TODO - [YOUR AUTHKEY]", StringComparison.OrdinalIgnoreCase))
{
Console.WriteLine("Please update your DocumentDB Account credentials in App.config");
Console.ReadKey();
}
else
{
try
{
// 1. It is recommended to create an instance of DocumentClient and reuse the same instance
// as opposed to creating, using and destroying the instance time and time again
// For this sample we are using the Defaults. There are optional parameters for things like ConnectionPolicy
// that allow you to change from Gateway to Direct or from HTTPS to TCP.
// For more information on this, please consult the Service Documentation page in Additional Resources
Console.WriteLine("1. Create an instance of DocumentClient");
using (client = new DocumentClient(new Uri(serviceEndpoint), authKey))
{
// 2.
Console.WriteLine("2. Getting reference to Database");
Database database = ReadOrCreateDatabase("documentDb");
// 3.
Console.WriteLine("3. Getting reference to a DocumentCollection");
DocumentCollection collection = ReadOrCreateCollection(database.SelfLink, "contoso");
// 4.
Console.WriteLine("4. Inserting Documents");
CreateDocuments(collection.SelfLink);
// 5.
Console.WriteLine("5. Querying for Documents");
QueryDocuments(collection.SelfLink);
// 6. Finally cleanup by deleting the Database
//Console.WriteLine("6. Cleaning Up");
//Cleanup(database.SelfLink);
}
}
catch (DocumentClientException docEx)
{
Exception baseException = docEx.GetBaseException();
Console.WriteLine("{0} StatusCode error occurred with activity id: {1}, Message: {2}",
docEx.StatusCode, docEx.ActivityId, docEx.Message, baseException.Message);
}
catch (AggregateException aggEx)
{
Console.WriteLine("One or more errors occurred during execution");
foreach (var ex in aggEx.InnerExceptions)
{
DocumentClientException dce = ex as DocumentClientException;
if (dce != null)
{
Console.WriteLine("A Document Client Exception with Status Code {0} occurred with activity id {1}, message: {2}",
dce.StatusCode, dce.ActivityId, dce.Message);
}
else
{
Console.WriteLine("An exception of type {0} occurred: {1}", ex.GetType(), ex.Message);
}
}
}
catch (Exception ex)
{
Console.WriteLine("An unexpected exception of type {0} occurred: {1}", ex.GetType(), ex.Message);
}
Console.WriteLine("\nSample complete. Press any key to exit.");
Console.ReadKey();
}
}
public ResourceNotFoundException(DocumentClientException ex)
{
this.ex = ex;
}
public static StorageException ToStorageException(this Exception exception, TableOperation tableOperation)
{
DocumentClientException docException = exception as DocumentClientException;
RequestResult reqResult = new RequestResult();
if (docException == null)
{
if (exception is TimeoutException)
{
return(StorageException.TranslateException(exception, new RequestResult()));
}
reqResult.HttpStatusCode = (int)500;
reqResult.HttpStatusMessage = TableErrorCodeMessageStrings.InternalServerError;
return(new StorageException(reqResult, exception.Message, exception));
}
else
{
reqResult.HttpStatusMessage = docException.Error.Code;
reqResult.HttpStatusCode = (int)docException.StatusCode;
reqResult.ExtendedErrorInformation = new StorageExtendedErrorInformation();
reqResult.ExtendedErrorInformation.ErrorCode = docException.Error.Code;
reqResult.ExtendedErrorInformation.ErrorMessage = docException.Message;
switch (docException.StatusCode)
{
case HttpStatusCode.Conflict:
if (tableOperation != null && tableOperation.IsTableEntity)
{
reqResult.ExtendedErrorInformation.ErrorCode = TableErrorCodeStrings.TableAlreadyExists;
reqResult.ExtendedErrorInformation.ErrorMessage = TableErrorCodeMessageStrings.TableAlreadyExistsMessage;
}
else
{
reqResult.ExtendedErrorInformation.ErrorCode = TableErrorCodeStrings.EntityAlreadyExists;
reqResult.ExtendedErrorInformation.ErrorMessage = TableErrorCodeMessageStrings.EntityAlreadyExistsMessage;
}
break;
case HttpStatusCode.NotFound:
if (tableOperation != null && tableOperation.IsTableEntity)
{
reqResult.ExtendedErrorInformation.ErrorCode = TableErrorCodeStrings.ResourceNotFound;
reqResult.ExtendedErrorInformation.ErrorMessage = TableErrorCodeMessageStrings.ResourceNotFoundMessage;
}
else
{
reqResult.ExtendedErrorInformation.ErrorCode = TableErrorCodeStrings.EntityNotFound;
reqResult.ExtendedErrorInformation.ErrorMessage = TableErrorCodeMessageStrings.EntityNotFoundMessage;
}
break;
case HttpStatusCode.PreconditionFailed:
reqResult.ExtendedErrorInformation.ErrorCode = TableErrorCodeStrings.UpdateConditionNotSatisfied;
reqResult.ExtendedErrorInformation.ErrorMessage = TableErrorCodeMessageStrings.UpdateConditionNotSatisfiedMessage;
break;
case HttpStatusCode.RequestEntityTooLarge:
reqResult.ExtendedErrorInformation.ErrorCode = TableErrorCodeStrings.EntityTooLarge;
reqResult.ExtendedErrorInformation.ErrorMessage = TableErrorCodeMessageStrings.EntityTooLargeMessage;
break;
}
return(new StorageException(reqResult, reqResult.ExtendedErrorInformation.ErrorMessage, docException));
}
}
private void TestRetryOnThrottled(int?numberOfRetries)
{
Mock <IStoreModel> mockStoreModel = new Mock <IStoreModel>();
mockStoreModel.Setup(model => model.ProcessMessageAsync(It.IsAny <DocumentServiceRequest>(), default(CancellationToken)))
.Throws(this.CreateTooManyRequestException(100));
ConnectionPolicy connectionPolicy = new ConnectionPolicy()
{
EnableEndpointDiscovery = false,
};
if (numberOfRetries != null)
{
connectionPolicy.RetryOptions = new RetryOptions {
MaxRetryAttemptsOnThrottledRequests = numberOfRetries.Value
};
}
DocumentClient client = new DocumentClient(
new Uri(ConfigurationManager.AppSettings["GatewayEndpoint"]),
ConfigurationManager.AppSettings["MasterKey"],
connectionPolicy);
client.GetDatabaseAccountAsync().Wait();
int expectedExecutionTimes = numberOfRetries + 1 ?? 10;
client.StoreModel = mockStoreModel.Object;
client.GatewayStoreModel = mockStoreModel.Object;
bool throttled = false;
try
{
CosmosDatabaseSettings db = new CosmosDatabaseSettings {
Id = "test db 1"
};
client.CreateDatabaseAsync(db).Wait();
}
catch (Exception exp)
{
DocumentClientException docExp = exp.InnerException as DocumentClientException;
Assert.AreEqual((HttpStatusCode)429, docExp.StatusCode);
throttled = true;
}
mockStoreModel.Verify(model => model.ProcessMessageAsync(It.IsAny <DocumentServiceRequest>(), default(CancellationToken)), Times.Exactly(expectedExecutionTimes));
Assert.IsTrue(throttled);
throttled = false;
try
{
client.ReadDatabaseAsync("/dbs/id1").Wait();
}
catch (Exception exp)
{
DocumentClientException docExp = exp.InnerException as DocumentClientException;
Assert.AreEqual((HttpStatusCode)429, docExp.StatusCode);
throttled = true;
}
mockStoreModel.Verify(model => model.ProcessMessageAsync(It.IsAny <DocumentServiceRequest>(), default(CancellationToken)), Times.Exactly(2 * expectedExecutionTimes));
Assert.IsTrue(throttled);
throttled = false;
try
{
client.DeleteDocumentCollectionAsync("dbs/db_rid/colls/col_rid/").Wait();
}
catch (Exception exp)
{
DocumentClientException docExp = exp.InnerException as DocumentClientException;
Assert.AreEqual((HttpStatusCode)429, docExp.StatusCode);
throttled = true;
}
mockStoreModel.Verify(model => model.ProcessMessageAsync(It.IsAny <DocumentServiceRequest>(), default(CancellationToken)), Times.Exactly(3 * expectedExecutionTimes));
Assert.IsTrue(throttled);
throttled = false;
try
{
client.CreateDatabaseQuery().AsDocumentQuery().ExecuteNextAsync().Wait();
}
catch (Exception exp)
{
DocumentClientException docExp = exp.InnerException as DocumentClientException;
Assert.AreEqual((HttpStatusCode)429, docExp.StatusCode);
throttled = true;
}
mockStoreModel.Verify(model => model.ProcessMessageAsync(It.IsAny <DocumentServiceRequest>(), default(CancellationToken)), Times.Exactly(4 * expectedExecutionTimes));
Assert.IsTrue(throttled);
}
protected bool NeedPartitionKeyRangeCacheRefresh(DocumentClientException ex)
{
return(ex.StatusCode == (HttpStatusCode)StatusCodes.Gone && ex.GetSubStatus() == SubStatusCodes.PartitionKeyRangeGone);
}
/// <summary>
/// When a javascript stored procedure throws an exception in code, such as
/// throw new Error(404, "Document not found");
/// the status code of the exception will be 400 bad request, but there will be a substatus header
/// with the error code specified in the Error object (404 in this example).
/// This method returns the substatus code if it is present.
/// </summary>
/// <param name="exception">The exception object</param>
/// <returns>The status code or null if not present (or not an integer)</returns>
public static HttpStatusCode?GetSubStatusCode(this DocumentClientException exception)
{
return((HttpStatusCode?)exception.GetSubStatusValue());
}
/// <summary>
/// Given a document client exception this function determines whether it was caused due to a split.
/// </summary>
/// <param name="ex">The document client exception</param>
/// <returns>Whether or not the exception was due to a split.</returns>
private static bool IsSplitException(DocumentClientException ex)
{
return(ex.StatusCode == (HttpStatusCode)StatusCodes.Gone && ex.GetSubStatus() == SubStatusCodes.PartitionKeyRangeGone);
}
/// <summary>
/// Azure DocumentDb concurrency exception
/// </summary>
/// <param name="etag">ETag used for the AccessCondition</param>
/// <param name="inner">Original DocumentClientException</param>
public ConcurrencyException(string etag, DocumentClientException inner) : base($"Azure DocumentDb concurrency exception for ETag {etag}", inner)
{
}
async Task IPartitionObserver <DocumentServiceLease> .OnPartitionAcquiredAsync(DocumentServiceLease lease)
{
Debug.Assert(lease != null && !string.IsNullOrEmpty(lease.Owner), "lease");
TraceLog.Informational(string.Format("Host '{0}' partition {1}: acquired!", this.HostName, lease.PartitionId));
#if DEBUG
Interlocked.Increment(ref this.partitionCount);
#endif
IChangeFeedObserver observer = this.observerFactory.CreateObserver();
ChangeFeedObserverContext context = new ChangeFeedObserverContext {
PartitionKeyRangeId = lease.PartitionId
};
CancellationTokenSource cancellation = new CancellationTokenSource();
// Create ChangeFeedOptions to use for this worker.
ChangeFeedOptions options = new ChangeFeedOptions
{
MaxItemCount = this.changeFeedOptions.MaxItemCount,
PartitionKeyRangeId = this.changeFeedOptions.PartitionKeyRangeId,
SessionToken = this.changeFeedOptions.SessionToken,
StartFromBeginning = this.changeFeedOptions.StartFromBeginning,
RequestContinuation = this.changeFeedOptions.RequestContinuation
};
var workerTask = await Task.Factory.StartNew(async() =>
{
ChangeFeedObserverCloseReason?closeReason = null;
try
{
try
{
await observer.OpenAsync(context);
}
catch (Exception ex)
{
TraceLog.Error(string.Format("IChangeFeedObserver.OpenAsync exception: {0}", ex));
closeReason = ChangeFeedObserverCloseReason.ObserverError;
throw;
}
options.PartitionKeyRangeId = lease.PartitionId;
if (!string.IsNullOrEmpty(lease.ContinuationToken))
{
options.RequestContinuation = lease.ContinuationToken;
}
IDocumentQuery <Document> query = this.documentClient.CreateDocumentChangeFeedQuery(this.collectionSelfLink, options);
TraceLog.Verbose(string.Format("Worker start: partition '{0}', continuation '{1}'", lease.PartitionId, lease.ContinuationToken));
try
{
while (this.isShutdown == 0)
{
do
{
DocumentClientException dcex = null;
FeedResponse <Document> response = null;
try
{
response = await query.ExecuteNextAsync <Document>();
}
catch (DocumentClientException ex)
{
if (StatusCode.NotFound != (StatusCode)ex.StatusCode &&
StatusCode.TooManyRequests != (StatusCode)ex.StatusCode &&
StatusCode.ServiceUnavailable != (StatusCode)ex.StatusCode)
{
throw;
}
dcex = ex;
}
if (dcex != null)
{
const int ReadSessionNotAvailable = 1002;
if (StatusCode.NotFound == (StatusCode)dcex.StatusCode && GetSubStatusCode(dcex) != ReadSessionNotAvailable)
{
// Most likely, the database or collection was removed while we were enumerating.
// Shut down. The user will need to start over.
// Note: this has to be a new task, can't await for shutdown here, as shudown awaits for all worker tasks.
await Task.Factory.StartNew(() => this.StopAsync(ChangeFeedObserverCloseReason.ResourceGone));
break;
}
else
{
Debug.Assert(StatusCode.TooManyRequests == (StatusCode)dcex.StatusCode || StatusCode.ServiceUnavailable == (StatusCode)dcex.StatusCode);
TraceLog.Warning(string.Format("Partition {0}: retriable exception : {1}", context.PartitionKeyRangeId, dcex.Message));
await Task.Delay(dcex.RetryAfter != TimeSpan.Zero ? dcex.RetryAfter : this.options.FeedPollDelay, cancellation.Token);
}
}
if (response != null)
{
if (response.Count > 0)
{
List <Document> docs = new List <Document>();
docs.AddRange(response);
try
{
await observer.ProcessChangesAsync(context, docs);
}
catch (Exception ex)
{
TraceLog.Error(string.Format("IChangeFeedObserver.ProcessChangesAsync exception: {0}", ex));
closeReason = ChangeFeedObserverCloseReason.ObserverError;
throw;
}
// Checkpoint after every successful delivery to the client.
lease = await CheckpointAsync(lease, response.ResponseContinuation, context);
}
else if (string.IsNullOrEmpty(lease.ContinuationToken))
{
// Checkpoint if we've never done that for this lease.
lease = await CheckpointAsync(lease, response.ResponseContinuation, context);
}
}
}while (query.HasMoreResults && this.isShutdown == 0);
if (this.isShutdown == 0)
{
await Task.Delay(this.options.FeedPollDelay, cancellation.Token);
}
} // Outer while (this.isShutdown == 0) loop.
}
catch (TaskCanceledException)
{
Debug.Assert(cancellation.IsCancellationRequested, "cancellation.IsCancellationRequested");
TraceLog.Informational(string.Format("Cancel signal received for partition {0} worker!", context.PartitionKeyRangeId));
}
}
catch (LeaseLostException)
{
closeReason = ChangeFeedObserverCloseReason.LeaseLost;
}
catch (Exception ex)
{
TraceLog.Error(string.Format("Partition {0} exception: {1}", context.PartitionKeyRangeId, ex));
if (!closeReason.HasValue)
{
closeReason = ChangeFeedObserverCloseReason.Unknown;
}
}
if (closeReason.HasValue)
{
TraceLog.Informational(string.Format("Releasing lease for partition {0} due to an error, reason: {1}!", context.PartitionKeyRangeId, closeReason.Value));
// Note: this has to be a new task, because OnPartitionReleasedAsync awaits for worker task.
await Task.Factory.StartNew(async() => await this.partitionManager.TryReleasePartitionAsync(context.PartitionKeyRangeId, true, closeReason.Value));
}
TraceLog.Informational(string.Format("Partition {0}: worker finished!", context.PartitionKeyRangeId));
});
var newWorkerData = new WorkerData(workerTask, observer, context, cancellation);
this.partitionKeyRangeIdToWorkerMap.AddOrUpdate(context.PartitionKeyRangeId, newWorkerData, (string id, WorkerData d) => { return(newWorkerData); });
}
public static IReadOnlyList <Range <string> > GetProvidedPartitionKeyRanges(
SqlQuerySpec querySpec,
bool enableCrossPartitionQuery,
bool parallelizeCrossPartitionQuery,
bool isContinuationExpected,
PartitionKeyDefinition partitionKeyDefinition,
QueryPartitionProvider queryPartitionProvider,
string clientApiVersion,
out QueryInfo queryInfo)
{
if (querySpec == null)
{
throw new ArgumentNullException("querySpec");
}
if (partitionKeyDefinition == null)
{
throw new ArgumentNullException("partitionKeyDefinition");
}
if (queryPartitionProvider == null)
{
throw new ArgumentNullException("queryPartitionProvider");
}
PartitionedQueryExecutionInfo queryExecutionInfo = null;
queryExecutionInfo = queryPartitionProvider.GetPartitionedQueryExecutionInfo(
querySpec,
partitionKeyDefinition,
VersionUtility.IsLaterThan(clientApiVersion, HttpConstants.Versions.v2016_11_14),
isContinuationExpected);
if (queryExecutionInfo == null ||
queryExecutionInfo.QueryRanges == null ||
queryExecutionInfo.QueryInfo == null ||
queryExecutionInfo.QueryRanges.Any(range => range.Min == null || range.Max == null))
{
DefaultTrace.TraceInformation("QueryPartitionProvider returned bad query info");
}
bool isSinglePartitionQuery = queryExecutionInfo.QueryRanges.Count == 1 && queryExecutionInfo.QueryRanges[0].IsSingleValue;
if (partitionKeyDefinition.Paths.Count > 0 && !isSinglePartitionQuery)
{
if (!enableCrossPartitionQuery)
{
throw new BadRequestException(RMResources.CrossPartitionQueryDisabled);
}
else
{
if (parallelizeCrossPartitionQuery ||
(queryExecutionInfo.QueryInfo != null && (queryExecutionInfo.QueryInfo.HasTop || queryExecutionInfo.QueryInfo.HasOrderBy || queryExecutionInfo.QueryInfo.HasAggregates)))
{
if (!IsSupportedPartitionedQueryExecutionInfo(queryExecutionInfo, clientApiVersion))
{
throw new BadRequestException(RMResources.UnsupportedCrossPartitionQuery);
}
else if (queryExecutionInfo.QueryInfo.HasAggregates && !IsAggregateSupportedApiVersion(clientApiVersion))
{
throw new BadRequestException(RMResources.UnsupportedCrossPartitionQueryWithAggregate);
}
else
{
DocumentClientException exception = new DocumentClientException(
RMResources.UnsupportedCrossPartitionQuery,
HttpStatusCode.BadRequest,
SubStatusCodes.CrossPartitionQueryNotServable);
exception.Error.AdditionalErrorInfo = JsonConvert.SerializeObject(queryExecutionInfo);
throw exception;
}
}
}
}
// For single partition query with aggregate functions and no continuation expected,
// we would try to accumulate the results for them on the SDK, if supported.
else if (queryExecutionInfo.QueryInfo.HasAggregates && !isContinuationExpected)
{
if (IsAggregateSupportedApiVersion(clientApiVersion))
{
DocumentClientException exception = new DocumentClientException(
RMResources.UnsupportedQueryWithFullResultAggregate,
HttpStatusCode.BadRequest,
SubStatusCodes.CrossPartitionQueryNotServable);
exception.Error.AdditionalErrorInfo = JsonConvert.SerializeObject(queryExecutionInfo);
throw exception;
}
else
{
throw new BadRequestException(RMResources.UnsupportedQueryWithFullResultAggregate);
}
}
queryInfo = queryExecutionInfo.QueryInfo;
return(queryExecutionInfo.QueryRanges);
}
public abstract void TraceUnauthorized(
DocumentClientException dce,
string authorizationToken,
string payload);
public QuerySyntaxException(DocumentClientException dce) : base(TryGetSyntaxErrorMessageFromException(dce))
{
}
internal CosmosResponse <TEntity> HandleDocumentClientException(TEntity entity, DocumentClientException exception)
{
if (exception.Message.Contains("Resource Not Found"))
{
return(new CosmosResponse <TEntity>(entity, CosmosOperationStatus.ResourceNotFound));
}
if (exception.Message.Contains("Request rate is large"))
{
return(new CosmosResponse <TEntity>(entity, CosmosOperationStatus.RequestRateIsLarge));
}
if (exception.Message.Contains("Resource with specified id or name already exists"))
{
return(new CosmosResponse <TEntity>(entity, CosmosOperationStatus.ResourceWithIdAlreadyExists));
}
throw exception;
}
public async Task RetryOnReadSessionNotAvailableMockTestAsync()
{
ConnectionPolicy connectionPolicy = new ConnectionPolicy
{
ConnectionMode = ConnectionMode.Direct,
ConnectionProtocol = Protocol.Tcp,
PreferredLocations = { "West US" },
};
DocumentClient client = new DocumentClient(
new Uri(ConfigurationManager.AppSettings["GatewayEndpoint"]),
ConfigurationManager.AppSettings["MasterKey"],
connectionPolicy,
ConsistencyLevel.Session);
await client.GetDatabaseAccountAsync();
// Set up the mock to throw exception on first call, test retry happens and request succeeds.
Mock <IStoreModel> mockStoreModel = new Mock <IStoreModel>();
mockStoreModel.Setup(model => model.ProcessMessageAsync(It.IsAny <DocumentServiceRequest>(), default(CancellationToken)))
.Returns <DocumentServiceRequest, CancellationToken>((r, cancellationToken) => this.ProcessMessageForRead(client, r));
client.StoreModel = mockStoreModel.Object;
client.GatewayStoreModel = mockStoreModel.Object;
ResourceResponse <CosmosDatabaseSettings> dbResponse = await client.ReadDatabaseAsync("/dbs/id1");
Assert.IsNotNull(dbResponse);
mockStoreModel.Verify(model => model.ProcessMessageAsync(It.IsAny <DocumentServiceRequest>(), default(CancellationToken)), Times.Exactly(2));
// Set up the mock to always throw exception, test retry happens only twice and request fails.
mockStoreModel = new Mock <IStoreModel>();
mockStoreModel.Setup(model => model.ProcessMessageAsync(It.IsAny <DocumentServiceRequest>(), default(CancellationToken)))
.Throws(this.CreateReadSessionNotAvailableException());
client.StoreModel = mockStoreModel.Object;
client.GatewayStoreModel = mockStoreModel.Object;
bool failed = false;
try
{
dbResponse = await client.ReadDatabaseAsync("/dbs/id1");
Assert.IsNull(dbResponse);
}
catch (DocumentClientException e)
{
failed = true;
Assert.AreEqual(HttpStatusCode.NotFound, e.StatusCode);
}
mockStoreModel.Verify(model => model.ProcessMessageAsync(It.IsAny <DocumentServiceRequest>(), default(CancellationToken)), Times.Exactly(2));
Assert.IsTrue(failed);
failed = false;
try
{
IQueryable <dynamic> dbIdQuery = client.CreateDatabaseQuery(@"select * from root r").AsQueryable();
Assert.AreEqual(0, dbIdQuery.AsEnumerable().Count());
}
catch (AggregateException e)
{
DocumentClientException docExp = e.InnerExceptions[0] as DocumentClientException;
Assert.IsNotNull(docExp);
Assert.AreEqual(HttpStatusCode.NotFound, docExp.StatusCode);
failed = true;
}
mockStoreModel.Verify(model => model.ProcessMessageAsync(It.IsAny <DocumentServiceRequest>(), default(CancellationToken)), Times.Exactly(4));
Assert.IsTrue(failed);
failed = false;
try
{
ResourceFeedReader <CosmosDatabaseSettings> dbFeed = client.CreateDatabaseFeedReader();
FeedResponse <CosmosDatabaseSettings> response = await dbFeed.ExecuteNextAsync();
Assert.AreEqual(1, response.Count);
Assert.AreEqual(false, dbFeed.HasMoreResults);
}
catch (DocumentClientException docExp)
{
Assert.IsNotNull(docExp);
Assert.AreEqual(HttpStatusCode.NotFound, docExp.StatusCode);
failed = true;
}
mockStoreModel.Verify(model => model.ProcessMessageAsync(It.IsAny <DocumentServiceRequest>(), default(CancellationToken)), Times.Exactly(6));
Assert.IsTrue(failed);
}
internal CosmosResponse <TEntity> HandleDocumentClientException(DocumentClientException exception)
{
return(HandleDocumentClientException(null, exception));
}
internal static CosmosResponse <TEntity> ToCosmosResponse <TEntity>(this DocumentClientException exception, TEntity entity) where TEntity : class
{
return(DocumentClientExceptionToCosmosResponse(exception, entity));
}
/// <summary>
/// When a javascript stored procedure throws an exception in code, such as
/// throw new Error(404, "Document not found");
/// the status code of the exception will be 400 bad request, but there will be a substatus header
/// with the error code specified in the Error object (404 in this example).
/// This method returns the value of substatus header if it is present.
/// </summary>
/// <param name="exception">The exception object</param>
/// <returns>The status code value or null if not present (or not an integer)</returns>
public static int?GetSubStatusValue(this DocumentClientException exception)
{
return(int.TryParse(exception.ResponseHeaders.Get(CosmosDbHeaders.SubStatus), NumberStyles.Integer, CultureInfo.InvariantCulture, out int subStatusCode)
? subStatusCode
: (int?)null);
}
internal static CosmosResponse <TEntity> DocumentClientExceptionToCosmosResponse <TEntity>(DocumentClientException exception, TEntity entity) where TEntity : class
{
switch (exception.StatusCode)
{
case HttpStatusCode.NotFound:
return(new CosmosResponse <TEntity>(entity, exception, CosmosOperationStatus.ResourceNotFound));
case (HttpStatusCode)CosmosConstants.TooManyRequestsStatusCode:
return(new CosmosResponse <TEntity>(entity, exception, CosmosOperationStatus.RequestRateIsLarge));
case HttpStatusCode.PreconditionFailed:
return(new CosmosResponse <TEntity>(entity, exception, CosmosOperationStatus.PreconditionFailed));
case HttpStatusCode.Conflict:
return(new CosmosResponse <TEntity>(entity, exception, CosmosOperationStatus.Conflict));
}
throw exception;
}
async Task IPartitionObserver <DocumentServiceLease> .OnPartitionAcquiredAsync(DocumentServiceLease lease)
{
Debug.Assert(lease != null && !string.IsNullOrEmpty(lease.Owner), "lease");
TraceLog.Informational(string.Format("Host '{0}' partition {1}: acquired!", this.HostName, lease.PartitionId));
#if DEBUG
Interlocked.Increment(ref this.partitionCount);
#endif
IChangeFeedObserver observer = this.observerFactory.CreateObserver();
ChangeFeedObserverContext context = new ChangeFeedObserverContext {
PartitionKeyRangeId = lease.PartitionId
};
CancellationTokenSource cancellation = new CancellationTokenSource();
// Create ChangeFeedOptions to use for this worker.
ChangeFeedOptions options = new ChangeFeedOptions
{
MaxItemCount = this.changeFeedOptions.MaxItemCount,
PartitionKeyRangeId = this.changeFeedOptions.PartitionKeyRangeId,
SessionToken = this.changeFeedOptions.SessionToken,
StartFromBeginning = this.changeFeedOptions.StartFromBeginning,
RequestContinuation = this.changeFeedOptions.RequestContinuation
};
var workerTask = await Task.Factory.StartNew(async() =>
{
ChangeFeedObserverCloseReason?closeReason = null;
try
{
try
{
await observer.OpenAsync(context);
}
catch (Exception ex)
{
TraceLog.Error(string.Format("IChangeFeedObserver.OpenAsync exception: {0}", ex));
closeReason = ChangeFeedObserverCloseReason.ObserverError;
throw;
}
options.PartitionKeyRangeId = lease.PartitionId;
if (!string.IsNullOrEmpty(lease.ContinuationToken))
{
options.RequestContinuation = lease.ContinuationToken;
}
CheckpointStats checkpointStats = null;
if (!this.statsSinceLastCheckpoint.TryGetValue(lease.PartitionId, out checkpointStats) || checkpointStats == null)
{
// It could be that the lease was created by different host and we picked it up.
checkpointStats = this.statsSinceLastCheckpoint.AddOrUpdate(
lease.PartitionId,
new CheckpointStats(),
(partitionId, existingStats) => existingStats);
Trace.TraceWarning(string.Format("Added stats for partition '{0}' for which the lease was picked up after the host was started.", lease.PartitionId));
}
IDocumentQuery <Document> query = this.documentClient.CreateDocumentChangeFeedQuery(this.collectionSelfLink, options);
TraceLog.Verbose(string.Format("Worker start: partition '{0}', continuation '{1}'", lease.PartitionId, lease.ContinuationToken));
string lastContinuation = options.RequestContinuation;
try
{
while (this.isShutdown == 0)
{
do
{
ExceptionDispatchInfo exceptionDispatchInfo = null;
FeedResponse <Document> response = null;
try
{
response = await query.ExecuteNextAsync <Document>();
lastContinuation = response.ResponseContinuation;
}
catch (DocumentClientException ex)
{
exceptionDispatchInfo = ExceptionDispatchInfo.Capture(ex);
}
if (exceptionDispatchInfo != null)
{
DocumentClientException dcex = (DocumentClientException)exceptionDispatchInfo.SourceException;
if (StatusCode.NotFound == (StatusCode)dcex.StatusCode && SubStatusCode.ReadSessionNotAvailable != (SubStatusCode)GetSubStatusCode(dcex))
{
// Most likely, the database or collection was removed while we were enumerating.
// Shut down. The user will need to start over.
// Note: this has to be a new task, can't await for shutdown here, as shudown awaits for all worker tasks.
TraceLog.Error(string.Format("Partition {0}: resource gone (subStatus={1}). Aborting.", context.PartitionKeyRangeId, GetSubStatusCode(dcex)));
await Task.Factory.StartNew(() => this.StopAsync(ChangeFeedObserverCloseReason.ResourceGone));
break;
}
else if (StatusCode.Gone == (StatusCode)dcex.StatusCode)
{
SubStatusCode subStatusCode = (SubStatusCode)GetSubStatusCode(dcex);
if (SubStatusCode.PartitionKeyRangeGone == subStatusCode)
{
bool isSuccess = await HandleSplitAsync(context.PartitionKeyRangeId, lastContinuation, lease.Id);
if (!isSuccess)
{
TraceLog.Error(string.Format("Partition {0}: HandleSplit failed! Aborting.", context.PartitionKeyRangeId));
await Task.Factory.StartNew(() => this.StopAsync(ChangeFeedObserverCloseReason.ResourceGone));
break;
}
// Throw LeaseLostException so that we take the lease down.
throw new LeaseLostException(lease, exceptionDispatchInfo.SourceException, true);
}
else if (SubStatusCode.Splitting == subStatusCode)
{
TraceLog.Warning(string.Format("Partition {0} is splitting. Will retry to read changes until split finishes. {1}", context.PartitionKeyRangeId, dcex.Message));
}
else
{
exceptionDispatchInfo.Throw();
}
}
else if (StatusCode.TooManyRequests == (StatusCode)dcex.StatusCode ||
StatusCode.ServiceUnavailable == (StatusCode)dcex.StatusCode)
{
TraceLog.Warning(string.Format("Partition {0}: retriable exception : {1}", context.PartitionKeyRangeId, dcex.Message));
}
else
{
exceptionDispatchInfo.Throw();
}
await Task.Delay(dcex.RetryAfter != TimeSpan.Zero ? dcex.RetryAfter : this.options.FeedPollDelay, cancellation.Token);
}
if (response != null)
{
if (response.Count > 0)
{
List <Document> docs = new List <Document>();
docs.AddRange(response);
try
{
context.FeedResponse = response;
await observer.ProcessChangesAsync(context, docs);
}
catch (Exception ex)
{
TraceLog.Error(string.Format("IChangeFeedObserver.ProcessChangesAsync exception: {0}", ex));
closeReason = ChangeFeedObserverCloseReason.ObserverError;
throw;
}
finally
{
context.FeedResponse = null;
}
}
checkpointStats.ProcessedDocCount += (uint)response.Count;
if (IsCheckpointNeeded(lease, checkpointStats))
{
lease = await CheckpointAsync(lease, response.ResponseContinuation, context);
checkpointStats.Reset();
}
else if (response.Count > 0)
{
TraceLog.Informational(string.Format("Checkpoint: not checkpointing for partition {0}, {1} docs, new continuation '{2}' as frequency condition is not met", lease.PartitionId, response.Count, response.ResponseContinuation));
}
}
}while (query.HasMoreResults && this.isShutdown == 0);
if (this.isShutdown == 0)
{
await Task.Delay(this.options.FeedPollDelay, cancellation.Token);
}
} // Outer while (this.isShutdown == 0) loop.
closeReason = ChangeFeedObserverCloseReason.Shutdown;
}
catch (TaskCanceledException)
{
Debug.Assert(cancellation.IsCancellationRequested, "cancellation.IsCancellationRequested");
TraceLog.Informational(string.Format("Cancel signal received for partition {0} worker!", context.PartitionKeyRangeId));
}
}
catch (LeaseLostException ex)
{
closeReason = ex.IsGone ? ChangeFeedObserverCloseReason.LeaseGone : ChangeFeedObserverCloseReason.LeaseLost;
}
catch (Exception ex)
{
TraceLog.Error(string.Format("Partition {0} exception: {1}", context.PartitionKeyRangeId, ex));
if (!closeReason.HasValue)
{
closeReason = ChangeFeedObserverCloseReason.Unknown;
}
}
if (closeReason.HasValue)
{
TraceLog.Informational(string.Format("Releasing lease for partition {0} due to an error, reason: {1}!", context.PartitionKeyRangeId, closeReason.Value));
// Note: this has to be a new task, because OnPartitionReleasedAsync awaits for worker task.
await Task.Factory.StartNew(async() => await this.partitionManager.TryReleasePartitionAsync(context.PartitionKeyRangeId, true, closeReason.Value));
}
TraceLog.Informational(string.Format("Partition {0}: worker finished!", context.PartitionKeyRangeId));
});
var newWorkerData = new WorkerData(workerTask, observer, context, cancellation);
this.partitionKeyRangeIdToWorkerMap.AddOrUpdate(context.PartitionKeyRangeId, newWorkerData, (string id, WorkerData d) => { return(newWorkerData); });
}
public static IReadOnlyList <Range <string> > GetProvidedPartitionKeyRanges(
Func <string, Exception> createBadRequestException,
SqlQuerySpec querySpec,
bool enableCrossPartitionQuery,
bool parallelizeCrossPartitionQuery,
bool isContinuationExpected,
bool hasLogicalPartitionKey,
PartitionKeyDefinition partitionKeyDefinition,
QueryPartitionProvider queryPartitionProvider,
string clientApiVersion,
out QueryInfo queryInfo)
{
if (querySpec == null)
{
throw new ArgumentNullException("querySpec");
}
if (partitionKeyDefinition == null)
{
throw new ArgumentNullException("partitionKeyDefinition");
}
if (queryPartitionProvider == null)
{
throw new ArgumentNullException("queryPartitionProvider");
}
TryCatch <PartitionedQueryExecutionInfo> tryGetPartitionQueryExecutionInfo = queryPartitionProvider.TryGetPartitionedQueryExecutionInfo(
querySpec: querySpec,
partitionKeyDefinition: partitionKeyDefinition,
requireFormattableOrderByQuery: VersionUtility.IsLaterThan(clientApiVersion, HttpConstants.Versions.v2016_11_14),
isContinuationExpected: isContinuationExpected,
allowNonValueAggregateQuery: false,
hasLogicalPartitionKey: hasLogicalPartitionKey);
if (!tryGetPartitionQueryExecutionInfo.Succeeded)
{
throw new BadRequestException(tryGetPartitionQueryExecutionInfo.Exception);
}
PartitionedQueryExecutionInfo queryExecutionInfo = tryGetPartitionQueryExecutionInfo.Result;
if (queryExecutionInfo == null ||
queryExecutionInfo.QueryRanges == null ||
queryExecutionInfo.QueryInfo == null ||
queryExecutionInfo.QueryRanges.Any(range => range.Min == null || range.Max == null))
{
DefaultTrace.TraceInformation("QueryPartitionProvider returned bad query info");
}
bool isSinglePartitionQuery = queryExecutionInfo.QueryRanges.Count == 1 && queryExecutionInfo.QueryRanges[0].IsSingleValue;
bool queryFansOutToMultiplePartitions = partitionKeyDefinition.Paths.Count > 0 && !isSinglePartitionQuery;
if (queryFansOutToMultiplePartitions)
{
if (!enableCrossPartitionQuery)
{
throw new BadRequestException(RMResources.CrossPartitionQueryDisabled);
}
else
{
bool queryNotServiceableByGateway = parallelizeCrossPartitionQuery ||
queryExecutionInfo.QueryInfo.HasTop ||
queryExecutionInfo.QueryInfo.HasOrderBy ||
queryExecutionInfo.QueryInfo.HasAggregates ||
queryExecutionInfo.QueryInfo.HasDistinct ||
queryExecutionInfo.QueryInfo.HasOffset ||
queryExecutionInfo.QueryInfo.HasLimit;
if (queryNotServiceableByGateway)
{
if (!IsSupportedPartitionedQueryExecutionInfo(queryExecutionInfo, clientApiVersion))
{
throw new BadRequestException(RMResources.UnsupportedCrossPartitionQuery);
}
else if (queryExecutionInfo.QueryInfo.HasAggregates && !IsAggregateSupportedApiVersion(clientApiVersion))
{
throw new BadRequestException(RMResources.UnsupportedCrossPartitionQueryWithAggregate);
}
else
{
DocumentClientException exception = new DocumentClientException(
RMResources.UnsupportedCrossPartitionQuery,
HttpStatusCode.BadRequest,
SubStatusCodes.CrossPartitionQueryNotServable);
exception.Error.AdditionalErrorInfo = JsonConvert.SerializeObject(queryExecutionInfo);
throw exception;
}
}
}
}
else
{
if (queryExecutionInfo.QueryInfo.HasAggregates && !isContinuationExpected)
{
// For single partition query with aggregate functions and no continuation expected,
// we would try to accumulate the results for them on the SDK, if supported.
if (IsAggregateSupportedApiVersion(clientApiVersion))
{
DocumentClientException exception = new DocumentClientException(
RMResources.UnsupportedQueryWithFullResultAggregate,
HttpStatusCode.BadRequest,
SubStatusCodes.CrossPartitionQueryNotServable);
exception.Error.AdditionalErrorInfo = JsonConvert.SerializeObject(queryExecutionInfo);
throw exception;
}
else
{
throw new BadRequestException(RMResources.UnsupportedQueryWithFullResultAggregate);
}
}
else if (queryExecutionInfo.QueryInfo.HasDistinct)
{
// If the query has distinct then we have to reject it since the backend only returns
// elements that are distinct within a page and we need the client to do post distinct processing
DocumentClientException exception = new DocumentClientException(
RMResources.UnsupportedCrossPartitionQuery,
HttpStatusCode.BadRequest,
SubStatusCodes.CrossPartitionQueryNotServable);
exception.Error.AdditionalErrorInfo = JsonConvert.SerializeObject(queryExecutionInfo);
throw exception;
}
}
queryInfo = queryExecutionInfo.QueryInfo;
return(queryExecutionInfo.QueryRanges);
}
internal static CosmosResponse <TEntity> ToCosmosResponse <TEntity>(this DocumentClientException exception) where TEntity : class
{
return(ToCosmosResponse <TEntity>(exception, null));
}
private async Task HandleDocumentClientException(ActionExecutingContext context, DocumentClientException documentClientException)
{
if (documentClientException.StatusCode == HttpStatusCode.TooManyRequests)
{
_logger.Warning("Received 429 from CosmosDb. Waiting for {Delay} then returning 429.", documentClientException.RetryAfter);
await Task.Delay(documentClientException.RetryAfter);
context.Result = new StatusCodeResult(429);
return;
}
throw documentClientException;
}
