public async Task NoSessionTokenCaptureForThrottledUpsertRequestsTest()
{
ToDoActivity testItem = ToDoActivity.CreateRandomToDoActivity();
ResetSessionToken(this.Container);
Assert.IsNull(await GetLSNFromSessionContainer(
this.Container, this.containerSettings, new PartitionKey(testItem.pk)));
ItemResponse <ToDoActivity> response = await this.Container.CreateItemAsync <ToDoActivity>(item : testItem);
Assert.IsNotNull(response);
Assert.IsNotNull(response.Resource);
Assert.IsNotNull(response.Diagnostics);
string sessionTokenHeaderValue = response.Headers[HttpConstants.HttpHeaders.SessionToken];
Assert.IsNotNull(sessionTokenHeaderValue);
long?lsnAfterCreate = await GetLSNFromSessionContainer(
this.Container, this.containerSettings, new PartitionKey(testItem.pk));
Assert.IsNotNull(lsnAfterCreate);
ResetSessionToken(this.Container);
Assert.IsNull(await GetLSNFromSessionContainer(
this.Container, this.containerSettings, new PartitionKey(testItem.pk)));
Container throttledContainer = TransportClientHelper.GetContainerWithIntercepter(
this.database.Id,
this.Container.Id,
(uri, resourceOperation, documentServiceRequest) => { },
useGatewayMode: false,
(uri, resourceOperation, documentServiceRequest) =>
{
StoreResponse throttledResponse = TransportClientHelper.ReturnThrottledStoreResponseOnItemOperation(
uri, resourceOperation, documentServiceRequest, Guid.NewGuid(), string.Empty);
throttledResponse.Headers.Add(HttpConstants.HttpHeaders.SessionToken, sessionTokenHeaderValue);
return(throttledResponse);
},
this.cosmosClient.DocumentClient.sessionContainer);
try
{
//Updated the taskNum field
testItem.taskNum = 9001;
response = await throttledContainer.UpsertItemAsync(testItem, partitionKey : new Cosmos.PartitionKey(testItem.pk));
}
catch (CosmosException cosmosException)
{
Assert.AreEqual(HttpStatusCode.TooManyRequests, cosmosException.StatusCode);
}
long?lsnAfterThrottledRequest = await GetLSNFromSessionContainer(
this.Container, this.containerSettings, new PartitionKey(testItem.pk));
Assert.IsNull(lsnAfterThrottledRequest);
}
public async Task PointOperationRequestTimeoutDiagnostic(bool disableDiagnostics)
{
ItemRequestOptions requestOptions = new ItemRequestOptions();
if (disableDiagnostics)
{
requestOptions.DiagnosticContextFactory = () => EmptyCosmosDiagnosticsContext.Singleton;
}
;
Guid exceptionActivityId = Guid.NewGuid();
string transportExceptionDescription = "transportExceptionDescription" + Guid.NewGuid();
Container containerWithTransportException = TransportClientHelper.GetContainerWithItemTransportException(
this.database.Id,
this.Container.Id,
exceptionActivityId,
transportExceptionDescription);
//Checking point operation diagnostics on typed operations
ToDoActivity testItem = ToDoActivity.CreateRandomToDoActivity();
try
{
ItemResponse <ToDoActivity> createResponse = await containerWithTransportException.CreateItemAsync <ToDoActivity>(
item : testItem,
requestOptions : requestOptions);
Assert.Fail("Should have thrown a request timeout exception");
}
catch (CosmosException ce) when(ce.StatusCode == System.Net.HttpStatusCode.RequestTimeout)
{
string exception = ce.ToString();
Assert.IsNotNull(exception);
Assert.IsTrue(exception.Contains(exceptionActivityId.ToString()));
Assert.IsTrue(exception.Contains(transportExceptionDescription));
string diagnosics = ce.Diagnostics.ToString();
if (disableDiagnostics)
{
Assert.IsTrue(string.IsNullOrEmpty(diagnosics));
}
else
{
Assert.IsFalse(string.IsNullOrEmpty(diagnosics));
Assert.IsTrue(exception.Contains(diagnosics));
DiagnosticValidator.ValidatePointOperationDiagnostics(ce.DiagnosticsContext);
}
}
}
internal static Container GetContainerWithItemTransportException(
string databaseId,
string containerId,
Guid activityId,
string transportExceptionSourceDescription)
{
return(GetContainerWithIntercepter(
databaseId,
containerId,
(uri, resourceOperation, request) => TransportClientHelper.ThrowTransportExceptionOnItemOperation(
uri,
resourceOperation,
request,
activityId,
transportExceptionSourceDescription)));
}
public async Task CheckCancellationWithTransportIntercepterTestAsync()
{
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
Container withCancellationToken = TransportClientHelper.GetContainerWithIntercepter(
this.database.Id,
this.Container.Id,
(uri, resourceOperation, documentServiceRequest) =>
{
if (documentServiceRequest.ResourceType == Documents.ResourceType.Document)
{
cancellationTokenSource.Cancel();
cancellationTokenSource.Token.ThrowIfCancellationRequested();
}
});
await this.CheckCancellationTokenTestAsync(withCancellationToken, cancellationTokenSource.Token);
}
public async Task CheckCancellationWithTransportIntercepterTestAsync()
{
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
Container withCancellationToken = TransportClientHelper.GetContainerWithIntercepter(
this.database.Id,
this.Container.Id,
(uri, resourceOperation, documentServiceRequest) =>
{
if (documentServiceRequest.ResourceType == Documents.ResourceType.Document)
{
cancellationTokenSource.Cancel();
}
},
useGatewayMode: false,
(uri, resourceOperation, documentServiceRequest)
=> TransportClientHelper.ReturnThrottledStoreResponseOnItemOperation(uri, resourceOperation, documentServiceRequest, Guid.NewGuid(), string.Empty));
await this.CheckCancellationTokenTestAsync(withCancellationToken, cancellationTokenSource.Token);
}
internal static Container GetContainerWithItemTransportException(
string databaseId,
string containerId,
Guid activityId,
string transportExceptionSourceDescription)
{
CosmosClient clientWithIntercepter = TestCommon.CreateCosmosClient(
builder =>
{
builder.WithTransportClientHandlerFactory(transportClient => new TransportClientWrapper(
transportClient,
(uri, resourceOperation, request) => TransportClientHelper.ThrowTransportExceptionOnItemOperation(
uri,
resourceOperation,
request,
activityId,
transportExceptionSourceDescription)));
});
return(clientWithIntercepter.GetContainer(databaseId, containerId));
}
public async Task PointOperationForbiddenDiagnostic()
{
List <int> stringLength = new List <int>();
foreach (int maxCount in new int[] { 1, 2, 4 })
{
int count = 0;
List <(string, string)> activityIdAndErrorMessage = new List <(string, string)>(maxCount);
Guid transportExceptionActivityId = Guid.NewGuid();
string transportErrorMessage = $"TransportErrorMessage{Guid.NewGuid()}";
Guid activityIdScope = Guid.Empty;
Action <Uri, Documents.ResourceOperation, Documents.DocumentServiceRequest> interceptor =
(uri, operation, request) =>
{
Assert.AreNotEqual(Trace.CorrelationManager.ActivityId, Guid.Empty, "Activity scope should be set");
if (request.ResourceType == Documents.ResourceType.Document)
{
if (activityIdScope == Guid.Empty)
{
activityIdScope = Trace.CorrelationManager.ActivityId;
}
else
{
Assert.AreEqual(Trace.CorrelationManager.ActivityId, activityIdScope, "Activity scope should match on retries");
}
if (count >= maxCount)
{
TransportClientHelper.ThrowTransportExceptionOnItemOperation(
uri,
operation,
request,
transportExceptionActivityId,
transportErrorMessage);
}
count++;
string activityId = Guid.NewGuid().ToString();
string errorMessage = $"Error{Guid.NewGuid()}";
activityIdAndErrorMessage.Add((activityId, errorMessage));
TransportClientHelper.ThrowForbiddendExceptionOnItemOperation(
uri,
request,
activityId,
errorMessage);
}
};
Container containerWithTransportException = TransportClientHelper.GetContainerWithIntercepter(
this.database.Id,
this.Container.Id,
interceptor);
//Checking point operation diagnostics on typed operations
ToDoActivity testItem = ToDoActivity.CreateRandomToDoActivity();
try
{
ItemResponse <ToDoActivity> createResponse = await containerWithTransportException.CreateItemAsync <ToDoActivity>(
item : testItem);
Assert.Fail("Should have thrown a request timeout exception");
}
catch (CosmosException ce) when(ce.StatusCode == System.Net.HttpStatusCode.RequestTimeout)
{
string exceptionToString = ce.ToString();
Assert.IsNotNull(exceptionToString);
stringLength.Add(exceptionToString.Length);
// Request timeout info will be in the exception message and in the diagnostic info.
Assert.AreEqual(2, Regex.Matches(exceptionToString, transportExceptionActivityId.ToString()).Count);
Assert.AreEqual(2, Regex.Matches(exceptionToString, transportErrorMessage).Count);
// Check to make sure the diagnostics does not include duplicate info.
foreach ((string activityId, string errorMessage) in activityIdAndErrorMessage)
{
Assert.AreEqual(1, Regex.Matches(exceptionToString, activityId).Count);
Assert.AreEqual(1, Regex.Matches(exceptionToString, errorMessage).Count);
}
}
}
// Check if the exception message is not growing exponentially
Assert.IsTrue(stringLength.Count > 2);
for (int i = 0; i < stringLength.Count - 1; i++)
{
int currLength = stringLength[i];
int nextLength = stringLength[i + 1];
Assert.IsTrue(nextLength < currLength * 2,
$"The diagnostic string is growing faster than linear. Length: {currLength}, Next Length: {nextLength}");
}
}
public async Task PointOperationThrottledDiagnostic(bool disableDiagnostics)
{
string errorMessage = "Mock throttle exception" + Guid.NewGuid().ToString();
Guid exceptionActivityId = Guid.NewGuid();
// Set a small retry count to reduce test time
CosmosClient throttleClient = TestCommon.CreateCosmosClient(builder =>
builder.WithThrottlingRetryOptions(TimeSpan.FromSeconds(5), 5)
.WithTransportClientHandlerFactory(transportClient => new TransportClientWrapper(
transportClient,
(uri, resourceOperation, request) => TransportClientHelper.ReturnThrottledStoreResponseOnItemOperation(
uri,
resourceOperation,
request,
exceptionActivityId,
errorMessage)))
);
ItemRequestOptions requestOptions = new ItemRequestOptions();
if (disableDiagnostics)
{
requestOptions.DiagnosticContextFactory = () => EmptyCosmosDiagnosticsContext.Singleton;
}
;
Container containerWithThrottleException = throttleClient.GetContainer(
this.database.Id,
this.Container.Id);
//Checking point operation diagnostics on typed operations
ToDoActivity testItem = ToDoActivity.CreateRandomToDoActivity();
try
{
ItemResponse <ToDoActivity> createResponse = await containerWithThrottleException.CreateItemAsync <ToDoActivity>(
item : testItem,
requestOptions : requestOptions);
Assert.Fail("Should have thrown a request timeout exception");
}
catch (CosmosException ce) when((int)ce.StatusCode == (int)Documents.StatusCodes.TooManyRequests)
{
string exception = ce.ToString();
Assert.IsNotNull(exception);
Assert.IsTrue(exception.Contains(exceptionActivityId.ToString()));
Assert.IsTrue(exception.Contains(errorMessage));
string diagnosics = ce.Diagnostics.ToString();
if (disableDiagnostics)
{
Assert.IsTrue(string.IsNullOrEmpty(diagnosics));
}
else
{
Assert.IsFalse(string.IsNullOrEmpty(diagnosics));
Assert.IsTrue(exception.Contains(diagnosics));
DiagnosticValidator.ValidatePointOperationDiagnostics(ce.DiagnosticsContext);
}
}
}
public async Task InvalidSessionTokenAfterContainerRecreationAndCollectionCacheRefreshReproTest()
{
// ingestionClinet is dedicated client simulating the writes / container recreation in
// the separate process - like Spark job
using CosmosClient ingestionClient = TestCommon.CreateCosmosClient();
Cosmos.Database ingestionDatabase = ingestionClient.GetDatabase(this.database.Id);
ContainerProperties multiPartitionContainerSettings =
new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPath: "/pk");
Container ingestionContainer =
await ingestionDatabase.CreateContainerAsync(multiPartitionContainerSettings);
const int itemCountToBeIngested = 10;
string pk = Guid.NewGuid().ToString("N");
long? latestLsn = null;
Console.WriteLine("INGEST DOCUMENTS");
for (int i = 0; i < itemCountToBeIngested; i++)
{
ToDoActivity testItem = ToDoActivity.CreateRandomToDoActivity();
testItem.pk = pk;
ItemResponse <ToDoActivity> response =
await ingestionContainer.CreateItemAsync <ToDoActivity>(item : testItem);
Assert.IsNotNull(response);
Assert.IsNotNull(response.Resource);
Assert.IsNotNull(response.Diagnostics);
long?lsnAfterCreate = await GetLSNFromSessionContainer(
ingestionContainer, multiPartitionContainerSettings, new PartitionKey(pk));
Assert.IsNotNull(lsnAfterCreate);
Assert.IsTrue(latestLsn == null || lsnAfterCreate.Value > latestLsn.Value);
latestLsn = lsnAfterCreate;
CosmosTraceDiagnostics diagnostics = (CosmosTraceDiagnostics)response.Diagnostics;
Assert.IsFalse(diagnostics.IsGoneExceptionHit());
Assert.IsFalse(string.IsNullOrEmpty(diagnostics.ToString()));
Assert.IsTrue(diagnostics.GetClientElapsedTime() > TimeSpan.Zero);
}
// Dedciated query client used only for queries simulating the customer's app
string lastRequestedSessionToken = null;
Container queryContainer = TransportClientHelper.GetContainerWithIntercepter(
this.database.Id,
ingestionContainer.Id,
(uri, operation, request) =>
{
if (request.ResourceType == ResourceType.Document &&
request.OperationType == OperationType.Query)
{
lastRequestedSessionToken = request.Headers[HttpConstants.HttpHeaders.SessionToken];
}
},
false,
null);
long?lsnAfterQueryOnOldContainer = null;
// Issueing two queries - first won't use session tokens yet
// second will provide session tokens captured from first request in the request to the backend
for (int i = 0; i < 2; i++)
{
Console.WriteLine("RUN QUERY ON OLD CONTAINER ({0})", i);
using FeedIterator <JObject> queryIteratorOldContainer = queryContainer.GetItemQueryIterator <JObject>(
new QueryDefinition("Select c.id FROM c"),
continuationToken: null,
new QueryRequestOptions
{
ConsistencyLevel = Cosmos.ConsistencyLevel.Session,
PartitionKey = new Cosmos.PartitionKey(pk)
});
int itemCountOldContainer = 0;
while (queryIteratorOldContainer.HasMoreResults)
{
FeedResponse <JObject> response = await queryIteratorOldContainer.ReadNextAsync();
if (i == 0)
{
string diagnosticString = response.Diagnostics.ToString();
Assert.IsTrue(diagnosticString.Contains("PKRangeCache Info("));
JObject diagnosticJobject = JObject.Parse(diagnosticString);
JToken actualToken = diagnosticJobject.SelectToken("$.children[0].children[?(@.name=='Get Partition Key Ranges')].children[?(@.name=='Try Get Overlapping Ranges')].data");
JToken actualNode = actualToken.Children().First().First();
Assert.IsTrue(actualNode["Previous Continuation Token"].ToString().Length == 0);
Assert.IsTrue(actualNode["Continuation Token"].ToString().Length > 0);
}
itemCountOldContainer += response.Count;
}
Assert.AreEqual(itemCountToBeIngested, itemCountOldContainer);
lsnAfterQueryOnOldContainer = await GetLSNFromSessionContainer(
queryContainer, multiPartitionContainerSettings, new PartitionKey(pk));
Assert.IsNotNull(lsnAfterQueryOnOldContainer);
Assert.AreEqual(latestLsn.Value, lsnAfterQueryOnOldContainer.Value);
if (i == 0)
{
Assert.IsNull(lastRequestedSessionToken);
}
else
{
Assert.IsNotNull(lastRequestedSessionToken);
Assert.AreEqual(latestLsn.Value, SessionTokenHelper.Parse(lastRequestedSessionToken).LSN);
}
}
Console.WriteLine(
"DELETE CONTAINER {0}",
(await queryContainer.ReadContainerAsync()).Resource.ResourceId);
await ingestionContainer.DeleteContainerAsync();
Console.WriteLine("RECREATING CONTAINER...");
ContainerResponse ingestionContainerResponse =
await ingestionDatabase.CreateContainerAsync(multiPartitionContainerSettings);
ingestionContainer = ingestionContainerResponse.Container;
string responseSessionTokenValue =
ingestionContainerResponse.Headers[HttpConstants.HttpHeaders.SessionToken];
long?lsnAfterRecreatingContainerFromIngestionClient = responseSessionTokenValue != null?
SessionTokenHelper.Parse(responseSessionTokenValue).LSN : null;
Console.WriteLine(
"RECREATED CONTAINER with new CollectionRid: {0} - LSN: {1}",
ingestionContainerResponse.Resource.ResourceId,
lsnAfterRecreatingContainerFromIngestionClient);
// validates that the query container still uses the LSN captured from the old LSN
long?lsnAfterCreatingNewContainerFromQueryClient = await GetLSNFromSessionContainer(
queryContainer, multiPartitionContainerSettings, new PartitionKey(pk));
Assert.IsNotNull(lsnAfterCreatingNewContainerFromQueryClient);
Assert.AreEqual(latestLsn.Value, lsnAfterCreatingNewContainerFromQueryClient.Value);
Console.WriteLine("GET FEED RANGES");
// this will force a CollectionCache refresh - because no pk ranegs can be identified
// for the old container anymore
_ = await queryContainer.GetFeedRangesAsync();
Console.WriteLine("RUN QUERY ON NEW CONTAINER");
int itemCountNewContainer = 0;
using FeedIterator <JObject> queryIteratorNewContainer = queryContainer.GetItemQueryIterator <JObject>(
new QueryDefinition("Select c.id FROM c"),
continuationToken: null,
new QueryRequestOptions
{
ConsistencyLevel = Cosmos.ConsistencyLevel.Session,
PartitionKey = new Cosmos.PartitionKey(pk),
});
Console.WriteLine("Query iterator created");
while (queryIteratorNewContainer.HasMoreResults)
{
Console.WriteLine("Retrieving first page");
try
{
FeedResponse <JObject> response = await queryIteratorNewContainer.ReadNextAsync();
Console.WriteLine("Request Diagnostics for query against new container: {0}",
response.Diagnostics.ToString());
itemCountNewContainer += response.Count;
}
catch (CosmosException cosmosException)
{
Console.WriteLine("COSMOS EXCEPTION: {0}", cosmosException);
throw;
}
}
Assert.AreEqual(0, itemCountNewContainer);
long?lsnAfterQueryOnNewContainer = await GetLSNFromSessionContainer(
queryContainer, multiPartitionContainerSettings, new PartitionKey(pk));
Assert.IsNotNull(lsnAfterQueryOnNewContainer);
Assert.IsTrue(
lastRequestedSessionToken == null ||
SessionTokenHelper.Parse(lastRequestedSessionToken).LSN ==
lsnAfterRecreatingContainerFromIngestionClient,
$"The requested session token {lastRequestedSessionToken} on the last query request should be null " +
$"or have LSN '{lsnAfterRecreatingContainerFromIngestionClient}' (which is the LSN after " +
"re-creating the container) if the session cache or the new CollectionName to Rid mapping was " +
"correctly populated in the SessionCache.");
}