private async Task GetIteratorResponse <T>(FeedIterator <T> feedIterator)
{
MetricsAccumulator metricsAccumulator = new MetricsAccumulator();
Documents.ValueStopwatch totalTime = new Documents.ValueStopwatch();
Documents.ValueStopwatch getTraceTime = new Documents.ValueStopwatch();
string diagnosticDataPath = Path.GetFullPath(DiagnosticsDataFileName);
while (feedIterator.HasMoreResults)
{
totalTime.Start();
FeedResponse <T> response = await feedIterator.ReadNextAsync();
getTraceTime.Start();
if (response.RequestCharge != 0)
{
using (StreamWriter outputFile = new StreamWriter(path: diagnosticDataPath, append: true))
{
outputFile.WriteLine(response.Diagnostics.ToString());
}
metricsAccumulator.ReadFromTrace(response, this.queryStatisticsDatumVisitor);
}
getTraceTime.Stop();
totalTime.Stop();
if (response.RequestCharge != 0)
{
this.queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - getTraceTime.ElapsedMilliseconds);
this.queryStatisticsDatumVisitor.PopulateMetrics();
}
totalTime.Reset();
getTraceTime.Reset();
}
}
public async Task VerifyPkRangeCacheRefreshOnSplitWithErrorsAsync()
{
this.loopBackgroundOperaitons = false;
int throwOnPkRefreshCount = 3;
int pkRangeCalls = 0;
bool causeSplitExceptionInRntbdCall = false;
HttpClientHandlerHelper httpHandlerHelper = new();
List <string> ifNoneMatchValues = new();
string failedIfNoneMatchValue = null;
httpHandlerHelper.RequestCallBack = (request, cancellationToken) =>
{
if (!request.RequestUri.ToString().EndsWith("pkranges"))
{
return(null);
}
ifNoneMatchValues.Add(request.Headers.IfNoneMatch.ToString());
pkRangeCalls++;
if (pkRangeCalls == throwOnPkRefreshCount)
{
failedIfNoneMatchValue = request.Headers.IfNoneMatch.ToString();
return(Task.FromResult(new HttpResponseMessage(HttpStatusCode.InternalServerError)));
}
return(null);
};
int countSplitExceptions = 0;
CosmosClientOptions clientOptions = new CosmosClientOptions()
{
HttpClientFactory = () => new HttpClient(httpHandlerHelper),
TransportClientHandlerFactory = (transportClient) => new TransportClientWrapper(
transportClient,
(uri, resource, dsr) =>
{
if (dsr.OperationType == Documents.OperationType.Read &&
dsr.ResourceType == Documents.ResourceType.Document &&
causeSplitExceptionInRntbdCall)
{
countSplitExceptions++;
causeSplitExceptionInRntbdCall = false;
throw new Documents.Routing.PartitionKeyRangeIsSplittingException("Test");
}
})
};
CosmosClient resourceClient = TestCommon.CreateCosmosClient(clientOptions);
string dbName = Guid.NewGuid().ToString();
string containerName = nameof(PartitionKeyRangeCacheTests);
Database db = await resourceClient.CreateDatabaseIfNotExistsAsync(dbName);
Container container = await db.CreateContainerIfNotExistsAsync(
containerName,
"/pk",
400);
// Start a background job that loops forever
List <Exception> exceptions = new();
Task backgroundItemOperatios = Task.Factory.StartNew(() => this.CreateAndReadItemBackgroundLoop(container, exceptions));
// Wait for the background job to start
Documents.ValueStopwatch stopwatch = Documents.ValueStopwatch.StartNew();
while (!this.loopBackgroundOperaitons && stopwatch.Elapsed.TotalSeconds < 30)
{
await Task.Delay(TimeSpan.FromSeconds(.5));
}
Assert.IsTrue(this.loopBackgroundOperaitons);
Assert.AreEqual(2, pkRangeCalls);
// Cause direct call to hit a split exception and wait for the background job to hit it
causeSplitExceptionInRntbdCall = true;
stopwatch = Documents.ValueStopwatch.StartNew();
while (causeSplitExceptionInRntbdCall && stopwatch.Elapsed.TotalSeconds < 10)
{
await Task.Delay(TimeSpan.FromSeconds(.5));
}
Assert.IsFalse(causeSplitExceptionInRntbdCall);
Assert.AreEqual(3, pkRangeCalls);
// Cause another direct call split exception
causeSplitExceptionInRntbdCall = true;
stopwatch = Documents.ValueStopwatch.StartNew();
while (causeSplitExceptionInRntbdCall && stopwatch.Elapsed.TotalSeconds < 10)
{
await Task.Delay(TimeSpan.FromSeconds(.5));
}
Assert.IsFalse(causeSplitExceptionInRntbdCall);
Assert.AreEqual(4, pkRangeCalls);
Assert.AreEqual(1, ifNoneMatchValues.Count(x => string.IsNullOrEmpty(x)));
Assert.AreEqual(3, ifNoneMatchValues.Count(x => x == failedIfNoneMatchValue), $"3 request with same if none value. 1 initial, 2 from the split errors. split exception count: {countSplitExceptions}; {string.Join(';', ifNoneMatchValues)}");
HashSet <string> verifyUniqueIfNoneHeaderValues = new HashSet <string>();
foreach (string ifNoneValue in ifNoneMatchValues)
{
if (!verifyUniqueIfNoneHeaderValues.Contains(ifNoneValue))
{
verifyUniqueIfNoneHeaderValues.Add(ifNoneValue);
}
else if (ifNoneValue != failedIfNoneMatchValue)
{
Assert.AreEqual(failedIfNoneMatchValue, ifNoneValue, $"Multiple duplicates; split exception count: {countSplitExceptions}; {string.Join(';', ifNoneMatchValues)}");
}
}
Assert.AreEqual(0, exceptions.Count, $"Unexpected exceptions: {string.Join(';', exceptions)}");
}
