private async Task GetIteratorResponse <T>(FeedIterator <T> feedIterator) { MetricsAccumulator metricsAccumulator = new MetricsAccumulator(); Documents.ValueStopwatch totalTime = new Documents.ValueStopwatch(); Documents.ValueStopwatch getTraceTime = new Documents.ValueStopwatch(); string diagnosticDataPath = Path.GetFullPath(DiagnosticsDataFileName); while (feedIterator.HasMoreResults) { totalTime.Start(); FeedResponse <T> response = await feedIterator.ReadNextAsync(); getTraceTime.Start(); if (response.RequestCharge != 0) { using (StreamWriter outputFile = new StreamWriter(path: diagnosticDataPath, append: true)) { outputFile.WriteLine(response.Diagnostics.ToString()); } metricsAccumulator.ReadFromTrace(response, this.queryStatisticsDatumVisitor); } getTraceTime.Stop(); totalTime.Stop(); if (response.RequestCharge != 0) { this.queryStatisticsDatumVisitor.AddEndToEndTime(totalTime.ElapsedMilliseconds - getTraceTime.ElapsedMilliseconds); this.queryStatisticsDatumVisitor.PopulateMetrics(); } totalTime.Reset(); getTraceTime.Reset(); } }
public async Task VerifyPkRangeCacheRefreshOnSplitWithErrorsAsync() { this.loopBackgroundOperaitons = false; int throwOnPkRefreshCount = 3; int pkRangeCalls = 0; bool causeSplitExceptionInRntbdCall = false; HttpClientHandlerHelper httpHandlerHelper = new(); List <string> ifNoneMatchValues = new(); string failedIfNoneMatchValue = null; httpHandlerHelper.RequestCallBack = (request, cancellationToken) => { if (!request.RequestUri.ToString().EndsWith("pkranges")) { return(null); } ifNoneMatchValues.Add(request.Headers.IfNoneMatch.ToString()); pkRangeCalls++; if (pkRangeCalls == throwOnPkRefreshCount) { failedIfNoneMatchValue = request.Headers.IfNoneMatch.ToString(); return(Task.FromResult(new HttpResponseMessage(HttpStatusCode.InternalServerError))); } return(null); }; int countSplitExceptions = 0; CosmosClientOptions clientOptions = new CosmosClientOptions() { HttpClientFactory = () => new HttpClient(httpHandlerHelper), TransportClientHandlerFactory = (transportClient) => new TransportClientWrapper( transportClient, (uri, resource, dsr) => { if (dsr.OperationType == Documents.OperationType.Read && dsr.ResourceType == Documents.ResourceType.Document && causeSplitExceptionInRntbdCall) { countSplitExceptions++; causeSplitExceptionInRntbdCall = false; throw new Documents.Routing.PartitionKeyRangeIsSplittingException("Test"); } }) }; CosmosClient resourceClient = TestCommon.CreateCosmosClient(clientOptions); string dbName = Guid.NewGuid().ToString(); string containerName = nameof(PartitionKeyRangeCacheTests); Database db = await resourceClient.CreateDatabaseIfNotExistsAsync(dbName); Container container = await db.CreateContainerIfNotExistsAsync( containerName, "/pk", 400); // Start a background job that loops forever List <Exception> exceptions = new(); Task backgroundItemOperatios = Task.Factory.StartNew(() => this.CreateAndReadItemBackgroundLoop(container, exceptions)); // Wait for the background job to start Documents.ValueStopwatch stopwatch = Documents.ValueStopwatch.StartNew(); while (!this.loopBackgroundOperaitons && stopwatch.Elapsed.TotalSeconds < 30) { await Task.Delay(TimeSpan.FromSeconds(.5)); } Assert.IsTrue(this.loopBackgroundOperaitons); Assert.AreEqual(2, pkRangeCalls); // Cause direct call to hit a split exception and wait for the background job to hit it causeSplitExceptionInRntbdCall = true; stopwatch = Documents.ValueStopwatch.StartNew(); while (causeSplitExceptionInRntbdCall && stopwatch.Elapsed.TotalSeconds < 10) { await Task.Delay(TimeSpan.FromSeconds(.5)); } Assert.IsFalse(causeSplitExceptionInRntbdCall); Assert.AreEqual(3, pkRangeCalls); // Cause another direct call split exception causeSplitExceptionInRntbdCall = true; stopwatch = Documents.ValueStopwatch.StartNew(); while (causeSplitExceptionInRntbdCall && stopwatch.Elapsed.TotalSeconds < 10) { await Task.Delay(TimeSpan.FromSeconds(.5)); } Assert.IsFalse(causeSplitExceptionInRntbdCall); Assert.AreEqual(4, pkRangeCalls); Assert.AreEqual(1, ifNoneMatchValues.Count(x => string.IsNullOrEmpty(x))); Assert.AreEqual(3, ifNoneMatchValues.Count(x => x == failedIfNoneMatchValue), $"3 request with same if none value. 1 initial, 2 from the split errors. split exception count: {countSplitExceptions}; {string.Join(';', ifNoneMatchValues)}"); HashSet <string> verifyUniqueIfNoneHeaderValues = new HashSet <string>(); foreach (string ifNoneValue in ifNoneMatchValues) { if (!verifyUniqueIfNoneHeaderValues.Contains(ifNoneValue)) { verifyUniqueIfNoneHeaderValues.Add(ifNoneValue); } else if (ifNoneValue != failedIfNoneMatchValue) { Assert.AreEqual(failedIfNoneMatchValue, ifNoneValue, $"Multiple duplicates; split exception count: {countSplitExceptions}; {string.Join(';', ifNoneMatchValues)}"); } } Assert.AreEqual(0, exceptions.Count, $"Unexpected exceptions: {string.Join(';', exceptions)}"); }