async Task ResetPerfCounterMonitorAsync() { _ctsPcounter?.Cancel(); if (_tcsPcounter != null) { await _tcsPcounter.Task; } lock (LstPerfCounterData) { measurementHolderInteractiveUser = new MeasurementHolder( TestNameOrTestContext: MeasurementHolder.InteractiveUser, new StressUtilOptions() { NumIterations = -1, logger = this, lstPerfCountersToUse = LstPerfCounterData }, sampleType: SampleType.SampleTypeNormal); _dataPoints.Clear(); _bufferIndex = 0; } if (UpdateInterval > 0) { AddStatusMsgAsync($"{nameof(ResetPerfCounterMonitorAsync)}").Forget(); DoPerfCounterMonitoring(); } else { AddStatusMsgAsync($"UpdateInterval = 0 auto sampling turned off").Forget(); } }
/// <summary> /// return true if regression found /// These tests will be affected by other tests running in the same instance of testhost because they share the same memory /// </summary> private async Task <bool> DoStressSimulation(int nIter, int nArraySize, float RatioThresholdSensitivity, Action action = null) { var lstPCs = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(), IsForStress: true); foreach (var ctr in lstPCs) { ctr.IsEnabledForMeasurement = true; } List <LeakAnalysisResult> lstRegResults; using (var measurementHolder = new MeasurementHolder( new TestContextWrapper(TestContext), new StressUtilOptions() { NumIterations = -1, Sensitivity = RatioThresholdSensitivity, logger = this, lstPerfCountersToUse = lstPCs }, SampleType.SampleTypeIteration)) { var lstBigStuff = new List <byte[]>(); LogMessage($"nIter={nIter:n0} ArraySize= {nArraySize:n0}"); for (int i = 0; i < nIter; i++) { if (action != null) { action(); } else { lstBigStuff.Add(new byte[nArraySize]); } // lstBigStuff.Add(new int[10000000]); var res = await measurementHolder.TakeMeasurementAsync($"iter {i}/{nIter}", DoForceGC : true); LogMessage(res); } var filename = measurementHolder.DumpOutMeasurementsToTxtFile(); LogMessage($"Results file name = {filename}"); lstRegResults = (await measurementHolder.CalculateLeaksAsync(showGraph: false, GraphsAsFilePrefix: "Graph")).Where(r => r.IsLeak).ToList(); } return(lstRegResults.Count > 0); }
public async Task TestMeasureRegressionVerifyGraph() { await Task.Yield(); var resultsFolder = string.Empty; using (var measurementHolder = new MeasurementHolder( new TestContextWrapper(TestContext), new StressUtilOptions() { NumIterations = -1, logger = this, lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(), IsForStress: true).Where(p => p.perfCounterType == PerfCounterType.KernelHandleCount).ToList() }, SampleType.SampleTypeIteration)) { resultsFolder = measurementHolder.ResultsFolder; for (int iter = 0; iter < 10; iter++) { foreach (var ctr in measurementHolder.LstPerfCounterData) { var val = 10000 + (uint)iter; if (iter == 5 && ctr.perfCounterType == PerfCounterType.KernelHandleCount) { val += 1; } measurementHolder.measurements[ctr.perfCounterType].Add(val); } } var res = await measurementHolder.CalculateLeaksAsync(showGraph : true, GraphsAsFilePrefix : "Graph"); } var strHtml = @" <a href=""file:\\C:\Users\calvinh\Source\repos\PerfGraphVSIX\TestResults\Deploy_calvinh 2019-11-19 11_00_13/Out/TestMeasureRegressionVerifyGraph/Graph Handle Count.png"">gr </a> "; var fileHtml = Path.Combine(resultsFolder, "IndexTest1.html"); File.WriteAllText(fileHtml, strHtml); TestContext.AddResultFile(fileHtml); Assert.Fail("failing test so results aren't deleted"); }
public async Task TestOutliers() { if (StressUtilOptions.IsRunningOnBuildMachine()) { return; } await Task.Yield(); // data from https://dev.azure.com/devdiv/DevDiv/_releaseProgress?_a=release-environment-extension&releaseId=548609&environmentId=2872682&extensionId=ms.vss-test-web.test-result-in-release-environment-editor-tab&runId=10533790&resultId=100000&paneView=attachments var testData = new uint[] { 1867008, 2713172, 2701928, 2701928, 2701800, 2701800, 2701700, 2701700, 2711368, 2711368, 2713228, 2713228, 2714876, 2714876, 2716588, 2716588, 2716588, 2732980, 2732980, 2734848, 2734848, 2736536, 2736536, 2738052, 2738052, 2739908, 2739908, 2741400, 2741400, 2742916, 2742916, 2744548, 2744548, 2744548, 2747340, 2747340, 2764696, 2764696, 2766384, 2766384, 2767888, 2767888, 2769756, 2769756, 2771260, 2771260, 2772764, 2772764, 2774268, 2774268, 2774268, 2776140, 2776140, 2777468, 2777468, 2779168, 2779168, 2779836, 2779836, 2797744, 2797744, 2799236, 2799236, 2800996, 2800996, 2803548, 2803548, 2899440, 2904492, 2904492, 2904492, }; var resultsFolder = string.Empty; using (var measurementHolder = new MeasurementHolder( new TestContextWrapper(TestContext), new StressUtilOptions() { NumIterations = -1, logger = this, pctOutliersToIgnore = 5, lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(), IsForStress: true).Where(p => p.perfCounterType == PerfCounterType.GCBytesInAllHeaps).ToList() }, SampleType.SampleTypeIteration)) { resultsFolder = measurementHolder.ResultsFolder; for (int iter = 0; iter < testData.Length; iter++) { foreach (var ctr in measurementHolder.LstPerfCounterData) { measurementHolder.measurements[ctr.perfCounterType].Add(testData[iter]); } } var leakAnalysisResults = await measurementHolder.CalculateLeaksAsync(showGraph : false, GraphsAsFilePrefix : "Graph"); LogMessage($"Yint = {leakAnalysisResults[0].yintercept:n0}"); foreach (var res in leakAnalysisResults[0].lstData) { LogMessage($"{res} dist = {res.distance} {res}"); } Assert.IsTrue(leakAnalysisResults[0].lstData[0].IsOutlier); Assert.IsTrue(leakAnalysisResults[0].lstData[1].IsOutlier); Assert.IsTrue(!leakAnalysisResults[0].lstData[2].IsOutlier); Assert.IsTrue(leakAnalysisResults[0].lstData[68].IsOutlier); } }
[ExpectedException(typeof(LeakException))] // to make the test pass, we need a LeakException. However, Pass deletes all the test results <sigh> public async Task StressVSOpenCloseWaitTilQuiet() { // we can't measure for quiet on the current process because the current process will be actively doing stuff. if (StressUtilOptions.IsRunningOnBuildMachine()) { throw new LeakException("Throwing expected exception so test passes", null); } string didGetLeakException = "didGetLeakException"; int numIter = 11; try { await StressUtil.DoIterationsAsync( this, new StressUtilOptions() { LoggerLogOutputToDestkop = true, NumIterations = numIter, ShowUI = false, actExecuteAfterEveryIterationAsync = async(nIter, measurementHolder) => { // we want to take measures in a circular buffer and wait til those are quiet var circBufferSize = 5; var numTimesToGetQuiet = 50; var quietMeasure = new MeasurementHolder( "Quiet", new StressUtilOptions() { NumIterations = 1, // we'll do 1 iteration pctOutliersToIgnore = 0, logger = measurementHolder.Logger, VSHandler = measurementHolder.stressUtilOptions.VSHandler, lstPerfCountersToUse = measurementHolder.stressUtilOptions.lstPerfCountersToUse, }, SampleType.SampleTypeIteration ); // We just took a measurement, so copy those values to init our buffer foreach (var pctrMeasure in measurementHolder.measurements.Keys) { var lastVal = measurementHolder.measurements[pctrMeasure][measurementHolder.nSamplesTaken - 1]; quietMeasure.measurements[pctrMeasure].Add(lastVal); } quietMeasure.nSamplesTaken++; var isQuiet = false; int nMeasurementsForQuiet = 0; while (!isQuiet && nMeasurementsForQuiet < numTimesToGetQuiet) { await quietMeasure.DoForceGCAsync(); await Task.Delay(TimeSpan.FromSeconds(1 * measurementHolder.stressUtilOptions.DelayMultiplier)); // after GC, wait 1 before taking measurements var sb = new StringBuilder($"Measure for Quiet iter = {nIter} QuietSamp#= {nMeasurementsForQuiet}"); quietMeasure.TakeRawMeasurement(sb); measurementHolder.Logger.LogMessage(sb.ToString()); ///xxxremove if (quietMeasure.nSamplesTaken == circBufferSize) { var lk = await quietMeasure.CalculateLeaksAsync( showGraph: false, GraphsAsFilePrefix: #if DEBUG "Graph" #else null #endif ); isQuiet = true; foreach (var k in lk.Where(p => !p.IsQuiet())) { measurementHolder.Logger.LogMessage($" !quiet {k}"); ///xxxremove isQuiet = false; } // isQuiet = !lk.Where(k => !k.IsQuiet()).Any(); foreach (var pctrMeasure in quietMeasure.measurements.Keys) // circular buffer: remove 1st item { quietMeasure.measurements[pctrMeasure].RemoveAt(0); } quietMeasure.nSamplesTaken--; } nMeasurementsForQuiet++; } if (isQuiet) // the counters have stabilized. We'll use the stabilized numbers as the sample value for the iteration { measurementHolder.Logger.LogMessage($"Gone quiet in {nMeasurementsForQuiet} measures"); } else { measurementHolder.Logger.LogMessage($"Didn't go quiet in {numTimesToGetQuiet}"); } // Whether or not it's quiet, we'll take the most recent measure as the iteration sample foreach (var pctrMeasure in measurementHolder.measurements.Keys) { var lastVal = quietMeasure.measurements[pctrMeasure][quietMeasure.nSamplesTaken - 1]; measurementHolder.measurements[pctrMeasure][measurementHolder.nSamplesTaken - 1] = lastVal; } return(true); // continue with normal iteration processing } }); await _VSHandler.OpenSolution(SolutionToLoad); await _VSHandler.CloseSolution(); }
public async Task IterateCode(int numIterations, double Sensitivity, int delayBetweenIterationsMsec) { try { using (var measurementHolder = new MeasurementHolder( TestName, new StressUtilOptions() { NumIterations = numIterations, ProcNamesToMonitor = string.Empty, ShowUI = this.ShowUI, logger = _logger, Sensitivity = Sensitivity, SecsBetweenIterations = SecsBetweenIterations, NumIterationsBeforeTotalToTakeBaselineSnapshot = NumIterationsBeforeTotalToTakeBaselineSnapshot, //actExecuteAfterEveryIterationAsync = async (nIter, mHolder) => // uncomment to suppress dump taking/processing. //{ // await Task.Yield(); // return false; //}, lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(System.Diagnostics.Process.GetCurrentProcess(), IsForStress: false) }, SampleType.SampleTypeIteration)) { var baseDumpFileName = string.Empty; for (int iteration = 0; iteration < numIterations && !_CancellationTokenExecuteCode.IsCancellationRequested; iteration++) { await DoIterationBodyAsync(iteration, _CancellationTokenExecuteCode); await Task.Delay(TimeSpan.FromMilliseconds(delayBetweenIterationsMsec *DelayMultiplier)); var desc = string.Format("Iter {0}/{1}", iteration + 1, numIterations); // we need to go thru the extension to get the measurement, so the vsix graph updates and adds to log await _itakeSample.DoSampleAsync(measurementHolder, DoForceGC : true, descriptionOverride : desc); if (_CancellationTokenExecuteCode.IsCancellationRequested) { break; } } if (!_CancellationTokenExecuteCode.IsCancellationRequested) { _logger.LogMessage(string.Format("Done all {0} iterations", numIterations)); } else { _logger.LogMessage("Cancelled Code Execution"); } } } catch (LeakException) { } catch (OperationCanceledException) { _logger.LogMessage("Cancelled"); } catch (Exception ex) { _logger.LogMessage(ex.ToString()); } }
async Task AddDataPointsAsync(MeasurementHolder measurementHolder) { var dictPerfCtrCurrentMeasurements = measurementHolder.GetLastMeasurements(); if (_dataPoints.Count == 0) // nothing yet { for (int i = 0; i < NumDataPoints; i++) { _dataPoints[i] = new List <uint>(dictPerfCtrCurrentMeasurements.Values); // let all init points be equal, so y axis scales IsStartedFromZero } } else { _dataPoints[_bufferIndex++] = new List <uint>(dictPerfCtrCurrentMeasurements.Values); if (_bufferIndex == _dataPoints.Count) // wraparound? { _bufferIndex = 0; } } await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); if (DoFullGCPerSample) { DoGC();// do a GC.Collect on main thread for every sample (the graphing uses memory) } // this needs to be done on UI thread _chart.Series.Clear(); _chart.ChartAreas.Clear(); ChartArea chartArea = new ChartArea("ChartArea"); chartArea.AxisY.LabelStyle.Format = "{0:n0}"; chartArea.AxisY.LabelStyle.Font = new System.Drawing.Font("Consolas", 12); _chart.ChartAreas.Add(chartArea); int ndxSeries = 0; chartArea.AxisY.IsStartedFromZero = false; if (SetMaxGraphTo100) { _chart.ChartAreas[0].AxisY.Maximum = 100; } foreach (var entry in dictPerfCtrCurrentMeasurements) { var series = new Series { ChartType = SeriesChartType.Line, Name = entry.Key.ToString() }; _chart.Series.Add(series); if (UpdateInterval == 0) // if we're not doing auto update on timer, we're iterating or doing manual measurement { series.MarkerSize = 10; series.MarkerStyle = MarkerStyle.Circle; } for (int i = 0; i < _dataPoints.Count; i++) { var ndx = _bufferIndex + i; if (ndx >= _dataPoints.Count) { ndx -= _dataPoints.Count; } var dp = new DataPoint(i + 1, _dataPoints[ndx][ndxSeries]); series.Points.Add(dp); } ndxSeries++; } _chart.Legends.Clear(); _chart.Legends.Add(new Legend()); _chart.DataBind(); if (_editorTracker != null) { var(openedViews, lstLeakedViews) = _editorTracker.GetCounts(); OpenedViews.Clear(); LeakedViews.Clear(); foreach (var dictEntry in openedViews) { var sp = new StackPanel() { Orientation = Orientation.Horizontal }; sp.Children.Add(new TextBlock() { Text = $"{ dictEntry.Key,-15} {dictEntry.Value,3}", FontFamily = FontFamilyMono }); OpenedViews.Add(sp); } foreach (var entry in lstLeakedViews) { var sp = new StackPanel() { Orientation = Orientation.Horizontal }; sp.Children.Add(new TextBlock() { Text = $"{ entry._contentType,-15} Ser#={entry._serialNo,3} {entry._dtCreated:hh:mm:ss} {entry._filename}", FontFamily = FontFamilyMono });
public async Task DoSampleAsync(MeasurementHolder measurementHolder, bool DoForceGC, string descriptionOverride = "") { var res = string.Empty; if (measurementHolder == null) { measurementHolder = measurementHolderInteractiveUser; } try { await TaskScheduler.Default; try { res = await measurementHolder.TakeMeasurementAsync(descriptionOverride, DoForceGC, IsForInteractiveGraph : UpdateInterval != 0); await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); await AddDataPointsAsync(measurementHolder); if (AutoDumpIsEnabled) { var lastmeasurements = measurementHolder.GetLastMeasurements(); if (lastmeasurements.Count == 1) { var val = lastmeasurements.Values.First(); if (val >= AutoDumpThresh) { await AddStatusMsgAsync($"Autodump Threshold triggered: {lastmeasurements.Keys.First()} {val} > {AutoDumpThresh}"); AutoDumpIsEnabled = false; var pathDump = await CreateDumpFileAsync(MemoryAnalysisType.JustCreateDump, "AutoDump", tspanDelayAfterGC : TimeSpan.FromSeconds(0.2)); } } } } catch (Exception ex) { res = ex.ToString(); } AddStatusMsgAsync($"{res}").Forget(); } catch (InvalidOperationException ex) { if (ex.Message.Contains("Instance 'devenv#")) // user changed # of instance of devenv runnning { await AddStatusMsgAsync($"Resetting perf counters due to devenv instances change"); lock (measurementHolder.LstPerfCounterData) { foreach (var ctr in measurementHolder.LstPerfCounterData) { ctr.ResetCounter(); } _dataPoints.Clear(); _bufferIndex = 0; } } } catch (Exception ex) { await AddStatusMsgAsync($"Exception in {nameof(DoSampleAsync)}" + ex.ToString()); _dataPoints.Clear(); _bufferIndex = 0; } }