public async Task TestLeakyNative() { if (StressUtilOptions.IsRunningOnBuildMachine()) { throw new LeakException("Throwing expected exception so test passes", null); } var lstperfCounterOverrideDataSettings = new List <PerfCounterOverrideThreshold> { new PerfCounterOverrideThreshold { perfCounterType = PerfCounterType.ProcessorPrivateBytes, regressionThreshold = 1024 * 1024 * .8f }, }; await StressUtil.DoIterationsAsync(this, new StressUtilOptions() { NumIterations = 17, ProcNamesToMonitor = string.Empty, PerfCounterOverrideSettings = lstperfCounterOverrideDataSettings }); _lst.Add(new BigStuffWithLongNameSoICanSeeItBetter(sizeToAllocate: 1024 * 1024)); }
[ExpectedException(typeof(LeakException))] // to make the test pass, we need a LeakException. However, Pass deletes all the test results <sigh> public async Task StressMultiSample() { if (StressUtilOptions.IsRunningOnBuildMachine()) { throw new LeakException("Throwing expected exception so test passes", null); } int numIter = 11; try { await StressUtil.DoIterationsAsync( this, new StressUtilOptions() { LoggerLogOutputToDestkop = true, NumIterations = numIter, ProcNamesToMonitor = string.Empty, ShowUI = false, actExecuteAfterEveryIterationAsync = async(nIter, measurementHolder) => { // this method yields a very nice stairstep in unit tests (where there's much less noise) int numAdditionalSamplesPerIteration = 5; // Since we're called after a sample, add 1 to get the actual # of samples/iteration var sb = new StringBuilder($"{nIter} ExtraIterations: {numAdditionalSamplesPerIteration}"); async Task CheckASampleAsync() { if (measurementHolder.nSamplesTaken == numAdditionalSamplesPerIteration * (measurementHolder.stressUtilOptions.NumIterations - measurementHolder.stressUtilOptions.NumIterationsBeforeTotalToTakeBaselineSnapshot)) { measurementHolder.baseDumpFileName = await measurementHolder.DoCreateDumpAsync($"Custom Code Taking base snapshot dump at Iter # {nIter} sample # {measurementHolder.nSamplesTaken}");; } if (measurementHolder.nSamplesTaken == numAdditionalSamplesPerIteration * measurementHolder.stressUtilOptions.NumIterations) { var lstLeakResults = (await measurementHolder.CalculateLeaksAsync(showGraph: measurementHolder.stressUtilOptions.ShowUI, GraphsAsFilePrefix: "Graph")) .Where(r => r.IsLeak).ToList(); var currentDumpFile = await measurementHolder.DoCreateDumpAsync($"Custom Code Taking final snapshot dump at iteration {measurementHolder.nSamplesTaken}"); if (!string.IsNullOrEmpty(measurementHolder.baseDumpFileName)) { var oDumpAnalyzer = new DumpAnalyzer(measurementHolder.Logger); foreach (var leak in lstLeakResults) { sb.AppendLine($"Custom code Leak Detected: {leak}"); } sb.AppendLine(); oDumpAnalyzer.GetDiff(sb, measurementHolder.baseDumpFileName, currentDumpFile, measurementHolder.stressUtilOptions.NumIterations, measurementHolder.stressUtilOptions.NumIterationsBeforeTotalToTakeBaselineSnapshot, measurementHolder.stressUtilOptions.TypesToReportStatisticsOn, out DumpAnalyzer.TypeStatistics _, out DumpAnalyzer.TypeStatistics _); var fname = Path.Combine(measurementHolder.ResultsFolder, $"{MeasurementHolder.DiffFileName}_{measurementHolder.nSamplesTaken}.txt"); File.WriteAllText(fname, sb.ToString()); if (measurementHolder.stressUtilOptions.ShowUI) { System.Diagnostics.Process.Start(fname); } measurementHolder.lstFileResults.Add(new FileResultsData() { filename = fname, description = $"Differences for Type and String counts at iter {measurementHolder.nSamplesTaken}" }); measurementHolder.Logger.LogMessage("Custom Code DumpDiff Analysis " + fname); } throw new LeakException($"Custom Code Leaks found: " + string.Join(",", lstLeakResults.Select(t => t.perfCounterData.perfCounterType).ToList()), lstLeakResults); //Leaks found: GCBytesInAllHeaps,ProcessorPrivateBytes,ProcessorVirtualBytes,KernelHandleCount } } await CheckASampleAsync(); for (int i = 0; i < numAdditionalSamplesPerIteration; i++) { await measurementHolder.DoForceGCAsync(); measurementHolder.TakeRawMeasurement(sb); await CheckASampleAsync(); } return(false); } } );; _lst.Add(new BigStuffWithLongNameSoICanSeeItBetter()); }
public async Task TestOutliers() { if (StressUtilOptions.IsRunningOnBuildMachine()) { return; } await Task.Yield(); // data from https://dev.azure.com/devdiv/DevDiv/_releaseProgress?_a=release-environment-extension&releaseId=548609&environmentId=2872682&extensionId=ms.vss-test-web.test-result-in-release-environment-editor-tab&runId=10533790&resultId=100000&paneView=attachments var testData = new uint[] { 1867008, 2713172, 2701928, 2701928, 2701800, 2701800, 2701700, 2701700, 2711368, 2711368, 2713228, 2713228, 2714876, 2714876, 2716588, 2716588, 2716588, 2732980, 2732980, 2734848, 2734848, 2736536, 2736536, 2738052, 2738052, 2739908, 2739908, 2741400, 2741400, 2742916, 2742916, 2744548, 2744548, 2744548, 2747340, 2747340, 2764696, 2764696, 2766384, 2766384, 2767888, 2767888, 2769756, 2769756, 2771260, 2771260, 2772764, 2772764, 2774268, 2774268, 2774268, 2776140, 2776140, 2777468, 2777468, 2779168, 2779168, 2779836, 2779836, 2797744, 2797744, 2799236, 2799236, 2800996, 2800996, 2803548, 2803548, 2899440, 2904492, 2904492, 2904492, }; var resultsFolder = string.Empty; using (var measurementHolder = new MeasurementHolder( new TestContextWrapper(TestContext), new StressUtilOptions() { NumIterations = -1, logger = this, pctOutliersToIgnore = 5, lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(), IsForStress: true).Where(p => p.perfCounterType == PerfCounterType.GCBytesInAllHeaps).ToList() }, SampleType.SampleTypeIteration)) { resultsFolder = measurementHolder.ResultsFolder; for (int iter = 0; iter < testData.Length; iter++) { foreach (var ctr in measurementHolder.LstPerfCounterData) { measurementHolder.measurements[ctr.perfCounterType].Add(testData[iter]); } } var leakAnalysisResults = await measurementHolder.CalculateLeaksAsync(showGraph : false, GraphsAsFilePrefix : "Graph"); LogMessage($"Yint = {leakAnalysisResults[0].yintercept:n0}"); foreach (var res in leakAnalysisResults[0].lstData) { LogMessage($"{res} dist = {res.distance} {res}"); } Assert.IsTrue(leakAnalysisResults[0].lstData[0].IsOutlier); Assert.IsTrue(leakAnalysisResults[0].lstData[1].IsOutlier); Assert.IsTrue(!leakAnalysisResults[0].lstData[2].IsOutlier); Assert.IsTrue(leakAnalysisResults[0].lstData[68].IsOutlier); } }
[ExpectedException(typeof(LeakException))] // to make the test pass, we need a LeakException. However, Pass deletes all the test results <sigh> public async Task StressVSOpenCloseWaitTilQuiet() { // we can't measure for quiet on the current process because the current process will be actively doing stuff. if (StressUtilOptions.IsRunningOnBuildMachine()) { throw new LeakException("Throwing expected exception so test passes", null); } string didGetLeakException = "didGetLeakException"; int numIter = 11; try { await StressUtil.DoIterationsAsync( this, new StressUtilOptions() { LoggerLogOutputToDestkop = true, NumIterations = numIter, ShowUI = false, actExecuteAfterEveryIterationAsync = async(nIter, measurementHolder) => { // we want to take measures in a circular buffer and wait til those are quiet var circBufferSize = 5; var numTimesToGetQuiet = 50; var quietMeasure = new MeasurementHolder( "Quiet", new StressUtilOptions() { NumIterations = 1, // we'll do 1 iteration pctOutliersToIgnore = 0, logger = measurementHolder.Logger, VSHandler = measurementHolder.stressUtilOptions.VSHandler, lstPerfCountersToUse = measurementHolder.stressUtilOptions.lstPerfCountersToUse, }, SampleType.SampleTypeIteration ); // We just took a measurement, so copy those values to init our buffer foreach (var pctrMeasure in measurementHolder.measurements.Keys) { var lastVal = measurementHolder.measurements[pctrMeasure][measurementHolder.nSamplesTaken - 1]; quietMeasure.measurements[pctrMeasure].Add(lastVal); } quietMeasure.nSamplesTaken++; var isQuiet = false; int nMeasurementsForQuiet = 0; while (!isQuiet && nMeasurementsForQuiet < numTimesToGetQuiet) { await quietMeasure.DoForceGCAsync(); await Task.Delay(TimeSpan.FromSeconds(1 * measurementHolder.stressUtilOptions.DelayMultiplier)); // after GC, wait 1 before taking measurements var sb = new StringBuilder($"Measure for Quiet iter = {nIter} QuietSamp#= {nMeasurementsForQuiet}"); quietMeasure.TakeRawMeasurement(sb); measurementHolder.Logger.LogMessage(sb.ToString()); ///xxxremove if (quietMeasure.nSamplesTaken == circBufferSize) { var lk = await quietMeasure.CalculateLeaksAsync( showGraph: false, GraphsAsFilePrefix: #if DEBUG "Graph" #else null #endif ); isQuiet = true; foreach (var k in lk.Where(p => !p.IsQuiet())) { measurementHolder.Logger.LogMessage($" !quiet {k}"); ///xxxremove isQuiet = false; } // isQuiet = !lk.Where(k => !k.IsQuiet()).Any(); foreach (var pctrMeasure in quietMeasure.measurements.Keys) // circular buffer: remove 1st item { quietMeasure.measurements[pctrMeasure].RemoveAt(0); } quietMeasure.nSamplesTaken--; } nMeasurementsForQuiet++; } if (isQuiet) // the counters have stabilized. We'll use the stabilized numbers as the sample value for the iteration { measurementHolder.Logger.LogMessage($"Gone quiet in {nMeasurementsForQuiet} measures"); } else { measurementHolder.Logger.LogMessage($"Didn't go quiet in {numTimesToGetQuiet}"); } // Whether or not it's quiet, we'll take the most recent measure as the iteration sample foreach (var pctrMeasure in measurementHolder.measurements.Keys) { var lastVal = quietMeasure.measurements[pctrMeasure][quietMeasure.nSamplesTaken - 1]; measurementHolder.measurements[pctrMeasure][measurementHolder.nSamplesTaken - 1] = lastVal; } return(true); // continue with normal iteration processing } }); await _VSHandler.OpenSolution(SolutionToLoad); await _VSHandler.CloseSolution(); }