public async Task TestLeakyDetectNativeVerySmallLeak() { var thresh = 1e3f; var lstperfCounterOverrideDataSettings = new List <PerfCounterOverrideThreshold> { new PerfCounterOverrideThreshold { perfCounterType = PerfCounterType.GCBytesInAllHeaps, regressionThreshold = 9 * thresh }, // use a very high thresh so this counter won't show as leak new PerfCounterOverrideThreshold { perfCounterType = PerfCounterType.ProcessorPrivateBytes, regressionThreshold = thresh }, new PerfCounterOverrideThreshold { perfCounterType = PerfCounterType.ProcessorVirtualBytes, regressionThreshold = 9 * thresh }, new PerfCounterOverrideThreshold { perfCounterType = PerfCounterType.KernelHandleCount, regressionThreshold = 9 * thresh }, }; await StressUtil.DoIterationsAsync(this, new StressUtilOptions() { NumIterations = 201, ProcNamesToMonitor = string.Empty, PerfCounterOverrideSettings = lstperfCounterOverrideDataSettings, ShowUI = false }); _lst.Add(new BigStuffWithLongNameSoICanSeeItBetter(sizeToAllocate: 100000)); }
public async Task StressOpenCloseSln() { try { // the only change to existing test required: call to static method await StressUtil.DoIterationsAsync(this, stressUtilOptions : new StressUtilOptions() { FailTestAsifLeaksFound = true, NumIterations = 17, actExecuteAfterEveryIterationAsync = async(nIter, measurementHolder) => { await Task.Yield(); return(true); //do the default action after iteration of checking iteration number and taking dumps, comparing. } }); await _VSHandler.OpenSolution(SolutionToLoad); await _VSHandler.CloseSolution(); } catch (Exception ex) { logger.LogMessage($"Exception {ex}"); throw; } }
[ExpectedException(typeof(LeakException))] // to make the test pass, we need a LeakException. However, Pass deletes all the test results <sigh> public async Task StressLeakyBadPerfCounterCat() { int numIter = 11; var stressOptions = new StressUtilOptions() { NumIterations = numIter, ProcNamesToMonitor = string.Empty, ShowUI = false }; stressOptions.lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(), IsForStress: true); stressOptions.lstPerfCountersToUse[0].PerfCounterCategory = "foobar"; // an invalid category try { await StressUtil.DoIterationsAsync( this, stressOptions ); _lst.Add(new BigStuffWithLongNameSoICanSeeItBetter()); } catch (LeakException ex) { TestContext.WriteLine($"Caught exception {ex.Message}"); var lstFileResults = (List <FileResultsData>)TestContext.Properties[StressUtil.PropNameListFileResults]; foreach (var result in lstFileResults.OrderBy(r => r.filename)) { TestContext.WriteLine($"File result {Path.GetFileName(result.filename)}"); } var expectedFiles = new[] { "Graph # Bytes in all Heaps.png", "Graph GDIHandles.png", "Graph Handle Count.png", "Graph Private Bytes.png", "Graph Thread Count.png", "Graph UserHandles.png", "Graph Virtual Bytes.png", "Measurements.txt", $"{TestContext.TestName}_{numIter}_0.dmp", $"{TestContext.TestName}_{numIter-4}_0.dmp", "StressTestLog.log", $"String and Type Count differences_{numIter}.txt", }; foreach (var itm in expectedFiles) { Assert.IsTrue(lstFileResults.Where(r => Path.GetFileName(r.filename) == itm).Count() == 1, $"Expected File attachment {itm}"); } var strAndTypeDiff = File.ReadAllText(lstFileResults.Where(r => Path.GetFileName(r.filename) == $"String and Type Count differences_{numIter}.txt").First().filename); TestContext.WriteLine($"String and Type Count differences_{numIter}.txt"); TestContext.WriteLine(strAndTypeDiff); Assert.IsTrue(strAndTypeDiff.Contains(nameof(BigStuffWithLongNameSoICanSeeItBetter)), $"Type must be in StringandTypeDiff"); Assert.IsTrue(strAndTypeDiff.Contains("leaking string"), $"'leaking string' must be in StringandTypeDiff"); Assert.AreEqual(expectedFiles.Length, lstFileResults.Count, $"# file results"); TestContext.Properties[didGetLeakException] = 1; throw; } }
public async Task TestLeakyNative() { if (StressUtilOptions.IsRunningOnBuildMachine()) { throw new LeakException("Throwing expected exception so test passes", null); } var lstperfCounterOverrideDataSettings = new List <PerfCounterOverrideThreshold> { new PerfCounterOverrideThreshold { perfCounterType = PerfCounterType.ProcessorPrivateBytes, regressionThreshold = 1024 * 1024 * .8f }, }; await StressUtil.DoIterationsAsync(this, new StressUtilOptions() { NumIterations = 17, ProcNamesToMonitor = string.Empty, PerfCounterOverrideSettings = lstperfCounterOverrideDataSettings }); _lst.Add(new BigStuffWithLongNameSoICanSeeItBetter(sizeToAllocate: 1024 * 1024)); }
public async Task StressMemSpectOpenCloseSln() { try { // 30 min for 7 iter await StressUtil.DoIterationsAsync(this, new StressUtilOptions() { ShowUI = true, NumIterations = 3 }); await _VSHandler.OpenSolution(StressVS.SolutionToLoad); await _VSHandler.CloseSolution(); } catch (Exception ex) { logger.LogMessage($"Exception {ex}"); throw; } }
[ExpectedException(typeof(LeakException))] // to make the test pass, we need a LeakException. However, Pass deletes all the test results <sigh> public async Task StressMultiSample() { if (StressUtilOptions.IsRunningOnBuildMachine()) { throw new LeakException("Throwing expected exception so test passes", null); } int numIter = 11; try { await StressUtil.DoIterationsAsync( this, new StressUtilOptions() { LoggerLogOutputToDestkop = true, NumIterations = numIter, ProcNamesToMonitor = string.Empty, ShowUI = false, actExecuteAfterEveryIterationAsync = async(nIter, measurementHolder) => { // this method yields a very nice stairstep in unit tests (where there's much less noise) int numAdditionalSamplesPerIteration = 5; // Since we're called after a sample, add 1 to get the actual # of samples/iteration var sb = new StringBuilder($"{nIter} ExtraIterations: {numAdditionalSamplesPerIteration}"); async Task CheckASampleAsync() { if (measurementHolder.nSamplesTaken == numAdditionalSamplesPerIteration * (measurementHolder.stressUtilOptions.NumIterations - measurementHolder.stressUtilOptions.NumIterationsBeforeTotalToTakeBaselineSnapshot)) { measurementHolder.baseDumpFileName = await measurementHolder.DoCreateDumpAsync($"Custom Code Taking base snapshot dump at Iter # {nIter} sample # {measurementHolder.nSamplesTaken}");; } if (measurementHolder.nSamplesTaken == numAdditionalSamplesPerIteration * measurementHolder.stressUtilOptions.NumIterations) { var lstLeakResults = (await measurementHolder.CalculateLeaksAsync(showGraph: measurementHolder.stressUtilOptions.ShowUI, GraphsAsFilePrefix: "Graph")) .Where(r => r.IsLeak).ToList(); var currentDumpFile = await measurementHolder.DoCreateDumpAsync($"Custom Code Taking final snapshot dump at iteration {measurementHolder.nSamplesTaken}"); if (!string.IsNullOrEmpty(measurementHolder.baseDumpFileName)) { var oDumpAnalyzer = new DumpAnalyzer(measurementHolder.Logger); foreach (var leak in lstLeakResults) { sb.AppendLine($"Custom code Leak Detected: {leak}"); } sb.AppendLine(); oDumpAnalyzer.GetDiff(sb, measurementHolder.baseDumpFileName, currentDumpFile, measurementHolder.stressUtilOptions.NumIterations, measurementHolder.stressUtilOptions.NumIterationsBeforeTotalToTakeBaselineSnapshot, measurementHolder.stressUtilOptions.TypesToReportStatisticsOn, out DumpAnalyzer.TypeStatistics _, out DumpAnalyzer.TypeStatistics _); var fname = Path.Combine(measurementHolder.ResultsFolder, $"{MeasurementHolder.DiffFileName}_{measurementHolder.nSamplesTaken}.txt"); File.WriteAllText(fname, sb.ToString()); if (measurementHolder.stressUtilOptions.ShowUI) { System.Diagnostics.Process.Start(fname); } measurementHolder.lstFileResults.Add(new FileResultsData() { filename = fname, description = $"Differences for Type and String counts at iter {measurementHolder.nSamplesTaken}" }); measurementHolder.Logger.LogMessage("Custom Code DumpDiff Analysis " + fname); } throw new LeakException($"Custom Code Leaks found: " + string.Join(",", lstLeakResults.Select(t => t.perfCounterData.perfCounterType).ToList()), lstLeakResults); //Leaks found: GCBytesInAllHeaps,ProcessorPrivateBytes,ProcessorVirtualBytes,KernelHandleCount } } await CheckASampleAsync(); for (int i = 0; i < numAdditionalSamplesPerIteration; i++) { await measurementHolder.DoForceGCAsync(); measurementHolder.TakeRawMeasurement(sb); await CheckASampleAsync(); } return(false); } } );; _lst.Add(new BigStuffWithLongNameSoICanSeeItBetter()); }
[ExpectedException(typeof(LeakException))] // to make the test pass, we need a LeakException. However, Pass deletes all the test results <sigh> public async Task StressLeaky() { if (TestContext.Properties.Contains(StressUtil.PropNameCurrentIteration) && // only do once, but after logger has been set (int)(TestContext.Properties[StressUtil.PropNameCurrentIteration]) == 0) { if (TestContext.Properties.Contains(StressUtil.PropNameLogger)) {/// put these in logger rather than TestContext.WriteLine so they show in build pipeline test results Logger logger = TestContext.Properties[StressUtil.PropNameLogger] as Logger; logger?.LogMessage($"Username="******"Username")); logger?.LogMessage($"Computername=" + Environment.GetEnvironmentVariable("Computername")); logger?.LogMessage($"UserDomain=" + Environment.GetEnvironmentVariable("userdomain")); } } int numIter = 11; try { await StressUtil.DoIterationsAsync( this, new StressUtilOptions() { NumIterations = numIter, ProcNamesToMonitor = string.Empty, ShowUI = false } ); _lst.Add(new BigStuffWithLongNameSoICanSeeItBetter()); } catch (LeakException ex) { TestContext.WriteLine($"Caught exception {ex.Message}"); var lstFileResults = (List <FileResultsData>)TestContext.Properties[StressUtil.PropNameListFileResults]; foreach (var result in lstFileResults.OrderBy(r => r.filename)) { TestContext.WriteLine($"File result {Path.GetFileName(result.filename)}"); } var expectedFiles = new[] { "Graph # Bytes in all Heaps.png", "Graph GDIHandles.png", "Graph Handle Count.png", "Graph Private Bytes.png", "Graph Thread Count.png", "Graph UserHandles.png", "Graph Virtual Bytes.png", "Measurements.txt", $"{TestContext.TestName}_{numIter}_0.dmp", $"{TestContext.TestName}_{numIter-4}_0.dmp", "StressTestLog.log", $"String and Type Count differences_{numIter}.txt", }; foreach (var itm in expectedFiles) { Assert.IsTrue(lstFileResults.Where(r => Path.GetFileName(r.filename) == itm).Count() == 1, $"Expected File attachment {itm}"); } var strAndTypeDiff = File.ReadAllText(lstFileResults.Where(r => Path.GetFileName(r.filename) == $"String and Type Count differences_{numIter}.txt").First().filename); TestContext.WriteLine($"String and Type Count differences_{numIter}.txt"); TestContext.WriteLine(strAndTypeDiff); Assert.IsTrue(strAndTypeDiff.Contains(nameof(BigStuffWithLongNameSoICanSeeItBetter)), $"Type must be in StringandTypeDiff"); Assert.IsTrue(strAndTypeDiff.Contains("leaking string"), $"'leaking string' must be in StringandTypeDiff"); Assert.AreEqual(expectedFiles.Length, lstFileResults.Count, $"# file results"); TestContext.Properties[didGetLeakException] = 1; throw; } }
[ExpectedException(typeof(LeakException))] // to make the test pass, we need a LeakException. However, Pass deletes all the test results <sigh> public async Task StressVSOpenCloseWaitTilQuiet() { // we can't measure for quiet on the current process because the current process will be actively doing stuff. if (StressUtilOptions.IsRunningOnBuildMachine()) { throw new LeakException("Throwing expected exception so test passes", null); } string didGetLeakException = "didGetLeakException"; int numIter = 11; try { await StressUtil.DoIterationsAsync( this, new StressUtilOptions() { LoggerLogOutputToDestkop = true, NumIterations = numIter, ShowUI = false, actExecuteAfterEveryIterationAsync = async(nIter, measurementHolder) => { // we want to take measures in a circular buffer and wait til those are quiet var circBufferSize = 5; var numTimesToGetQuiet = 50; var quietMeasure = new MeasurementHolder( "Quiet", new StressUtilOptions() { NumIterations = 1, // we'll do 1 iteration pctOutliersToIgnore = 0, logger = measurementHolder.Logger, VSHandler = measurementHolder.stressUtilOptions.VSHandler, lstPerfCountersToUse = measurementHolder.stressUtilOptions.lstPerfCountersToUse, }, SampleType.SampleTypeIteration ); // We just took a measurement, so copy those values to init our buffer foreach (var pctrMeasure in measurementHolder.measurements.Keys) { var lastVal = measurementHolder.measurements[pctrMeasure][measurementHolder.nSamplesTaken - 1]; quietMeasure.measurements[pctrMeasure].Add(lastVal); } quietMeasure.nSamplesTaken++; var isQuiet = false; int nMeasurementsForQuiet = 0; while (!isQuiet && nMeasurementsForQuiet < numTimesToGetQuiet) { await quietMeasure.DoForceGCAsync(); await Task.Delay(TimeSpan.FromSeconds(1 * measurementHolder.stressUtilOptions.DelayMultiplier)); // after GC, wait 1 before taking measurements var sb = new StringBuilder($"Measure for Quiet iter = {nIter} QuietSamp#= {nMeasurementsForQuiet}"); quietMeasure.TakeRawMeasurement(sb); measurementHolder.Logger.LogMessage(sb.ToString()); ///xxxremove if (quietMeasure.nSamplesTaken == circBufferSize) { var lk = await quietMeasure.CalculateLeaksAsync( showGraph: false, GraphsAsFilePrefix: #if DEBUG "Graph" #else null #endif ); isQuiet = true; foreach (var k in lk.Where(p => !p.IsQuiet())) { measurementHolder.Logger.LogMessage($" !quiet {k}"); ///xxxremove isQuiet = false; } // isQuiet = !lk.Where(k => !k.IsQuiet()).Any(); foreach (var pctrMeasure in quietMeasure.measurements.Keys) // circular buffer: remove 1st item { quietMeasure.measurements[pctrMeasure].RemoveAt(0); } quietMeasure.nSamplesTaken--; } nMeasurementsForQuiet++; } if (isQuiet) // the counters have stabilized. We'll use the stabilized numbers as the sample value for the iteration { measurementHolder.Logger.LogMessage($"Gone quiet in {nMeasurementsForQuiet} measures"); } else { measurementHolder.Logger.LogMessage($"Didn't go quiet in {numTimesToGetQuiet}"); } // Whether or not it's quiet, we'll take the most recent measure as the iteration sample foreach (var pctrMeasure in measurementHolder.measurements.Keys) { var lastVal = quietMeasure.measurements[pctrMeasure][quietMeasure.nSamplesTaken - 1]; measurementHolder.measurements[pctrMeasure][measurementHolder.nSamplesTaken - 1] = lastVal; } return(true); // continue with normal iteration processing } }); await _VSHandler.OpenSolution(SolutionToLoad); await _VSHandler.CloseSolution(); }