[ExpectedException(typeof(LeakException))] // to make the test pass, we need a LeakException. However, Pass deletes all the test results <sigh>
public async Task StressLeakyBadPerfCounterCat()
{
int numIter = 11;
var stressOptions = new StressUtilOptions()
{
NumIterations = numIter, ProcNamesToMonitor = string.Empty, ShowUI = false
};
stressOptions.lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(), IsForStress: true);
stressOptions.lstPerfCountersToUse[0].PerfCounterCategory = "foobar"; // an invalid category
try
{
await StressUtil.DoIterationsAsync(
this,
stressOptions
);
_lst.Add(new BigStuffWithLongNameSoICanSeeItBetter());
}
catch (LeakException ex)
{
TestContext.WriteLine($"Caught exception {ex.Message}");
var lstFileResults = (List <FileResultsData>)TestContext.Properties[StressUtil.PropNameListFileResults];
foreach (var result in lstFileResults.OrderBy(r => r.filename))
{
TestContext.WriteLine($"File result {Path.GetFileName(result.filename)}");
}
var expectedFiles = new[] {
"Graph # Bytes in all Heaps.png",
"Graph GDIHandles.png",
"Graph Handle Count.png",
"Graph Private Bytes.png",
"Graph Thread Count.png",
"Graph UserHandles.png",
"Graph Virtual Bytes.png",
"Measurements.txt",
$"{TestContext.TestName}_{numIter}_0.dmp",
$"{TestContext.TestName}_{numIter-4}_0.dmp",
"StressTestLog.log",
$"String and Type Count differences_{numIter}.txt",
};
foreach (var itm in expectedFiles)
{
Assert.IsTrue(lstFileResults.Where(r => Path.GetFileName(r.filename) == itm).Count() == 1, $"Expected File attachment {itm}");
}
var strAndTypeDiff = File.ReadAllText(lstFileResults.Where(r => Path.GetFileName(r.filename) == $"String and Type Count differences_{numIter}.txt").First().filename);
TestContext.WriteLine($"String and Type Count differences_{numIter}.txt");
TestContext.WriteLine(strAndTypeDiff);
Assert.IsTrue(strAndTypeDiff.Contains(nameof(BigStuffWithLongNameSoICanSeeItBetter)), $"Type must be in StringandTypeDiff");
Assert.IsTrue(strAndTypeDiff.Contains("leaking string"), $"'leaking string' must be in StringandTypeDiff");
Assert.AreEqual(expectedFiles.Length, lstFileResults.Count, $"# file results");
TestContext.Properties[didGetLeakException] = 1;
throw;
}
}
public async Task TestXmlSerializeOptions()
{
var thresh = 1e6f;
var stressUtilOptions = new StressUtilOptions()
{
PerfCounterOverrideSettings = new List <PerfCounterOverrideThreshold>
{
new PerfCounterOverrideThreshold {
perfCounterType = PerfCounterType.GCBytesInAllHeaps, regressionThreshold = thresh
},
new PerfCounterOverrideThreshold {
perfCounterType = PerfCounterType.ProcessorPrivateBytes, regressionThreshold = 9 * thresh
}, // use a very high thresh so this counter won't show as leak
new PerfCounterOverrideThreshold {
perfCounterType = PerfCounterType.ProcessorVirtualBytes, regressionThreshold = 9 * thresh
},
new PerfCounterOverrideThreshold {
perfCounterType = PerfCounterType.KernelHandleCount, regressionThreshold = 9 * thresh
},
},
NumIterations = 7,
ShowUI = false
};
var filename = Path.Combine(TestContext.DeploymentDirectory, "opts.xml");
stressUtilOptions.WriteOptionsToFile(filename);
LogMessage($"Output to {filename}");
LogMessage(File.ReadAllText(filename));
var didlambda = false;
var newopts = new StressUtilOptions()
{
NumIterations = 321,
actExecuteAfterEveryIterationAsync = async(nIter, measurementHolder) => // test that lambda doesn't get overridden
{
await Task.Yield();
didlambda = true;
return(false);
}
};
newopts.ReadOptionsFromFile(filename);
Assert.AreEqual(stressUtilOptions.NumIterations, 7);
await newopts.actExecuteAfterEveryIterationAsync(0, null);
Assert.IsTrue(didlambda);
}
public async Task TestLeakyNative()
{
if (StressUtilOptions.IsRunningOnBuildMachine())
{
throw new LeakException("Throwing expected exception so test passes", null);
}
var lstperfCounterOverrideDataSettings = new List <PerfCounterOverrideThreshold>
{
new PerfCounterOverrideThreshold {
perfCounterType = PerfCounterType.ProcessorPrivateBytes, regressionThreshold = 1024 * 1024 * .8f
},
};
await StressUtil.DoIterationsAsync(this, new StressUtilOptions()
{
NumIterations = 17, ProcNamesToMonitor = string.Empty, PerfCounterOverrideSettings = lstperfCounterOverrideDataSettings
});
_lst.Add(new BigStuffWithLongNameSoICanSeeItBetter(sizeToAllocate: 1024 * 1024));
}
[ExpectedException(typeof(LeakException))] // to make the test pass, we need a LeakException. However, Pass deletes all the test results <sigh>
public async Task StressMultiSample()
{
if (StressUtilOptions.IsRunningOnBuildMachine())
{
throw new LeakException("Throwing expected exception so test passes", null);
}
int numIter = 11;
try
{
await StressUtil.DoIterationsAsync(
this,
new StressUtilOptions()
{
LoggerLogOutputToDestkop = true,
NumIterations = numIter,
ProcNamesToMonitor = string.Empty,
ShowUI = false,
actExecuteAfterEveryIterationAsync = async(nIter, measurementHolder) =>
{
// this method yields a very nice stairstep in unit tests (where there's much less noise)
int numAdditionalSamplesPerIteration = 5; // Since we're called after a sample, add 1 to get the actual # of samples/iteration
var sb = new StringBuilder($"{nIter} ExtraIterations: {numAdditionalSamplesPerIteration}");
async Task CheckASampleAsync()
{
if (measurementHolder.nSamplesTaken == numAdditionalSamplesPerIteration * (measurementHolder.stressUtilOptions.NumIterations - measurementHolder.stressUtilOptions.NumIterationsBeforeTotalToTakeBaselineSnapshot))
{
measurementHolder.baseDumpFileName = await measurementHolder.DoCreateDumpAsync($"Custom Code Taking base snapshot dump at Iter # {nIter} sample # {measurementHolder.nSamplesTaken}");;
}
if (measurementHolder.nSamplesTaken == numAdditionalSamplesPerIteration * measurementHolder.stressUtilOptions.NumIterations)
{
var lstLeakResults = (await measurementHolder.CalculateLeaksAsync(showGraph: measurementHolder.stressUtilOptions.ShowUI, GraphsAsFilePrefix: "Graph"))
.Where(r => r.IsLeak).ToList();
var currentDumpFile = await measurementHolder.DoCreateDumpAsync($"Custom Code Taking final snapshot dump at iteration {measurementHolder.nSamplesTaken}");
if (!string.IsNullOrEmpty(measurementHolder.baseDumpFileName))
{
var oDumpAnalyzer = new DumpAnalyzer(measurementHolder.Logger);
foreach (var leak in lstLeakResults)
{
sb.AppendLine($"Custom code Leak Detected: {leak}");
}
sb.AppendLine();
oDumpAnalyzer.GetDiff(sb,
measurementHolder.baseDumpFileName,
currentDumpFile,
measurementHolder.stressUtilOptions.NumIterations,
measurementHolder.stressUtilOptions.NumIterationsBeforeTotalToTakeBaselineSnapshot,
measurementHolder.stressUtilOptions.TypesToReportStatisticsOn,
out DumpAnalyzer.TypeStatistics _,
out DumpAnalyzer.TypeStatistics _);
var fname = Path.Combine(measurementHolder.ResultsFolder, $"{MeasurementHolder.DiffFileName}_{measurementHolder.nSamplesTaken}.txt");
File.WriteAllText(fname, sb.ToString());
if (measurementHolder.stressUtilOptions.ShowUI)
{
System.Diagnostics.Process.Start(fname);
}
measurementHolder.lstFileResults.Add(new FileResultsData()
{
filename = fname, description = $"Differences for Type and String counts at iter {measurementHolder.nSamplesTaken}"
});
measurementHolder.Logger.LogMessage("Custom Code DumpDiff Analysis " + fname);
}
throw new LeakException($"Custom Code Leaks found: " + string.Join(",", lstLeakResults.Select(t => t.perfCounterData.perfCounterType).ToList()), lstLeakResults); //Leaks found: GCBytesInAllHeaps,ProcessorPrivateBytes,ProcessorVirtualBytes,KernelHandleCount
}
}
await CheckASampleAsync();
for (int i = 0; i < numAdditionalSamplesPerIteration; i++)
{
await measurementHolder.DoForceGCAsync();
measurementHolder.TakeRawMeasurement(sb);
await CheckASampleAsync();
}
return(false);
}
}
);;
_lst.Add(new BigStuffWithLongNameSoICanSeeItBetter());
}
public async Task TestOutliers()
{
if (StressUtilOptions.IsRunningOnBuildMachine())
{
return;
}
await Task.Yield();
// data from https://dev.azure.com/devdiv/DevDiv/_releaseProgress?_a=release-environment-extension&releaseId=548609&environmentId=2872682&extensionId=ms.vss-test-web.test-result-in-release-environment-editor-tab&runId=10533790&resultId=100000&paneView=attachments
var testData = new uint[]
{
1867008,
2713172,
2701928,
2701928,
2701800,
2701800,
2701700,
2701700,
2711368,
2711368,
2713228,
2713228,
2714876,
2714876,
2716588,
2716588,
2716588,
2732980,
2732980,
2734848,
2734848,
2736536,
2736536,
2738052,
2738052,
2739908,
2739908,
2741400,
2741400,
2742916,
2742916,
2744548,
2744548,
2744548,
2747340,
2747340,
2764696,
2764696,
2766384,
2766384,
2767888,
2767888,
2769756,
2769756,
2771260,
2771260,
2772764,
2772764,
2774268,
2774268,
2774268,
2776140,
2776140,
2777468,
2777468,
2779168,
2779168,
2779836,
2779836,
2797744,
2797744,
2799236,
2799236,
2800996,
2800996,
2803548,
2803548,
2899440,
2904492,
2904492,
2904492,
};
var resultsFolder = string.Empty;
using (var measurementHolder = new MeasurementHolder(
new TestContextWrapper(TestContext),
new StressUtilOptions()
{
NumIterations = -1,
logger = this,
pctOutliersToIgnore = 5,
lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(),
IsForStress: true).Where(p => p.perfCounterType == PerfCounterType.GCBytesInAllHeaps).ToList()
},
SampleType.SampleTypeIteration))
{
resultsFolder = measurementHolder.ResultsFolder;
for (int iter = 0; iter < testData.Length; iter++)
{
foreach (var ctr in measurementHolder.LstPerfCounterData)
{
measurementHolder.measurements[ctr.perfCounterType].Add(testData[iter]);
}
}
var leakAnalysisResults = await measurementHolder.CalculateLeaksAsync(showGraph : false, GraphsAsFilePrefix : "Graph");
LogMessage($"Yint = {leakAnalysisResults[0].yintercept:n0}");
foreach (var res in leakAnalysisResults[0].lstData)
{
LogMessage($"{res} dist = {res.distance} {res}");
}
Assert.IsTrue(leakAnalysisResults[0].lstData[0].IsOutlier);
Assert.IsTrue(leakAnalysisResults[0].lstData[1].IsOutlier);
Assert.IsTrue(!leakAnalysisResults[0].lstData[2].IsOutlier);
Assert.IsTrue(leakAnalysisResults[0].lstData[68].IsOutlier);
}
}
[ExpectedException(typeof(LeakException))] // to make the test pass, we need a LeakException. However, Pass deletes all the test results <sigh>
public async Task StressVSOpenCloseWaitTilQuiet()
{
// we can't measure for quiet on the current process because the current process will be actively doing stuff.
if (StressUtilOptions.IsRunningOnBuildMachine())
{
throw new LeakException("Throwing expected exception so test passes", null);
}
string didGetLeakException = "didGetLeakException";
int numIter = 11;
try
{
await StressUtil.DoIterationsAsync(
this,
new StressUtilOptions()
{
LoggerLogOutputToDestkop = true,
NumIterations = numIter,
ShowUI = false,
actExecuteAfterEveryIterationAsync = async(nIter, measurementHolder) =>
{
// we want to take measures in a circular buffer and wait til those are quiet
var circBufferSize = 5;
var numTimesToGetQuiet = 50;
var quietMeasure = new MeasurementHolder(
"Quiet",
new StressUtilOptions()
{
NumIterations = 1, // we'll do 1 iteration
pctOutliersToIgnore = 0,
logger = measurementHolder.Logger,
VSHandler = measurementHolder.stressUtilOptions.VSHandler,
lstPerfCountersToUse = measurementHolder.stressUtilOptions.lstPerfCountersToUse,
}, SampleType.SampleTypeIteration
);
// We just took a measurement, so copy those values to init our buffer
foreach (var pctrMeasure in measurementHolder.measurements.Keys)
{
var lastVal = measurementHolder.measurements[pctrMeasure][measurementHolder.nSamplesTaken - 1];
quietMeasure.measurements[pctrMeasure].Add(lastVal);
}
quietMeasure.nSamplesTaken++;
var isQuiet = false;
int nMeasurementsForQuiet = 0;
while (!isQuiet && nMeasurementsForQuiet < numTimesToGetQuiet)
{
await quietMeasure.DoForceGCAsync();
await Task.Delay(TimeSpan.FromSeconds(1 * measurementHolder.stressUtilOptions.DelayMultiplier)); // after GC, wait 1 before taking measurements
var sb = new StringBuilder($"Measure for Quiet iter = {nIter} QuietSamp#= {nMeasurementsForQuiet}");
quietMeasure.TakeRawMeasurement(sb);
measurementHolder.Logger.LogMessage(sb.ToString()); ///xxxremove
if (quietMeasure.nSamplesTaken == circBufferSize)
{
var lk = await quietMeasure.CalculateLeaksAsync(
showGraph: false,
GraphsAsFilePrefix:
#if DEBUG
"Graph"
#else
null
#endif
);
isQuiet = true;
foreach (var k in lk.Where(p => !p.IsQuiet()))
{
measurementHolder.Logger.LogMessage($" !quiet {k}"); ///xxxremove
isQuiet = false;
}
// isQuiet = !lk.Where(k => !k.IsQuiet()).Any();
foreach (var pctrMeasure in quietMeasure.measurements.Keys) // circular buffer: remove 1st item
{
quietMeasure.measurements[pctrMeasure].RemoveAt(0);
}
quietMeasure.nSamplesTaken--;
}
nMeasurementsForQuiet++;
}
if (isQuiet) // the counters have stabilized. We'll use the stabilized numbers as the sample value for the iteration
{
measurementHolder.Logger.LogMessage($"Gone quiet in {nMeasurementsForQuiet} measures");
}
else
{
measurementHolder.Logger.LogMessage($"Didn't go quiet in {numTimesToGetQuiet}");
}
// Whether or not it's quiet, we'll take the most recent measure as the iteration sample
foreach (var pctrMeasure in measurementHolder.measurements.Keys)
{
var lastVal = quietMeasure.measurements[pctrMeasure][quietMeasure.nSamplesTaken - 1];
measurementHolder.measurements[pctrMeasure][measurementHolder.nSamplesTaken - 1] = lastVal;
}
return(true); // continue with normal iteration processing
}
});
await _VSHandler.OpenSolution(SolutionToLoad);
await _VSHandler.CloseSolution();
}
