async Task ResetPerfCounterMonitorAsync()
{
_ctsPcounter?.Cancel();
if (_tcsPcounter != null)
{
await _tcsPcounter.Task;
}
lock (LstPerfCounterData)
{
measurementHolderInteractiveUser = new MeasurementHolder(
TestNameOrTestContext: MeasurementHolder.InteractiveUser,
new StressUtilOptions()
{
NumIterations = -1, logger = this, lstPerfCountersToUse = LstPerfCounterData
},
sampleType: SampleType.SampleTypeNormal);
_dataPoints.Clear();
_bufferIndex = 0;
}
if (UpdateInterval > 0)
{
AddStatusMsgAsync($"{nameof(ResetPerfCounterMonitorAsync)}").Forget();
DoPerfCounterMonitoring();
}
else
{
AddStatusMsgAsync($"UpdateInterval = 0 auto sampling turned off").Forget();
}
}
/// <summary>
/// return true if regression found
/// These tests will be affected by other tests running in the same instance of testhost because they share the same memory
/// </summary>
private async Task <bool> DoStressSimulation(int nIter, int nArraySize, float RatioThresholdSensitivity, Action action = null)
{
var lstPCs = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(), IsForStress: true);
foreach (var ctr in lstPCs)
{
ctr.IsEnabledForMeasurement = true;
}
List <LeakAnalysisResult> lstRegResults;
using (var measurementHolder = new MeasurementHolder(
new TestContextWrapper(TestContext),
new StressUtilOptions()
{
NumIterations = -1, Sensitivity = RatioThresholdSensitivity, logger = this, lstPerfCountersToUse = lstPCs
},
SampleType.SampleTypeIteration))
{
var lstBigStuff = new List <byte[]>();
LogMessage($"nIter={nIter:n0} ArraySize= {nArraySize:n0}");
for (int i = 0; i < nIter; i++)
{
if (action != null)
{
action();
}
else
{
lstBigStuff.Add(new byte[nArraySize]);
}
// lstBigStuff.Add(new int[10000000]);
var res = await measurementHolder.TakeMeasurementAsync($"iter {i}/{nIter}", DoForceGC : true);
LogMessage(res);
}
var filename = measurementHolder.DumpOutMeasurementsToTxtFile();
LogMessage($"Results file name = {filename}");
lstRegResults = (await measurementHolder.CalculateLeaksAsync(showGraph: false, GraphsAsFilePrefix: "Graph")).Where(r => r.IsLeak).ToList();
}
return(lstRegResults.Count > 0);
}
public async Task TestMeasureRegressionVerifyGraph()
{
await Task.Yield();
var resultsFolder = string.Empty;
using (var measurementHolder = new MeasurementHolder(
new TestContextWrapper(TestContext),
new StressUtilOptions()
{
NumIterations = -1, logger = this, lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(), IsForStress: true).Where(p => p.perfCounterType == PerfCounterType.KernelHandleCount).ToList()
},
SampleType.SampleTypeIteration))
{
resultsFolder = measurementHolder.ResultsFolder;
for (int iter = 0; iter < 10; iter++)
{
foreach (var ctr in measurementHolder.LstPerfCounterData)
{
var val = 10000 + (uint)iter;
if (iter == 5 && ctr.perfCounterType == PerfCounterType.KernelHandleCount)
{
val += 1;
}
measurementHolder.measurements[ctr.perfCounterType].Add(val);
}
}
var res = await measurementHolder.CalculateLeaksAsync(showGraph : true, GraphsAsFilePrefix : "Graph");
}
var strHtml = @"
<a href=""file:\\C:\Users\calvinh\Source\repos\PerfGraphVSIX\TestResults\Deploy_calvinh 2019-11-19 11_00_13/Out/TestMeasureRegressionVerifyGraph/Graph Handle Count.png"">gr </a>
";
var fileHtml = Path.Combine(resultsFolder, "IndexTest1.html");
File.WriteAllText(fileHtml, strHtml);
TestContext.AddResultFile(fileHtml);
Assert.Fail("failing test so results aren't deleted");
}
public async Task TestOutliers()
{
if (StressUtilOptions.IsRunningOnBuildMachine())
{
return;
}
await Task.Yield();
// data from https://dev.azure.com/devdiv/DevDiv/_releaseProgress?_a=release-environment-extension&releaseId=548609&environmentId=2872682&extensionId=ms.vss-test-web.test-result-in-release-environment-editor-tab&runId=10533790&resultId=100000&paneView=attachments
var testData = new uint[]
{
1867008,
2713172,
2701928,
2701928,
2701800,
2701800,
2701700,
2701700,
2711368,
2711368,
2713228,
2713228,
2714876,
2714876,
2716588,
2716588,
2716588,
2732980,
2732980,
2734848,
2734848,
2736536,
2736536,
2738052,
2738052,
2739908,
2739908,
2741400,
2741400,
2742916,
2742916,
2744548,
2744548,
2744548,
2747340,
2747340,
2764696,
2764696,
2766384,
2766384,
2767888,
2767888,
2769756,
2769756,
2771260,
2771260,
2772764,
2772764,
2774268,
2774268,
2774268,
2776140,
2776140,
2777468,
2777468,
2779168,
2779168,
2779836,
2779836,
2797744,
2797744,
2799236,
2799236,
2800996,
2800996,
2803548,
2803548,
2899440,
2904492,
2904492,
2904492,
};
var resultsFolder = string.Empty;
using (var measurementHolder = new MeasurementHolder(
new TestContextWrapper(TestContext),
new StressUtilOptions()
{
NumIterations = -1,
logger = this,
pctOutliersToIgnore = 5,
lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(),
IsForStress: true).Where(p => p.perfCounterType == PerfCounterType.GCBytesInAllHeaps).ToList()
},
SampleType.SampleTypeIteration))
{
resultsFolder = measurementHolder.ResultsFolder;
for (int iter = 0; iter < testData.Length; iter++)
{
foreach (var ctr in measurementHolder.LstPerfCounterData)
{
measurementHolder.measurements[ctr.perfCounterType].Add(testData[iter]);
}
}
var leakAnalysisResults = await measurementHolder.CalculateLeaksAsync(showGraph : false, GraphsAsFilePrefix : "Graph");
LogMessage($"Yint = {leakAnalysisResults[0].yintercept:n0}");
foreach (var res in leakAnalysisResults[0].lstData)
{
LogMessage($"{res} dist = {res.distance} {res}");
}
Assert.IsTrue(leakAnalysisResults[0].lstData[0].IsOutlier);
Assert.IsTrue(leakAnalysisResults[0].lstData[1].IsOutlier);
Assert.IsTrue(!leakAnalysisResults[0].lstData[2].IsOutlier);
Assert.IsTrue(leakAnalysisResults[0].lstData[68].IsOutlier);
}
}
[ExpectedException(typeof(LeakException))] // to make the test pass, we need a LeakException. However, Pass deletes all the test results <sigh>
public async Task StressVSOpenCloseWaitTilQuiet()
{
// we can't measure for quiet on the current process because the current process will be actively doing stuff.
if (StressUtilOptions.IsRunningOnBuildMachine())
{
throw new LeakException("Throwing expected exception so test passes", null);
}
string didGetLeakException = "didGetLeakException";
int numIter = 11;
try
{
await StressUtil.DoIterationsAsync(
this,
new StressUtilOptions()
{
LoggerLogOutputToDestkop = true,
NumIterations = numIter,
ShowUI = false,
actExecuteAfterEveryIterationAsync = async(nIter, measurementHolder) =>
{
// we want to take measures in a circular buffer and wait til those are quiet
var circBufferSize = 5;
var numTimesToGetQuiet = 50;
var quietMeasure = new MeasurementHolder(
"Quiet",
new StressUtilOptions()
{
NumIterations = 1, // we'll do 1 iteration
pctOutliersToIgnore = 0,
logger = measurementHolder.Logger,
VSHandler = measurementHolder.stressUtilOptions.VSHandler,
lstPerfCountersToUse = measurementHolder.stressUtilOptions.lstPerfCountersToUse,
}, SampleType.SampleTypeIteration
);
// We just took a measurement, so copy those values to init our buffer
foreach (var pctrMeasure in measurementHolder.measurements.Keys)
{
var lastVal = measurementHolder.measurements[pctrMeasure][measurementHolder.nSamplesTaken - 1];
quietMeasure.measurements[pctrMeasure].Add(lastVal);
}
quietMeasure.nSamplesTaken++;
var isQuiet = false;
int nMeasurementsForQuiet = 0;
while (!isQuiet && nMeasurementsForQuiet < numTimesToGetQuiet)
{
await quietMeasure.DoForceGCAsync();
await Task.Delay(TimeSpan.FromSeconds(1 * measurementHolder.stressUtilOptions.DelayMultiplier)); // after GC, wait 1 before taking measurements
var sb = new StringBuilder($"Measure for Quiet iter = {nIter} QuietSamp#= {nMeasurementsForQuiet}");
quietMeasure.TakeRawMeasurement(sb);
measurementHolder.Logger.LogMessage(sb.ToString()); ///xxxremove
if (quietMeasure.nSamplesTaken == circBufferSize)
{
var lk = await quietMeasure.CalculateLeaksAsync(
showGraph: false,
GraphsAsFilePrefix:
#if DEBUG
"Graph"
#else
null
#endif
);
isQuiet = true;
foreach (var k in lk.Where(p => !p.IsQuiet()))
{
measurementHolder.Logger.LogMessage($" !quiet {k}"); ///xxxremove
isQuiet = false;
}
// isQuiet = !lk.Where(k => !k.IsQuiet()).Any();
foreach (var pctrMeasure in quietMeasure.measurements.Keys) // circular buffer: remove 1st item
{
quietMeasure.measurements[pctrMeasure].RemoveAt(0);
}
quietMeasure.nSamplesTaken--;
}
nMeasurementsForQuiet++;
}
if (isQuiet) // the counters have stabilized. We'll use the stabilized numbers as the sample value for the iteration
{
measurementHolder.Logger.LogMessage($"Gone quiet in {nMeasurementsForQuiet} measures");
}
else
{
measurementHolder.Logger.LogMessage($"Didn't go quiet in {numTimesToGetQuiet}");
}
// Whether or not it's quiet, we'll take the most recent measure as the iteration sample
foreach (var pctrMeasure in measurementHolder.measurements.Keys)
{
var lastVal = quietMeasure.measurements[pctrMeasure][quietMeasure.nSamplesTaken - 1];
measurementHolder.measurements[pctrMeasure][measurementHolder.nSamplesTaken - 1] = lastVal;
}
return(true); // continue with normal iteration processing
}
});
await _VSHandler.OpenSolution(SolutionToLoad);
await _VSHandler.CloseSolution();
}
public async Task IterateCode(int numIterations, double Sensitivity, int delayBetweenIterationsMsec)
{
try
{
using (var measurementHolder = new MeasurementHolder(
TestName,
new StressUtilOptions()
{
NumIterations = numIterations,
ProcNamesToMonitor = string.Empty,
ShowUI = this.ShowUI,
logger = _logger,
Sensitivity = Sensitivity,
SecsBetweenIterations = SecsBetweenIterations,
NumIterationsBeforeTotalToTakeBaselineSnapshot = NumIterationsBeforeTotalToTakeBaselineSnapshot,
//actExecuteAfterEveryIterationAsync = async (nIter, mHolder) => // uncomment to suppress dump taking/processing.
//{
// await Task.Yield();
// return false;
//},
lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(System.Diagnostics.Process.GetCurrentProcess(), IsForStress: false)
},
SampleType.SampleTypeIteration))
{
var baseDumpFileName = string.Empty;
for (int iteration = 0; iteration < numIterations && !_CancellationTokenExecuteCode.IsCancellationRequested; iteration++)
{
await DoIterationBodyAsync(iteration, _CancellationTokenExecuteCode);
await Task.Delay(TimeSpan.FromMilliseconds(delayBetweenIterationsMsec *DelayMultiplier));
var desc = string.Format("Iter {0}/{1}", iteration + 1, numIterations);
// we need to go thru the extension to get the measurement, so the vsix graph updates and adds to log
await _itakeSample.DoSampleAsync(measurementHolder, DoForceGC : true, descriptionOverride : desc);
if (_CancellationTokenExecuteCode.IsCancellationRequested)
{
break;
}
}
if (!_CancellationTokenExecuteCode.IsCancellationRequested)
{
_logger.LogMessage(string.Format("Done all {0} iterations", numIterations));
}
else
{
_logger.LogMessage("Cancelled Code Execution");
}
}
}
catch (LeakException)
{
}
catch (OperationCanceledException)
{
_logger.LogMessage("Cancelled");
}
catch (Exception ex)
{
_logger.LogMessage(ex.ToString());
}
}
async Task AddDataPointsAsync(MeasurementHolder measurementHolder)
{
var dictPerfCtrCurrentMeasurements = measurementHolder.GetLastMeasurements();
if (_dataPoints.Count == 0) // nothing yet
{
for (int i = 0; i < NumDataPoints; i++)
{
_dataPoints[i] = new List <uint>(dictPerfCtrCurrentMeasurements.Values); // let all init points be equal, so y axis scales IsStartedFromZero
}
}
else
{
_dataPoints[_bufferIndex++] = new List <uint>(dictPerfCtrCurrentMeasurements.Values);
if (_bufferIndex == _dataPoints.Count) // wraparound?
{
_bufferIndex = 0;
}
}
await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync();
if (DoFullGCPerSample)
{
DoGC();// do a GC.Collect on main thread for every sample (the graphing uses memory)
}
// this needs to be done on UI thread
_chart.Series.Clear();
_chart.ChartAreas.Clear();
ChartArea chartArea = new ChartArea("ChartArea");
chartArea.AxisY.LabelStyle.Format = "{0:n0}";
chartArea.AxisY.LabelStyle.Font = new System.Drawing.Font("Consolas", 12);
_chart.ChartAreas.Add(chartArea);
int ndxSeries = 0;
chartArea.AxisY.IsStartedFromZero = false;
if (SetMaxGraphTo100)
{
_chart.ChartAreas[0].AxisY.Maximum = 100;
}
foreach (var entry in dictPerfCtrCurrentMeasurements)
{
var series = new Series
{
ChartType = SeriesChartType.Line,
Name = entry.Key.ToString()
};
_chart.Series.Add(series);
if (UpdateInterval == 0) // if we're not doing auto update on timer, we're iterating or doing manual measurement
{
series.MarkerSize = 10;
series.MarkerStyle = MarkerStyle.Circle;
}
for (int i = 0; i < _dataPoints.Count; i++)
{
var ndx = _bufferIndex + i;
if (ndx >= _dataPoints.Count)
{
ndx -= _dataPoints.Count;
}
var dp = new DataPoint(i + 1, _dataPoints[ndx][ndxSeries]);
series.Points.Add(dp);
}
ndxSeries++;
}
_chart.Legends.Clear();
_chart.Legends.Add(new Legend());
_chart.DataBind();
if (_editorTracker != null)
{
var(openedViews, lstLeakedViews) = _editorTracker.GetCounts();
OpenedViews.Clear();
LeakedViews.Clear();
foreach (var dictEntry in openedViews)
{
var sp = new StackPanel()
{
Orientation = Orientation.Horizontal
};
sp.Children.Add(new TextBlock()
{
Text = $"{ dictEntry.Key,-15} {dictEntry.Value,3}", FontFamily = FontFamilyMono
});
OpenedViews.Add(sp);
}
foreach (var entry in lstLeakedViews)
{
var sp = new StackPanel()
{
Orientation = Orientation.Horizontal
};
sp.Children.Add(new TextBlock()
{
Text = $"{ entry._contentType,-15} Ser#={entry._serialNo,3} {entry._dtCreated:hh:mm:ss} {entry._filename}", FontFamily = FontFamilyMono
});
public async Task DoSampleAsync(MeasurementHolder measurementHolder, bool DoForceGC, string descriptionOverride = "")
{
var res = string.Empty;
if (measurementHolder == null)
{
measurementHolder = measurementHolderInteractiveUser;
}
try
{
await TaskScheduler.Default;
try
{
res = await measurementHolder.TakeMeasurementAsync(descriptionOverride, DoForceGC, IsForInteractiveGraph : UpdateInterval != 0);
await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync();
await AddDataPointsAsync(measurementHolder);
if (AutoDumpIsEnabled)
{
var lastmeasurements = measurementHolder.GetLastMeasurements();
if (lastmeasurements.Count == 1)
{
var val = lastmeasurements.Values.First();
if (val >= AutoDumpThresh)
{
await AddStatusMsgAsync($"Autodump Threshold triggered: {lastmeasurements.Keys.First()} {val} > {AutoDumpThresh}");
AutoDumpIsEnabled = false;
var pathDump = await CreateDumpFileAsync(MemoryAnalysisType.JustCreateDump, "AutoDump", tspanDelayAfterGC : TimeSpan.FromSeconds(0.2));
}
}
}
}
catch (Exception ex)
{
res = ex.ToString();
}
AddStatusMsgAsync($"{res}").Forget();
}
catch (InvalidOperationException ex)
{
if (ex.Message.Contains("Instance 'devenv#")) // user changed # of instance of devenv runnning
{
await AddStatusMsgAsync($"Resetting perf counters due to devenv instances change");
lock (measurementHolder.LstPerfCounterData)
{
foreach (var ctr in measurementHolder.LstPerfCounterData)
{
ctr.ResetCounter();
}
_dataPoints.Clear();
_bufferIndex = 0;
}
}
}
catch (Exception ex)
{
await AddStatusMsgAsync($"Exception in {nameof(DoSampleAsync)}" + ex.ToString());
_dataPoints.Clear();
_bufferIndex = 0;
}
}
