[ExpectedException(typeof(LeakException))] // to make the test pass, we need a LeakException. However, Pass deletes all the test results <sigh> public async Task StressLeakyBadPerfCounterCat() { int numIter = 11; var stressOptions = new StressUtilOptions() { NumIterations = numIter, ProcNamesToMonitor = string.Empty, ShowUI = false }; stressOptions.lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(), IsForStress: true); stressOptions.lstPerfCountersToUse[0].PerfCounterCategory = "foobar"; // an invalid category try { await StressUtil.DoIterationsAsync( this, stressOptions ); _lst.Add(new BigStuffWithLongNameSoICanSeeItBetter()); } catch (LeakException ex) { TestContext.WriteLine($"Caught exception {ex.Message}"); var lstFileResults = (List <FileResultsData>)TestContext.Properties[StressUtil.PropNameListFileResults]; foreach (var result in lstFileResults.OrderBy(r => r.filename)) { TestContext.WriteLine($"File result {Path.GetFileName(result.filename)}"); } var expectedFiles = new[] { "Graph # Bytes in all Heaps.png", "Graph GDIHandles.png", "Graph Handle Count.png", "Graph Private Bytes.png", "Graph Thread Count.png", "Graph UserHandles.png", "Graph Virtual Bytes.png", "Measurements.txt", $"{TestContext.TestName}_{numIter}_0.dmp", $"{TestContext.TestName}_{numIter-4}_0.dmp", "StressTestLog.log", $"String and Type Count differences_{numIter}.txt", }; foreach (var itm in expectedFiles) { Assert.IsTrue(lstFileResults.Where(r => Path.GetFileName(r.filename) == itm).Count() == 1, $"Expected File attachment {itm}"); } var strAndTypeDiff = File.ReadAllText(lstFileResults.Where(r => Path.GetFileName(r.filename) == $"String and Type Count differences_{numIter}.txt").First().filename); TestContext.WriteLine($"String and Type Count differences_{numIter}.txt"); TestContext.WriteLine(strAndTypeDiff); Assert.IsTrue(strAndTypeDiff.Contains(nameof(BigStuffWithLongNameSoICanSeeItBetter)), $"Type must be in StringandTypeDiff"); Assert.IsTrue(strAndTypeDiff.Contains("leaking string"), $"'leaking string' must be in StringandTypeDiff"); Assert.AreEqual(expectedFiles.Length, lstFileResults.Count, $"# file results"); TestContext.Properties[didGetLeakException] = 1; throw; } }
/// <summary> /// return true if regression found /// These tests will be affected by other tests running in the same instance of testhost because they share the same memory /// </summary> private async Task <bool> DoStressSimulation(int nIter, int nArraySize, float RatioThresholdSensitivity, Action action = null) { var lstPCs = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(), IsForStress: true); foreach (var ctr in lstPCs) { ctr.IsEnabledForMeasurement = true; } List <LeakAnalysisResult> lstRegResults; using (var measurementHolder = new MeasurementHolder( new TestContextWrapper(TestContext), new StressUtilOptions() { NumIterations = -1, Sensitivity = RatioThresholdSensitivity, logger = this, lstPerfCountersToUse = lstPCs }, SampleType.SampleTypeIteration)) { var lstBigStuff = new List <byte[]>(); LogMessage($"nIter={nIter:n0} ArraySize= {nArraySize:n0}"); for (int i = 0; i < nIter; i++) { if (action != null) { action(); } else { lstBigStuff.Add(new byte[nArraySize]); } // lstBigStuff.Add(new int[10000000]); var res = await measurementHolder.TakeMeasurementAsync($"iter {i}/{nIter}", DoForceGC : true); LogMessage(res); } var filename = measurementHolder.DumpOutMeasurementsToTxtFile(); LogMessage($"Results file name = {filename}"); lstRegResults = (await measurementHolder.CalculateLeaksAsync(showGraph: false, GraphsAsFilePrefix: "Graph")).Where(r => r.IsLeak).ToList(); } return(lstRegResults.Count > 0); }
public async Task TestMeasureRegressionVerifyGraph() { await Task.Yield(); var resultsFolder = string.Empty; using (var measurementHolder = new MeasurementHolder( new TestContextWrapper(TestContext), new StressUtilOptions() { NumIterations = -1, logger = this, lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(), IsForStress: true).Where(p => p.perfCounterType == PerfCounterType.KernelHandleCount).ToList() }, SampleType.SampleTypeIteration)) { resultsFolder = measurementHolder.ResultsFolder; for (int iter = 0; iter < 10; iter++) { foreach (var ctr in measurementHolder.LstPerfCounterData) { var val = 10000 + (uint)iter; if (iter == 5 && ctr.perfCounterType == PerfCounterType.KernelHandleCount) { val += 1; } measurementHolder.measurements[ctr.perfCounterType].Add(val); } } var res = await measurementHolder.CalculateLeaksAsync(showGraph : true, GraphsAsFilePrefix : "Graph"); } var strHtml = @" <a href=""file:\\C:\Users\calvinh\Source\repos\PerfGraphVSIX\TestResults\Deploy_calvinh 2019-11-19 11_00_13/Out/TestMeasureRegressionVerifyGraph/Graph Handle Count.png"">gr </a> "; var fileHtml = Path.Combine(resultsFolder, "IndexTest1.html"); File.WriteAllText(fileHtml, strHtml); TestContext.AddResultFile(fileHtml); Assert.Fail("failing test so results aren't deleted"); }
public async Task TestOutliers() { if (StressUtilOptions.IsRunningOnBuildMachine()) { return; } await Task.Yield(); // data from https://dev.azure.com/devdiv/DevDiv/_releaseProgress?_a=release-environment-extension&releaseId=548609&environmentId=2872682&extensionId=ms.vss-test-web.test-result-in-release-environment-editor-tab&runId=10533790&resultId=100000&paneView=attachments var testData = new uint[] { 1867008, 2713172, 2701928, 2701928, 2701800, 2701800, 2701700, 2701700, 2711368, 2711368, 2713228, 2713228, 2714876, 2714876, 2716588, 2716588, 2716588, 2732980, 2732980, 2734848, 2734848, 2736536, 2736536, 2738052, 2738052, 2739908, 2739908, 2741400, 2741400, 2742916, 2742916, 2744548, 2744548, 2744548, 2747340, 2747340, 2764696, 2764696, 2766384, 2766384, 2767888, 2767888, 2769756, 2769756, 2771260, 2771260, 2772764, 2772764, 2774268, 2774268, 2774268, 2776140, 2776140, 2777468, 2777468, 2779168, 2779168, 2779836, 2779836, 2797744, 2797744, 2799236, 2799236, 2800996, 2800996, 2803548, 2803548, 2899440, 2904492, 2904492, 2904492, }; var resultsFolder = string.Empty; using (var measurementHolder = new MeasurementHolder( new TestContextWrapper(TestContext), new StressUtilOptions() { NumIterations = -1, logger = this, pctOutliersToIgnore = 5, lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(Process.GetCurrentProcess(), IsForStress: true).Where(p => p.perfCounterType == PerfCounterType.GCBytesInAllHeaps).ToList() }, SampleType.SampleTypeIteration)) { resultsFolder = measurementHolder.ResultsFolder; for (int iter = 0; iter < testData.Length; iter++) { foreach (var ctr in measurementHolder.LstPerfCounterData) { measurementHolder.measurements[ctr.perfCounterType].Add(testData[iter]); } } var leakAnalysisResults = await measurementHolder.CalculateLeaksAsync(showGraph : false, GraphsAsFilePrefix : "Graph"); LogMessage($"Yint = {leakAnalysisResults[0].yintercept:n0}"); foreach (var res in leakAnalysisResults[0].lstData) { LogMessage($"{res} dist = {res.distance} {res}"); } Assert.IsTrue(leakAnalysisResults[0].lstData[0].IsOutlier); Assert.IsTrue(leakAnalysisResults[0].lstData[1].IsOutlier); Assert.IsTrue(!leakAnalysisResults[0].lstData[2].IsOutlier); Assert.IsTrue(leakAnalysisResults[0].lstData[68].IsOutlier); } }
public async Task IterateCode(int numIterations, double Sensitivity, int delayBetweenIterationsMsec) { try { using (var measurementHolder = new MeasurementHolder( TestName, new StressUtilOptions() { NumIterations = numIterations, ProcNamesToMonitor = string.Empty, ShowUI = this.ShowUI, logger = _logger, Sensitivity = Sensitivity, SecsBetweenIterations = SecsBetweenIterations, NumIterationsBeforeTotalToTakeBaselineSnapshot = NumIterationsBeforeTotalToTakeBaselineSnapshot, //actExecuteAfterEveryIterationAsync = async (nIter, mHolder) => // uncomment to suppress dump taking/processing. //{ // await Task.Yield(); // return false; //}, lstPerfCountersToUse = PerfCounterData.GetPerfCountersToUse(System.Diagnostics.Process.GetCurrentProcess(), IsForStress: false) }, SampleType.SampleTypeIteration)) { var baseDumpFileName = string.Empty; for (int iteration = 0; iteration < numIterations && !_CancellationTokenExecuteCode.IsCancellationRequested; iteration++) { await DoIterationBodyAsync(iteration, _CancellationTokenExecuteCode); await Task.Delay(TimeSpan.FromMilliseconds(delayBetweenIterationsMsec *DelayMultiplier)); var desc = string.Format("Iter {0}/{1}", iteration + 1, numIterations); // we need to go thru the extension to get the measurement, so the vsix graph updates and adds to log await _itakeSample.DoSampleAsync(measurementHolder, DoForceGC : true, descriptionOverride : desc); if (_CancellationTokenExecuteCode.IsCancellationRequested) { break; } } if (!_CancellationTokenExecuteCode.IsCancellationRequested) { _logger.LogMessage(string.Format("Done all {0} iterations", numIterations)); } else { _logger.LogMessage("Cancelled Code Execution"); } } } catch (LeakException) { } catch (OperationCanceledException) { _logger.LogMessage("Cancelled"); } catch (Exception ex) { _logger.LogMessage(ex.ToString()); } }
public PerfGraphToolWindowControl() { this.InitializeComponent(); g_PerfGraphToolWindowControl = this; try { #if DEBUG LogMessage($"Starting {TipString}"); #endif var tspanDesiredLeaseLifetime = TimeSpan.FromSeconds(2); var oldval = System.Runtime.Remoting.Lifetime.LifetimeServices.LeaseTime; if (oldval == tspanDesiredLeaseLifetime) { LogMessage($"System.Runtime.Remoting.Lifetime.LifetimeServices.LeaseTime.TotalSeconds already set at {oldval.TotalSeconds} secs"); } else { try { System.Runtime.Remoting.Lifetime.LifetimeServices.LeaseTime = tspanDesiredLeaseLifetime; LogMessage($"Success Change System.Runtime.Remoting.Lifetime.LifetimeServices.LeaseTime.TotalSeconds from {oldval.TotalSeconds} secs to {System.Runtime.Remoting.Lifetime.LifetimeServices.LeaseTime.TotalSeconds}"); } catch (System.Runtime.Remoting.RemotingException) { LogMessage($"Failed to Change System.Runtime.Remoting.Lifetime.LifetimeServices.LeaseTime.TotalSeconds from {oldval.TotalSeconds} secs to {tspanDesiredLeaseLifetime.TotalSeconds} secs"); } } LstPerfCounterData = PerfCounterData.GetPerfCountersToUse(System.Diagnostics.Process.GetCurrentProcess(), IsForStress: false); async Task RefreshCodeToRunAsync() { await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); FileInfo mostRecentFileInfo = null; foreach (var file in Directory.GetFiles(CodeSampleDirectory, "*.*", SearchOption.AllDirectories) .Where(f => ".vb|.cs".Contains(Path.GetExtension(f).ToLower())) .OrderByDescending(f => new FileInfo(f).LastWriteTime)) { if (!file.Contains(@"\Util\"))// utility folder doesn't contain code with Main program { var finfo = new FileInfo(file); if (mostRecentFileInfo == null || finfo.LastWriteTime > mostRecentFileInfo.LastWriteTime) { mostRecentFileInfo = finfo; } } } _codeSampleControl = new CodeSamples(CodeSampleDirectory, mostRecentFileInfo?.Name); this.spCodeSamples.Children.Clear(); this.spCodeSamples.Children.Add(_codeSampleControl); } _ = Task.Run(() => { _ = RefreshCodeToRunAsync(); }); _fileSystemWatcher = new FileSystemWatcher(CodeSampleDirectory); FileSystemEventHandler h = new FileSystemEventHandler( (o, e) => { // LogMessage($"FileWatcher {e.ChangeType} '{e.FullPath}'"); _ = RefreshCodeToRunAsync(); } ); // we don't handle Rename here: just save the newly renamed file to trigger the Changed event. _fileSystemWatcher.Changed += h; _fileSystemWatcher.Created += h; _fileSystemWatcher.Deleted += h; _fileSystemWatcher.EnableRaisingEvents = true; ThreadHelper.JoinableTaskFactory.StartOnIdle(async() => { await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); if (PerfGraphToolWindowCommand.Instance.g_dte == null) // if the toolwindow was already opened, this is set in InitializeToolWindowAsync. 1st time opening set it here { EnvDTE.DTE dte = (EnvDTE.DTE) await PerfGraphToolWindowCommand.Instance.package.GetServiceAsync(typeof(EnvDTE.DTE)); PerfGraphToolWindowCommand.Instance.g_dte = dte; // ?? throw new InvalidOperationException(nameof(dte)); } _objTracker = new ObjTracker(this); _editorTracker = PerfGraphToolWindowPackage.ComponentModel.GetService <EditorTracker>(); _editorTracker.Initialize(this, _objTracker); _openFolderTracker = PerfGraphToolWindowPackage.ComponentModel.GetService <OpenFolderTracker>(); _openFolderTracker.Initialize(this, _objTracker); if (this.IsLeakTrackerServiceSupported()) { this.inProcLeakTracerTabItem.Visibility = Visibility.Visible; this.inProcLeakTracker.Content = new InProcLeakTracker(); } await TaskScheduler.Default; var telEvent = new TelemetryEvent(TelemetryEventBaseName + "Start"); TelemetryService.DefaultSession.PostEvent(telEvent); await DoProcessAutoexecAsync(); }); txtUpdateInterval.LostFocus += (o, e) => { _ = ResetPerfCounterMonitorAsync(); }; btnDoSample.Click += (o, e) => { ThreadHelper.JoinableTaskFactory.Run(async() => { await WaitForInitializationCompleteAsync(); await DoSampleAsync(measurementHolderInteractiveUser, DoForceGC: true, descriptionOverride: "Manual"); } ); }; lbPCounters.ItemsSource = LstPerfCounterData.Select(s => s.perfCounterType); lbPCounters.SelectedIndex = 0; LstPerfCounterData.Where(s => s.perfCounterType == PerfCounterType.GCBytesInAllHeaps).Single().IsEnabledForGraph = true; #pragma warning disable VSTHRD101 // Avoid unsupported async delegates lbPCounters.SelectionChanged += async(ol, el) => { try { lbPCounters.IsEnabled = false; // cancel the perf monitoring _ctsPcounter?.Cancel(); // before we wait for cancel to finish we can do some work PerfCounterType pctrEnum = PerfCounterType.None; foreach (var itm in lbPCounters.SelectedItems) { pctrEnum |= (PerfCounterType)Enum.Parse(typeof(PerfCounterType), itm.ToString()); } AddStatusMsgAsync($"Setting counters to {pctrEnum}").Forget(); // wait for it to be done cancelling if (_tskDoPerfMonitoring != null) { await _tskDoPerfMonitoring; } await Task.Run(async() => { // run on threadpool thread lock (LstPerfCounterData) { foreach (var itm in LstPerfCounterData) { itm.IsEnabledForGraph = pctrEnum.HasFlag(itm.perfCounterType); } } await ResetPerfCounterMonitorAsync(); }); AddStatusMsgAsync($"SelectionChanged done").Forget(); lbPCounters.IsEnabled = true; el.Handled = true; } catch (Exception) { } }; _chart = new Chart(); wfhost.Child = _chart; txtStatus.ContextMenu = new ContextMenu(); txtStatus.ContextMenu.AddMenuItem((o, e) => { txtStatus.Clear(); }, "_Clear All", "Clear the current contents"); _ = Task.Run(async() => { //await AddStatusMsgAsync("Waiting 15 seconds to initialize graph"); //await Task.Delay(TimeSpan.FromSeconds(15));// delay samples til VS started await ResetPerfCounterMonitorAsync(); }); #if DEBUG var tsk = AddStatusMsgAsync($"PerfGraphVsix curdir= {Environment.CurrentDirectory}"); #endif Microsoft.VisualStudio.Shell.Events.SolutionEvents.OnAfterOpenProject += (o, e) => { Microsoft.VisualStudio.Shell.ThreadHelper.ThrowIfNotOnUIThread(); if (this.TrackProjectObjects) { var hier = e.Hierarchy; if (hier.GetProperty((uint)Microsoft.VisualStudio.VSConstants.VSITEMID.Root, (int)Microsoft.VisualStudio.Shell.Interop.__VSHPROPID.VSHPROPID_ExtObject, out var extObject) == Microsoft.VisualStudio.VSConstants.S_OK) { var proj = extObject as EnvDTE.Project; // comobj or Microsoft.VisualStudio.ProjectSystem.VS.Implementation.Package.Automation.OAProject var name = proj.Name; var context = proj as IVsBrowseObjectContext; // Microsoft.VisualStudio.ProjectSystem.VS.Implementation.Package.Automation.OAProject if (context == null && proj != null) { context = proj.Object as IVsBrowseObjectContext; // {Microsoft.VisualStudio.Project.VisualC.VCProjectEngine.VCProjectShim} } if (context != null) { // var task = AddStatusMsgAsync($"{nameof(Microsoft.VisualStudio.Shell.Events.SolutionEvents.OnAfterOpenProject)} {proj.Name} Context = {context}"); _objTracker.AddObjectToTrack(context, ObjSource.FromProject, description: proj.Name); //var x = proj.Object as Microsoft.VisualStudio.ProjectSystem.Properties.IVsBrowseObjectContext; } } } }; } catch (Exception ex) { this.Content = ex.ToString(); } }