/// <summary> /// Analyzes the results of the test run and outputs the results file. /// </summary> /// <param name="testInfo">Contains the test results.</param> private void AnalyzeTestResults(TestInfo testInfo) { bool allTestsAreParserTests = true; // Check each of the code files to see whether we should compare the // parser object models. foreach (TestCodeFileInfo codeFileInfo in testInfo.TestCodeFiles) { if (codeFileInfo.TestObjectModel) { this.AnalyzeCodeFileObjectModel(testInfo, codeFileInfo.CodeFile); } else { allTestsAreParserTests = false; } } if (!allTestsAreParserTests) { // Load the expected violations. List<ViolationInfo> expectedViolations = LoadExpectedViolations(testInfo.TestDescription); if (expectedViolations != null) { this.AnalyzeViolations(testInfo, expectedViolations); } } }
/// <summary> /// Sets up a StyleCopConsole to run the given test. /// </summary> /// <param name="testInfo">Information about the test to run.</param> /// <param name="autoFix">Indicates whether StyleCop should automatically fix the violations it finds.</param> /// <returns>Returns the StyleCopConsole.</returns> private static StyleCopConsole PrepareStyleCopConsoleForTest(TestInfo testInfo, bool autoFix) { Debug.Assert(testInfo != null, "The parameter must not be null"); // Create a list of addin paths and add the path to the currently executing test assembly. List<string> addinPaths = new List<string>(1); string assemblyLocation = Assembly.GetExecutingAssembly().Location; addinPaths.Add(Path.GetDirectoryName(assemblyLocation)); addinPaths.Add(Path.GetDirectoryName(Path.Combine(testInfo.TestOutputPath, @"..\..\"))); // Create the StyleCop console. StyleCopConsole console = new StyleCopConsole( testInfo.StyleCopSettingsFileLocation, false, testInfo.StyleCopOutputLocation, addinPaths, false, testInfo.TestOutputPath); return console; }
/// <summary> /// Runs StyleCop in auto-fix mode to clean up violations in the files. /// </summary> /// <param name="testInfo">The test information.</param> /// <returns>Returns false if StyleCop encountered an error.</returns> private bool PrepareFixedFiles(TestInfo testInfo) { // Set up the source analyis settings file to use for the test. if (this.PrepareStyleCopSettingsFile(testInfo)) { // Set up the StyleCop console which will run the test. StyleCopConsole testConsole = PrepareStyleCopConsoleForTest(testInfo, true); CodeProject project = PrepareCodeProjectForTest(testInfo, testConsole, true, true, this.frameworkVersion); // Run the test and capture any exceptions. Exception testException = null; try { // Run StyleCop. testConsole.Start(new CodeProject[] { project }, true); } catch (OutOfMemoryException) { throw; } catch (ThreadAbortException) { // The thread is being aborted. } catch (Exception ex) { // Catch and log all other exceptions thrown during the test. testException = ex; } // If an exception occurred, log it; otherwise analyze the test results. if (testException != null) { this.AddTestResult(false, testInfo.TestName, null, Strings.TestException, testException.Message); return false; } return true; } return false; }
/// <summary> /// Sets up a CodeProject for use during the test. /// </summary> /// <param name="testInfo">The test information.</param> /// <param name="console">The console which will run the test.</param> /// <param name="autoFix">Indicates whether the test is running in auto-fix mode.</param> /// <param name="copy">Indicates whether to create the file copy.</param> /// <param name="simulationFrameworkVersion">The framework version to simulate.</param> /// <returns> /// Returns the CodeProject. /// </returns> private static CodeProject PrepareCodeProjectForTest(TestInfo testInfo, StyleCopConsole console, bool autoFix, bool copy, double simulationFrameworkVersion) { // Create an empty configuration. Configuration configuration = new Configuration(null); // Create a CodeProject for the test file. CodeProject project = new CodeProject( "TheProject".GetHashCode(), Path.GetDirectoryName(testInfo.StyleCopSettingsFileLocation), configuration, simulationFrameworkVersion); // Add each source file to this project. foreach (TestCodeFileInfo sourceFile in testInfo.TestCodeFiles) { if (autoFix) { string autoFixFile = testInfo.AutoFixFileName(sourceFile.CodeFile); console.Core.Environment.AddSourceCode(project, autoFixFile, null); if (copy) { Directory.CreateDirectory(Path.GetDirectoryName(autoFixFile)); File.Copy(sourceFile.CodeFile, autoFixFile); } } else { console.Core.Environment.AddSourceCode(project, sourceFile.CodeFile, null); } } return project; }
/// <summary> /// Sets up a StyleCop settings file for use during the test. /// </summary> /// <param name="testInfo">The test information.</param> /// <returns>Returns true on success; false on error.</returns> private bool PrepareStyleCopSettingsFile(TestInfo testInfo) { XmlDocument settings = new XmlDocument(); // Add the root node. XmlElement root = settings.CreateElement("StyleCopSettings"); XmlAttribute attribute = settings.CreateAttribute("Version"); attribute.Value = "4.3"; root.Attributes.Append(attribute); settings.AppendChild(root); // Add any settings provided in the test description document. XmlNode additionalSettings = testInfo.TestDescription.SelectSingleNode("Settings"); if (additionalSettings != null) { foreach (XmlNode child in additionalSettings.ChildNodes) { XmlNode importedChild = settings.ImportNode(child, true); root.AppendChild(importedChild); } } // Try to save the settings file. Exception exception = null; try { settings.Save(testInfo.StyleCopSettingsFileLocation); } catch (XmlException xmlex) { exception = xmlex; } catch (IOException ioex) { exception = ioex; } catch (SecurityException secex) { exception = secex; } catch (UnauthorizedAccessException unauthex) { exception = unauthex; } if (exception != null) { this.AddTestResult( false, testInfo.TestName, null, Strings.CouldNotSaveSettingsFile, testInfo.StyleCopSettingsFileLocation, exception.Message); return false; } return true; }
/// <summary> /// Saves the given violation detail log. /// </summary> /// <param name="testInfo">Contains information about the test.</param> /// <param name="detailLog">The detail log to save.</param> /// <returns>Returns the path to the log file.</returns> private string SaveViolationDetailLog(TestInfo testInfo, XmlDocument detailLog) { Debug.Assert(detailLog != null, "The parameter must not be null"); string path = Path.Combine( Path.GetDirectoryName(this.resultsOutputLocation), string.Concat(testInfo.TestName, "_ExpectedViolationMismatchDetail.xml")); try { detailLog.Save(path); } catch (XmlException xmlex) { Console.WriteLine(Strings.CouldNotWriteDetailLog, path, xmlex.Message); } catch (IOException ioex) { Console.WriteLine(Strings.CouldNotWriteDetailLog, path, ioex.Message); } catch (SecurityException secex) { Console.WriteLine(Strings.CouldNotWriteDetailLog, path, secex.Message); } catch (UnauthorizedAccessException unauthex) { Console.WriteLine(Strings.CouldNotWriteDetailLog, path, unauthex.Message); } catch (ArgumentException argex) { Console.WriteLine(Strings.CouldNotWriteDetailLog, path, argex.Message); } return path; }
/// <summary> /// Analyzes the resulting object model for the given code file against the expected /// object model, to test the parser output. /// </summary> /// <param name="testInfo">The test information.</param> /// <param name="codeFile">The the code file to check.</param> private void AnalyzeCodeFileObjectModel(TestInfo testInfo, string codeFile) { // Extract the name of the code file. string fileName = Path.GetFileNameWithoutExtension(codeFile); // Figure out the path to the expected object model file and the actual // object model file. string expectedObjectModelFilePath = Path.Combine(this.testDataLocation, fileName + "ObjectModel.xml"); string actualObjectModelFilePath = Path.Combine(testInfo.TestOutputPath, fileName + "ObjectModelResults.xml"); // Load both of the files. string errorMessage; XmlDocument expectedObjectModel = LoadXmlDocument(expectedObjectModelFilePath, out errorMessage); if (expectedObjectModel == null) { this.AddTestResult(false, testInfo.TestName, null, Strings.MissingExpectedObjectModelFile, expectedObjectModelFilePath); return; } XmlDocument actualObjectModel = LoadXmlDocument(actualObjectModelFilePath, out errorMessage); if (actualObjectModel == null) { this.AddTestResult(false, testInfo.TestName, null, Strings.MissingActualObjectModelFile, actualObjectModelFilePath); return; } // Compare the two documents to see if they are equal. if (!this.CompareXmlDocuments(expectedObjectModel, actualObjectModel)) { this.AddTestResult(false, testInfo.TestName, actualObjectModelFilePath, Strings.ParserObjectModelMismatch); } }
/// <summary> /// Compares the list of expected violations against the actual violations produced by the test. /// </summary> /// <param name="testInfo">The test information.</param> /// <param name="expectedViolations">The list of expected violations.</param> private void AnalyzeViolations(TestInfo testInfo, List<ViolationInfo> expectedViolations) { Param.AssertNotNull(testInfo, "testInfo"); Param.AssertNotNull(expectedViolations, "expectedViolations"); // Open the results file. string errorMessage; XmlDocument styleCopAnalysisResultsDocument = LoadXmlDocument(testInfo.StyleCopOutputLocation, out errorMessage); if (styleCopAnalysisResultsDocument == null) { this.AddTestResult(false, testInfo.TestName, null, Strings.MissingResultsFile, testInfo.StyleCopOutputLocation); } else { // Load the actual violations. List<ViolationInfo> actualViolations = LoadActualViolations(testInfo.StyleCopOutputLocation); // Manually verify that the violations are the same. int expectedViolationIndex = 0; while (expectedViolations.Count > 0 && expectedViolationIndex < expectedViolations.Count) { ViolationInfo expectedViolation = expectedViolations[expectedViolationIndex]; bool found = false; foreach (ViolationInfo actualViolation in actualViolations) { if (MatchesExpectedViolation(expectedViolation, actualViolation)) { expectedViolations.Remove(expectedViolation); actualViolations.Remove(actualViolation); found = true; break; } } if (!found) { ++expectedViolationIndex; } } XmlDocument detailLog = null; if (expectedViolations.Count > 0 || actualViolations.Count > 0) { detailLog = new XmlDocument(); detailLog.AppendChild(detailLog.CreateElement("StyleCopTestFailureDetails")); if (expectedViolations.Count > 0) { // There were more expected violations than actual violations. XmlElement root = detailLog.CreateElement("MissingExpectedViolations"); detailLog.DocumentElement.AppendChild(root); foreach (ViolationInfo violationInfo in expectedViolations) { WriteViolationDetailToLog(violationInfo, root); } } if (actualViolations.Count > 0) { // There were more actual violations than expected violations. XmlElement root = detailLog.CreateElement("ExtraActualViolations"); detailLog.DocumentElement.AppendChild(root); foreach (ViolationInfo violationInfo in actualViolations) { WriteViolationDetailToLog(violationInfo, root); } } string detailLogPath = this.SaveViolationDetailLog(testInfo, detailLog); this.AddTestResult(false, testInfo.TestName, detailLogPath, Strings.MismatchedViolations); } } }