/// <summary>
/// Analyzes the results of the test run and outputs the results file.
/// </summary>
/// <param name="testInfo">Contains the test results.</param>
private void AnalyzeTestResults(TestInfo testInfo)
{
bool allTestsAreParserTests = true;
// Check each of the code files to see whether we should compare the
// parser object models.
foreach (TestCodeFileInfo codeFileInfo in testInfo.TestCodeFiles)
{
if (codeFileInfo.TestObjectModel)
{
this.AnalyzeCodeFileObjectModel(testInfo, codeFileInfo.CodeFile);
}
else
{
allTestsAreParserTests = false;
}
}
if (!allTestsAreParserTests)
{
// Load the expected violations.
List<ViolationInfo> expectedViolations = LoadExpectedViolations(testInfo.TestDescription);
if (expectedViolations != null)
{
this.AnalyzeViolations(testInfo, expectedViolations);
}
}
}
/// <summary>
/// Sets up a StyleCopConsole to run the given test.
/// </summary>
/// <param name="testInfo">Information about the test to run.</param>
/// <param name="autoFix">Indicates whether StyleCop should automatically fix the violations it finds.</param>
/// <returns>Returns the StyleCopConsole.</returns>
private static StyleCopConsole PrepareStyleCopConsoleForTest(TestInfo testInfo, bool autoFix)
{
Debug.Assert(testInfo != null, "The parameter must not be null");
// Create a list of addin paths and add the path to the currently executing test assembly.
List<string> addinPaths = new List<string>(1);
string assemblyLocation = Assembly.GetExecutingAssembly().Location;
addinPaths.Add(Path.GetDirectoryName(assemblyLocation));
addinPaths.Add(Path.GetDirectoryName(Path.Combine(testInfo.TestOutputPath, @"..\..\")));
// Create the StyleCop console.
StyleCopConsole console = new StyleCopConsole(
testInfo.StyleCopSettingsFileLocation,
false,
testInfo.StyleCopOutputLocation,
addinPaths,
false,
testInfo.TestOutputPath);
return console;
}
/// <summary>
/// Runs StyleCop in auto-fix mode to clean up violations in the files.
/// </summary>
/// <param name="testInfo">The test information.</param>
/// <returns>Returns false if StyleCop encountered an error.</returns>
private bool PrepareFixedFiles(TestInfo testInfo)
{
// Set up the source analyis settings file to use for the test.
if (this.PrepareStyleCopSettingsFile(testInfo))
{
// Set up the StyleCop console which will run the test.
StyleCopConsole testConsole = PrepareStyleCopConsoleForTest(testInfo, true);
CodeProject project = PrepareCodeProjectForTest(testInfo, testConsole, true, true, this.frameworkVersion);
// Run the test and capture any exceptions.
Exception testException = null;
try
{
// Run StyleCop.
testConsole.Start(new CodeProject[] { project }, true);
}
catch (OutOfMemoryException)
{
throw;
}
catch (ThreadAbortException)
{
// The thread is being aborted.
}
catch (Exception ex)
{
// Catch and log all other exceptions thrown during the test.
testException = ex;
}
// If an exception occurred, log it; otherwise analyze the test results.
if (testException != null)
{
this.AddTestResult(false, testInfo.TestName, null, Strings.TestException, testException.Message);
return false;
}
return true;
}
return false;
}
/// <summary>
/// Sets up a CodeProject for use during the test.
/// </summary>
/// <param name="testInfo">The test information.</param>
/// <param name="console">The console which will run the test.</param>
/// <param name="autoFix">Indicates whether the test is running in auto-fix mode.</param>
/// <param name="copy">Indicates whether to create the file copy.</param>
/// <param name="simulationFrameworkVersion">The framework version to simulate.</param>
/// <returns>
/// Returns the CodeProject.
/// </returns>
private static CodeProject PrepareCodeProjectForTest(TestInfo testInfo, StyleCopConsole console, bool autoFix, bool copy, double simulationFrameworkVersion)
{
// Create an empty configuration.
Configuration configuration = new Configuration(null);
// Create a CodeProject for the test file.
CodeProject project = new CodeProject(
"TheProject".GetHashCode(),
Path.GetDirectoryName(testInfo.StyleCopSettingsFileLocation),
configuration,
simulationFrameworkVersion);
// Add each source file to this project.
foreach (TestCodeFileInfo sourceFile in testInfo.TestCodeFiles)
{
if (autoFix)
{
string autoFixFile = testInfo.AutoFixFileName(sourceFile.CodeFile);
console.Core.Environment.AddSourceCode(project, autoFixFile, null);
if (copy)
{
Directory.CreateDirectory(Path.GetDirectoryName(autoFixFile));
File.Copy(sourceFile.CodeFile, autoFixFile);
}
}
else
{
console.Core.Environment.AddSourceCode(project, sourceFile.CodeFile, null);
}
}
return project;
}
/// <summary>
/// Sets up a StyleCop settings file for use during the test.
/// </summary>
/// <param name="testInfo">The test information.</param>
/// <returns>Returns true on success; false on error.</returns>
private bool PrepareStyleCopSettingsFile(TestInfo testInfo)
{
XmlDocument settings = new XmlDocument();
// Add the root node.
XmlElement root = settings.CreateElement("StyleCopSettings");
XmlAttribute attribute = settings.CreateAttribute("Version");
attribute.Value = "4.3";
root.Attributes.Append(attribute);
settings.AppendChild(root);
// Add any settings provided in the test description document.
XmlNode additionalSettings = testInfo.TestDescription.SelectSingleNode("Settings");
if (additionalSettings != null)
{
foreach (XmlNode child in additionalSettings.ChildNodes)
{
XmlNode importedChild = settings.ImportNode(child, true);
root.AppendChild(importedChild);
}
}
// Try to save the settings file.
Exception exception = null;
try
{
settings.Save(testInfo.StyleCopSettingsFileLocation);
}
catch (XmlException xmlex)
{
exception = xmlex;
}
catch (IOException ioex)
{
exception = ioex;
}
catch (SecurityException secex)
{
exception = secex;
}
catch (UnauthorizedAccessException unauthex)
{
exception = unauthex;
}
if (exception != null)
{
this.AddTestResult(
false,
testInfo.TestName,
null,
Strings.CouldNotSaveSettingsFile,
testInfo.StyleCopSettingsFileLocation,
exception.Message);
return false;
}
return true;
}
/// <summary>
/// Saves the given violation detail log.
/// </summary>
/// <param name="testInfo">Contains information about the test.</param>
/// <param name="detailLog">The detail log to save.</param>
/// <returns>Returns the path to the log file.</returns>
private string SaveViolationDetailLog(TestInfo testInfo, XmlDocument detailLog)
{
Debug.Assert(detailLog != null, "The parameter must not be null");
string path = Path.Combine(
Path.GetDirectoryName(this.resultsOutputLocation),
string.Concat(testInfo.TestName, "_ExpectedViolationMismatchDetail.xml"));
try
{
detailLog.Save(path);
}
catch (XmlException xmlex)
{
Console.WriteLine(Strings.CouldNotWriteDetailLog, path, xmlex.Message);
}
catch (IOException ioex)
{
Console.WriteLine(Strings.CouldNotWriteDetailLog, path, ioex.Message);
}
catch (SecurityException secex)
{
Console.WriteLine(Strings.CouldNotWriteDetailLog, path, secex.Message);
}
catch (UnauthorizedAccessException unauthex)
{
Console.WriteLine(Strings.CouldNotWriteDetailLog, path, unauthex.Message);
}
catch (ArgumentException argex)
{
Console.WriteLine(Strings.CouldNotWriteDetailLog, path, argex.Message);
}
return path;
}
/// <summary>
/// Analyzes the resulting object model for the given code file against the expected
/// object model, to test the parser output.
/// </summary>
/// <param name="testInfo">The test information.</param>
/// <param name="codeFile">The the code file to check.</param>
private void AnalyzeCodeFileObjectModel(TestInfo testInfo, string codeFile)
{
// Extract the name of the code file.
string fileName = Path.GetFileNameWithoutExtension(codeFile);
// Figure out the path to the expected object model file and the actual
// object model file.
string expectedObjectModelFilePath = Path.Combine(this.testDataLocation, fileName + "ObjectModel.xml");
string actualObjectModelFilePath = Path.Combine(testInfo.TestOutputPath, fileName + "ObjectModelResults.xml");
// Load both of the files.
string errorMessage;
XmlDocument expectedObjectModel = LoadXmlDocument(expectedObjectModelFilePath, out errorMessage);
if (expectedObjectModel == null)
{
this.AddTestResult(false, testInfo.TestName, null, Strings.MissingExpectedObjectModelFile, expectedObjectModelFilePath);
return;
}
XmlDocument actualObjectModel = LoadXmlDocument(actualObjectModelFilePath, out errorMessage);
if (actualObjectModel == null)
{
this.AddTestResult(false, testInfo.TestName, null, Strings.MissingActualObjectModelFile, actualObjectModelFilePath);
return;
}
// Compare the two documents to see if they are equal.
if (!this.CompareXmlDocuments(expectedObjectModel, actualObjectModel))
{
this.AddTestResult(false, testInfo.TestName, actualObjectModelFilePath, Strings.ParserObjectModelMismatch);
}
}
/// <summary>
/// Compares the list of expected violations against the actual violations produced by the test.
/// </summary>
/// <param name="testInfo">The test information.</param>
/// <param name="expectedViolations">The list of expected violations.</param>
private void AnalyzeViolations(TestInfo testInfo, List<ViolationInfo> expectedViolations)
{
Param.AssertNotNull(testInfo, "testInfo");
Param.AssertNotNull(expectedViolations, "expectedViolations");
// Open the results file.
string errorMessage;
XmlDocument styleCopAnalysisResultsDocument = LoadXmlDocument(testInfo.StyleCopOutputLocation, out errorMessage);
if (styleCopAnalysisResultsDocument == null)
{
this.AddTestResult(false, testInfo.TestName, null, Strings.MissingResultsFile, testInfo.StyleCopOutputLocation);
}
else
{
// Load the actual violations.
List<ViolationInfo> actualViolations = LoadActualViolations(testInfo.StyleCopOutputLocation);
// Manually verify that the violations are the same.
int expectedViolationIndex = 0;
while (expectedViolations.Count > 0 && expectedViolationIndex < expectedViolations.Count)
{
ViolationInfo expectedViolation = expectedViolations[expectedViolationIndex];
bool found = false;
foreach (ViolationInfo actualViolation in actualViolations)
{
if (MatchesExpectedViolation(expectedViolation, actualViolation))
{
expectedViolations.Remove(expectedViolation);
actualViolations.Remove(actualViolation);
found = true;
break;
}
}
if (!found)
{
++expectedViolationIndex;
}
}
XmlDocument detailLog = null;
if (expectedViolations.Count > 0 || actualViolations.Count > 0)
{
detailLog = new XmlDocument();
detailLog.AppendChild(detailLog.CreateElement("StyleCopTestFailureDetails"));
if (expectedViolations.Count > 0)
{
// There were more expected violations than actual violations.
XmlElement root = detailLog.CreateElement("MissingExpectedViolations");
detailLog.DocumentElement.AppendChild(root);
foreach (ViolationInfo violationInfo in expectedViolations)
{
WriteViolationDetailToLog(violationInfo, root);
}
}
if (actualViolations.Count > 0)
{
// There were more actual violations than expected violations.
XmlElement root = detailLog.CreateElement("ExtraActualViolations");
detailLog.DocumentElement.AppendChild(root);
foreach (ViolationInfo violationInfo in actualViolations)
{
WriteViolationDetailToLog(violationInfo, root);
}
}
string detailLogPath = this.SaveViolationDetailLog(testInfo, detailLog);
this.AddTestResult(false, testInfo.TestName, detailLogPath, Strings.MismatchedViolations);
}
}
}