/// <summary>
/// Ends the test module.
/// </summary>
/// <param name="testModule">The test module.</param>
/// <param name="result">The result.</param>
/// <param name="exception">The exception.</param>
/// <param name="statisticsByPriority">The statistics for the module, broken down by priority.</param>
/// <param name="globalStatistics">The global statistics for the module.</param>
public void EndTestModule(TestModuleData testModule, TestResult result, ExceptionDetails exception, IDictionary <int, RunStatistics> statisticsByPriority, RunStatistics globalStatistics)
{
TestModuleData currentTestModule = this.testItemStack.Pop() as TestModuleData;
ExceptionUtilities.Assert(currentTestModule == testModule, "Invalid test module on stack.");
this.WriteResult(result, exception);
this.writer.WriteEndElement();
this.writer.WriteStartElement("Summary");
this.OutputCounts(globalStatistics);
foreach (var kvp in statisticsByPriority.OrderBy(c => c.Key))
{
this.writer.WriteStartElement("Property");
this.OutputCounts(kvp.Value);
this.writer.WriteAttributeString("Name", "Pri");
this.writer.WriteAttributeString("Value", XmlConvert.ToString(kvp.Key));
this.writer.WriteEndElement();
}
this.writer.WriteEndElement();
this.writer.Flush();
}
/// <summary>
/// Gets the included test items for the given module and suite
/// </summary>
/// <param name="module">The test module data</param>
/// <param name="suite">The test suite</param>
/// <param name="logWriter">optional log writer in case something goes wrong</param>
/// <returns>The included test items without the excluded items</returns>
public static IEnumerable <TestItemData> GetIncludedVariations(TestModuleData module, TestSuite suite, ITestLogWriter logWriter)
{
ExceptionUtilities.CheckArgumentNotNull(suite, "suite");
var included = suite.Items.Where(i => i.IsIncluded).Select(i => i.Name);
var excluded = suite.Items.Where(i => !i.IsIncluded).Select(i => i.Name);
return(GetIncludedVariations(module, included, excluded, logWriter));
}
/// <summary>
/// Begins the test module.
/// </summary>
/// <param name="testModule">The test module.</param>
public void BeginTestModule(TestModuleData testModule)
{
this.testItemStack.Push(testModule);
var metadata = testModule.Metadata;
// <Module Name="System.Data.BVT" Desc=string.Empty Version="00.00.0000.00" Pri="0" Variations="2" Owner="nvalluri" Inheritance="True" Implemented="True" Skipped="False" Error="False" Manual="False" Security="0" Stress="False" Timeout="0" Threads="1" Repeat="0" >
this.writer.WriteStartElement("Module");
this.writer.WriteAttributeString("Name", metadata.Name);
this.writer.WriteAttributeString("Desc", metadata.Description);
this.writer.WriteAttributeString("Version", metadata.Version);
this.writer.WriteAttributeString("Pri", XmlConvert.ToString(metadata.Priority));
this.writer.WriteAttributeString("Owner", metadata.Owner);
this.writer.WriteString("\r\n");
this.writer.Flush();
}
/// <summary>
/// Runs the specified module.
/// </summary>
/// <param name="testModuleData">Data about the test module to run.</param>
/// <param name="logWriter">The log writer.</param>
/// <param name="variationsToRun">The variations to run.</param>
/// <param name="parameters">The parameters.</param>
public void Run(TestModuleData testModuleData, ITestLogWriter logWriter, IEnumerable <TestItemData> variationsToRun, IDictionary <string, string> parameters)
{
ExceptionUtilities.CheckArgumentNotNull(testModuleData, "testModuleData");
ExceptionUtilities.CheckArgumentNotNull(logWriter, "logWriter");
if (parameters == null)
{
parameters = new Dictionary <string, string>();
}
var testModule = (TestModule)Activator.CreateInstance(testModuleData.TestItemType);
testModule.ExplorationSeed = testModuleData.ExplorationSeed;
testModule.ExplorationKind = testModuleData.ExplorationKind;
foreach (var param in parameters)
{
testModule.TestParameters[param.Key] = param.Value;
}
if (variationsToRun == null)
{
variationsToRun = testModuleData.GetAllChildrenRecursive().Where(c => c.IsVariation);
}
lock (this.lockObject)
{
var logger = new TestLogWriterLogger(logWriter);
logger.OnWarning += this.OnWarning;
testModule.Log = logger;
this.executionThreadAborted = false;
this.runCompleted = false;
this.currentLogWriter = logWriter;
this.scheduledVariations = variationsToRun.Distinct().ToDictionary <TestItemData, TestItemData>(iteration => iteration);
this.completedItems.Clear();
this.statisticsByPriority.Clear();
this.globalStatistics = new RunStatistics();
this.currentLogWriter.BeginTestSuite();
this.executionThread = new Thread(this.ExecutionThreadStart)
{
IsBackground = true,
};
this.executionThread.Start();
this.BeginExecution(testModule, testModuleData);
}
}
/// <summary>
/// Gets the included variations for the given module
/// </summary>
/// <param name="module">The test module data</param>
/// <param name="includedPaths">The included paths</param>
/// <param name="excludedPaths">The excluded paths</param>
/// <param name="logWriter">optional log writer in case something goes wrong</param>
/// <returns>The included test items without the excluded items</returns>
internal static IEnumerable <TestItemData> GetIncludedVariations(TestModuleData module, IEnumerable <string> includedPaths, IEnumerable <string> excludedPaths, ITestLogWriter logWriter)
{
ExceptionUtilities.CheckArgumentNotNull(module, "module");
ExceptionUtilities.CheckArgumentNotNull(includedPaths, "includedPaths");
ExceptionUtilities.CheckArgumentNotNull(excludedPaths, "excludedPaths");
var allPossibleItems = module.GetAllChildrenRecursive().Where(i => i.IsVariation).ToList();
var includedItems = new List <TestItemData>();
bool anyIncludedPath = false;
foreach (var includedPath in includedPaths)
{
anyIncludedPath = true;
var toAdd = FilterItemsByPath(allPossibleItems, includedPath).ToList();
if (toAdd.Count > 0)
{
includedItems.AddRange(toAdd);
// TODO: should we remove them?
// It seems like it would make subsequent lookups faster and avoid the possibility of double-including something
toAdd.ForEach(i => allPossibleItems.Remove(i));
}
else if (logWriter != null)
{
var safeString = includedPath.Replace('/', '\\');
logWriter.WriteLine(LogLevel.Warning, string.Format(CultureInfo.InvariantCulture, "No test items found for path '{0}' in module '{1}'", safeString, module.Metadata.Name));
}
}
// special case, if no includes are specified, just take everything
if (!anyIncludedPath)
{
includedItems = allPossibleItems;
}
foreach (var excludedPath in excludedPaths)
{
var toRemove = FilterItemsByPath(includedItems, excludedPath).ToList();
toRemove.ForEach(i => includedItems.Remove(i));
}
return(includedItems);
}
/// <summary>
/// Finds the test module to execute in the loaded assemblies
/// </summary>
/// <param name="loadedAssemblies">The assemblies loaded in the current application</param>
/// <returns>The test module to execute</returns>
protected TestModuleData FindTestModule(IEnumerable <Assembly> loadedAssemblies)
{
ExceptionUtilities.CheckArgumentNotNull(loadedAssemblies, "loadedAssemblies");
ExceptionUtilities.Assert(loadedAssemblies.Any(), "No loaded assemblies to run tests from");
TestModuleData testModuleData = null;
ITestModuleLoader testModuleLoader = this.loaders.FirstOrDefault(testLoader => testLoader.CanLoad(loadedAssemblies));
ExceptionUtilities.CheckObjectNotNull(testModuleLoader, "could not find a suitable test loader");
int explorationSeed = this.GetExplorationSeed();
var explorationKind = this.GetExplorationKind();
this.WriteStatusText(string.Format(CultureInfo.InvariantCulture, "{0}={1}", ExplorationSeedParameterName, explorationSeed));
if (explorationKind.HasValue)
{
this.WriteStatusText(string.Format(CultureInfo.InvariantCulture, "{0}={1}", ExplorationKindParameterName, explorationKind));
}
testModuleData = testModuleLoader.Load(loadedAssemblies, explorationSeed, explorationKind);
return(testModuleData);
}
/// <summary>
/// Ends the test module.
/// </summary>
/// <param name="testModule">The test module.</param>
/// <param name="result">The result.</param>
/// <param name="exception">The exception.</param>
/// <param name="statisticsByPriority">The statistics for the module, broken down by priority.</param>
/// <param name="globalStatistics">The global statistics for the module.</param>
public void EndTestModule(TestModuleData testModule, TestResult result, ExceptionDetails exception, IDictionary <int, RunStatistics> statisticsByPriority, RunStatistics globalStatistics)
{
this.ForAll(i => i.EndTestModule(testModule, result, exception, statisticsByPriority, globalStatistics));
}
/// <summary>
/// Begins the test module.
/// </summary>
/// <param name="testModule">The test module.</param>
public void BeginTestModule(TestModuleData testModule)
{
this.ForAll(i => i.BeginTestModule(testModule));
}
/// <summary>
/// Begins the test module.
/// </summary>
/// <param name="testModule">The test module.</param>
public void BeginTestModule(TestModuleData testModule)
{
this.WriteBegin(testModule);
}
/// <summary>
/// Runs the specified module.
/// </summary>
/// <param name="testModuleData">Data about the test module to run.</param>
/// <param name="logWriter">The log writer.</param>
/// <param name="variationsToRun">The variations to run.</param>
/// <param name="parameters">The test parameters with which to invoke the test module.</param>
public void Run(TestModuleData testModuleData, ITestLogWriter logWriter, IEnumerable <TestItemData> variationsToRun, IDictionary <string, string> parameters)
{
this.Runner.Run(testModuleData, logWriter, variationsToRun, parameters);
}
private void EndTestItem()
{
var testItemData = this.currentContext.TestItemData;
var result = this.currentContext.CurrentResult;
var exceptionDetails = (this.currentContext.Exception == null) ? (ExceptionDetails)null : new ExceptionDetails(this.currentContext.Exception);
this.completedItems[testItemData] = result;
if (result == TestResult.Failed || result == TestResult.Timeout)
{
if (exceptionDetails != null)
{
foreach (var bug in testItemData.Bugs)
{
this.currentLogWriter.WriteLine(LogLevel.Verbose, bug.ToString());
}
}
}
bool updatedCounters = false;
if (testItemData.IsVariation)
{
this.IncrementResultCounter(testItemData, result);
updatedCounters = true;
this.currentLogWriter.EndVariation(testItemData, result, exceptionDetails);
}
else if (result == TestResult.Failed || result == TestResult.Skipped || result == TestResult.Timeout)
{
// If this is a failure/skip/timeout for a test item that's not a variation,
// mark all its variations that were scheduled to run but have
// not already run as that result.
foreach (var tid in testItemData.GetAllChildrenRecursive().Where(
t => t.IsVariation && !this.completedItems.ContainsKey(t) && this.scheduledVariations.ContainsKey(t)))
{
this.IncrementResultCounter(tid, result);
updatedCounters = true;
}
}
if (updatedCounters)
{
this.OnRunStatisticsUpdated(new RunStatisticsUpdatedEventArgs(this.globalStatistics));
}
if (testItemData.IsTestCase)
{
this.currentLogWriter.EndTestCase(testItemData, result, exceptionDetails);
}
TestModuleData testModuleData = testItemData as TestModuleData;
if (testModuleData != null)
{
this.currentLogWriter.EndTestModule(testModuleData, result, exceptionDetails, this.statisticsByPriority, this.globalStatistics);
}
this.ReportItemCompleted(result);
this.GoToNextParentItem();
}
/// <summary>
/// Ends the test module.
/// </summary>
/// <param name="testModule">The test module.</param>
/// <param name="result">The result.</param>
/// <param name="exception">The exception.</param>
/// <param name="statisticsByPriority">The statistics for the module, broken down by priority.</param>
/// <param name="globalStatistics">The global statistics for the module.</param>
public void EndTestModule(TestModuleData testModule, TestResult result, ExceptionDetails exception, IDictionary <int, RunStatistics> statisticsByPriority, RunStatistics globalStatistics)
{
}
/// <summary>
/// Begins the test module.
/// </summary>
/// <param name="testModule">The test module.</param>
public void BeginTestModule(TestModuleData testModule)
{
}
/// <summary>
/// Ends the test module.
/// </summary>
/// <param name="testModule">The test module.</param>
/// <param name="result">The result.</param>
/// <param name="exception">The exception.</param>
/// <param name="statisticsByPriority">The statistics for the module, broken down by priority.</param>
/// <param name="globalStatistics">The global statistics for the module.</param>
public void EndTestModule(TestModuleData testModule, TestResult result, ExceptionDetails exception, IDictionary <int, RunStatistics> statisticsByPriority, RunStatistics globalStatistics)
{
this.WriteTestItemDataDurationResult(testModule);
}
/// <summary>
/// Begins the test module.
/// </summary>
/// <param name="testModule">The test module.</param>
public void BeginTestModule(TestModuleData testModule)
{
this.testItemTimeStack.Push(new KeyValuePair <TestItemData, DateTimeOffset>(testModule, this.GetCurrentTime()));
this.testItemRunningTotalLookup.Add(testModule, 0);
}
