public static void SubmitResult(Uri testVaultServer, string session, string project, string buildname, string testgroup, string testname, TestOutcome outcome, bool personal)
{
try
{
using ( var client = new WebClient() ){
client.BaseAddress = testVaultServer.ToString();
client.CachePolicy = new System.Net.Cache.RequestCachePolicy( System.Net.Cache.RequestCacheLevel.NoCacheNoStore );
client.Headers.Add("Content-Type", "text/xml");
var result = new TestResult()
{
Group = new TestGroup() { Name = testgroup, Project = new TestProject() { Project = project } },
Name = testname,
Outcome = outcome,
TestSession = session,
IsPersonal = personal,
BuildID = buildname,
};
var xc = new XmlSerializer(result.GetType());
var io = new System.IO.MemoryStream();
xc.Serialize( io, result );
client.UploadData( testVaultServer.ToString(), io.ToArray() );
}
} catch ( Exception e )
{
Console.Error.WriteLine( e );
}
}
private TestStepRun CreateFakeRun(string id, TestOutcome outcome, params TestStepRun[] children)
{
var run = new TestStepRun(new TestStepData(id, id, id, id));
run.Result = new TestResult(outcome);
run.Children.AddRange(children);
return run;
}
/// <summary>
/// Creates a result record.
/// </summary>
/// <param name="method">Test method metadata object.</param>
/// <param name="testClass">Test class metadata object.</param>
/// <param name="result">Test result object.</param>
/// <param name="exception">Exception instance, if any.</param>
public ScenarioResult(ITestMethod method, ITestClass testClass, TestOutcome result, Exception exception)
{
TestClass = testClass;
TestMethod = method;
Exception = exception;
Result = result;
}
public TestResult ToTestResult(string qualifiedTestCaseName, TestOutcome outcome, int duration, string executable = DummyExecutable)
{
return new TestResult(ToTestCase(qualifiedTestCaseName, executable))
{
Outcome = outcome,
Duration = TimeSpan.FromMilliseconds(duration)
};
}
public int TestOutcomeTotalsAreCorrect(TestOutcome outcome)
{
return testLog.Events
.FindAll(e => e.EventType == FakeFrameworkHandle.EventType.RecordResult)
.ConvertAll(e => e.TestResult)
.FindAll(r => r.Outcome == outcome)
.Count;
}
public static TestInfo FromRelativePaths(string className, string methodName, string projectFilePath, string sourceCodeFilePath, int sourceCodeLineNumber, TestOutcome outcome, string classFilePath = null) {
return FromAbsolutePaths(className,
methodName,
TestData.GetPath(projectFilePath),
TestData.GetPath(sourceCodeFilePath),
sourceCodeLineNumber,
outcome,
classFilePath != null ? TestData.GetPath(classFilePath) : null);
}
void ITestExecutionRecorder.RecordEnd(TestCase testCase, TestOutcome outcome)
{
Events.Add(new Event()
{
EventType = EventType.RecordEnd,
TestCase = testCase,
TestOutcome = outcome
});
}
public static TestInfo FromAbsolutePaths(string className, string methodName, string projectFilePath, string sourceCodeFilePath, int sourceCodeLineNumber, TestOutcome outcome, string classFilePath = null) {
TestInfo ti = new TestInfo();
ti.ClassName = className;
ti.MethodName = methodName;
ti.ProjectFilePath = projectFilePath;
ti.SourceCodeFilePath = sourceCodeFilePath;
ti.SourceCodeLineNumber = sourceCodeLineNumber;
ti.Outcome = outcome;
ti.ClassFilePath = classFilePath ?? sourceCodeFilePath;
ti.RelativeClassFilePath = CommonUtils.GetRelativeFilePath(Path.GetDirectoryName(ti.ProjectFilePath), ti.ClassFilePath);
return ti;
}
public GallioRunListener(ISpecificationRunListener listener, IProgressMonitor progressMonitor,
ITestContext context, IEnumerable<ITestCommand> specificationCommands)
{
_listener = listener;
_progressMonitor = progressMonitor;
_testContext = context;
_outcome = TestOutcome.Passed;
_contextsBySpec = new Dictionary<string, ITestContext>();
_commandsBySpec = (from command in specificationCommands
let specificationTest = command.Test as MachineSpecificationTest
where specificationTest != null
select new { Name = specificationTest.Specification.Name, Command = command })
.ToDictionary(x => x.Name, x => x.Command);
}
public static async Task FinishTestAsync(string testName, TestOutcome outcome, string message = null, string filename = null, string stdOut = null, bool printFailuresOnly = false)
{
var endTime = DateTimeOffset.Now.Ticks;
TimeSpan duration;
long startTime;
if (TestStartTimes.TryGetValue(testName, out startTime))
{
duration = new TimeSpan(endTime - startTime);
TestStartTimes.Remove(testName);
}
else
{
duration = TimeSpan.Zero;
}
if (null == filename)
{
TestStartFilename.TryGetValue(testName, out filename);
TestStartFilename.Remove(testName);
}
if (!printFailuresOnly || outcome != TestOutcome.Passed)
{
FancyConsole.Write("Test {0} complete.", testName);
switch (outcome)
{
case TestOutcome.Failed:
FancyConsole.Write(ConsoleColor.Red, " Failed: {0}", message);
break;
case TestOutcome.Passed:
FancyConsole.Write(ConsoleColor.Green, " Passed: {0}", message);
break;
default:
FancyConsole.Write(" {0}: {1}", outcome, message);
break;
}
FancyConsole.WriteLine(" duration: {0}", duration);
}
await BuildWorkerApi.RecordTestAsync(testName, TestFrameworkName, outcome: outcome, durationInMilliseconds: (long)duration.TotalMilliseconds, errorMessage: message, filename: filename, stdOut: stdOut);
}
/// <summary>
/// Initializes a new instance of the <see cref="MSTestAllureAdapter.MSTestResult"/> class.
/// </summary>
/// <param name="name">Name.</param>
/// <param name="outcome">Outcome.</param>
/// <param name="start">Start time.</param>
/// <param name="end">End time.</param>
/// <param name="suits">List of test suits this test belongs to.</param>
/// <param name="innerResults">List of inner test results this test might have.</param>
public MSTestResult(string name, TestOutcome outcome, DateTime start, DateTime end, IEnumerable<string> suits, IEnumerable<MSTestResult> innerResults)
{
Name = name;
mSuits = suits ?? Enumerable.Empty<string>();
Outcome = outcome;
Start = start;
End = end;
List<MSTestResult> results = new List<MSTestResult>();
if (innerResults != null)
{
foreach (MSTestResult result in innerResults)
{
results.Add(result.Clone());
}
}
InnerTests = results;
}
/// <summary>
/// Increment the number of passing results.
/// </summary>
/// <param name="outcome">The test outcome.</param>
public void IncrementResults(TestOutcome outcome)
{
if (_total == 0)
{
RunOutcome = outcome;
}
++_total;
Counters.SetAttributeValue("total", _total.ToString(CultureInfo.InvariantCulture));
if (!_outcomes.ContainsKey(outcome))
{
_outcomes[outcome] = 0;
}
++_outcomes[outcome];
string value = outcome.ToString();
string outcomeName = value.Substring(0, 1).ToLower(CultureInfo.InvariantCulture) + value.Substring(1);
Counters.SetAttributeValue(outcomeName, _outcomes[outcome].ToString(CultureInfo.InvariantCulture));
}
private int GetCount(TestOutcome outcome)
{
return(summary.ContainsKey(outcome)
? summary[outcome]
: 0);
}
public TestResult(TestOutcome outcome, string output = "", long duration = 0)
{
Outcome = outcome;
Output = output;
Duration = duration;
}
public void RecordEnd(TestCase testCase, TestOutcome outcome)
{
}
public void PostTestCleanup(TestOutcome testOutcome, MethodInfo memberInfo)
{
_testExecutionPlugins.ForEach(p => p.PostTestCleanup(testOutcome, memberInfo));
}
private void HostProcessErrorHandler(string errorMessage, TestOutcome outcome, bool abortTestRun)
{
Debug.Assert(m_runContext != null);
SendResult(errorMessage, outcome, abortTestRun);
}
/// <summary>
/// Creates a silent test exception with the specified outcome.
/// </summary>
/// <param name="outcome">The test outcome.</param>
public TestTerminatedException(TestOutcome outcome)
: this(outcome, null, null)
{
}
/// <summary>
/// Finishes the step represented by the context.
/// </summary>
/// <param name="outcome">The outcome.</param>
/// <returns>The final test result.</returns>
internal TestResult FinishStep(TestOutcome outcome)
{
return(inner.FinishStep(outcome, null));
}
/// <summary>
/// Sets the step's interim <see cref="Outcome" />.
/// </summary>
/// <remarks>
/// <para>
/// The interim outcome is used to communicate the anticipated outcome
/// of the step to later phases of execution.
/// </para>
/// <para>
/// The value set here will be overridden by whatever final outcome the step
/// returns. Consequently the actual outcome may still differ from the anticipated outcome
/// that was set using this method.
/// </para>
/// </remarks>
/// <exception cref="InvalidOperationException">Thrown if attempting to set the outcome while the test is not running.</exception>
/// <seealso cref="Outcome"/>
public void SetInterimOutcome(TestOutcome outcome)
{
inner.SetInterimOutcome(outcome);
}
/// <summary>
/// Performs an action as a new step within the current context and associates it
/// with the specified code reference. Verifies that the step produced the expected outcome.
/// </summary>
/// <remarks>
/// <para>
/// This method creates a new child context with a new nested <see cref="Model.Tree.TestStep" />,
/// enters the child context, performs the action, then exits the child context.
/// </para>
/// <para>
/// This method may be called recursively to create nested steps or concurrently
/// to create parallel steps.
/// </para>
/// <para>
/// This method verifies that the step produced the expected outcome. If a different outcome
/// was obtained, then raises an assertion failure.
/// </para>
/// </remarks>
/// <param name="name">The name of the step.</param>
/// <param name="action">The action to perform.</param>
/// <param name="timeout">The step execution timeout, or null if none.</param>
/// <param name="isTestCase">True if the step represents an independent test case.</param>
/// <param name="codeElement">The associated code element, or null if none.</param>
/// <param name="expectedOutcome">The expected outcome of the step.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="name"/> or
/// <paramref name="action"/> is null.</exception>
/// <returns>The context of the step that ran.</returns>
/// <exception cref="ArgumentException">Thrown if <paramref name="name"/> is the empty string.</exception>
/// <exception cref="ArgumentOutOfRangeException">Thrown if <paramref name="timeout"/> is negative.</exception>
/// <exception cref="AssertionFailureException">Thrown if the expected outcome was not obtained.</exception>
public TestContext RunStepAndVerifyOutcome(string name, GallioAction action, TimeSpan?timeout, bool isTestCase, ICodeElementInfo codeElement, TestOutcome expectedOutcome)
{
TestContext childContext = RunStep(name, action, timeout, isTestCase, codeElement);
AssertionHelper.Verify(() =>
{
TestOutcome actualOutcome = childContext.Outcome;
if (actualOutcome == expectedOutcome)
{
return(null);
}
return(new AssertionFailureBuilder("The test step did not produce the expected outcome.")
.AddLabeledValue("Expected Outcome", expectedOutcome.ToString())
.AddLabeledValue("Actual Outcome", actualOutcome.ToString())
.ToAssertionFailure());
});
return(childContext);
}
VsTestResult MakeVsTestResult(TestOutcome outcome, ITestResultMessage testResult)
{
return(MakeVsTestResult(outcome, testResult.TestCase, testResult.Test.DisplayName, (double)testResult.ExecutionTime, testResult.Output));
}
public void RunByOutcome(TestOutcome outcome)
{
Run(_knownOutcomes.Where(test => test.Value == outcome).Select(test => test.Key));
}
public void Ended(TestCaseToRun testCase, TestOutcome outcome)
{
Console.WriteLine("Ended");
}
public void RecordEnd(TestCase testCase, TestOutcome outcome) => throw new NotImplementedException();
public TestExecutionEventArgs(TestOutcome testOutcome, string testName, MemberInfo memberInfo)
{
this.testOutcome = testOutcome;
this.testName = testName;
this.memberInfo = memberInfo;
}
/// <summary>
/// Notify the framework about completion of the test case.
/// Framework sends this event to data collectors enabled in the run. If no data collector is enabled, then the event is ignored.
/// </summary>
/// <param name="testCase">test case which has completed.</param>
/// <param name="outcome">outcome of the test case.</param>
public void RecordEnd(TestCase testCase, TestOutcome outcome)
{
EqtTrace.Verbose("TestExecutionRecorder.RecordEnd: test: {0} execution completed.", testCase?.FullyQualifiedName);
this.testRunCache.OnTestCompletion(testCase);
this.SendTestCaseEnd(testCase, outcome);
}
/// <summary>
/// Called when a test run is completed.
/// </summary>
private void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e)
{
// Stop the progress indicator as we are about to print the summary
this.progressIndicator?.Stop();
var passedTests = 0;
var failedTests = 0;
var skippedTests = 0;
var totalTests = 0;
Output.WriteLine(string.Empty, OutputLevel.Information);
// Printing Run-level Attachments
var runLevelAttachementCount = (e.AttachmentSets == null) ? 0 : e.AttachmentSets.Sum(attachmentSet => attachmentSet.Attachments.Count);
if (runLevelAttachementCount > 0)
{
Output.Information(false, CommandLineResources.AttachmentsBanner);
foreach (var attachmentSet in e.AttachmentSets)
{
foreach (var uriDataAttachment in attachmentSet.Attachments)
{
var attachmentOutput = string.Format(CultureInfo.CurrentCulture, CommandLineResources.AttachmentOutputFormat, uriDataAttachment.Uri.LocalPath);
Output.Information(false, attachmentOutput);
}
}
}
var leafTestResultsPerSource = this.leafTestResults.Select(p => p.Value).GroupBy(r => r.TestCase.Source);
foreach (var sd in leafTestResultsPerSource)
{
var source = sd.Key;
var sourceSummary = new SourceSummary();
var results = sd.ToArray();
// duration of the whole source is the difference between the test that ended last and the one that started first
sourceSummary.Duration = !results.Any() ? TimeSpan.Zero : results.Max(r => r.EndTime) - results.Min(r => r.StartTime);
foreach (var result in results)
{
switch (result.Outcome)
{
case TestOutcome.Passed:
sourceSummary.TotalTests++;
sourceSummary.PassedTests++;
break;
case TestOutcome.Failed:
sourceSummary.TotalTests++;
sourceSummary.FailedTests++;
break;
case TestOutcome.Skipped:
sourceSummary.TotalTests++;
sourceSummary.SkippedTests++;
break;
default:
break;
}
}
if (verbosityLevel == Verbosity.Quiet || verbosityLevel == Verbosity.Minimal)
{
TestOutcome sourceOutcome = TestOutcome.None;
if (sourceSummary.FailedTests > 0)
{
sourceOutcome = TestOutcome.Failed;
}
else if (sourceSummary.PassedTests > 0)
{
sourceOutcome = TestOutcome.Passed;
}
else if (sourceSummary.SkippedTests > 0)
{
sourceOutcome = TestOutcome.Skipped;
}
string resultString;
switch (sourceOutcome)
{
case TestOutcome.Failed:
resultString = (CommandLineResources.FailedTestIndicator + "!").PadRight(LongestResultIndicator);
break;
case TestOutcome.Passed:
resultString = (CommandLineResources.PassedTestIndicator + "!").PadRight(LongestResultIndicator);
break;
case TestOutcome.Skipped:
resultString = (CommandLineResources.SkippedTestIndicator + "!").PadRight(LongestResultIndicator);
break;
default:
resultString = CommandLineResources.None.PadRight(LongestResultIndicator);
break;
}
;
var failed = sourceSummary.FailedTests.ToString().PadLeft(5);
var passed = sourceSummary.PassedTests.ToString().PadLeft(5);
var skipped = sourceSummary.SkippedTests.ToString().PadLeft(5);
var total = sourceSummary.TotalTests.ToString().PadLeft(5);
var frameworkString = string.IsNullOrEmpty(targetFramework)
? string.Empty
: $"({targetFramework})";
var duration = GetFormattedDurationString(sourceSummary.Duration);
var sourceName = sd.Key.Split('\\').Last();
var outputLine = string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummary,
resultString,
failed,
passed,
skipped,
total,
duration,
sourceName,
frameworkString);
ConsoleColor?color = null;
if (sourceOutcome == TestOutcome.Failed)
{
color = ConsoleColor.Red;
}
else if (sourceOutcome == TestOutcome.Passed)
{
color = ConsoleColor.Green;
}
else if (sourceOutcome == TestOutcome.Skipped)
{
color = ConsoleColor.Yellow;
}
if (color != null)
{
Output.Write(outputLine, OutputLevel.Information, color.Value);
}
else
{
Output.Write(outputLine, OutputLevel.Information);
}
Output.Information(false, CommandLineResources.TestRunSummaryAssemblyAndFramework,
sourceName,
frameworkString);
}
passedTests += sourceSummary.PassedTests;
failedTests += sourceSummary.FailedTests;
skippedTests += sourceSummary.SkippedTests;
totalTests += sourceSummary.TotalTests;
}
if (verbosityLevel == Verbosity.Quiet || verbosityLevel == Verbosity.Minimal)
{
if (e.IsCanceled)
{
Output.Error(false, CommandLineResources.TestRunCanceled);
}
else if (e.IsAborted)
{
Output.Error(false, CommandLineResources.TestRunAborted);
}
return;
}
if (e.IsCanceled)
{
Output.Error(false, CommandLineResources.TestRunCanceled);
}
else if (e.IsAborted)
{
Output.Error(false, CommandLineResources.TestRunAborted);
}
else if (failedTests > 0 || this.testRunHasErrorMessages)
{
Output.Error(false, CommandLineResources.TestRunFailed);
}
else if (totalTests > 0)
{
Output.Information(false, ConsoleColor.Green, CommandLineResources.TestRunSuccessful);
}
// Output a summary.
if (totalTests > 0)
{
string totalTestsformat = (e.IsAborted || e.IsCanceled) ? CommandLineResources.TestRunSummaryForCanceledOrAbortedRun : CommandLineResources.TestRunSummaryTotalTests;
Output.Information(false, string.Format(CultureInfo.CurrentCulture, totalTestsformat, totalTests));
if (passedTests > 0)
{
Output.Information(false, ConsoleColor.Green, string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryPassedTests, passedTests));
}
if (failedTests > 0)
{
Output.Information(false, ConsoleColor.Red, string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryFailedTests, failedTests));
}
if (skippedTests > 0)
{
Output.Information(false, ConsoleColor.Yellow, string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummarySkippedTests, skippedTests));
}
}
if (totalTests > 0)
{
if (e.ElapsedTimeInRunningTests.Equals(TimeSpan.Zero))
{
EqtTrace.Info("Skipped printing test execution time on console because it looks like the test run had faced some errors");
}
else
{
PrintTimeSpan(e.ElapsedTimeInRunningTests);
}
}
}
/// <summary>
/// Creates an exception.
/// </summary>
/// <param name="outcome">The test outcome.</param>
/// <param name="message">The message, or null if none.</param>
/// <param name="innerException">The inner exception, or null if none.</param>
public TestTerminatedException(TestOutcome outcome, string message, Exception innerException)
: base(message, innerException)
{
this.outcome = outcome;
}
private bool CheckSingleResultHasOutcome(IEnumerable <TestResult> testResults, TestOutcome outcome)
{
return(testResults.SingleOrDefault()?.Outcome == outcome);
}
public static TestInfo FromAbsolutePaths(string className, string methodName, string projectFilePath, string sourceCodeFilePath, int sourceCodeLineNumber, TestOutcome outcome, string classFilePath = null, TimeSpan?minDuration = null, string containedErrorMessage = null)
{
TestInfo ti = new TestInfo();
ti.ClassName = className;
ti.MethodName = methodName;
ti.ProjectFilePath = projectFilePath;
ti.SourceCodeFilePath = sourceCodeFilePath;
ti.SourceCodeLineNumber = sourceCodeLineNumber;
ti.Outcome = outcome;
ti.ClassFilePath = classFilePath ?? sourceCodeFilePath;
ti.RelativeClassFilePath = CommonUtils.GetRelativeFilePath(Path.GetDirectoryName(ti.ProjectFilePath), ti.ClassFilePath);
ti.MinDuration = minDuration ?? TimeSpan.Zero;
ti.ContainedErrorMessage = containedErrorMessage;
return(ti);
}
public void PostTestCleanup(TestOutcome testOutcome, string testName, MemberInfo testMethodMemberInfo, Type testClassType, List <string> categories, List <string> authors, List <string> descriptions, string message, string stackTrace, Exception exception) => RaiseTestEvent(PostTestCleanupEvent, testOutcome, testName, testMethodMemberInfo, testClassType, categories, authors, descriptions, message, stackTrace, exception);
public void RecordEnd(TestCase testCase, TestOutcome outcome)
{
_monitor.RecordEnd(testCase.Convert(), outcome.Convert());
}
private void RaiseClassTestEvent(EventHandler <PluginEventArgs> eventHandler, TestOutcome testOutcome, Type testClassType)
{
var args = new PluginEventArgs(testOutcome, testClassType);
eventHandler?.Invoke(this, args);
}
public void RecordEnd(TestCase testCase, TestOutcome outcome) => NotImplemented();
private void SendResult(string messageText, TestOutcome outcome)
{
SendResult(messageText, outcome, false);
}
public static MSVST4U_UnitTestResult New_MSVST4U_UnitTestResult_DataDriven(Guid runId, ITestElement test, TestOutcome outcome, TestResultCounter counters, MSVSTTC_TestResult[] innerResults)
{
return MSVST4U_Tunnels.CreateAggregateDataTestResult(runId, test, outcome, counters, innerResults);
}
/// <summary>
/// Performs an action as a new step within the current context and associates it
/// with the specified code reference. Verifies that the step produced the expected outcome.
/// </summary>
/// <remarks>
/// <para>
/// This method creates a new child context with a new nested <see cref="Model.Tree.TestStep" />,
/// enters the child context, performs the action, then exits the child context.
/// </para>
/// <para>
/// This method may be called recursively to create nested steps or concurrently
/// to create parallel steps.
/// </para>
/// <para>
/// This method verifies that the step produced the expected outcome. If a different outcome
/// was obtained, then raises an assertion failure.
/// </para>
/// </remarks>
/// <param name="name">The name of the step.</param>
/// <param name="action">The action to perform.</param>
/// <param name="timeout">The step execution timeout, or null if none.</param>
/// <param name="isTestCase">True if the step represents an independent test case.</param>
/// <param name="codeElement">The associated code element, or null if none.</param>
/// <param name="expectedOutcome">The expected outcome of the step.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="name"/> or
/// <paramref name="action"/> is null.</exception>
/// <returns>The context of the step that ran.</returns>
/// <exception cref="ArgumentException">Thrown if <paramref name="name"/> is the empty string.</exception>
/// <exception cref="ArgumentOutOfRangeException">Thrown if <paramref name="timeout"/> is negative.</exception>
/// <exception cref="AssertionFailureException">Thrown if the expected outcome was not obtained.</exception>
public TestContext RunStepAndVerifyOutcome(string name, Action action, TimeSpan? timeout, bool isTestCase, ICodeElementInfo codeElement, TestOutcome expectedOutcome)
{
TestContext childContext = RunStep(name, action, timeout, isTestCase, codeElement);
AssertionHelper.Verify(() =>
{
TestOutcome actualOutcome = childContext.Outcome;
if (actualOutcome == expectedOutcome)
return null;
return new AssertionFailureBuilder("The test step did not produce the expected outcome.")
.AddLabeledValue("Expected Outcome", expectedOutcome.ToString())
.AddLabeledValue("Actual Outcome", actualOutcome.ToString())
.ToAssertionFailure();
});
return childContext;
}
public void TestFinished_OutcomesAreCorrectlyTranslated(ResultState resultState, TestOutcome outcome, string message)
{
fakeNUnitResult.SetResult(resultState, message, null);
listener.TestFinished(fakeNUnitResult);
Assume.That(testLog.Events.Count, Is.EqualTo(2));
Assume.That(
testLog.Events[0].EventType,
Is.EqualTo(FakeFrameworkHandle.EventType.RecordEnd));
Assume.That(
testLog.Events[1].EventType,
Is.EqualTo(FakeFrameworkHandle.EventType.RecordResult));
Assert.AreEqual(outcome, testLog.Events[0].TestOutcome);
Assert.AreEqual(outcome, testLog.Events[1].TestResult.Outcome);
Assert.AreEqual(message, testLog.Events[1].TestResult.ErrorMessage);
}
/// <summary>
/// Finishes the step represented by the context.
/// </summary>
/// <param name="outcome">The outcome.</param>
/// <returns>The final test result.</returns>
internal TestResult FinishStep(TestOutcome outcome)
{
return inner.FinishStep(outcome, null);
}
/// <summary>
/// Adds the result of a test method into the log.
/// </summary>
/// <param name="test">The test metadata.</param>
/// <param name="storage">The storage value.</param>
/// <param name="codeBase">The code base value.</param>
/// <param name="adapterTypeName">The adapter type name.</param>
/// <param name="className">The class name.</param>
/// <param name="testListName">The test list name.</param>
/// <param name="computerName">The computer name.</param>
/// <param name="startTime">The start time.</param>
/// <param name="endTime">The end time.</param>
/// <param name="outcome">The outcome.</param>
public void AddTestMethodResult(
ITestMethod test,
string storage,
string codeBase,
string adapterTypeName,
string className,
string testListName,
string computerName,
DateTime startTime,
DateTime endTime,
TestOutcome outcome)
{
if (test == null)
{
throw new ArgumentNullException("test");
}
// Friendly name of the test
string name = test.Name;
// Generate GUIDs.
string testId = Guid.NewGuid().ToString();
string executionId = Guid.NewGuid().ToString();
string testListId = GetTestListGuid(testListName);
// UnitTest element.
SimpleXElement unitTest = CreateElement("UnitTest");
unitTest.SetAttributeValue("name", name);
unitTest.SetAttributeValue("storage", storage);
unitTest.SetAttributeValue("id", testId);
SimpleXElement owners = CreateElement("Owners");
SimpleXElement owner = CreateElement("Owner");
string ownerString = test.Owner ?? string.Empty;
owner.SetAttributeValue("name", ownerString);
owners.Add(owner);
unitTest.Add(owners);
if (!string.IsNullOrEmpty(test.Description))
{
SimpleXElement description = CreateElement("Description");
description.SetValue(test.Description);
unitTest.Add(description);
}
SimpleXElement execution = CreateElement("Execution");
execution.SetAttributeValue("id", executionId);
unitTest.Add(execution);
// TestMethod element.
SimpleXElement testMethod = CreateElement("TestMethod");
testMethod.SetAttributeValue("codeBase", codeBase);
testMethod.SetAttributeValue("adapterTypeName", adapterTypeName);
testMethod.SetAttributeValue("className", className);
testMethod.SetAttributeValue("name", name);
unitTest.Add(testMethod);
TestDefinitions.Add(unitTest);
// TestEntry element.
SimpleXElement testEntry = CreateElement("TestEntry");
testEntry.SetAttributeValue("testId", testId);
testEntry.SetAttributeValue("executionId", executionId);
testEntry.SetAttributeValue("testListId", testListId);
TestEntries.Add(testEntry);
// UnitTestResult element.
SimpleXElement unitTestResult = CreateElement("UnitTestResult");
unitTestResult.SetAttributeValue("executionId", executionId);
unitTestResult.SetAttributeValue("testId", testId);
unitTestResult.SetAttributeValue("testName", name);
unitTestResult.SetAttributeValue("computerName", computerName);
TimeSpan duration = endTime.Subtract(startTime);
unitTestResult.SetAttributeValue("duration", duration.ToString());
unitTestResult.SetAttributeValue("startTime", ToDateString(startTime));
unitTestResult.SetAttributeValue("endTime", ToDateString(endTime));
unitTestResult.SetAttributeValue("testType", UnitTestTestTypeId);
unitTestResult.SetAttributeValue("outcome", outcome.ToString());
unitTestResult.SetAttributeValue("testListId", testListId.ToString());
// Add any pending items
foreach (SimpleXElement pending in _pendingElements)
{
unitTestResult.Add(pending);
}
_pendingElements.Clear();
Results.Add(unitTestResult);
}
public TestResult(string name, TestOutcome testOutcome, string message = null)
{
Name = name;
TestOutcome = testOutcome;
Message = message;
}
public override void OnSpecificationEnd(SpecificationInfo specification, Result result)
{
_listener.OnSpecificationEnd(specification, result);
ITestContext testContext = _contextsBySpec[specification.Name];
testContext.LifecyclePhase = LifecyclePhases.Finishing;
TestOutcome outcome;
TimeSpan? span = null;
if (result.Status == Status.NotImplemented)
{
TestLog.Warnings.WriteLine("{0} ({1})", specification.Name, "NOT IMPLEMENTED");
TestLog.Warnings.Flush();
outcome = TestOutcome.Pending;
span = new TimeSpan(0);
}
else if (result.Status == Status.Ignored)
{
TestLog.Warnings.WriteLine("{0} ({1})", specification.Name, "IGNORED");
TestLog.Warnings.Flush();
outcome = TestOutcome.Ignored;
span = new TimeSpan(0);
}
else if (result.Passed)
{
outcome = TestOutcome.Passed;
}
else
{
TestLog.Failures.WriteException( Convert( result.Exception));
TestLog.Failures.Flush();
outcome = TestOutcome.Failed;
}
testContext.FinishStep(outcome, span);
_outcome = _outcome.CombineWith(outcome);
_progressMonitor.Worked(1);
}
/// <summary>
/// Decorate the log message object with a test outcome object.
/// </summary>
/// <param name="message">The log message object.</param>
/// <param name="outcome">Test outcome object.</param>
protected static void DecorateTestOutcome(LogMessage message, TestOutcome outcome)
{
Decorate(message, LogDecorator.TestOutcome, outcome);
}
/// <summary>
/// Sets the step's interim <see cref="Outcome" />.
/// </summary>
/// <remarks>
/// <para>
/// The interim outcome is used to communicate the anticipated outcome
/// of the step to later phases of execution.
/// </para>
/// <para>
/// The value set here will be overridden by whatever final outcome the step
/// returns. Consequently the actual outcome may still differ from the anticipated outcome
/// that was set using this method.
/// </para>
/// </remarks>
/// <exception cref="InvalidOperationException">Thrown if attempting to set the outcome while the test is not running.</exception>
/// <seealso cref="Outcome"/>
public void SetInterimOutcome(TestOutcome outcome)
{
inner.SetInterimOutcome(outcome);
}
/// <summary>
/// Terminates the test and reports a specific test outcome.
/// </summary>
/// <remarks>
/// <para>
/// The test is terminated with by throwing a <see cref="TestTerminatedException" />
/// with the specified outcome. If other code in the test case catches
/// this exception and does not rethrow it then the test might not terminate correctly.
/// </para>
/// </remarks>
/// <param name="outcome">The desired test outcome.</param>
/// <exception cref="TestTerminatedException">Thrown always.</exception>
public static void Terminate(TestOutcome outcome)
{
Terminate(outcome, null, null);
}
private void CloseSharedStep(TestOutcome testOutcome)
{
SetTestStepResultAndSaveToIteration(_currentSharedTestStepResult, testOutcome);
}
/// <summary>
/// Terminates the test silently and reports a specific test outcome.
/// </summary>
/// <remarks>
/// <para>
/// Unlike <see cref="Terminate(TestOutcome)" /> this method does not report the
/// stack trace. It also does not include a termination reason unless one is explicitly
/// specified by the caller.
/// </para>
/// <para>
/// The test is terminated with by throwing a <see cref="SilentTestException" />
/// with the specified outcome. If other code in the test case catches
/// this exception and does not rethrow it then the test might not terminate correctly.
/// </para>
/// </remarks>
/// <param name="outcome">The desired test outcome.</param>
/// <exception cref="SilentTestException">Thrown always.</exception>
public static void TerminateSilently(TestOutcome outcome)
{
TerminateSilently(outcome, null, null);
}
public void RecordEnd(TestCase testCase, TestOutcome outcome) {
}
private void SetTestStepResultAndSaveToIteration(ITestActionResult testResult, TestOutcome testOutcome)
{
SetTestStepResult(testResult, testOutcome);
_iteration.Actions.Add(testResult);
}
/// <summary>
/// Record a test outcome.
/// </summary>
/// <param name="message">The accompanying message.</param>
/// <param name="outcome">The outcome value.</param>
public void TestResult(string message, TestOutcome outcome)
{
LogMessage m = Create(LogMessageType.TestResult, message);
DecorateTestOutcome(m, outcome);
}
private void SetTestStepResult(ITestActionResult testResult, TestOutcome testOutcome)
{
testResult.Outcome = testOutcome;
testResult.DateCompleted = DateTime.Now;
testResult.Duration = testResult.DateCompleted.Subtract(testResult.DateStarted);
}
/// <summary>
/// Terminates the test and reports a specific test outcome.
/// </summary>
/// <remarks>
/// <para>
/// The test is terminated with by throwing a <see cref="TestTerminatedException" />
/// with the specified outcome. If other code in the test case catches
/// this exception and does not rethrow it then the test might not terminate correctly.
/// </para>
/// </remarks>
/// <param name="outcome">The desired test outcome.</param>
/// <param name="messageFormat">The custom message format string, or null if none.</param>
/// <param name="messageArgs">The custom message arguments, or null if none.</param>
/// <exception cref="TestTerminatedException">Thrown always.</exception>
public static void Terminate(TestOutcome outcome, string messageFormat, params object[] messageArgs)
{
throw new TestTerminatedException(outcome, messageFormat != null && messageArgs != null
? String.Format(messageFormat, messageArgs)
: messageFormat ?? "The test was terminated.");
}
public int TestOutcomeTotalsAreCorrect(TestOutcome outcome)
{
return(testLog.Events
.Count(e => e.EventType == FakeFrameworkHandle.EventType.RecordResult && e.TestResult.Outcome == outcome));
}
/// <summary>
/// Terminates the test silently and reports a specific test outcome.
/// </summary>
/// <remarks>
/// <para>
/// Unlike <see cref="Terminate(TestOutcome, string, object[])" /> this method does not report the
/// stack trace. It also does not include a termination reason unless one is explicitly
/// specified by the caller.
/// </para>
/// <para>
/// The test is terminated with by throwing a <see cref="TestTerminatedException" />
/// with the specified outcome. If other code in the test case catches
/// this exception and does not rethrow it then the test might not terminate correctly.
/// </para>
/// </remarks>
/// <param name="outcome">The desired test outcome.</param>
/// <param name="messageFormat">The custom message format string, or null if none.</param>
/// <param name="messageArgs">The custom message arguments, or null if none.</param>
/// <exception cref="SilentTestException">Thrown always.</exception>
public static void TerminateSilently(TestOutcome outcome, string messageFormat, params object[] messageArgs)
{
throw new SilentTestException(outcome, messageFormat != null && messageArgs != null
? String.Format(messageFormat, messageArgs)
: null);
}
public void UpdateResult(TestOutcome outcome, string message = "", long duration = 0, DateTime?startTime = null, DateTime?endTime = null)
{
Result.SetValues(outcome, message, duration, startTime, endTime);
OnPropertyChanged("Result");
}
/// <summary>
/// Creates a silent test exception with the specified outcome.
/// </summary>
/// <param name="outcome">The test outcome.</param>
/// <param name="message">The message, or null if none.</param>
public TestTerminatedException(TestOutcome outcome, string message)
: this(outcome, message, null)
{
}
public async Task RecordTestAsync(string testName, string testFramework = null, string filename = null, TestOutcome outcome = TestOutcome.None, long durationInMilliseconds = 0, string errorMessage = null, string errorStackTrace = null, string stdOut = null, string stdErr = null)
{
try
{
var body = new
{
testName = testName,
testFramework = testFramework,
fileName = filename,
outcome = outcome,
durationMilliseconds = durationInMilliseconds,
ErrorMessage = errorMessage,
ErrorStackTrace = errorStackTrace,
StdOut = stdOut,
StdErr = stdErr
};
await this.PostToApiAsync("api/tests", body);
}
catch
{
// ignored
}
}
/// <summary>
/// Creates a exception from serialization info.
/// </summary>
/// <param name="info">The serialization info.</param>
/// <param name="context">The streaming context.</param>
protected TestTerminatedException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
outcome = (TestOutcome) info.GetValue(OutcomeKey, typeof(TestOutcome));
}
public static TestInfo FromRelativePaths(string className, string methodName, string projectFilePath, string sourceCodeFilePath, int sourceCodeLineNumber, TestOutcome outcome, string classFilePath = null, TimeSpan?minDuration = null, string containedErrorMessage = null)
{
return(FromAbsolutePaths(className,
methodName,
TestData.GetPath(projectFilePath),
TestData.GetPath(sourceCodeFilePath),
sourceCodeLineNumber,
outcome,
classFilePath != null ? TestData.GetPath(classFilePath) : null,
minDuration,
containedErrorMessage));
}
