public void DeserializedTestCaseContainsSameDataAsOriginalTestCase()
{
var testCollection = new XunitTestCollection();
var type = typeof(ClassUnderTest);
var method = type.GetMethod("FactMethod");
var fact = CustomAttributeData.GetCustomAttributes(method).Single(cad => cad.AttributeType == typeof(FactAttribute));
var testCase = new XunitTestCase(testCollection, Reflector.Wrap(type.Assembly), Reflector.Wrap(type), Reflector.Wrap(method), Reflector.Wrap(fact));
var serialized = SerializationHelper.Serialize(testCase);
var result = SerializationHelper.Deserialize <XunitTestCase>(serialized);
Assert.Equal(testCase.Assembly.AssemblyPath, result.Assembly.AssemblyPath);
Assert.Equal(testCase.Assembly.Name, result.Assembly.Name);
Assert.Equal(testCase.Class.Name, result.Class.Name);
Assert.Equal(testCase.Method.Name, result.Method.Name);
Assert.Equal(testCase.DisplayName, result.DisplayName);
Assert.Null(result.Arguments);
Assert.Equal(testCase.SkipReason, result.SkipReason);
Assert.Collection(result.Traits.Keys,
key =>
{
Assert.Equal("name", key);
Assert.Equal("value", Assert.Single(result.Traits[key]));
});
}
public static async ValueTask <bool> TrySkipAsync(XunitTestCase testCase, IMessageBus messageBus)
{
var method = testCase.Method;
var type = testCase.TestMethod.TestClass.Class;
var assembly = type.Assembly;
var skipReasons = new List <string>();
var attributes =
_assemblyAttributes.GetOrAdd(
assembly.Name,
a => assembly.GetCustomAttributes(typeof(ITestCondition)).ToList())
.Concat(
_typeAttributes.GetOrAdd(
type.Name,
t => type.GetCustomAttributes(typeof(ITestCondition)).ToList()))
.Concat(method.GetCustomAttributes(typeof(ITestCondition)))
.OfType <ReflectionAttributeInfo>()
.Select(attributeInfo => (ITestCondition)attributeInfo.Attribute);
foreach (var attribute in attributes)
{
if (!await attribute.IsMetAsync())
{
skipReasons.Add(attribute.SkipReason);
}
}
if (skipReasons.Count > 0)
{
messageBus.QueueMessage(new TestSkipped(new XunitTest(testCase, testCase.DisplayName), string.Join(Environment.NewLine, skipReasons)));
return(true);
}
return(false);
}
internal static async Task <RunSummary> PerformRetry(
Func <IMessageSink,
IMessageBus,
object[],
ExceptionAggregator,
CancellationTokenSource, Task <RunSummary> > executer,
int maxRetries,
IMessageSink diagnosticMessageSink,
IMessageBus messageBus,
object[] constructorArguments,
ExceptionAggregator aggregator,
XunitTestCase testCase,
CancellationTokenSource cancellationTokenSource)
{
var testRetryCount = 0;
var rnd = new Random();
using (var delayedMessageBus = new DelayedMessageBus(messageBus))
{
while (true)
{
var summary = await executer(diagnosticMessageSink, delayedMessageBus, constructorArguments, aggregator, cancellationTokenSource);
var lastAssert = delayedMessageBus.GetLastFailure();
if (aggregator.HasExceptions || summary.Failed == 0 || testRetryCount + 1 > maxRetries)
{
delayedMessageBus.Complete();
if (testRetryCount > 0)
{
LogMessage($"test '{testCase.DisplayName}' retry finished. Number of failed tests of last run: {summary.Failed}, retry state: {testRetryCount}/{maxRetries}");
if (summary.Failed == 0)
{
LogMessage($"test '{testCase.DisplayName}' succeeded after {testRetryCount}/{maxRetries} executions.");
}
else if (testRetryCount == maxRetries)
{
LogMessage($"test '{testCase.DisplayName}' failed after {testRetryCount}/{maxRetries} executions.", lastAssert);
}
}
if (aggregator.HasExceptions)
{
LogMessage($"test '{testCase.DisplayName}' failed with exception. {aggregator.ToException()}");
}
return(summary);
}
testRetryCount++;
var retryDelay = (int)Math.Min(180_000, Math.Pow(2, testRetryCount) * rnd.Next(5000, 30_000));
var msg = $"performing retry number {testRetryCount}/{maxRetries} in {retryDelay}ms for Test '{testCase.DisplayName}'";
LogMessage(msg, lastAssert);
await Task.Delay(retryDelay, cancellationTokenSource.Token);
}
}
}
public static void SkipReason()
{
var testMethod = Mocks.TestMethod("MockType", "MockMethod", skip: "Skip Reason");
var testCase = new XunitTestCase(testMethod);
Assert.Equal("Skip Reason", testCase.SkipReason);
}
public static void Timeout()
{
var testMethod = Mocks.TestMethod(timeout: 42);
var testCase = new XunitTestCase(SpyMessageSink.Create(), TestMethodDisplay.ClassAndMethod, TestMethodDisplayOptions.None, testMethod);
Assert.Equal(42, testCase.Timeout);
}
public static void CustomDisplayName()
{
var testMethod = Mocks.TestMethod(displayName: "Custom Display Name");
var testCase = new XunitTestCase(SpyMessageSink.Create(), TestMethodDisplay.ClassAndMethod, testMethod);
Assert.Equal("Custom Display Name", testCase.DisplayName);
}
public static void SkipReason()
{
var testMethod = Mocks.TestMethod("MockType", "MockMethod", skip: "Skip Reason");
var testCase = new XunitTestCase(SpyMessageSink.Create(), TestMethodDisplay.ClassAndMethod, testMethod);
Assert.Equal("Skip Reason", testCase.SkipReason);
}
public static void CustomDisplayName()
{
var testMethod = Mocks.TestMethod(displayName: "Custom Display Name");
var testCase = new XunitTestCase(testMethod);
Assert.Equal("Custom Display Name", testCase.DisplayName);
}
public void XunitFactWithColonsGetsEscaped()
{
var testMethod = Mocks.TestMethod("TESTS:TESTS", "a:b");
var testCase = new XunitTestCase(null, Xunit.Sdk.TestMethodDisplay.ClassAndMethod, Xunit.Sdk.TestMethodDisplayOptions.None, testMethod);
var serializedTestCase = discoverer.Serialize(testCase);
Assert.StartsWith(":F:TESTS::TESTS:a::b:1:0:", serializedTestCase);
}
public static void NotEnoughTestArguments()
{
var param = Mocks.ParameterInfo("p1");
var testMethod = Mocks.TestMethod(parameters: new[] { param });
var testCase = new XunitTestCase(testMethod, new object[0]);
Assert.Equal("MockType.MockMethod(p1: ???)", testCase.DisplayName);
}
public static void TooManyTestArguments()
{
var param = Mocks.ParameterInfo("p1");
var testMethod = Mocks.TestMethod(parameters: new[] { param });
var arguments = new object[] { 42, 21.12M };
var testCase = new XunitTestCase(testMethod, arguments);
Assert.Equal(String.Format("MockType.MockMethod(p1: 42, ???: {0})", 21.12), testCase.DisplayName);
}
public static void DefaultBehavior()
{
var testMethod = Mocks.TestMethod("MockType", "MockMethod");
var testCase = new XunitTestCase(testMethod);
Assert.Equal("MockType.MockMethod", testCase.DisplayName);
Assert.Null(testCase.SkipReason);
Assert.Empty(testCase.Traits);
}
public void CanSerializeFactBasedTestCase()
{
var testCollection = new XunitTestCollection();
var type = typeof(ClassUnderTest);
var method = type.GetMethod("FactMethod");
var fact = CustomAttributeData.GetCustomAttributes(method).Single(cad => cad.AttributeType == typeof(FactAttribute));
var testCase = new XunitTestCase(testCollection, Reflector.Wrap(type.Assembly), Reflector.Wrap(type), Reflector.Wrap(method), Reflector.Wrap(fact));
Assert.DoesNotThrow(() => SerializationHelper.Serialize(testCase));
}
public static void DefaultBehavior()
{
var testMethod = Mocks.TestMethod("MockType", "MockMethod");
var testCase = new XunitTestCase(SpyMessageSink.Create(), TestMethodDisplay.ClassAndMethod, testMethod);
Assert.Equal("MockType.MockMethod", testCase.DisplayName);
Assert.Null(testCase.SkipReason);
Assert.Empty(testCase.Traits);
}
internal WcfTestCase(XunitTestCase testCase,
TestMethodDisplay defaultMethodDisplay,
string skippedReason = null,
bool isTheory = false,
IMessageSink diagnosticMessageSink = null)
: base(diagnosticMessageSink, defaultMethodDisplay, TestMethodDisplayOptions.None, testCase.TestMethod, testCase.TestMethodArguments)
{
_skippedReason = skippedReason;
_isTheory = isTheory;
_diagnosticMessageSink = diagnosticMessageSink;
}
public static void TraitsOnTestClass()
{
var trait1 = Mocks.TraitAttribute("Trait1", "Value1");
var trait2 = Mocks.TraitAttribute("Trait2", "Value2");
var testMethod = Mocks.TestMethod(classAttributes: new[] { trait1, trait2 });
var testCase = new XunitTestCase(SpyMessageSink.Create(), TestMethodDisplay.ClassAndMethod, testMethod);
Assert.Equal("Value1", Assert.Single(testCase.Traits["Trait1"]));
Assert.Equal("Value2", Assert.Single(testCase.Traits["Trait2"]));
}
public static void TraitsOnTestMethod()
{
var trait1 = Mocks.TraitAttribute("Trait1", "Value1");
var trait2 = Mocks.TraitAttribute("Trait2", "Value2");
var testMethod = Mocks.TestMethod(methodAttributes: new[] { trait1, trait2 });
var testCase = new XunitTestCase(testMethod);
Assert.Equal("Value1", Assert.Single(testCase.Traits["Trait1"]));
Assert.Equal("Value2", Assert.Single(testCase.Traits["Trait2"]));
}
public static void CustomDisplayNameWithArguments()
{
var param1 = Mocks.ParameterInfo("p1");
var param2 = Mocks.ParameterInfo("p2");
var param3 = Mocks.ParameterInfo("p3");
var testMethod = Mocks.TestMethod(displayName: "Custom Display Name", parameters: new[] { param1, param2, param3 });
var arguments = new object[] { 42, "Hello, world!", 'A' };
var testCase = new XunitTestCase(SpyMessageSink.Create(), TestMethodDisplay.ClassAndMethod, testMethod, arguments);
Assert.Equal("Custom Display Name(p1: 42, p2: \"Hello, world!\", p3: 'A')", testCase.DisplayName);
}
public void DeserializedTestWithNonSerializableArgumentsThrows()
{
var testCollection = new XunitTestCollection();
var type = typeof(ClassUnderTest);
var method = type.GetMethod("FactMethod");
var fact = CustomAttributeData.GetCustomAttributes(method).Single(cad => cad.AttributeType == typeof(FactAttribute));
var testCase = new XunitTestCase(testCollection, Reflector.Wrap(type.Assembly), Reflector.Wrap(type), Reflector.Wrap(method), Reflector.Wrap(fact), new object[] { new ClassUnderTest() });
var ex = Record.Exception(() => SerializationHelper.Serialize(testCase));
Assert.IsType <SerializationException>(ex);
}
public static void CorrectNumberOfTestArguments()
{
var param1 = Mocks.ParameterInfo("p1");
var param2 = Mocks.ParameterInfo("p2");
var param3 = Mocks.ParameterInfo("p3");
var testMethod = Mocks.TestMethod(parameters: new[] { param1, param2, param3 });
var arguments = new object[] { 42, "Hello, world!", 'A' };
var testCase = new XunitTestCase(testMethod, arguments);
Assert.Equal("MockType.MockMethod(p1: 42, p2: \"Hello, world!\", p3: 'A')", testCase.DisplayName);
}
public static void DisposesArguments()
{
var disposable1 = Substitute.For <IDisposable>();
var disposable2 = Substitute.For <IDisposable>();
var testMethod = Mocks.TestMethod();
var testCase = new XunitTestCase(testMethod, new[] { disposable1, disposable2 });
testCase.Dispose();
disposable1.Received(1).Dispose();
disposable2.Received(1).Dispose();
}
public void SerializationTestsForXunitTestCase()
{
var messageSink = SpyMessageSink.Create();
var testMethod = Mocks.TestMethod <ClassWithSingleTest>(nameof(ClassWithSingleTest.TestMethod));
var testCase = new XunitTestCase(messageSink, TestMethodDisplay.ClassAndMethod, TestMethodDisplayOptions.None, testMethod);
framework.ReportDiscoveredTestCase_Public(testCase, includeSourceInformation: true, messageBus);
var msg = Assert.Single(messageBus.Messages);
var discoveryMsg = Assert.IsAssignableFrom <_TestCaseDiscovered>(msg);
Assert.Equal(":F:XunitTestFrameworkDiscovererTests+ClassWithSingleTest:TestMethod:1:0:0:(null)", discoveryMsg.Serialization);
}
public static void CustomTraitWithoutDiscoverer()
{
var trait = Mocks.TraitAttribute <BadTraitAttribute>();
var testMethod = Mocks.TestMethod(classAttributes: new[] { trait });
var messages = new List <IMessageSinkMessage>();
var spy = SpyMessageSink.Create(messages: messages);
var testCase = new XunitTestCase(spy, TestMethodDisplay.ClassAndMethod, testMethod);
Assert.Empty(testCase.Traits);
var diagnosticMessages = messages.OfType <IDiagnosticMessage>();
var diagnosticMessage = Assert.Single(diagnosticMessages);
Assert.Equal("Trait attribute on 'MockType.MockMethod' did not have [TraitDiscoverer]", diagnosticMessage.Message);
}
public void DeserializedTestWithSerializableArgumentsPreservesArguments()
{
var testCollection = new XunitTestCollection();
var type = typeof(ClassUnderTest);
var method = type.GetMethod("FactMethod");
var fact = CustomAttributeData.GetCustomAttributes(method).Single(cad => cad.AttributeType == typeof(FactAttribute));
var testCase = new XunitTestCase(testCollection, Reflector.Wrap(type.Assembly), Reflector.Wrap(type), Reflector.Wrap(method), Reflector.Wrap(fact), new object[] { 42, 21.12, "Hello world" });
var serialized = SerializationHelper.Serialize(testCase);
var result = SerializationHelper.Deserialize <XunitTestCase>(serialized);
Assert.Collection(result.Arguments,
arg => Assert.Equal(42, arg),
arg => Assert.Equal(21.12, arg),
arg => Assert.Equal("Hello world", arg));
}
public static bool TrySkip(XunitTestCase testCase, IMessageBus messageBus)
{
var method = testCase.Method;
var type = testCase.TestMethod.TestClass.Class;
var assembly = type.Assembly;
var key = $"{method.Name}<<{type.Name}<<{assembly.Name}";
var skipReason = _resolvedConditions.GetOrAdd(
key,
k =>
{
var skipReasons = method
.GetCustomAttributes(typeof(ITestCondition))
.Concat(
_typeAttributes.GetOrAdd(
type.Name,
t => type.GetCustomAttributes(typeof(ITestCondition)).ToList()))
.Concat(
_assemblyAttributes.GetOrAdd(
assembly.Name,
a => assembly.GetCustomAttributes(typeof(ITestCondition)).ToList()))
.OfType <ReflectionAttributeInfo>()
.Select(attributeInfo => (ITestCondition)attributeInfo.Attribute)
.Where(condition => !condition.IsMet)
.Select(condition => condition.SkipReason)
.ToList();
return(skipReasons.Count > 0 ? string.Join(Environment.NewLine, skipReasons) : null);
});
if (skipReason != null)
{
messageBus.QueueMessage(new TestSkipped(new XunitTest(testCase, testCase.DisplayName), skipReason));
return(true);
}
return(false);
}
static void ComputeTraits(Assembly assembly, XunitTestCase tc)
{
// Traits are not set because of some assembly loading problems i.e. Assembly.Load (new AssemblyName ("Microsoft.DotNet.XUnitExtensions")) fails
// So load them manually
foreach (ReflectionAttributeInfo attr in ((ReflectionMethodInfo)tc.Method).GetCustomAttributes(typeof(ITraitAttribute)))
{
var discovererAttr = attr.GetCustomAttributes(typeof(TraitDiscovererAttribute)).FirstOrDefault();
if (discovererAttr != null)
{
var discoverer_args = discovererAttr.GetConstructorArguments().Cast <string>().ToList();
var disc_assembly = Assembly.LoadFrom(Path.Combine(Path.GetDirectoryName(assembly.Location), discoverer_args [1]) + ".dll");
var disc_type = disc_assembly.GetType(discoverer_args [0]);
var disc_obj = (ITraitDiscoverer)Activator.CreateInstance(disc_type);
foreach (var trait in disc_obj.GetTraits(attr))
{
if (!tc.Traits.ContainsKey(trait.Key))
{
tc.Traits [trait.Key] = new List <string> ();
}
tc.Traits [trait.Key].Add(trait.Value);
}
}
}
}
public void RunTests(XunitTestCase testCase)
{
testCase.Run();
}