private async Task <decimal> InvokeTestMethodAsync(ExceptionAggregator aggregator, ITestOutputHelper output)
{
var collectDump = TestMethod.GetCustomAttribute <CollectDumpAttribute>() != null;
var repeatAttribute = GetRepeatAttribute(TestMethod);
if (!typeof(LoggedTestBase).IsAssignableFrom(TestClass) || repeatAttribute == null)
{
return(await new LoggedTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, aggregator, CancellationTokenSource, output, null, collectDump).RunAsync());
}
return(await RunRepeatTestInvoker(aggregator, output, collectDump, repeatAttribute));
}
protected override object CallTestMethod(object testClassInstance)
{
var asyncSyncContext = (AsyncTestSyncContext)SynchronizationContext.Current;
//
// Run the test method inside of our iterator. Note that BenchmarkIterator.Run ensures that only one test
// method is running at any given time, so we don't need extra synchronization here.
//
var benchmarkAttr = (BenchmarkAttribute)TestMethod.GetCustomAttribute(typeof(BenchmarkAttribute));
var iterator = new BenchmarkIteratorImpl(DisplayName, benchmarkAttr.InnerIterationCount);
return(iterator.RunAsync(async() =>
{
var success = false;
BenchmarkEventSource.Log.BenchmarkStart(BenchmarkConfiguration.Instance.RunId, DisplayName);
try
{
var result = TestMethod.Invoke(testClassInstance, TestMethodArguments);
var task = result as Task;
if (task != null)
{
await task;
success = true;
}
else
{
var ex = await asyncSyncContext.WaitForCompletionAsync();
if (ex == null)
{
success = true;
}
else
{
Aggregator.Add(ex);
}
}
if (iterator.IterationStopReason == "NoIterations")
{
success = false;
throw new Exception("Benchmark did not execute any iterations. Please use one of the iteration methods in Microsoft.Xunit.Performance.Benchmark");
}
}
finally
{
var stopReason = success ? iterator.IterationStopReason : "TestFailed";
BenchmarkEventSource.Log.BenchmarkStop(BenchmarkConfiguration.Instance.RunId, DisplayName, stopReason);
BenchmarkEventSource.Log.Flush();
}
}));
}
private async Task <decimal> InvokeTestMethodAsync(ExceptionAggregator aggregator, ITestOutputHelper output)
{
var retryAttribute = GetRetryAttribute(TestMethod);
var collectDump = TestMethod.GetCustomAttribute <CollectDumpAttribute>() != null;
if (!typeof(LoggedTestBase).IsAssignableFrom(TestClass) || retryAttribute == null)
{
return(await new LoggedTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, aggregator, CancellationTokenSource, output, null, collectDump).RunAsync());
}
var retryPredicateMethodName = retryAttribute.RetryPredicateName;
var retryPredicateMethod = TestClass.GetMethod(retryPredicateMethodName,
BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static,
null,
new Type[] { typeof(Exception) },
null)
?? throw new InvalidOperationException($"No valid static retry predicate method {retryPredicateMethodName} was found on the type {TestClass.FullName}.");
if (retryPredicateMethod.ReturnType != typeof(bool))
{
throw new InvalidOperationException($"Retry predicate method {retryPredicateMethodName} on {TestClass.FullName} does not return bool.");
}
var retryContext = new RetryContext()
{
Limit = retryAttribute.RetryLimit,
Reason = retryAttribute.RetryReason,
};
var retryAggregator = new ExceptionAggregator();
var loggedTestInvoker = new LoggedTestInvoker(Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, retryAggregator, CancellationTokenSource, output, retryContext, collectDump);
var totalTime = 0.0M;
do
{
retryAggregator.Clear();
totalTime += await loggedTestInvoker.RunAsync();
retryContext.CurrentIteration++;
}while (retryAggregator.HasExceptions &&
retryContext.CurrentIteration < retryContext.Limit &&
(retryPredicateMethod.IsStatic
? (bool)retryPredicateMethod.Invoke(null, new object[] { retryAggregator.ToException() })
: (bool)retryPredicateMethod.Invoke(retryContext.TestClassInstance, new object[] { retryAggregator.ToException() }))
);
aggregator.Aggregate(retryAggregator);
return(totalTime);
}
protected override object CreateTestClass()
{
var testClass = base.CreateTestClass();
if (testClass is LoggedTest loggedTestClass)
{
var classType = loggedTestClass.GetType();
var logLevelAttribute = TestMethod.GetCustomAttribute <LogLevelAttribute>() as LogLevelAttribute;
var testName = TestMethodArguments.Aggregate(TestMethod.Name, (a, b) => $"{a}-{(b ?? "null")}");
// Try resolving ITestOutputHelper from constructor arguments
loggedTestClass.TestOutputHelper = ConstructorArguments?.SingleOrDefault(a => typeof(ITestOutputHelper).IsAssignableFrom(a.GetType())) as ITestOutputHelper;
var useShortClassName = TestMethod.DeclaringType.GetCustomAttribute <ShortClassNameAttribute>()
?? TestMethod.DeclaringType.Assembly.GetCustomAttribute <ShortClassNameAttribute>();
var resolvedClassName = useShortClassName == null ? classType.FullName : classType.Name;
// None resolved so create a new one and retain a reference to it for initialization/uninitialization
if (loggedTestClass.TestOutputHelper == null)
{
loggedTestClass.TestOutputHelper = _output = new TestOutputHelper();
_output.Initialize(MessageBus, Test);
}
AssemblyTestLog
.ForAssembly(classType.GetTypeInfo().Assembly)
.StartTestLog(
loggedTestClass.TestOutputHelper,
resolvedClassName,
out var loggerFactory,
logLevelAttribute?.LogLevel ?? LogLevel.Trace,
out var resolvedTestName,
testName);
// internal for testing
loggedTestClass.ResolvedTestMethodName = resolvedTestName;
loggedTestClass.ResolvedTestClassName = resolvedClassName;
loggedTestClass.LoggerFactory = loggerFactory;
loggedTestClass.Logger = loggerFactory.CreateLogger(classType);
loggedTestClass.TestSink = new TestSink();
loggerFactory.AddProvider(new TestLoggerProvider(loggedTestClass.TestSink));
loggedTestClass.AdditionalSetup();
}
return(testClass);
}
