private void TestBuiltins()
{
// Find all backends that can create a headless graphics device on this system.
IReadOnlyList <ToolChain> toolChains = ToolChain.Requires(ToolFeatures.HeadlessGraphicsDevice, false);
if (toolChains.Count < 1)
{
throw new RequiredToolFeatureMissingException(
$"At least one tool chain capable of creating headless graphics devices is required for this test!");
}
/*
* Auto-generate C# code for testing methods.
*/
IReadOnlyList <MethodInfo> methods = null;
Mappings mappings;
Compilation compilation;
using (new TestTimer(_output, () => $"Generating C# shader code to test {methods.Count} methods"))
{
// Get all the methods we wish to test
methods = MethodsToTest.ToArray();
mappings = CreateMethodTestCompilation(methods, out compilation);
// Note, you could use compilation.Emit(...) at this point to compile the auto-generated code!
// however, for now we'll invoke methods directly rather than executing the C# code that has been
// generated, as loading emitted code into a test is currently much more difficult.
}
// Allocate enough space to store the result sets for each backend!
TestSets testSets = null;
using (new TestTimer(
_output,
() =>
$"Generating random test data ({(mappings.BufferSize * testSets.TestLoops).ToMemorySize()}) for {testSets.TestLoops} iterations of {mappings.Methods} methods")
)
{
testSets = new TestSets(toolChains, compilation, mappings, TestLoops, MinMantissa, MaxMantissa);
}
/*
* Transpile shaders
*/
ShaderGenerationResult generationResult;
using (new TestTimer(
_output,
t =>
$"Generated shader sets for {string.Join(", ", toolChains.Select(tc => tc.Name))} backends in {t * 1000:#.##}ms.")
)
{
ShaderGenerator sg = new ShaderGenerator(
compilation,
testSets.Select(t => t.Backend).Where(b => b != null).ToArray(),
null,
null,
"ComputeShader.CS");
generationResult = sg.GenerateShaders();
}
/*
* Loop through each backend to run tests.
*/
bool first = true;
using (new TestTimer(_output, "Executing all tests on all backends"))
{
foreach (TestSet testSet in testSets)
{
_output.WriteLine(string.Empty);
if (first)
{
// This is the first test set, so we use Space1 instead of Spacer 2.
first = false;
_output.WriteLine(TestUtil.Spacer1);
}
else
{
_output.WriteLine(TestUtil.Spacer2);
}
_output.WriteLine(String.Empty);
testSet.Execute(generationResult, "CS", _output);
}
_output.WriteLine(string.Empty);
}
_output.WriteLine(string.Empty);
_output.WriteLine(TestUtil.Spacer1);
_output.WriteLine(string.Empty);
Assert.True(testSets.Count(t => t.Executed) > 1,
"At least 2 test sets are required for comparison.");
/*
* Finally, evaluate differences between results
*/
IReadOnlyList <(MethodMap MethodMap, IReadOnlyList <Failure> Failures)> failures;
using (new TestTimer(_output, "Analysing results for failures"))
{
failures = testSets.GetFailures(FloatEpsilon)
.Select(kvp => (MethodMap: kvp.Key, Failures: kvp.Value))
.OrderByDescending(kvp => kvp.Failures.Count)
.ToArray();
}
if (!failures.Any())
{
_output.WriteLine("No failures detected!");
return;
}
_output.WriteLine(
$"{failures.Count} methods had failures out of {mappings.Methods} ({100.0 * failures.Count / mappings.Methods:#.##}%).");
_output.WriteLine(string.Empty);
// Get pointer array
string lastMethodName = null;
foreach ((MethodMap methodMap, IReadOnlyList <Failure> methodFailures) in failures)
{
if (lastMethodName != methodMap.Method.Name)
{
if (lastMethodName != null)
{
// Seperate methods of different names with spacer 1
_output.WriteLine(string.Empty);
_output.WriteLine(TestUtil.Spacer1);
_output.WriteLine(string.Empty);
}
lastMethodName = methodMap.Method.Name;
}
else
{
_output.WriteLine(string.Empty);
_output.WriteLine(TestUtil.Spacer2);
_output.WriteLine(string.Empty);
}
int failureCount = methodFailures.Count;
_output.WriteLine(
$"{TestUtil.GetUnicodePieChart((double)failureCount / testSets.TestLoops)} {methodMap.Signature} failed {failureCount}/{testSets.TestLoops} ({failureCount * 100.0 / testSets.TestLoops:#.##}%).");
// Output examples!
int example = 0;
foreach (Failure failure in methodFailures)
{
_output.WriteLine(string.Empty);
if (example++ >= FailureExamples)
{
_output.WriteLine("…");
break;
}
_output.WriteLine(failure.ToString());
}
}
_output.WriteLine(string.Empty);
_output.WriteLine(TestUtil.Spacer2);
_output.WriteLine(string.Empty);
}
