/// <summary>
/// Initializes a new instance of the <see cref="TestSet"/> class.
/// </summary>
/// <param name="testSets">The test sets.</param>
/// <param name="toolChain">The tool chain (if any).</param>
public TestSet(TestSets testSets, ToolChain toolChain = null)
{
TestSets = testSets;
Name = toolChain?.GraphicsBackend.ToString() ?? "CPU";
ToolChain = toolChain;
Backend = toolChain?.CreateBackend(testSets.Compilation);
}
/// <summary>
/// Executes the specified test.
/// </summary>
/// <param name="generationResult">The generation result.</param>
/// <param name="csFunctionName">Name of the cs function.</param>
/// <param name="output">The output.</param>
public void Execute(
ShaderGenerationResult generationResult,
string csFunctionName,
ITestOutputHelper output)
{
if (Executed)
{
output.WriteLine(
$"The {Name} tests have already been executed!");
return;
}
TestSets testSets = TestSets;
Mappings mappings = testSets.Mappings;
if (ToolChain == null)
{
/*
* Generate the test data and the result set data for the CPU.
*/
AllocateResults(output);
using (new TestTimer(output,
$"Running {testSets.TestLoops} iterations on the {Name} backend"))
{
for (int test = 0; test < testSets.TestLoops; test++)
{
foreach (MethodMap method in mappings.MethodMaps)
{
method.ExecuteCPU(TestSets.TestData, Results, test);
}
}
return;
}
}
GeneratedShaderSet set;
CompileResult compilationResult;
// Compile shader for this backend.
using (new TestTimer(output, $"Compiling Compute Shader for {ToolChain.GraphicsBackend}"))
{
set = generationResult.GetOutput(Backend).Single();
compilationResult =
ToolChain.Compile(set.ComputeShaderCode, Stage.Compute, set.ComputeFunction.Name);
}
if (compilationResult.HasError)
{
output.WriteLine($"Failed to compile Compute Shader from set \"{set.Name}\"!");
output.WriteLine(compilationResult.ToString());
return;
}
Assert.NotNull(compilationResult.CompiledOutput);
using (GraphicsDevice graphicsDevice = ToolChain.CreateHeadless())
{
if (!graphicsDevice.Features.ComputeShader)
{
output.WriteLine(
$"The {ToolChain.GraphicsBackend} backend does not support compute shaders, skipping!");
return;
}
ResourceFactory factory = graphicsDevice.ResourceFactory;
using (DeviceBuffer inOutBuffer = factory.CreateBuffer(
new BufferDescription(
(uint)mappings.BufferSize,
BufferUsage.StructuredBufferReadWrite,
(uint)mappings.StructSize)))
using (Shader computeShader = factory.CreateShader(
new ShaderDescription(
ShaderStages.Compute,
compilationResult.CompiledOutput,
csFunctionName)))
using (ResourceLayout inOutStorageLayout = factory.CreateResourceLayout(
new ResourceLayoutDescription(
new ResourceLayoutElementDescription("InOutBuffer", ResourceKind.StructuredBufferReadWrite,
ShaderStages.Compute))))
using (Pipeline computePipeline = factory.CreateComputePipeline(new ComputePipelineDescription(
computeShader,
new[] { inOutStorageLayout },
1, 1, 1)))
using (ResourceSet computeResourceSet = factory.CreateResourceSet(
new ResourceSetDescription(inOutStorageLayout, inOutBuffer)))
using (CommandList commandList = factory.CreateCommandList())
{
// Ensure the headless graphics device is the backend we expect.
Assert.Equal(ToolChain.GraphicsBackend, graphicsDevice.BackendType);
output.WriteLine($"Created compute pipeline for {Name} backend.");
// Allocate the results buffer
AllocateResults(output);
using (new TestTimer(output,
$"Running {testSets.TestLoops} iterations on the {Name} backend"))
{
// Loop for each test
for (int test = 0; test < testSets.TestLoops; test++)
{
// Update parameter buffer
graphicsDevice.UpdateBuffer(
inOutBuffer,
0,
// Get the portion of test data for the current test loop
Marshal.UnsafeAddrOfPinnedArrayElement(testSets.TestData, mappings.BufferSize * test),
(uint)mappings.BufferSize);
graphicsDevice.WaitForIdle();
// Execute compute shaders
commandList.Begin();
commandList.SetPipeline(computePipeline);
commandList.SetComputeResourceSet(0, computeResourceSet);
commandList.Dispatch((uint)mappings.Methods, 1, 1);
commandList.End();
graphicsDevice.SubmitCommands(commandList);
graphicsDevice.WaitForIdle();
// Read back parameters using a staging buffer
using (DeviceBuffer stagingBuffer =
factory.CreateBuffer(
new BufferDescription(inOutBuffer.SizeInBytes, BufferUsage.Staging)))
{
commandList.Begin();
commandList.CopyBuffer(inOutBuffer, 0, stagingBuffer, 0, stagingBuffer.SizeInBytes);
commandList.End();
graphicsDevice.SubmitCommands(commandList);
graphicsDevice.WaitForIdle();
// Read back test results
MappedResource map = graphicsDevice.Map(stagingBuffer, MapMode.Read);
mappings.SetResults(map.Data, Results, test);
graphicsDevice.Unmap(stagingBuffer);
}
}
}
}
}
}
private void TestBuiltins()
{
// Find all backends that can create a headless graphics device on this system.
IReadOnlyList <ToolChain> toolChains = ToolChain.Requires(ToolFeatures.HeadlessGraphicsDevice, false);
if (toolChains.Count < 1)
{
throw new RequiredToolFeatureMissingException(
$"At least one tool chain capable of creating headless graphics devices is required for this test!");
}
/*
* Auto-generate C# code for testing methods.
*/
IReadOnlyList <MethodInfo> methods = null;
Mappings mappings;
Compilation compilation;
using (new TestTimer(_output, () => $"Generating C# shader code to test {methods.Count} methods"))
{
// Get all the methods we wish to test
methods = MethodsToTest.ToArray();
mappings = CreateMethodTestCompilation(methods, out compilation);
// Note, you could use compilation.Emit(...) at this point to compile the auto-generated code!
// however, for now we'll invoke methods directly rather than executing the C# code that has been
// generated, as loading emitted code into a test is currently much more difficult.
}
// Allocate enough space to store the result sets for each backend!
TestSets testSets = null;
using (new TestTimer(
_output,
() =>
$"Generating random test data ({(mappings.BufferSize * testSets.TestLoops).ToMemorySize()}) for {testSets.TestLoops} iterations of {mappings.Methods} methods")
)
{
testSets = new TestSets(toolChains, compilation, mappings, TestLoops, MinMantissa, MaxMantissa);
}
/*
* Transpile shaders
*/
ShaderGenerationResult generationResult;
using (new TestTimer(
_output,
t =>
$"Generated shader sets for {string.Join(", ", toolChains.Select(tc => tc.Name))} backends in {t * 1000:#.##}ms.")
)
{
ShaderGenerator sg = new ShaderGenerator(
compilation,
testSets.Select(t => t.Backend).Where(b => b != null).ToArray(),
null,
null,
"ComputeShader.CS");
generationResult = sg.GenerateShaders();
}
/*
* Loop through each backend to run tests.
*/
bool first = true;
using (new TestTimer(_output, "Executing all tests on all backends"))
{
foreach (TestSet testSet in testSets)
{
_output.WriteLine(string.Empty);
if (first)
{
// This is the first test set, so we use Space1 instead of Spacer 2.
first = false;
_output.WriteLine(TestUtil.Spacer1);
}
else
{
_output.WriteLine(TestUtil.Spacer2);
}
_output.WriteLine(String.Empty);
testSet.Execute(generationResult, "CS", _output);
}
_output.WriteLine(string.Empty);
}
_output.WriteLine(string.Empty);
_output.WriteLine(TestUtil.Spacer1);
_output.WriteLine(string.Empty);
Assert.True(testSets.Count(t => t.Executed) > 1,
"At least 2 test sets are required for comparison.");
/*
* Finally, evaluate differences between results
*/
IReadOnlyList <(MethodMap MethodMap, IReadOnlyList <Failure> Failures)> failures;
using (new TestTimer(_output, "Analysing results for failures"))
{
failures = testSets.GetFailures(FloatEpsilon)
.Select(kvp => (MethodMap: kvp.Key, Failures: kvp.Value))
.OrderByDescending(kvp => kvp.Failures.Count)
.ToArray();
}
if (!failures.Any())
{
_output.WriteLine("No failures detected!");
return;
}
_output.WriteLine(
$"{failures.Count} methods had failures out of {mappings.Methods} ({100.0 * failures.Count / mappings.Methods:#.##}%).");
_output.WriteLine(string.Empty);
// Get pointer array
string lastMethodName = null;
foreach ((MethodMap methodMap, IReadOnlyList <Failure> methodFailures) in failures)
{
if (lastMethodName != methodMap.Method.Name)
{
if (lastMethodName != null)
{
// Seperate methods of different names with spacer 1
_output.WriteLine(string.Empty);
_output.WriteLine(TestUtil.Spacer1);
_output.WriteLine(string.Empty);
}
lastMethodName = methodMap.Method.Name;
}
else
{
_output.WriteLine(string.Empty);
_output.WriteLine(TestUtil.Spacer2);
_output.WriteLine(string.Empty);
}
int failureCount = methodFailures.Count;
_output.WriteLine(
$"{TestUtil.GetUnicodePieChart((double)failureCount / testSets.TestLoops)} {methodMap.Signature} failed {failureCount}/{testSets.TestLoops} ({failureCount * 100.0 / testSets.TestLoops:#.##}%).");
// Output examples!
int example = 0;
foreach (Failure failure in methodFailures)
{
_output.WriteLine(string.Empty);
if (example++ >= FailureExamples)
{
_output.WriteLine("…");
break;
}
_output.WriteLine(failure.ToString());
}
}
_output.WriteLine(string.Empty);
_output.WriteLine(TestUtil.Spacer2);
_output.WriteLine(string.Empty);
}