/// <summary> /// Initializes a new instance of the <see cref="TestSet"/> class. /// </summary> /// <param name="testSets">The test sets.</param> /// <param name="toolChain">The tool chain (if any).</param> public TestSet(TestSets testSets, ToolChain toolChain = null) { TestSets = testSets; Name = toolChain?.GraphicsBackend.ToString() ?? "CPU"; ToolChain = toolChain; Backend = toolChain?.CreateBackend(testSets.Compilation); }
/// <summary> /// Executes the specified test. /// </summary> /// <param name="generationResult">The generation result.</param> /// <param name="csFunctionName">Name of the cs function.</param> /// <param name="output">The output.</param> public void Execute( ShaderGenerationResult generationResult, string csFunctionName, ITestOutputHelper output) { if (Executed) { output.WriteLine( $"The {Name} tests have already been executed!"); return; } TestSets testSets = TestSets; Mappings mappings = testSets.Mappings; if (ToolChain == null) { /* * Generate the test data and the result set data for the CPU. */ AllocateResults(output); using (new TestTimer(output, $"Running {testSets.TestLoops} iterations on the {Name} backend")) { for (int test = 0; test < testSets.TestLoops; test++) { foreach (MethodMap method in mappings.MethodMaps) { method.ExecuteCPU(TestSets.TestData, Results, test); } } return; } } GeneratedShaderSet set; CompileResult compilationResult; // Compile shader for this backend. using (new TestTimer(output, $"Compiling Compute Shader for {ToolChain.GraphicsBackend}")) { set = generationResult.GetOutput(Backend).Single(); compilationResult = ToolChain.Compile(set.ComputeShaderCode, Stage.Compute, set.ComputeFunction.Name); } if (compilationResult.HasError) { output.WriteLine($"Failed to compile Compute Shader from set \"{set.Name}\"!"); output.WriteLine(compilationResult.ToString()); return; } Assert.NotNull(compilationResult.CompiledOutput); using (GraphicsDevice graphicsDevice = ToolChain.CreateHeadless()) { if (!graphicsDevice.Features.ComputeShader) { output.WriteLine( $"The {ToolChain.GraphicsBackend} backend does not support compute shaders, skipping!"); return; } ResourceFactory factory = graphicsDevice.ResourceFactory; using (DeviceBuffer inOutBuffer = factory.CreateBuffer( new BufferDescription( (uint)mappings.BufferSize, BufferUsage.StructuredBufferReadWrite, (uint)mappings.StructSize))) using (Shader computeShader = factory.CreateShader( new ShaderDescription( ShaderStages.Compute, compilationResult.CompiledOutput, csFunctionName))) using (ResourceLayout inOutStorageLayout = factory.CreateResourceLayout( new ResourceLayoutDescription( new ResourceLayoutElementDescription("InOutBuffer", ResourceKind.StructuredBufferReadWrite, ShaderStages.Compute)))) using (Pipeline computePipeline = factory.CreateComputePipeline(new ComputePipelineDescription( computeShader, new[] { inOutStorageLayout }, 1, 1, 1))) using (ResourceSet computeResourceSet = factory.CreateResourceSet( new ResourceSetDescription(inOutStorageLayout, inOutBuffer))) using (CommandList commandList = factory.CreateCommandList()) { // Ensure the headless graphics device is the backend we expect. Assert.Equal(ToolChain.GraphicsBackend, graphicsDevice.BackendType); output.WriteLine($"Created compute pipeline for {Name} backend."); // Allocate the results buffer AllocateResults(output); using (new TestTimer(output, $"Running {testSets.TestLoops} iterations on the {Name} backend")) { // Loop for each test for (int test = 0; test < testSets.TestLoops; test++) { // Update parameter buffer graphicsDevice.UpdateBuffer( inOutBuffer, 0, // Get the portion of test data for the current test loop Marshal.UnsafeAddrOfPinnedArrayElement(testSets.TestData, mappings.BufferSize * test), (uint)mappings.BufferSize); graphicsDevice.WaitForIdle(); // Execute compute shaders commandList.Begin(); commandList.SetPipeline(computePipeline); commandList.SetComputeResourceSet(0, computeResourceSet); commandList.Dispatch((uint)mappings.Methods, 1, 1); commandList.End(); graphicsDevice.SubmitCommands(commandList); graphicsDevice.WaitForIdle(); // Read back parameters using a staging buffer using (DeviceBuffer stagingBuffer = factory.CreateBuffer( new BufferDescription(inOutBuffer.SizeInBytes, BufferUsage.Staging))) { commandList.Begin(); commandList.CopyBuffer(inOutBuffer, 0, stagingBuffer, 0, stagingBuffer.SizeInBytes); commandList.End(); graphicsDevice.SubmitCommands(commandList); graphicsDevice.WaitForIdle(); // Read back test results MappedResource map = graphicsDevice.Map(stagingBuffer, MapMode.Read); mappings.SetResults(map.Data, Results, test); graphicsDevice.Unmap(stagingBuffer); } } } } } }
private void TestBuiltins() { // Find all backends that can create a headless graphics device on this system. IReadOnlyList <ToolChain> toolChains = ToolChain.Requires(ToolFeatures.HeadlessGraphicsDevice, false); if (toolChains.Count < 1) { throw new RequiredToolFeatureMissingException( $"At least one tool chain capable of creating headless graphics devices is required for this test!"); } /* * Auto-generate C# code for testing methods. */ IReadOnlyList <MethodInfo> methods = null; Mappings mappings; Compilation compilation; using (new TestTimer(_output, () => $"Generating C# shader code to test {methods.Count} methods")) { // Get all the methods we wish to test methods = MethodsToTest.ToArray(); mappings = CreateMethodTestCompilation(methods, out compilation); // Note, you could use compilation.Emit(...) at this point to compile the auto-generated code! // however, for now we'll invoke methods directly rather than executing the C# code that has been // generated, as loading emitted code into a test is currently much more difficult. } // Allocate enough space to store the result sets for each backend! TestSets testSets = null; using (new TestTimer( _output, () => $"Generating random test data ({(mappings.BufferSize * testSets.TestLoops).ToMemorySize()}) for {testSets.TestLoops} iterations of {mappings.Methods} methods") ) { testSets = new TestSets(toolChains, compilation, mappings, TestLoops, MinMantissa, MaxMantissa); } /* * Transpile shaders */ ShaderGenerationResult generationResult; using (new TestTimer( _output, t => $"Generated shader sets for {string.Join(", ", toolChains.Select(tc => tc.Name))} backends in {t * 1000:#.##}ms.") ) { ShaderGenerator sg = new ShaderGenerator( compilation, testSets.Select(t => t.Backend).Where(b => b != null).ToArray(), null, null, "ComputeShader.CS"); generationResult = sg.GenerateShaders(); } /* * Loop through each backend to run tests. */ bool first = true; using (new TestTimer(_output, "Executing all tests on all backends")) { foreach (TestSet testSet in testSets) { _output.WriteLine(string.Empty); if (first) { // This is the first test set, so we use Space1 instead of Spacer 2. first = false; _output.WriteLine(TestUtil.Spacer1); } else { _output.WriteLine(TestUtil.Spacer2); } _output.WriteLine(String.Empty); testSet.Execute(generationResult, "CS", _output); } _output.WriteLine(string.Empty); } _output.WriteLine(string.Empty); _output.WriteLine(TestUtil.Spacer1); _output.WriteLine(string.Empty); Assert.True(testSets.Count(t => t.Executed) > 1, "At least 2 test sets are required for comparison."); /* * Finally, evaluate differences between results */ IReadOnlyList <(MethodMap MethodMap, IReadOnlyList <Failure> Failures)> failures; using (new TestTimer(_output, "Analysing results for failures")) { failures = testSets.GetFailures(FloatEpsilon) .Select(kvp => (MethodMap: kvp.Key, Failures: kvp.Value)) .OrderByDescending(kvp => kvp.Failures.Count) .ToArray(); } if (!failures.Any()) { _output.WriteLine("No failures detected!"); return; } _output.WriteLine( $"{failures.Count} methods had failures out of {mappings.Methods} ({100.0 * failures.Count / mappings.Methods:#.##}%)."); _output.WriteLine(string.Empty); // Get pointer array string lastMethodName = null; foreach ((MethodMap methodMap, IReadOnlyList <Failure> methodFailures) in failures) { if (lastMethodName != methodMap.Method.Name) { if (lastMethodName != null) { // Seperate methods of different names with spacer 1 _output.WriteLine(string.Empty); _output.WriteLine(TestUtil.Spacer1); _output.WriteLine(string.Empty); } lastMethodName = methodMap.Method.Name; } else { _output.WriteLine(string.Empty); _output.WriteLine(TestUtil.Spacer2); _output.WriteLine(string.Empty); } int failureCount = methodFailures.Count; _output.WriteLine( $"{TestUtil.GetUnicodePieChart((double)failureCount / testSets.TestLoops)} {methodMap.Signature} failed {failureCount}/{testSets.TestLoops} ({failureCount * 100.0 / testSets.TestLoops:#.##}%)."); // Output examples! int example = 0; foreach (Failure failure in methodFailures) { _output.WriteLine(string.Empty); if (example++ >= FailureExamples) { _output.WriteLine("…"); break; } _output.WriteLine(failure.ToString()); } } _output.WriteLine(string.Empty); _output.WriteLine(TestUtil.Spacer2); _output.WriteLine(string.Empty); }