Beispiel #1
0
 private async Task <EffectBytecodeCompilerResult> CompileAsync(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters)
 {
     return(await remoteEffectCompilerClient.Compile(mixinTree, effectParameters));
 }
Beispiel #2
0
        public override TaskOrResult <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixin, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters)
        {
            var database = FileProvider as DatabaseFileProvider ?? throw new NotSupportedException("Using the cache requires to ContentManager.FileProvider to be valid.");

            var usedParameters = compilerParameters;
            var mixinObjectId  = ShaderMixinObjectId.Compute(mixin, usedParameters.EffectParameters);

            // Final url of the compiled bytecode
            var compiledUrl = string.Format("{0}/{1}", CompiledShadersKey, mixinObjectId);

            var bytecode = new KeyValuePair <EffectBytecode, EffectBytecodeCacheLoadSource>(null, EffectBytecodeCacheLoadSource.JustCompiled);

            lock (bytecodes)
            {
                // ------------------------------------------------------------------------------------------------------------
                // 1) Try to load latest bytecode
                // ------------------------------------------------------------------------------------------------------------
                ObjectId bytecodeId;
                if (database.ContentIndexMap.TryGetValue(compiledUrl, out bytecodeId))
                {
                    bytecode = LoadEffectBytecode(database, bytecodeId);
                }

                // ------------------------------------------------------------------------------------------------------------
                // 2) Try to load from database cache
                // ------------------------------------------------------------------------------------------------------------
                if (bytecode.Key == null && database.ObjectDatabase.Exists(mixinObjectId))
                {
                    using (var stream = database.ObjectDatabase.OpenStream(mixinObjectId))
                    {
                        // We have an existing stream, make sure the shader is compiled
                        var objectIdBuffer = new byte[ObjectId.HashSize];
                        if (stream.Read(objectIdBuffer, 0, ObjectId.HashSize) == ObjectId.HashSize)
                        {
                            var newBytecodeId = new ObjectId(objectIdBuffer);
                            bytecode = LoadEffectBytecode(database, newBytecodeId);

                            if (bytecode.Key != null)
                            {
                                // If we successfully retrieved it from cache, add it to index map so that it won't be collected and available for faster lookup
                                database.ContentIndexMap[compiledUrl] = newBytecodeId;
                            }
                        }
                    }
                }
            }

            if (bytecode.Key != null)
            {
                return(new EffectBytecodeCompilerResult(bytecode.Key, bytecode.Value));
            }
            else if (Platform.IsWindowsDesktop == false || Compiler is NullEffectCompiler)
            {
                // On non Windows platform, we are expecting to have the bytecode stored directly or in database cache
                var stringBuilder = new StringBuilder();
                stringBuilder.AppendFormat("Unable to find compiled shaders [{0}] for mixin [{1}] with parameters [{2}]", compiledUrl, mixin, usedParameters.ToStringPermutationsDetailed());
                Log.Error(stringBuilder.ToString());
                throw new InvalidOperationException(stringBuilder.ToString());
            }

            // ------------------------------------------------------------------------------------------------------------
            // 3) Compile the shader
            // ------------------------------------------------------------------------------------------------------------
            lock (compilingShaders)
            {
                Task <EffectBytecodeCompilerResult> compilingShaderTask;
                if (compilingShaders.TryGetValue(mixinObjectId, out compilingShaderTask))
                {
                    // Note: Task might still be compiling
                    return(compilingShaderTask);
                }

                // Compile the mixin in a Task
                if (CompileEffectAsynchronously)
                {
                    var compilerParametersCopy = compilerParameters != null ? new CompilerParameters(compilerParameters) : null;
                    var resultTask             = Task.Factory.StartNew(() => CompileBytecode(mixin, effectParameters, compilerParametersCopy, mixinObjectId, database, compiledUrl), CancellationToken.None, TaskCreationOptions.None, taskSchedulerSelector != null ? taskSchedulerSelector(mixin, compilerParametersCopy.EffectParameters) : TaskScheduler.Default);

                    compilingShaders.Add(mixinObjectId, resultTask);

                    return(resultTask);
                }
                else
                {
                    return(CompileBytecode(mixin, effectParameters, compilerParameters, mixinObjectId, database, compiledUrl));
                }
            }
        }
Beispiel #3
0
 /// <inheritdoc/>
 public override TaskOrResult <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters = null)
 {
     return(CompileAsync(mixinTree, effectParameters));
 }
Beispiel #4
0
        private EffectBytecodeCompilerResult CompileBytecode(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters, ObjectId mixinObjectId, DatabaseFileProvider database, string compiledUrl)
        {
            // Open the database for writing
            var log       = new LoggerResult();
            var effectLog = GlobalLogger.GetLogger("EffectCompilerCache");

            if (CompileSynchronization.IsSet)
            {
                CompileSynchronization.Reset(1);
            }
            else
            {
                CompileSynchronization.AddCount();
            }

            // Note: this compiler is expected to not be async and directly write stuff in localLogger
            var compiledShader = base.Compile(mixinTree, effectParameters, compilerParameters).WaitForResult();

            compiledShader.CompilationLog.CopyTo(log);

            // If there are any errors, return immediately
            if (log.HasErrors)
            {
                lock (compilingShaders)
                {
                    compilingShaders.Remove(mixinObjectId);
                }

                log.CopyTo(effectLog);
                return(new EffectBytecodeCompilerResult(null, log));
            }

            // Compute the bytecodeId
            var newBytecodeId = compiledShader.Bytecode.ComputeId();

            // Check if we really need to store the bytecode
            lock (bytecodes)
            {
                // Using custom serialization to the database to store an object with a custom id
                // TODO: Check if we really need to write the bytecode everytime even if id is not changed
                var memoryStream = new MemoryStream();
                compiledShader.Bytecode.WriteTo(memoryStream);

                // Write current cache at the end (not part of the pure bytecode, but we use this as meta info)
                var writer = new BinarySerializationWriter(memoryStream);
                writer.Write(CurrentCache);

                memoryStream.Position = 0;
                database.ObjectDatabase.Write(memoryStream, newBytecodeId, true);
                database.ContentIndexMap[compiledUrl] = newBytecodeId;

                // Save bytecode Id to the database cache as well
                memoryStream.SetLength(0);
                memoryStream.Write((byte[])newBytecodeId, 0, ObjectId.HashSize);
                memoryStream.Position = 0;
                database.ObjectDatabase.Write(memoryStream, mixinObjectId, true);

                if (!bytecodes.ContainsKey(newBytecodeId))
                {
                    log.Verbose($"New effect compiled #{effectCompileCount} [{mixinObjectId}] (db: {newBytecodeId})\r\n{compilerParameters?.ToStringPermutationsDetailed()}");
                    Interlocked.Increment(ref effectCompileCount);

                    // Replace or add new bytecode
                    bytecodes[newBytecodeId] = new KeyValuePair <EffectBytecode, EffectBytecodeCacheLoadSource>(compiledShader.Bytecode, EffectBytecodeCacheLoadSource.JustCompiled);
                }
            }

            CompileSynchronization.Signal();

            lock (compilingShaders)
            {
                compilingShaders.Remove(mixinObjectId);
            }

            log.CopyTo(effectLog);
            return(compiledShader);
        }
Beispiel #5
0
        public override TaskOrResult <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters)
        {
            var log = new LoggerResult();

            // Load D3D compiler dll
            // Note: No lock, it's probably fine if it gets called from multiple threads at the same time.
            if (Platform.IsWindowsDesktop && !d3dCompilerLoaded)
            {
                NativeLibrary.PreloadLibrary("d3dcompiler_47.dll");
                d3dCompilerLoaded = true;
            }

            var shaderMixinSource = mixinTree;
            var fullEffectName    = mixinTree.Name;

            // Make a copy of shaderMixinSource. Use deep clone since shaderMixinSource can be altered during compilation (e.g. macros)
            var shaderMixinSourceCopy = new ShaderMixinSource();

            shaderMixinSourceCopy.DeepCloneFrom(shaderMixinSource);
            shaderMixinSource = shaderMixinSourceCopy;

            // Generate platform-specific macros
            switch (effectParameters.Platform)
            {
            case GraphicsPlatform.Direct3D11:
                shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_DIRECT3D", 1);
                shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_DIRECT3D11", 1);
                break;

            case GraphicsPlatform.Direct3D12:
                shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_DIRECT3D", 1);
                shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_DIRECT3D12", 1);
                break;

            case GraphicsPlatform.OpenGL:
                shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_OPENGL", 1);
                shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_OPENGLCORE", 1);
                break;

            case GraphicsPlatform.OpenGLES:
                shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_OPENGL", 1);
                shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_OPENGLES", 1);
                break;

            case GraphicsPlatform.Vulkan:
                shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_VULKAN", 1);
                break;

            default:
                throw new NotSupportedException();
            }

            // Generate profile-specific macros
            shaderMixinSource.AddMacro("XENKO_GRAPHICS_PROFILE", (int)effectParameters.Profile);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_1", (int)GraphicsProfile.Level_9_1);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_2", (int)GraphicsProfile.Level_9_2);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_3", (int)GraphicsProfile.Level_9_3);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_0", (int)GraphicsProfile.Level_10_0);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_1", (int)GraphicsProfile.Level_10_1);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_0", (int)GraphicsProfile.Level_11_0);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_1", (int)GraphicsProfile.Level_11_1);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_2", (int)GraphicsProfile.Level_11_2);

            // In .xksl, class has been renamed to shader to avoid ambiguities with HLSL
            shaderMixinSource.AddMacro("class", "shader");

            var parsingResult = GetMixinParser().Parse(shaderMixinSource, shaderMixinSource.Macros.ToArray());

            // Copy log from parser results to output
            CopyLogs(parsingResult, log);

            // Return directly if there are any errors
            if (parsingResult.HasErrors)
            {
                return(new EffectBytecodeCompilerResult(null, log));
            }

            // Convert the AST to HLSL
            var writer = new Xenko.Core.Shaders.Writer.Hlsl.HlslWriter
            {
                EnablePreprocessorLine = false // Allow to output links to original pdxsl via #line pragmas
            };

            writer.Visit(parsingResult.Shader);
            var shaderSourceText = writer.Text;

            if (string.IsNullOrEmpty(shaderSourceText))
            {
                log.Error($"No code generated for effect [{fullEffectName}]");
                return(new EffectBytecodeCompilerResult(null, log));
            }

            // -------------------------------------------------------
            // Save shader log
            // TODO: TEMP code to allow debugging generated shaders on Windows Desktop
#if XENKO_PLATFORM_WINDOWS_DESKTOP
            var shaderId = ObjectId.FromBytes(Encoding.UTF8.GetBytes(shaderSourceText));

            var logDir = Path.Combine(PlatformFolders.ApplicationBinaryDirectory, "log");
            if (!Directory.Exists(logDir))
            {
                Directory.CreateDirectory(logDir);
            }
            var shaderSourceFilename = Path.Combine(logDir, "shader_" + fullEffectName.Replace('.', '_') + "_" + shaderId + ".hlsl");
            lock (WriterLock) // protect write in case the same shader is created twice
            {
                // Write shader before generating to make sure that we are having a trace before compiling it (compiler may crash...etc.)
                if (!File.Exists(shaderSourceFilename))
                {
                    File.WriteAllText(shaderSourceFilename, shaderSourceText);
                }
            }
#else
            string shaderSourceFilename = null;
#endif
            // -------------------------------------------------------

            var bytecode = new EffectBytecode {
                Reflection = parsingResult.Reflection, HashSources = parsingResult.HashSources
            };

            // Select the correct backend compiler
            IShaderCompiler compiler;
            switch (effectParameters.Platform)
            {
#if XENKO_PLATFORM_WINDOWS
            case GraphicsPlatform.Direct3D11:
            case GraphicsPlatform.Direct3D12:
                compiler = new Direct3D.ShaderCompiler();
                break;
#endif
            case GraphicsPlatform.OpenGL:
            case GraphicsPlatform.OpenGLES:
            case GraphicsPlatform.Vulkan:
                // get the number of render target outputs
                var rtOutputs = 0;
                var psOutput  = parsingResult.Shader.Declarations.OfType <StructType>().FirstOrDefault(x => x.Name.Text == "PS_OUTPUT");
                if (psOutput != null)
                {
                    foreach (var rto in psOutput.Fields)
                    {
                        var sem = rto.Qualifiers.OfType <Semantic>().FirstOrDefault();
                        if (sem != null)
                        {
                            // special case SV_Target
                            if (rtOutputs == 0 && sem.Name.Text == "SV_Target")
                            {
                                rtOutputs = 1;
                                break;
                            }
                            for (var i = rtOutputs; i < 8; ++i)
                            {
                                if (sem.Name.Text == ("SV_Target" + i))
                                {
                                    rtOutputs = i + 1;
                                    break;
                                }
                            }
                        }
                    }
                }
                compiler = new OpenGL.ShaderCompiler(rtOutputs);
                break;

            default:
                throw new NotSupportedException();
            }

            var shaderStageBytecodes = new List <ShaderBytecode>();

#if XENKO_PLATFORM_WINDOWS_DESKTOP
            var stageStringBuilder = new StringBuilder();
#endif
            // if the shader (non-compute) does not have a pixel shader, we should add it for OpenGL and OpenGL ES.
            if ((effectParameters.Platform == GraphicsPlatform.OpenGL || effectParameters.Platform == GraphicsPlatform.OpenGLES) && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Pixel) && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Compute))
            {
                parsingResult.EntryPoints.Add(ShaderStage.Pixel, null);
            }

            foreach (var stageBinding in parsingResult.EntryPoints)
            {
                // Compile
                // TODO: We could compile stages in different threads to improve compiler throughput?
                var result = compiler.Compile(shaderSourceText, stageBinding.Value, stageBinding.Key, effectParameters, bytecode.Reflection, shaderSourceFilename);
                result.CopyTo(log);

                if (result.HasErrors)
                {
                    continue;
                }

                // -------------------------------------------------------
                // Append bytecode id to shader log
#if XENKO_PLATFORM_WINDOWS_DESKTOP
                stageStringBuilder.AppendLine("@G    {0} => {1}".ToFormat(stageBinding.Key, result.Bytecode.Id));
                if (result.DisassembleText != null)
                {
                    stageStringBuilder.Append(result.DisassembleText);
                }
#endif
                // -------------------------------------------------------

                shaderStageBytecodes.Add(result.Bytecode);

                // When this is a compute shader, there is no need to scan other stages
                if (stageBinding.Key == ShaderStage.Compute)
                {
                    break;
                }
            }

            // Remove unused reflection data, as it is entirely resolved at compile time.
            CleanupReflection(bytecode.Reflection);
            bytecode.Stages = shaderStageBytecodes.ToArray();

#if XENKO_PLATFORM_WINDOWS_DESKTOP
            int    shaderSourceLineOffset      = 0;
            int    shaderSourceCharacterOffset = 0;
            string outputShaderLog;
            lock (WriterLock) // protect write in case the same shader is created twice
            {
                var builder = new StringBuilder();
                builder.AppendLine("/**************************");
                builder.AppendLine("***** Compiler Parameters *****");
                builder.AppendLine("***************************");
                builder.Append("@P EffectName: ");
                builder.AppendLine(fullEffectName ?? "");
                builder.Append(compilerParameters?.ToStringPermutationsDetailed());
                builder.AppendLine("***************************");

                if (bytecode.Reflection.ConstantBuffers.Count > 0)
                {
                    builder.AppendLine("****  ConstantBuffers  ****");
                    builder.AppendLine("***************************");
                    foreach (var cBuffer in bytecode.Reflection.ConstantBuffers)
                    {
                        builder.AppendFormat("cbuffer {0} [Size: {1}]", cBuffer.Name, cBuffer.Size).AppendLine();
                        foreach (var parameter in cBuffer.Members)
                        {
                            builder.AppendFormat("@C    {0} => {1}", parameter.RawName, parameter.KeyInfo.KeyName).AppendLine();
                        }
                    }
                    builder.AppendLine("***************************");
                }

                if (bytecode.Reflection.ResourceBindings.Count > 0)
                {
                    builder.AppendLine("******  Resources    ******");
                    builder.AppendLine("***************************");
                    foreach (var resource in bytecode.Reflection.ResourceBindings)
                    {
                        builder.AppendFormat("@R    {0} => {1} [Stage: {2}, Slot: ({3}-{4})]", resource.RawName, resource.KeyInfo.KeyName, resource.Stage, resource.SlotStart, resource.SlotStart + resource.SlotCount - 1).AppendLine();
                    }
                    builder.AppendLine("***************************");
                }

                if (bytecode.HashSources.Count > 0)
                {
                    builder.AppendLine("*****     Sources     *****");
                    builder.AppendLine("***************************");
                    foreach (var hashSource in bytecode.HashSources)
                    {
                        builder.AppendFormat("@S    {0} => {1}", hashSource.Key, hashSource.Value).AppendLine();
                    }
                    builder.AppendLine("***************************");
                }

                if (bytecode.Stages.Length > 0)
                {
                    builder.AppendLine("*****     Stages      *****");
                    builder.AppendLine("***************************");
                    builder.Append(stageStringBuilder);
                    builder.AppendLine("***************************");
                }
                builder.AppendLine("*************************/");

                shaderSourceCharacterOffset = builder.Length;

                // Re-append the shader with all informations
                builder.Append(shaderSourceText);

                outputShaderLog = builder.ToString();
                File.WriteAllText(shaderSourceFilename, outputShaderLog);
            }

            // Count lines till source start
            for (int i = 0; i < shaderSourceCharacterOffset - 1;)
            {
                if (outputShaderLog[i] == '\r' && outputShaderLog[i + 1] == '\n')
                {
                    shaderSourceLineOffset++;
                    i += 2;
                }
                else
                {
                    i++;
                }
            }

            // Rewrite shader log
            Regex shaderLogReplace = new Regex(@"\.hlsl\((\d+),[0-9\-]+\):");
            foreach (var msg in log.Messages)
            {
                var match = shaderLogReplace.Match(msg.Text);
                if (match.Success)
                {
                    int line = int.Parse(match.Groups[1].Value);
                    line += shaderSourceLineOffset;

                    msg.Text = msg.Text.Remove(match.Groups[1].Index, match.Groups[1].Length)
                               .Insert(match.Groups[1].Index, line.ToString());
                }
            }
#endif

            return(new EffectBytecodeCompilerResult(bytecode, log));
        }
Beispiel #6
0
        public async Task <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters)
        {
            // Make sure we are connected
            // TODO: Handle reconnections, etc...
            var socketMessageLayer = await GetOrCreateConnection(cancellationTokenSource.Token);

            var shaderCompilerAnswer = (RemoteEffectCompilerEffectAnswer)await socketMessageLayer.SendReceiveAsync(new RemoteEffectCompilerEffectRequest
            {
                MixinTree        = mixinTree,
                EffectParameters = effectParameters,
            });

            var result = new EffectBytecodeCompilerResult(shaderCompilerAnswer.EffectBytecode, EffectBytecodeCacheLoadSource.JustCompiled);

            foreach (var message in shaderCompilerAnswer.LogMessages)
            {
                result.CompilationLog.Messages.Add(message);
            }

            result.CompilationLog.HasErrors = shaderCompilerAnswer.LogHasErrors;

            return(result);
        }
Beispiel #7
0
 /// <summary>
 /// Compiles the ShaderMixinSource into a platform bytecode.
 /// </summary>
 /// <param name="mixinTree">The mixin tree.</param>
 /// <param name="effectParameters"></param>
 /// <param name="compilerParameters"></param>
 /// <returns>The platform-dependent bytecode.</returns>
 public abstract TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters);
Beispiel #8
0
 public override TaskOrResult <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters = null)
 {
     throw new NotSupportedException("Shader Compilation is not allowed at run time on this platform.");
 }