private EffectBytecodeCompilerResult CompileBytecode(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters, ObjectId mixinObjectId, DatabaseFileProvider database, string compiledUrl)
{
// Open the database for writing
var log = new LoggerResult();
var effectLog = GlobalLogger.GetLogger("EffectCompilerCache");
// Note: this compiler is expected to not be async and directly write stuff in localLogger
var compiledShader = base.Compile(mixinTree, effectParameters, compilerParameters).WaitForResult();
compiledShader.CompilationLog.CopyTo(log);
// If there are any errors, return immediately
if (log.HasErrors)
{
lock (compilingShaders)
{
compilingShaders.Remove(mixinObjectId);
}
log.CopyTo(effectLog);
return(new EffectBytecodeCompilerResult(null, log));
}
// Compute the bytecodeId
var newBytecodeId = compiledShader.Bytecode.ComputeId();
// Check if we really need to store the bytecode
lock (bytecodes)
{
// Using custom serialization to the database to store an object with a custom id
// TODO: Check if we really need to write the bytecode everytime even if id is not changed
var memoryStream = new MemoryStream();
compiledShader.Bytecode.WriteTo(memoryStream);
// Write current cache at the end (not part of the pure bytecode, but we use this as meta info)
var writer = new BinarySerializationWriter(memoryStream);
writer.Write(CurrentCache);
memoryStream.Position = 0;
database.ObjectDatabase.Write(memoryStream, newBytecodeId, true);
database.ContentIndexMap[compiledUrl] = newBytecodeId;
// Save bytecode Id to the database cache as well
memoryStream.SetLength(0);
memoryStream.Write((byte[])newBytecodeId, 0, ObjectId.HashSize);
memoryStream.Position = 0;
database.ObjectDatabase.Write(memoryStream, mixinObjectId, true);
if (!bytecodes.ContainsKey(newBytecodeId))
{
log.Verbose($"New effect compiled #{effectCompileCount} [{mixinObjectId}] (db: {newBytecodeId})\r\n{compilerParameters?.ToStringPermutationsDetailed()}");
Interlocked.Increment(ref effectCompileCount);
// Replace or add new bytecode
bytecodes[newBytecodeId] = new KeyValuePair <EffectBytecode, EffectBytecodeCacheLoadSource>(compiledShader.Bytecode, EffectBytecodeCacheLoadSource.JustCompiled);
}
}
lock (compilingShaders)
{
compilingShaders.Remove(mixinObjectId);
}
log.CopyTo(effectLog);
return(compiledShader);
}
public override TaskOrResult <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters)
{
var log = new LoggerResult();
// Load D3D compiler dll
// Note: No lock, it's probably fine if it gets called from multiple threads at the same time.
if (Platform.IsWindowsDesktop && !d3dCompilerLoaded)
{
NativeLibraryHelper.PreloadLibrary("d3dcompiler_47.dll", typeof(EffectCompiler));
d3dCompilerLoaded = true;
}
var shaderMixinSource = mixinTree;
var fullEffectName = mixinTree.Name;
// Make a copy of shaderMixinSource. Use deep clone since shaderMixinSource can be altered during compilation (e.g. macros)
var shaderMixinSourceCopy = new ShaderMixinSource();
shaderMixinSourceCopy.DeepCloneFrom(shaderMixinSource);
shaderMixinSource = shaderMixinSourceCopy;
// Generate platform-specific macros
switch (effectParameters.Platform)
{
case GraphicsPlatform.Direct3D11:
shaderMixinSource.AddMacro("STRIDE_GRAPHICS_API_DIRECT3D", 1);
shaderMixinSource.AddMacro("STRIDE_GRAPHICS_API_DIRECT3D11", 1);
break;
case GraphicsPlatform.Direct3D12:
shaderMixinSource.AddMacro("STRIDE_GRAPHICS_API_DIRECT3D", 1);
shaderMixinSource.AddMacro("STRIDE_GRAPHICS_API_DIRECT3D12", 1);
break;
case GraphicsPlatform.OpenGL:
shaderMixinSource.AddMacro("STRIDE_GRAPHICS_API_OPENGL", 1);
shaderMixinSource.AddMacro("STRIDE_GRAPHICS_API_OPENGLCORE", 1);
break;
case GraphicsPlatform.OpenGLES:
shaderMixinSource.AddMacro("STRIDE_GRAPHICS_API_OPENGL", 1);
shaderMixinSource.AddMacro("STRIDE_GRAPHICS_API_OPENGLES", 1);
break;
case GraphicsPlatform.Vulkan:
shaderMixinSource.AddMacro("STRIDE_GRAPHICS_API_VULKAN", 1);
break;
default:
throw new NotSupportedException();
}
// Generate profile-specific macros
shaderMixinSource.AddMacro("STRIDE_GRAPHICS_PROFILE", (int)effectParameters.Profile);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_1", (int)GraphicsProfile.Level_9_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_2", (int)GraphicsProfile.Level_9_2);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_3", (int)GraphicsProfile.Level_9_3);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_0", (int)GraphicsProfile.Level_10_0);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_1", (int)GraphicsProfile.Level_10_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_0", (int)GraphicsProfile.Level_11_0);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_1", (int)GraphicsProfile.Level_11_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_2", (int)GraphicsProfile.Level_11_2);
// In .sdsl, class has been renamed to shader to avoid ambiguities with HLSL
shaderMixinSource.AddMacro("class", "shader");
var parsingResult = GetMixinParser().Parse(shaderMixinSource, shaderMixinSource.Macros.ToArray());
// Copy log from parser results to output
CopyLogs(parsingResult, log);
// Return directly if there are any errors
if (parsingResult.HasErrors)
{
return(new EffectBytecodeCompilerResult(null, log));
}
// Convert the AST to HLSL
var writer = new Stride.Core.Shaders.Writer.Hlsl.HlslWriter
{
EnablePreprocessorLine = false, // Allow to output links to original pdxsl via #line pragmas
};
writer.Visit(parsingResult.Shader);
var shaderSourceText = writer.Text;
if (string.IsNullOrEmpty(shaderSourceText))
{
log.Error($"No code generated for effect [{fullEffectName}]");
return(new EffectBytecodeCompilerResult(null, log));
}
// -------------------------------------------------------
// Save shader log
// TODO: TEMP code to allow debugging generated shaders on Windows Desktop
#if STRIDE_PLATFORM_DESKTOP
var shaderId = ObjectId.FromBytes(Encoding.UTF8.GetBytes(shaderSourceText));
var logDir = Path.Combine(PlatformFolders.ApplicationBinaryDirectory, "log");
if (!Directory.Exists(logDir))
{
Directory.CreateDirectory(logDir);
}
var shaderSourceFilename = Path.Combine(logDir, "shader_" + fullEffectName.Replace('.', '_') + "_" + shaderId + ".hlsl");
lock (WriterLock) // protect write in case the same shader is created twice
{
// Write shader before generating to make sure that we are having a trace before compiling it (compiler may crash...etc.)
if (!File.Exists(shaderSourceFilename))
{
File.WriteAllText(shaderSourceFilename, shaderSourceText);
}
}
#else
string shaderSourceFilename = null;
#endif
// -------------------------------------------------------
var bytecode = new EffectBytecode {
Reflection = parsingResult.Reflection, HashSources = parsingResult.HashSources
};
// Select the correct backend compiler
IShaderCompiler compiler;
switch (effectParameters.Platform)
{
#if STRIDE_PLATFORM_DESKTOP
case GraphicsPlatform.Direct3D11:
case GraphicsPlatform.Direct3D12:
if (Platform.Type != PlatformType.Windows && Platform.Type != PlatformType.UWP)
{
throw new NotSupportedException();
}
compiler = new Direct3D.ShaderCompiler();
break;
#endif
case GraphicsPlatform.OpenGL:
case GraphicsPlatform.OpenGLES:
case GraphicsPlatform.Vulkan:
// get the number of render target outputs
var rtOutputs = 0;
var psOutput = parsingResult.Shader.Declarations.OfType <StructType>().FirstOrDefault(x => x.Name.Text == "PS_OUTPUT");
if (psOutput != null)
{
foreach (var rto in psOutput.Fields)
{
var sem = rto.Qualifiers.OfType <Semantic>().FirstOrDefault();
if (sem != null)
{
// special case SV_Target
if (rtOutputs == 0 && sem.Name.Text == "SV_Target")
{
rtOutputs = 1;
break;
}
for (var i = rtOutputs; i < 8; ++i)
{
if (sem.Name.Text == ("SV_Target" + i))
{
rtOutputs = i + 1;
break;
}
}
}
}
}
compiler = new OpenGL.ShaderCompiler(rtOutputs);
break;
default:
throw new NotSupportedException();
}
var shaderStageBytecodes = new List <ShaderBytecode>();
#if STRIDE_PLATFORM_DESKTOP
var stageStringBuilder = new StringBuilder();
#endif
// if the shader (non-compute) does not have a pixel shader, we should add it for OpenGL and OpenGL ES.
if ((effectParameters.Platform == GraphicsPlatform.OpenGL || effectParameters.Platform == GraphicsPlatform.OpenGLES) && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Pixel) && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Compute))
{
parsingResult.EntryPoints.Add(ShaderStage.Pixel, null);
}
foreach (var stageBinding in parsingResult.EntryPoints)
{
// Compile
// TODO: We could compile stages in different threads to improve compiler throughput?
var result = compiler.Compile(shaderSourceText, stageBinding.Value, stageBinding.Key, effectParameters, bytecode.Reflection, shaderSourceFilename);
result.CopyTo(log);
if (result.HasErrors)
{
continue;
}
// -------------------------------------------------------
// Append bytecode id to shader log
#if STRIDE_PLATFORM_DESKTOP
stageStringBuilder.AppendLine("@G {0} => {1}".ToFormat(stageBinding.Key, result.Bytecode.Id));
if (result.DisassembleText != null)
{
stageStringBuilder.Append(result.DisassembleText);
}
#endif
// -------------------------------------------------------
shaderStageBytecodes.Add(result.Bytecode);
// When this is a compute shader, there is no need to scan other stages
if (stageBinding.Key == ShaderStage.Compute)
{
break;
}
}
// Remove unused reflection data, as it is entirely resolved at compile time.
CleanupReflection(bytecode.Reflection);
bytecode.Stages = shaderStageBytecodes.ToArray();
#if STRIDE_PLATFORM_DESKTOP
int shaderSourceLineOffset = 0;
int shaderSourceCharacterOffset = 0;
string outputShaderLog;
lock (WriterLock) // protect write in case the same shader is created twice
{
var builder = new StringBuilder();
builder.AppendLine("/**************************");
builder.AppendLine("***** Compiler Parameters *****");
builder.AppendLine("***************************");
builder.Append("@P EffectName: ");
builder.AppendLine(fullEffectName ?? "");
builder.Append(compilerParameters?.ToStringPermutationsDetailed());
builder.AppendLine("***************************");
if (bytecode.Reflection.ConstantBuffers.Count > 0)
{
builder.AppendLine("**** ConstantBuffers ****");
builder.AppendLine("***************************");
foreach (var cBuffer in bytecode.Reflection.ConstantBuffers)
{
builder.AppendFormat("cbuffer {0} [Size: {1}]", cBuffer.Name, cBuffer.Size).AppendLine();
foreach (var parameter in cBuffer.Members)
{
builder.AppendFormat("@C {0} => {1}", parameter.RawName, parameter.KeyInfo.KeyName).AppendLine();
}
}
builder.AppendLine("***************************");
}
if (bytecode.Reflection.ResourceBindings.Count > 0)
{
builder.AppendLine("****** Resources ******");
builder.AppendLine("***************************");
foreach (var resource in bytecode.Reflection.ResourceBindings)
{
builder.AppendFormat("@R {0} => {1} [Stage: {2}, Slot: ({3}-{4})]", resource.RawName, resource.KeyInfo.KeyName, resource.Stage, resource.SlotStart, resource.SlotStart + resource.SlotCount - 1).AppendLine();
}
builder.AppendLine("***************************");
}
if (bytecode.HashSources.Count > 0)
{
builder.AppendLine("***** Sources *****");
builder.AppendLine("***************************");
foreach (var hashSource in bytecode.HashSources)
{
builder.AppendFormat("@S {0} => {1}", hashSource.Key, hashSource.Value).AppendLine();
}
builder.AppendLine("***************************");
}
if (bytecode.Stages.Length > 0)
{
builder.AppendLine("***** Stages *****");
builder.AppendLine("***************************");
builder.Append(stageStringBuilder);
builder.AppendLine("***************************");
}
builder.AppendLine("*************************/");
shaderSourceCharacterOffset = builder.Length;
// Re-append the shader with all informations
builder.Append(shaderSourceText);
outputShaderLog = builder.ToString();
File.WriteAllText(shaderSourceFilename, outputShaderLog);
}
// Count lines till source start
for (int i = 0; i < shaderSourceCharacterOffset - 1;)
{
if (outputShaderLog[i] == '\r' && outputShaderLog[i + 1] == '\n')
{
shaderSourceLineOffset++;
i += 2;
}
else
{
i++;
}
}
// Rewrite shader log
Regex shaderLogReplace = new Regex(@"\.hlsl\((\d+),[0-9\-]+\):");
foreach (var msg in log.Messages)
{
var match = shaderLogReplace.Match(msg.Text);
if (match.Success)
{
int line = int.Parse(match.Groups[1].Value);
line += shaderSourceLineOffset;
msg.Text = msg.Text.Remove(match.Groups[1].Index, match.Groups[1].Length)
.Insert(match.Groups[1].Index, line.ToString());
}
}
#endif
return(new EffectBytecodeCompilerResult(bytecode, log));
}