private async Task <EffectBytecodeCompilerResult> CompileAsync(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters)
{
return(await remoteEffectCompilerClient.Compile(mixinTree, effectParameters));
}
public override TaskOrResult <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixin, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters)
{
var database = FileProvider as DatabaseFileProvider ?? throw new NotSupportedException("Using the cache requires to ContentManager.FileProvider to be valid.");
var usedParameters = compilerParameters;
var mixinObjectId = ShaderMixinObjectId.Compute(mixin, usedParameters.EffectParameters);
// Final url of the compiled bytecode
var compiledUrl = string.Format("{0}/{1}", CompiledShadersKey, mixinObjectId);
var bytecode = new KeyValuePair <EffectBytecode, EffectBytecodeCacheLoadSource>(null, EffectBytecodeCacheLoadSource.JustCompiled);
lock (bytecodes)
{
// ------------------------------------------------------------------------------------------------------------
// 1) Try to load latest bytecode
// ------------------------------------------------------------------------------------------------------------
ObjectId bytecodeId;
if (database.ContentIndexMap.TryGetValue(compiledUrl, out bytecodeId))
{
bytecode = LoadEffectBytecode(database, bytecodeId);
}
// ------------------------------------------------------------------------------------------------------------
// 2) Try to load from database cache
// ------------------------------------------------------------------------------------------------------------
if (bytecode.Key == null && database.ObjectDatabase.Exists(mixinObjectId))
{
using (var stream = database.ObjectDatabase.OpenStream(mixinObjectId))
{
// We have an existing stream, make sure the shader is compiled
var objectIdBuffer = new byte[ObjectId.HashSize];
if (stream.Read(objectIdBuffer, 0, ObjectId.HashSize) == ObjectId.HashSize)
{
var newBytecodeId = new ObjectId(objectIdBuffer);
bytecode = LoadEffectBytecode(database, newBytecodeId);
if (bytecode.Key != null)
{
// If we successfully retrieved it from cache, add it to index map so that it won't be collected and available for faster lookup
database.ContentIndexMap[compiledUrl] = newBytecodeId;
}
}
}
}
}
if (bytecode.Key != null)
{
return(new EffectBytecodeCompilerResult(bytecode.Key, bytecode.Value));
}
else if (Platform.IsWindowsDesktop == false || Compiler is NullEffectCompiler)
{
// On non Windows platform, we are expecting to have the bytecode stored directly or in database cache
var stringBuilder = new StringBuilder();
stringBuilder.AppendFormat("Unable to find compiled shaders [{0}] for mixin [{1}] with parameters [{2}]", compiledUrl, mixin, usedParameters.ToStringPermutationsDetailed());
Log.Error(stringBuilder.ToString());
throw new InvalidOperationException(stringBuilder.ToString());
}
// ------------------------------------------------------------------------------------------------------------
// 3) Compile the shader
// ------------------------------------------------------------------------------------------------------------
lock (compilingShaders)
{
Task <EffectBytecodeCompilerResult> compilingShaderTask;
if (compilingShaders.TryGetValue(mixinObjectId, out compilingShaderTask))
{
// Note: Task might still be compiling
return(compilingShaderTask);
}
// Compile the mixin in a Task
if (CompileEffectAsynchronously)
{
var compilerParametersCopy = compilerParameters != null ? new CompilerParameters(compilerParameters) : null;
var resultTask = Task.Factory.StartNew(() => CompileBytecode(mixin, effectParameters, compilerParametersCopy, mixinObjectId, database, compiledUrl), CancellationToken.None, TaskCreationOptions.None, taskSchedulerSelector != null ? taskSchedulerSelector(mixin, compilerParametersCopy.EffectParameters) : TaskScheduler.Default);
compilingShaders.Add(mixinObjectId, resultTask);
return(resultTask);
}
else
{
return(CompileBytecode(mixin, effectParameters, compilerParameters, mixinObjectId, database, compiledUrl));
}
}
}
/// <inheritdoc/>
public override TaskOrResult <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters = null)
{
return(CompileAsync(mixinTree, effectParameters));
}
private EffectBytecodeCompilerResult CompileBytecode(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters, ObjectId mixinObjectId, DatabaseFileProvider database, string compiledUrl)
{
// Open the database for writing
var log = new LoggerResult();
var effectLog = GlobalLogger.GetLogger("EffectCompilerCache");
if (CompileSynchronization.IsSet)
{
CompileSynchronization.Reset(1);
}
else
{
CompileSynchronization.AddCount();
}
// Note: this compiler is expected to not be async and directly write stuff in localLogger
var compiledShader = base.Compile(mixinTree, effectParameters, compilerParameters).WaitForResult();
compiledShader.CompilationLog.CopyTo(log);
// If there are any errors, return immediately
if (log.HasErrors)
{
lock (compilingShaders)
{
compilingShaders.Remove(mixinObjectId);
}
log.CopyTo(effectLog);
return(new EffectBytecodeCompilerResult(null, log));
}
// Compute the bytecodeId
var newBytecodeId = compiledShader.Bytecode.ComputeId();
// Check if we really need to store the bytecode
lock (bytecodes)
{
// Using custom serialization to the database to store an object with a custom id
// TODO: Check if we really need to write the bytecode everytime even if id is not changed
var memoryStream = new MemoryStream();
compiledShader.Bytecode.WriteTo(memoryStream);
// Write current cache at the end (not part of the pure bytecode, but we use this as meta info)
var writer = new BinarySerializationWriter(memoryStream);
writer.Write(CurrentCache);
memoryStream.Position = 0;
database.ObjectDatabase.Write(memoryStream, newBytecodeId, true);
database.ContentIndexMap[compiledUrl] = newBytecodeId;
// Save bytecode Id to the database cache as well
memoryStream.SetLength(0);
memoryStream.Write((byte[])newBytecodeId, 0, ObjectId.HashSize);
memoryStream.Position = 0;
database.ObjectDatabase.Write(memoryStream, mixinObjectId, true);
if (!bytecodes.ContainsKey(newBytecodeId))
{
log.Verbose($"New effect compiled #{effectCompileCount} [{mixinObjectId}] (db: {newBytecodeId})\r\n{compilerParameters?.ToStringPermutationsDetailed()}");
Interlocked.Increment(ref effectCompileCount);
// Replace or add new bytecode
bytecodes[newBytecodeId] = new KeyValuePair <EffectBytecode, EffectBytecodeCacheLoadSource>(compiledShader.Bytecode, EffectBytecodeCacheLoadSource.JustCompiled);
}
}
CompileSynchronization.Signal();
lock (compilingShaders)
{
compilingShaders.Remove(mixinObjectId);
}
log.CopyTo(effectLog);
return(compiledShader);
}
public override TaskOrResult <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters)
{
var log = new LoggerResult();
// Load D3D compiler dll
// Note: No lock, it's probably fine if it gets called from multiple threads at the same time.
if (Platform.IsWindowsDesktop && !d3dCompilerLoaded)
{
NativeLibrary.PreloadLibrary("d3dcompiler_47.dll");
d3dCompilerLoaded = true;
}
var shaderMixinSource = mixinTree;
var fullEffectName = mixinTree.Name;
// Make a copy of shaderMixinSource. Use deep clone since shaderMixinSource can be altered during compilation (e.g. macros)
var shaderMixinSourceCopy = new ShaderMixinSource();
shaderMixinSourceCopy.DeepCloneFrom(shaderMixinSource);
shaderMixinSource = shaderMixinSourceCopy;
// Generate platform-specific macros
switch (effectParameters.Platform)
{
case GraphicsPlatform.Direct3D11:
shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_DIRECT3D", 1);
shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_DIRECT3D11", 1);
break;
case GraphicsPlatform.Direct3D12:
shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_DIRECT3D", 1);
shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_DIRECT3D12", 1);
break;
case GraphicsPlatform.OpenGL:
shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_OPENGL", 1);
shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_OPENGLCORE", 1);
break;
case GraphicsPlatform.OpenGLES:
shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_OPENGL", 1);
shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_OPENGLES", 1);
break;
case GraphicsPlatform.Vulkan:
shaderMixinSource.AddMacro("XENKO_GRAPHICS_API_VULKAN", 1);
break;
default:
throw new NotSupportedException();
}
// Generate profile-specific macros
shaderMixinSource.AddMacro("XENKO_GRAPHICS_PROFILE", (int)effectParameters.Profile);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_1", (int)GraphicsProfile.Level_9_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_2", (int)GraphicsProfile.Level_9_2);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_3", (int)GraphicsProfile.Level_9_3);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_0", (int)GraphicsProfile.Level_10_0);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_1", (int)GraphicsProfile.Level_10_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_0", (int)GraphicsProfile.Level_11_0);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_1", (int)GraphicsProfile.Level_11_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_2", (int)GraphicsProfile.Level_11_2);
// In .xksl, class has been renamed to shader to avoid ambiguities with HLSL
shaderMixinSource.AddMacro("class", "shader");
var parsingResult = GetMixinParser().Parse(shaderMixinSource, shaderMixinSource.Macros.ToArray());
// Copy log from parser results to output
CopyLogs(parsingResult, log);
// Return directly if there are any errors
if (parsingResult.HasErrors)
{
return(new EffectBytecodeCompilerResult(null, log));
}
// Convert the AST to HLSL
var writer = new Xenko.Core.Shaders.Writer.Hlsl.HlslWriter
{
EnablePreprocessorLine = false // Allow to output links to original pdxsl via #line pragmas
};
writer.Visit(parsingResult.Shader);
var shaderSourceText = writer.Text;
if (string.IsNullOrEmpty(shaderSourceText))
{
log.Error($"No code generated for effect [{fullEffectName}]");
return(new EffectBytecodeCompilerResult(null, log));
}
// -------------------------------------------------------
// Save shader log
// TODO: TEMP code to allow debugging generated shaders on Windows Desktop
#if XENKO_PLATFORM_WINDOWS_DESKTOP
var shaderId = ObjectId.FromBytes(Encoding.UTF8.GetBytes(shaderSourceText));
var logDir = Path.Combine(PlatformFolders.ApplicationBinaryDirectory, "log");
if (!Directory.Exists(logDir))
{
Directory.CreateDirectory(logDir);
}
var shaderSourceFilename = Path.Combine(logDir, "shader_" + fullEffectName.Replace('.', '_') + "_" + shaderId + ".hlsl");
lock (WriterLock) // protect write in case the same shader is created twice
{
// Write shader before generating to make sure that we are having a trace before compiling it (compiler may crash...etc.)
if (!File.Exists(shaderSourceFilename))
{
File.WriteAllText(shaderSourceFilename, shaderSourceText);
}
}
#else
string shaderSourceFilename = null;
#endif
// -------------------------------------------------------
var bytecode = new EffectBytecode {
Reflection = parsingResult.Reflection, HashSources = parsingResult.HashSources
};
// Select the correct backend compiler
IShaderCompiler compiler;
switch (effectParameters.Platform)
{
#if XENKO_PLATFORM_WINDOWS
case GraphicsPlatform.Direct3D11:
case GraphicsPlatform.Direct3D12:
compiler = new Direct3D.ShaderCompiler();
break;
#endif
case GraphicsPlatform.OpenGL:
case GraphicsPlatform.OpenGLES:
case GraphicsPlatform.Vulkan:
// get the number of render target outputs
var rtOutputs = 0;
var psOutput = parsingResult.Shader.Declarations.OfType <StructType>().FirstOrDefault(x => x.Name.Text == "PS_OUTPUT");
if (psOutput != null)
{
foreach (var rto in psOutput.Fields)
{
var sem = rto.Qualifiers.OfType <Semantic>().FirstOrDefault();
if (sem != null)
{
// special case SV_Target
if (rtOutputs == 0 && sem.Name.Text == "SV_Target")
{
rtOutputs = 1;
break;
}
for (var i = rtOutputs; i < 8; ++i)
{
if (sem.Name.Text == ("SV_Target" + i))
{
rtOutputs = i + 1;
break;
}
}
}
}
}
compiler = new OpenGL.ShaderCompiler(rtOutputs);
break;
default:
throw new NotSupportedException();
}
var shaderStageBytecodes = new List <ShaderBytecode>();
#if XENKO_PLATFORM_WINDOWS_DESKTOP
var stageStringBuilder = new StringBuilder();
#endif
// if the shader (non-compute) does not have a pixel shader, we should add it for OpenGL and OpenGL ES.
if ((effectParameters.Platform == GraphicsPlatform.OpenGL || effectParameters.Platform == GraphicsPlatform.OpenGLES) && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Pixel) && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Compute))
{
parsingResult.EntryPoints.Add(ShaderStage.Pixel, null);
}
foreach (var stageBinding in parsingResult.EntryPoints)
{
// Compile
// TODO: We could compile stages in different threads to improve compiler throughput?
var result = compiler.Compile(shaderSourceText, stageBinding.Value, stageBinding.Key, effectParameters, bytecode.Reflection, shaderSourceFilename);
result.CopyTo(log);
if (result.HasErrors)
{
continue;
}
// -------------------------------------------------------
// Append bytecode id to shader log
#if XENKO_PLATFORM_WINDOWS_DESKTOP
stageStringBuilder.AppendLine("@G {0} => {1}".ToFormat(stageBinding.Key, result.Bytecode.Id));
if (result.DisassembleText != null)
{
stageStringBuilder.Append(result.DisassembleText);
}
#endif
// -------------------------------------------------------
shaderStageBytecodes.Add(result.Bytecode);
// When this is a compute shader, there is no need to scan other stages
if (stageBinding.Key == ShaderStage.Compute)
{
break;
}
}
// Remove unused reflection data, as it is entirely resolved at compile time.
CleanupReflection(bytecode.Reflection);
bytecode.Stages = shaderStageBytecodes.ToArray();
#if XENKO_PLATFORM_WINDOWS_DESKTOP
int shaderSourceLineOffset = 0;
int shaderSourceCharacterOffset = 0;
string outputShaderLog;
lock (WriterLock) // protect write in case the same shader is created twice
{
var builder = new StringBuilder();
builder.AppendLine("/**************************");
builder.AppendLine("***** Compiler Parameters *****");
builder.AppendLine("***************************");
builder.Append("@P EffectName: ");
builder.AppendLine(fullEffectName ?? "");
builder.Append(compilerParameters?.ToStringPermutationsDetailed());
builder.AppendLine("***************************");
if (bytecode.Reflection.ConstantBuffers.Count > 0)
{
builder.AppendLine("**** ConstantBuffers ****");
builder.AppendLine("***************************");
foreach (var cBuffer in bytecode.Reflection.ConstantBuffers)
{
builder.AppendFormat("cbuffer {0} [Size: {1}]", cBuffer.Name, cBuffer.Size).AppendLine();
foreach (var parameter in cBuffer.Members)
{
builder.AppendFormat("@C {0} => {1}", parameter.RawName, parameter.KeyInfo.KeyName).AppendLine();
}
}
builder.AppendLine("***************************");
}
if (bytecode.Reflection.ResourceBindings.Count > 0)
{
builder.AppendLine("****** Resources ******");
builder.AppendLine("***************************");
foreach (var resource in bytecode.Reflection.ResourceBindings)
{
builder.AppendFormat("@R {0} => {1} [Stage: {2}, Slot: ({3}-{4})]", resource.RawName, resource.KeyInfo.KeyName, resource.Stage, resource.SlotStart, resource.SlotStart + resource.SlotCount - 1).AppendLine();
}
builder.AppendLine("***************************");
}
if (bytecode.HashSources.Count > 0)
{
builder.AppendLine("***** Sources *****");
builder.AppendLine("***************************");
foreach (var hashSource in bytecode.HashSources)
{
builder.AppendFormat("@S {0} => {1}", hashSource.Key, hashSource.Value).AppendLine();
}
builder.AppendLine("***************************");
}
if (bytecode.Stages.Length > 0)
{
builder.AppendLine("***** Stages *****");
builder.AppendLine("***************************");
builder.Append(stageStringBuilder);
builder.AppendLine("***************************");
}
builder.AppendLine("*************************/");
shaderSourceCharacterOffset = builder.Length;
// Re-append the shader with all informations
builder.Append(shaderSourceText);
outputShaderLog = builder.ToString();
File.WriteAllText(shaderSourceFilename, outputShaderLog);
}
// Count lines till source start
for (int i = 0; i < shaderSourceCharacterOffset - 1;)
{
if (outputShaderLog[i] == '\r' && outputShaderLog[i + 1] == '\n')
{
shaderSourceLineOffset++;
i += 2;
}
else
{
i++;
}
}
// Rewrite shader log
Regex shaderLogReplace = new Regex(@"\.hlsl\((\d+),[0-9\-]+\):");
foreach (var msg in log.Messages)
{
var match = shaderLogReplace.Match(msg.Text);
if (match.Success)
{
int line = int.Parse(match.Groups[1].Value);
line += shaderSourceLineOffset;
msg.Text = msg.Text.Remove(match.Groups[1].Index, match.Groups[1].Length)
.Insert(match.Groups[1].Index, line.ToString());
}
}
#endif
return(new EffectBytecodeCompilerResult(bytecode, log));
}
public async Task <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters)
{
// Make sure we are connected
// TODO: Handle reconnections, etc...
var socketMessageLayer = await GetOrCreateConnection(cancellationTokenSource.Token);
var shaderCompilerAnswer = (RemoteEffectCompilerEffectAnswer)await socketMessageLayer.SendReceiveAsync(new RemoteEffectCompilerEffectRequest
{
MixinTree = mixinTree,
EffectParameters = effectParameters,
});
var result = new EffectBytecodeCompilerResult(shaderCompilerAnswer.EffectBytecode, EffectBytecodeCacheLoadSource.JustCompiled);
foreach (var message in shaderCompilerAnswer.LogMessages)
{
result.CompilationLog.Messages.Add(message);
}
result.CompilationLog.HasErrors = shaderCompilerAnswer.LogHasErrors;
return(result);
}
/// <summary> /// Compiles the ShaderMixinSource into a platform bytecode. /// </summary> /// <param name="mixinTree">The mixin tree.</param> /// <param name="effectParameters"></param> /// <param name="compilerParameters"></param> /// <returns>The platform-dependent bytecode.</returns> public abstract TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters);
public override TaskOrResult <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters = null)
{
throw new NotSupportedException("Shader Compilation is not allowed at run time on this platform.");
}