public EffectCompileCommand(AssetCompilerContext context, UDirectory baseUrl, string effectName, CompilerParameters compilerParameters)
{
this.context = context;
this.baseUrl = baseUrl;
this.effectName = effectName;
this.compilerParameters = compilerParameters;
}
/// <summary>
/// Initializes a new instance of the <see cref="DynamicEffectCompiler" /> class.
/// </summary>
/// <param name="services">The services.</param>
/// <param name="effectName">Name of the effect.</param>
/// <param name="asyncDynamicEffectCompiler">if set to <c>true</c> it can compile effect asynchronously.</param>
/// <exception cref="System.ArgumentNullException">services
/// or
/// effectName</exception>
public DynamicEffectCompiler(IServiceRegistry services, string effectName, int taskPriority = 0)
{
if (services == null) throw new ArgumentNullException("services");
if (effectName == null) throw new ArgumentNullException("effectName");
Services = services;
this.effectName = effectName;
this.taskPriority = taskPriority;
EffectSystem = Services.GetSafeServiceAs<EffectSystem>();
GraphicsDevice = Services.GetSafeServiceAs<IGraphicsDeviceService>().GraphicsDevice;
parameterCollections = new FastList<ParameterCollection>();
// Default behavior for fallback effect: load effect with same name but empty compiler parameters
ComputeFallbackEffect = (dynamicEffectCompiler, type, name, parameters) =>
{
ParameterCollection usedParameters;
var compilerParameters = new CompilerParameters();
// We want high priority
compilerParameters.TaskPriority = -1;
var effect = dynamicEffectCompiler.EffectSystem.LoadEffect(effectName, compilerParameters, out usedParameters).WaitForResult();
return new ComputeFallbackEffectResult(effect, usedParameters);
};
}
private async Task<EffectBytecodeCompilerResult> CompileAsync(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
{
// Make sure we are connected
// TODO: Handle reconnections, etc...
await shaderCompilerConnected;
return await shaderCompilerTarget.Compile(mixinTree, compilerParameters);
}
public CompilerResults Compile(ShaderSource shaderSource, CompilerParameters compilerParameters)
{
ShaderMixinSource mixinToCompile;
var shaderMixinGeneratorSource = shaderSource as ShaderMixinGeneratorSource;
if (shaderMixinGeneratorSource != null)
{
mixinToCompile = ShaderMixinManager.Generate(shaderMixinGeneratorSource.Name, compilerParameters);
}
else
{
mixinToCompile = shaderSource as ShaderMixinSource;
var shaderClassSource = shaderSource as ShaderClassSource;
if (shaderClassSource != null)
{
mixinToCompile = new ShaderMixinSource { Name = shaderClassSource.ClassName };
mixinToCompile.Mixins.Add(shaderClassSource);
}
if (mixinToCompile == null)
{
throw new ArgumentException("Unsupported ShaderSource type [{0}]. Supporting only ShaderMixinSource/pdxfx, ShaderClassSource", "shaderSource");
}
if (string.IsNullOrEmpty(mixinToCompile.Name))
{
throw new ArgumentException("ShaderMixinSource must have a name", "shaderSource");
}
}
// Copy global parameters to used Parameters by default, as it is used by the compiler
mixinToCompile.UsedParameters.Set(CompilerParameters.GraphicsPlatformKey, compilerParameters.Platform);
mixinToCompile.UsedParameters.Set(CompilerParameters.GraphicsProfileKey, compilerParameters.Profile);
mixinToCompile.UsedParameters.Set(CompilerParameters.DebugKey, compilerParameters.Debug);
// Compile the whole mixin tree
var compilerResults = new CompilerResults { Module = string.Format("EffectCompile [{0}]", mixinToCompile.Name) };
var bytecode = Compile(mixinToCompile, compilerParameters);
// Since bytecode.Result is a struct, we check if any of its member has been set to know if it's valid
if (bytecode.Result.CompilationLog != null || bytecode.Task != null)
{
if (bytecode.Result.CompilationLog != null)
{
bytecode.Result.CompilationLog.CopyTo(compilerResults);
}
compilerResults.Bytecode = bytecode;
compilerResults.UsedParameters = mixinToCompile.UsedParameters;
}
return compilerResults;
}
public async Task<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
{
// Make sure we are connected
// TODO: Handle reconnections, etc...
var socketMessageLayer = await GetOrCreateConnection();
var shaderCompilerAnswer = (RemoteEffectCompilerEffectAnswer)await socketMessageLayer.SendReceiveAsync(new RemoteEffectCompilerEffectRequest
{
MixinTree = mixinTree,
UsedParameters = mixinTree.UsedParameters,
});
// TODO: Get LoggerResult as well
return new EffectBytecodeCompilerResult(shaderCompilerAnswer.EffectBytecode);
}
public override TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
{
throw new NotSupportedException("Shader Compilation is not allowed at run time on this platform.");
}
/// <summary>
/// Loads the effect.
/// </summary>
/// <param name="effectName">Name of the effect.</param>
/// <param name="compilerParameters">The compiler parameters.</param>
/// <param name="usedParameters">The used parameters.</param>
/// <returns>A new instance of an effect.</returns>
/// <exception cref="System.InvalidOperationException">Could not compile shader. Need fallback.</exception>
public TaskOrResult<Effect> LoadEffect(string effectName, CompilerParameters compilerParameters, out ParameterCollection usedParameters)
{
if (effectName == null) throw new ArgumentNullException("effectName");
if (compilerParameters == null) throw new ArgumentNullException("compilerParameters");
// Get the compiled result
var compilerResult = GetCompilerResults(effectName, compilerParameters);
CheckResult(compilerResult);
// Only take the sub-effect
var bytecode = compilerResult.Bytecode;
usedParameters = compilerResult.UsedParameters;
if (bytecode.Task != null && !bytecode.Task.IsCompleted)
{
// Result was async, keep it async
return bytecode.Task.ContinueWith(x => CreateEffect(effectName, x.Result, compilerResult));
}
else
{
return CreateEffect(effectName, bytecode.WaitForResult(), compilerResult);
}
}
public override TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
{
var log = new LoggerResult();
// Load D3D compiler dll
// Note: No lock, it's probably fine if it gets called from multiple threads at the same time.
if (Platform.IsWindowsDesktop && !d3dCompilerLoaded)
{
NativeLibrary.PreloadLibrary("d3dcompiler_47.dll");
d3dCompilerLoaded = true;
}
var shaderMixinSource = mixinTree;
var fullEffectName = mixinTree.Name;
var usedParameters = mixinTree.UsedParameters;
// Make a copy of shaderMixinSource. Use deep clone since shaderMixinSource can be altered during compilation (e.g. macros)
var shaderMixinSourceCopy = new ShaderMixinSource();
shaderMixinSourceCopy.DeepCloneFrom(shaderMixinSource);
shaderMixinSource = shaderMixinSourceCopy;
// Generate platform-specific macros
var platform = usedParameters.Get(CompilerParameters.GraphicsPlatformKey);
switch (platform)
{
case GraphicsPlatform.Direct3D11:
shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_API_DIRECT3D", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_API_DIRECT3D11", 1);
break;
case GraphicsPlatform.OpenGL:
shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGL", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGLCORE", 1);
break;
case GraphicsPlatform.OpenGLES:
shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGL", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGLES", 1);
break;
default:
throw new NotSupportedException();
}
// Generate profile-specific macros
var profile = usedParameters.Get(CompilerParameters.GraphicsProfileKey);
shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_PROFILE", (int)profile);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_1", (int)GraphicsProfile.Level_9_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_2", (int)GraphicsProfile.Level_9_2);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_3", (int)GraphicsProfile.Level_9_3);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_0", (int)GraphicsProfile.Level_10_0);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_1", (int)GraphicsProfile.Level_10_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_0", (int)GraphicsProfile.Level_11_0);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_1", (int)GraphicsProfile.Level_11_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_2", (int)GraphicsProfile.Level_11_2);
var parsingResult = GetMixinParser().Parse(shaderMixinSource, shaderMixinSource.Macros.ToArray());
// Copy log from parser results to output
CopyLogs(parsingResult, log);
// Return directly if there are any errors
if (parsingResult.HasErrors)
{
return new EffectBytecodeCompilerResult(null, log);
}
// Convert the AST to HLSL
var writer = new SiliconStudio.Shaders.Writer.Hlsl.HlslWriter
{
EnablePreprocessorLine = true // Allow to output links to original pdxsl via #line pragmas
};
writer.Visit(parsingResult.Shader);
var shaderSourceText = writer.Text;
if (string.IsNullOrEmpty(shaderSourceText))
{
log.Error("No code generated for effect [{0}]", fullEffectName);
return new EffectBytecodeCompilerResult(null, log);
}
// -------------------------------------------------------
// Save shader log
// TODO: TEMP code to allow debugging generated shaders on Windows Desktop
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
var shaderId = ObjectId.FromBytes(Encoding.UTF8.GetBytes(shaderSourceText));
var logDir = "log";
if (!Directory.Exists(logDir))
{
Directory.CreateDirectory(logDir);
}
var shaderSourceFilename = Path.Combine(logDir, "shader_" + fullEffectName.Replace('.', '_') + "_" + shaderId + ".hlsl");
lock (WriterLock) // protect write in case the same shader is created twice
{
// Write shader before generating to make sure that we are having a trace before compiling it (compiler may crash...etc.)
if (!File.Exists(shaderSourceFilename))
{
File.WriteAllText(shaderSourceFilename, shaderSourceText);
}
}
#else
string shaderSourceFilename = null;
#endif
// -------------------------------------------------------
var bytecode = new EffectBytecode { Reflection = parsingResult.Reflection, HashSources = parsingResult.HashSources };
// Select the correct backend compiler
IShaderCompiler compiler;
switch (platform)
{
#if SILICONSTUDIO_PLATFORM_WINDOWS
case GraphicsPlatform.Direct3D11:
compiler = new Direct3D.ShaderCompiler();
break;
#endif
case GraphicsPlatform.OpenGL:
case GraphicsPlatform.OpenGLES:
// get the number of render target outputs
var rtOutputs = 0;
var psOutput = parsingResult.Shader.Declarations.OfType<StructType>().FirstOrDefault(x => x.Name.Text == "PS_OUTPUT");
if (psOutput != null)
{
foreach (var rto in psOutput.Fields)
{
var sem = rto.Qualifiers.OfType<Semantic>().FirstOrDefault();
if (sem != null)
{
// special case SV_Target
if (rtOutputs == 0 && sem.Name.Text == "SV_Target")
{
rtOutputs = 1;
break;
}
for (var i = rtOutputs; i < 8; ++i)
{
if (sem.Name.Text == ("SV_Target" + i))
{
rtOutputs = i + 1;
break;
}
}
}
}
}
compiler = new OpenGL.ShaderCompiler(rtOutputs);
break;
default:
throw new NotSupportedException();
}
var shaderStageBytecodes = new List<ShaderBytecode>();
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
var stageStringBuilder = new StringBuilder();
#endif
// if the shader (non-compute) does not have a pixel shader, we should add it on OpenGL ES.
if (platform == GraphicsPlatform.OpenGLES && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Pixel) && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Compute))
{
parsingResult.EntryPoints.Add(ShaderStage.Pixel, null);
}
foreach (var stageBinding in parsingResult.EntryPoints)
{
// Compile
// TODO: We could compile stages in different threads to improve compiler throughput?
var result = compiler.Compile(shaderSourceText, stageBinding.Value, stageBinding.Key, usedParameters, bytecode.Reflection, shaderSourceFilename);
result.CopyTo(log);
if (result.HasErrors)
{
continue;
}
// -------------------------------------------------------
// Append bytecode id to shader log
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
stageStringBuilder.AppendLine("@G {0} => {1}".ToFormat(stageBinding.Key, result.Bytecode.Id));
#endif
// -------------------------------------------------------
shaderStageBytecodes.Add(result.Bytecode);
// When this is a compute shader, there is no need to scan other stages
if (stageBinding.Key == ShaderStage.Compute)
break;
}
// In case of Direct3D, we can safely remove reflection data as it is entirely resolved at compile time.
if (platform == GraphicsPlatform.Direct3D11)
{
CleanupReflection(bytecode.Reflection);
}
bytecode.Stages = shaderStageBytecodes.ToArray();
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
lock (WriterLock) // protect write in case the same shader is created twice
{
var builder = new StringBuilder();
builder.AppendLine("/**************************");
builder.AppendLine("***** Used Parameters *****");
builder.AppendLine("***************************");
builder.Append("@P EffectName: ");
builder.AppendLine(fullEffectName ?? "");
builder.Append(usedParameters.ToStringDetailed());
builder.AppendLine("***************************");
if (bytecode.Reflection.ConstantBuffers.Count > 0)
{
builder.AppendLine("**** ConstantBuffers ****");
builder.AppendLine("***************************");
foreach (var cBuffer in bytecode.Reflection.ConstantBuffers)
{
builder.AppendFormat("cbuffer {0} [Stage: {1}, Size: {2}]", cBuffer.Name, cBuffer.Stage, cBuffer.Size).AppendLine();
foreach (var parameter in cBuffer.Members)
{
builder.AppendFormat("@C {0} => {1}", parameter.Param.RawName, parameter.Param.KeyName).AppendLine();
}
}
builder.AppendLine("***************************");
}
if (bytecode.Reflection.ResourceBindings.Count > 0)
{
builder.AppendLine("****** Resources ******");
builder.AppendLine("***************************");
foreach (var resource in bytecode.Reflection.ResourceBindings)
{
var parameter = resource.Param;
builder.AppendFormat("@R {0} => {1} [Stage: {2}, Slot: ({3}-{4})]", parameter.RawName, parameter.KeyName, resource.Stage, resource.SlotStart, resource.SlotStart + resource.SlotCount - 1).AppendLine();
}
builder.AppendLine("***************************");
}
if (bytecode.HashSources.Count > 0)
{
builder.AppendLine("***** Sources *****");
builder.AppendLine("***************************");
foreach (var hashSource in bytecode.HashSources)
{
builder.AppendFormat("@S {0} => {1}", hashSource.Key, hashSource.Value).AppendLine();
}
builder.AppendLine("***************************");
}
if (bytecode.Stages.Length > 0)
{
builder.AppendLine("***** Stages *****");
builder.AppendLine("***************************");
builder.Append(stageStringBuilder);
builder.AppendLine("***************************");
}
builder.AppendLine("*************************/");
// Re-append the shader with all informations
builder.Append(shaderSourceText);
File.WriteAllText(shaderSourceFilename, builder.ToString());
}
#endif
return new EffectBytecodeCompilerResult(bytecode, log);
}
public void TestMaterial()
{
var compiler = new EffectCompiler { UseFileSystem = true };
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Graphics\Shaders");
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Engine\Shaders");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Core");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Lights");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Materials");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Shadows");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\ComputeColor");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Skinning");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Shading");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Transformation");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Utils");
var compilerParameters = new CompilerParameters { Platform = GraphicsPlatform.Direct3D11 };
var layers = new MaterialBlendLayers();
layers.Add(new MaterialBlendLayer
{
BlendMap = new ComputeFloat(0.5f),
Material = AttachedReferenceManager.CreateSerializableVersion<Material>(Guid.Empty, "fake")
});
var materialAsset = new MaterialAsset
{
Attributes = new MaterialAttributes()
{
Diffuse = new MaterialDiffuseMapFeature()
{
DiffuseMap = new ComputeColor(Color4.White)
},
DiffuseModel = new MaterialDiffuseLambertModelFeature()
},
Layers = layers
};
var fakeAsset = new MaterialAsset
{
Attributes = new MaterialAttributes()
{
Diffuse = new MaterialDiffuseMapFeature()
{
DiffuseMap = new ComputeColor(Color.Blue)
},
}
};
var context = new MaterialGeneratorContext { FindAsset = reference => fakeAsset };
var result = MaterialGenerator.Generate(new MaterialDescriptor { Attributes = materialAsset.Attributes, Layers = materialAsset.Layers }, context, "TestMaterial");
compilerParameters.Set(MaterialKeys.PixelStageSurfaceShaders, result.Material.Parameters.Get(MaterialKeys.PixelStageSurfaceShaders));
var directionalLightGroup = new ShaderClassSource("LightDirectionalGroup", 1);
compilerParameters.Set(LightingKeys.DirectLightGroups, new List<ShaderSource> { directionalLightGroup });
//compilerParameters.Set(LightingKeys.CastShadows, false);
//compilerParameters.Set(MaterialParameters.HasSkinningPosition, true);
//compilerParameters.Set(MaterialParameters.HasSkinningNormal, true);
compilerParameters.Set(MaterialKeys.HasNormalMap, true);
var results = compiler.Compile(new ShaderMixinGeneratorSource("ParadoxEffectBase"), compilerParameters);
Assert.IsFalse(results.HasErrors);
}
public void TestNoClean(out CompilerResults left, out CompilerResults right)
{
// Create and mount database file system
var objDatabase = ObjectDatabase.CreateDefaultDatabase();
using (var assetIndexMap = AssetIndexMap.Load(VirtualFileSystem.ApplicationDatabaseIndexPath))
{
var database = new DatabaseFileProvider(assetIndexMap, objDatabase);
AssetManager.GetFileProvider = () => database;
foreach (var shaderName in Directory.EnumerateFiles(@"..\..\sources\shaders", "*.pdxsl"))
CopyStream(database, shaderName);
foreach (var shaderName in Directory.EnumerateFiles(@"..\..\sources\engine\SiliconStudio.Paradox.Shaders.Tests\GameAssets\Compiler", "*.pdxsl"))
CopyStream(database, shaderName);
var compiler = new EffectCompiler();
compiler.SourceDirectories.Add("assets/shaders");
var compilerCache = new EffectCompilerCache(compiler);
var compilerParameters = new CompilerParameters {Platform = GraphicsPlatform.Direct3D11};
left = compilerCache.Compile(new ShaderMixinGeneratorSource("SimpleEffect"), compilerParameters);
right = compilerCache.Compile(new ShaderMixinGeneratorSource("SimpleEffect"), compilerParameters);
}
}
public override TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixin, CompilerParameters compilerParameters)
{
var database = (FileProvider ?? AssetManager.FileProvider) as DatabaseFileProvider;
if (database == null)
{
throw new NotSupportedException("Using the cache requires to AssetManager.FileProvider to be valid.");
}
// Forward DatabaseFileProvider to actual compiler here
// Since we might be in a Task, it has to be forwarded manually (otherwise MicroThreadLocal ones wouldn't work during build)
// Note: this system might need an overhaul... (too many states?)
base.FileProvider = database;
var usedParameters = mixin.UsedParameters;
var mixinObjectId = ShaderMixinObjectId.Compute(mixin, usedParameters);
// Final url of the compiled bytecode
var compiledUrl = string.Format("{0}/{1}", CompiledShadersKey, mixinObjectId);
EffectBytecode bytecode = null;
lock (bytecodes)
{
// ------------------------------------------------------------------------------------------------------------
// 1) Try to load latest bytecode
// ------------------------------------------------------------------------------------------------------------
ObjectId bytecodeId;
if (database.AssetIndexMap.TryGetValue(compiledUrl, out bytecodeId))
{
bytecode = LoadEffectBytecode(database, bytecodeId);
}
// On non Windows platform, we are expecting to have the bytecode stored directly
if (Compiler is NullEffectCompiler && bytecode == null)
{
var stringBuilder = new StringBuilder();
stringBuilder.AppendFormat("Unable to find compiled shaders [{0}] for mixin [{1}] with parameters [{2}]", compiledUrl, mixin, usedParameters.ToStringDetailed());
Log.Error(stringBuilder.ToString());
throw new InvalidOperationException(stringBuilder.ToString());
}
// ------------------------------------------------------------------------------------------------------------
// 2) Try to load from database cache
// ------------------------------------------------------------------------------------------------------------
if (bytecode == null && database.ObjectDatabase.Exists(mixinObjectId))
{
using (var stream = database.ObjectDatabase.OpenStream(mixinObjectId))
{
// We have an existing stream, make sure the shader is compiled
var objectIdBuffer = new byte[ObjectId.HashSize];
if (stream.Read(objectIdBuffer, 0, ObjectId.HashSize) == ObjectId.HashSize)
{
var newBytecodeId = new ObjectId(objectIdBuffer);
bytecode = LoadEffectBytecode(database, newBytecodeId);
if (bytecode != null)
{
// If we successfully retrieved it from cache, add it to index map so that it won't be collected and available for faster lookup
database.AssetIndexMap[compiledUrl] = newBytecodeId;
}
}
}
}
}
if (bytecode != null)
{
return new EffectBytecodeCompilerResult(bytecode);
}
// ------------------------------------------------------------------------------------------------------------
// 3) Compile the shader
// ------------------------------------------------------------------------------------------------------------
lock (compilingShaders)
{
Task<EffectBytecodeCompilerResult> compilingShaderTask;
if (compilingShaders.TryGetValue(mixinObjectId, out compilingShaderTask))
{
// Note: Task might still be compiling
return compilingShaderTask;
}
// Compile the mixin in a Task
if (CompileEffectAsynchronously)
{
var resultTask = Task.Factory.StartNew(() => CompileBytecode(mixin, compilerParameters, mixinObjectId, database, compiledUrl, usedParameters), CancellationToken.None, TaskCreationOptions.None, taskSchedulerSelector != null ? taskSchedulerSelector(mixin, compilerParameters) : TaskScheduler.Default);
compilingShaders.Add(mixinObjectId, resultTask);
return resultTask;
}
else
{
return CompileBytecode(mixin, compilerParameters, mixinObjectId, database, compiledUrl, usedParameters);
}
}
}
private EffectBytecodeCompilerResult CompileBytecode(ShaderMixinSource mixinTree, CompilerParameters compilerParameters, ObjectId mixinObjectId, DatabaseFileProvider database, string compiledUrl, ShaderMixinParameters usedParameters)
{
// Open the database for writing
var log = new LoggerResult();
// Note: this compiler is expected to not be async and directly write stuff in localLogger
var compiledShader = base.Compile(mixinTree, compilerParameters).WaitForResult();
compiledShader.CompilationLog.CopyTo(log);
// If there are any errors, return immediately
if (log.HasErrors)
{
lock (compilingShaders)
{
compilingShaders.Remove(mixinObjectId);
}
return new EffectBytecodeCompilerResult(null, log);
}
// Compute the bytecodeId
var newBytecodeId = compiledShader.Bytecode.ComputeId();
// Check if we really need to store the bytecode
lock (bytecodes)
{
// Using custom serialization to the database to store an object with a custom id
// TODO: Check if we really need to write the bytecode everytime even if id is not changed
var memoryStream = new MemoryStream();
compiledShader.Bytecode.WriteTo(memoryStream);
memoryStream.Position = 0;
database.ObjectDatabase.Write(memoryStream, newBytecodeId, true);
database.AssetIndexMap[compiledUrl] = newBytecodeId;
// Save bytecode Id to the database cache as well
memoryStream.SetLength(0);
memoryStream.Write((byte[])newBytecodeId, 0, ObjectId.HashSize);
memoryStream.Position = 0;
database.ObjectDatabase.Write(memoryStream, mixinObjectId, true);
if (!bytecodes.ContainsKey(newBytecodeId))
{
log.Info("New effect compiled #{0} [{1}] (db: {2})\r\n{3}", effectCompileCount, mixinObjectId, newBytecodeId, usedParameters.ToStringDetailed());
Interlocked.Increment(ref effectCompileCount);
// Replace or add new bytecode
bytecodes[newBytecodeId] = compiledShader.Bytecode;
}
}
lock (compilingShaders)
{
compilingShaders.Remove(mixinObjectId);
}
return compiledShader;
}
private async Task<EffectBytecodeCompilerResult> CompileAsync(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
{
return await remoteEffectCompilerClient.Compile(mixinTree, compilerParameters);
}
private Effect GetOrCreateSelectedSpriteEffect(ref Effect effect, bool isSRgb)
{
if (effect == null)
{
var compilerParameters = new CompilerParameters { [SpriteBaseKeys.ColorIsSRgb] = isSRgb};
effect = EffectSystem.LoadEffect("SelectedSprite", compilerParameters).WaitForResult();
}
return effect;
}
/// <summary>
/// Get the shader from the database based on the parameters used for its compilation.
/// </summary>
/// <param name="effectName">Name of the effect.</param>
/// <param name="parameters">The parameters.</param>
/// <returns>The EffectBytecode if found.</returns>
protected CompilerResults GetShaderFromParameters(string effectName, CompilerParameters parameters)
{
lock (earlyCompilerCache)
{
List<CompilerResults> compilerResultsList;
if (!earlyCompilerCache.TryGetValue(effectName, out compilerResultsList))
return null;
// TODO: Optimize it so that search is not linear?
// Probably not trivial for subset testing
foreach (var compiledResults in compilerResultsList)
{
if (parameters.Contains(compiledResults.UsedParameters))
{
return compiledResults;
}
}
}
return null;
}
private CompilerResults GetCompilerResults(string effectName, CompilerParameters compilerParameters)
{
compilerParameters.Profile = GraphicsDevice.ShaderProfile.HasValue ? GraphicsDevice.ShaderProfile.Value : GraphicsDevice.Features.Profile;
#if SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGLCORE
compilerParameters.Platform = GraphicsPlatform.OpenGL;
#endif
#if SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGLES
compilerParameters.Platform = GraphicsPlatform.OpenGLES;
#endif
// Compile shader
var isPdxfx = ShaderMixinManager.Contains(effectName);
// getting the effect from the used parameters only makes sense when the source files are the same
// TODO: improve this by updating earlyCompilerCache - cache can still be relevant
CompilerResults compilerResult = null;
if (isPdxfx)
{
// perform an early test only based on the parameters
compilerResult = GetShaderFromParameters(effectName, compilerParameters);
}
if (compilerResult == null)
{
var source = isPdxfx ? new ShaderMixinGeneratorSource(effectName) : (ShaderSource)new ShaderClassSource(effectName);
compilerResult = compiler.Compile(source, compilerParameters);
var effectRequested = EffectUsed;
if (effectRequested != null)
{
effectRequested(new EffectCompileRequest(effectName, compilerResult.UsedParameters));
}
if (!compilerResult.HasErrors && isPdxfx)
{
lock (earlyCompilerCache)
{
List<CompilerResults> effectCompilerResults;
if (!earlyCompilerCache.TryGetValue(effectName, out effectCompilerResults))
{
effectCompilerResults = new List<CompilerResults>();
earlyCompilerCache.Add(effectName, effectCompilerResults);
}
// Register bytecode used parameters so that they are checked when another effect is instanced
effectCompilerResults.Add(compilerResult);
}
}
}
foreach (var message in compilerResult.Messages)
{
Log.Log(message);
}
return compilerResult;
}
public void TestStream()
{
var compiler = new EffectCompiler { UseFileSystem = true };
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Shaders.Tests\GameAssets\Compiler");
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Graphics\Shaders");
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Engine\Shaders");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Core");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Lights");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Materials");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Shadows");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\ComputeColor");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Skinning");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Shading");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Transformation");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Utils");
var compilerParameters = new CompilerParameters { Platform = GraphicsPlatform.Direct3D11 };
var results = compiler.Compile(new ShaderClassSource("TestStream"), compilerParameters);
Assert.IsFalse(results.HasErrors);
}
public void TestMixinAndComposeKeys()
{
var compiler = new EffectCompiler { UseFileSystem = true };
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Graphics\Shaders");
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Shaders.Tests\GameAssets\Mixins");
var compilerParameters = new CompilerParameters {Platform = GraphicsPlatform.Direct3D11};
var subCompute1Key = TestABC.TestParameters.UseComputeColor2.ComposeWith("SubCompute1");
var subCompute2Key = TestABC.TestParameters.UseComputeColor2.ComposeWith("SubCompute2");
var subComputesKey = TestABC.TestParameters.UseComputeColorRedirect.ComposeWith("SubComputes[0]");
compilerParameters.Set(subCompute1Key, true);
compilerParameters.Set(subComputesKey, true);
var results = compiler.Compile(new ShaderMixinGeneratorSource("test_mixin_compose_keys"), compilerParameters);
Assert.IsFalse(results.HasErrors);
var mainBytecode = results.Bytecode.WaitForResult();
Assert.IsFalse(mainBytecode.CompilationLog.HasErrors);
Assert.NotNull(mainBytecode.Bytecode.Reflection.ConstantBuffers);
Assert.AreEqual(1, mainBytecode.Bytecode.Reflection.ConstantBuffers.Count);
var cbuffer = mainBytecode.Bytecode.Reflection.ConstantBuffers[0];
Assert.NotNull(cbuffer.Members);
Assert.AreEqual(2, cbuffer.Members.Length);
// Check that ComputeColor2.Color is correctly composed for variables
var computeColorSubCompute2 = ComputeColor2Keys.Color.ComposeWith("SubCompute1");
var computeColorSubComputes = ComputeColor2Keys.Color.ComposeWith("ColorRedirect.SubComputes[0]");
var members = cbuffer.Members.Select(member => member.Param.KeyName).ToList();
Assert.IsTrue(members.Contains(computeColorSubCompute2.Name));
Assert.IsTrue(members.Contains(computeColorSubComputes.Name));
}
public static void Write(string name, CompilerParameters parameters, EffectBytecode effectData, TextWriter writer)
{
const string codeTemplate = @"//------------------------------------------------------------------------------
// <auto-generated>
// Paradox Effect Compiler File Generated:
{0}//
// Command Line: {7}
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
namespace {1}
{{
{2} class {3}
{{
{4} static readonly byte[] {5} = new byte[] {{
{6}
}};
}}
}}
";
var effectToGenerateText = new StringBuilder();
effectToGenerateText.AppendFormat("// Effect [{0}]\r\n", name);
var buffer = new MemoryStream();
effectData.WriteTo(buffer);
var bufferAsText = new StringBuilder();
var bufferArray = buffer.ToArray();
for (int i = 0; i < bufferArray.Length; i++)
{
bufferAsText.Append(bufferArray[i]).Append(", ");
if (i > 0 && (i % 64) == 0)
{
bufferAsText.AppendLine();
}
}
var classDeclaration = parameters.Get(EffectSourceCodeKeys.ClassDeclaration);
var fieldDeclaration = parameters.Get(EffectSourceCodeKeys.FieldDeclaration);
var nameSpace = parameters.Get(EffectSourceCodeKeys.Namespace);
var className = parameters.Get(EffectSourceCodeKeys.ClassName) ?? name;
var fieldName = parameters.Get(EffectSourceCodeKeys.FieldName);
var commandLine = string.Join(" ", Environment.GetCommandLineArgs());
var graphicsPlatform = parameters.Get(CompilerParameters.GraphicsPlatformKey);
string paradoxDefine = "undefined";
switch (graphicsPlatform)
{
case GraphicsPlatform.Direct3D11:
paradoxDefine = "SILICONSTUDIO_PARADOX_GRAPHICS_API_DIRECT3D11";
break;
case GraphicsPlatform.OpenGL:
paradoxDefine = "SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGLCORE";
break;
case GraphicsPlatform.OpenGLES:
paradoxDefine = "SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGLES";
break;
}
writer.WriteLine("#if {0}", paradoxDefine);
writer.Write(codeTemplate,
effectToGenerateText, // {0}
nameSpace, // {1}
classDeclaration, // {2}
className, // {3}
fieldDeclaration, // {4}
fieldName, // {5}
bufferAsText, // {6}
commandLine); // {7}
writer.WriteLine("#endif");
writer.Flush();
}
public void TestGlslESCompiler()
{
VirtualFileSystem.RemountFileSystem("/shaders", "../../../../shaders");
VirtualFileSystem.RemountFileSystem("/baseShaders", "../../../../engine/SiliconStudio.Paradox.Graphics/Shaders");
VirtualFileSystem.RemountFileSystem("/compiler", "Compiler");
var compiler = new EffectCompiler();
compiler.SourceDirectories.Add("shaders");
compiler.SourceDirectories.Add("compiler");
compiler.SourceDirectories.Add("baseShaders");
var compilerParameters = new CompilerParameters { Platform = GraphicsPlatform.OpenGLES };
var results = compiler.Compile(new ShaderMixinGeneratorSource("ToGlslEffect"), compilerParameters);
}
/// <summary> /// Compiles the ShaderMixinSource into a platform bytecode. /// </summary> /// <param name="mixinTree">The mixin tree.</param> /// <param name="compilerParameters">The compiler parameters.</param> /// <returns>The platform-dependent bytecode.</returns> public abstract TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters);
public override TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
{
return CompileAsync(mixinTree, compilerParameters);
}
private void CreateEffect(DynamicEffectInstance effectInstance, ParameterCollection passParameters)
{
var compilerParameters = new CompilerParameters();
parameterCollections.Clear(true);
if (passParameters != null)
{
parameterCollections.Add(passParameters);
}
effectInstance.FillParameterCollections(parameterCollections);
foreach (var parameterCollection in parameterCollections)
{
if (parameterCollection != null)
{
foreach (var parameter in parameterCollection.InternalValues)
{
compilerParameters.SetObject(parameter.Key, parameter.Value.Object);
}
}
}
compilerParameters.TaskPriority = taskPriority;
foreach (var parameter in GraphicsDevice.Parameters.InternalValues)
{
compilerParameters.SetObject(parameter.Key, parameter.Value.Object);
}
// Compile shader
// possible exception in LoadEffect
TaskOrResult<Effect> effect;
ParameterCollection usedParameters;
try
{
effect = EffectSystem.LoadEffect(EffectName, compilerParameters, out usedParameters);
}
catch (Exception)
{
effectInstance.HasErrors = true;
// Fallback to error effect
var fallbackEffect = ComputeFallbackEffect(this, FallbackEffectType.Error, EffectName, compilerParameters);
UpdateEffect(effectInstance, fallbackEffect.Effect, fallbackEffect.UsedParameters, passParameters);
return;
}
// Do we have an async compilation?
if (asyncEffectCompiler && effect.Task != null)
{
effectInstance.CurrentlyCompilingEffect = effect.Task;
effectInstance.CurrentlyCompilingUsedParameters = usedParameters;
if (!effectInstance.HasErrors) // If there was an error, stay in that state (we don't want to switch between reloading and error states)
{
// Fallback to default effect
var fallbackEffect = ComputeFallbackEffect(this, FallbackEffectType.Compiling, EffectName, compilerParameters);
UpdateEffect(effectInstance, fallbackEffect.Effect, fallbackEffect.UsedParameters, passParameters);
}
return;
}
var compiledEffect = effect.WaitForResult();
UpdateEffect(effectInstance, compiledEffect, usedParameters, passParameters);
// Effect has been updated
effectInstance.CurrentlyCompilingEffect = null;
effectInstance.CurrentlyCompilingUsedParameters = null;
}
public async Task<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
{
var socketContextClient = await socketContextClientTCS.Task;
var shaderCompilerAnswer = (ShaderCompilerAnswer)await socketContextClient.SendReceiveAsync(new ShaderCompilerRequest
{
MixinTree = mixinTree,
});
// TODO: Get LoggerResult as well
return new EffectBytecodeCompilerResult(shaderCompilerAnswer.EffectBytecode);
}
private CompilerParameters BuildCompilerParameters(DynamicEffectInstance effectInstance, ParameterCollection passParameters)
{
var compilerParameters = new CompilerParameters();
parameterCollections.Clear();
if (passParameters != null)
{
parameterCollections.Add(passParameters);
}
effectInstance.FillParameterCollections(ref parameterCollections);
foreach (var parameterCollection in parameterCollections)
{
if (parameterCollection != null)
{
foreach (var parameter in parameterCollection.InternalValues)
{
compilerParameters.SetObject(parameter.Key, parameter.Value.Object);
}
}
}
compilerParameters.TaskPriority = taskPriority;
foreach (var parameter in GraphicsDevice.Parameters.InternalValues)
{
compilerParameters.SetObject(parameter.Key, parameter.Value.Object);
}
return compilerParameters;
}
/// <summary>
/// Update a dynamic effect instance based on its parameters.
/// </summary>
/// <param name="effectInstance">A dynmaic effect instance</param>
/// <param name="passParameters">The pass parameters.</param>
/// <returns><c>true</c> if the effect was recomiled on the effect instance, <c>false</c> otherwise.</returns>
public bool Update(DynamicEffectInstance effectInstance, ParameterCollection passParameters)
{
bool effectChanged = false;
var currentlyCompilingEffect = effectInstance.CurrentlyCompilingEffect;
if (currentlyCompilingEffect != null)
{
if (currentlyCompilingEffect.IsCompleted)
{
if (currentlyCompilingEffect.IsFaulted)
{
var compilerParameters = new CompilerParameters();
effectInstance.CurrentlyCompilingUsedParameters.CopyTo(compilerParameters);
effectInstance.HasErrors = true;
// Fallback for errors
var fallbackEffect = ComputeFallbackEffect(this, FallbackEffectType.Error, EffectName, compilerParameters);
UpdateEffect(effectInstance, fallbackEffect.Effect, fallbackEffect.UsedParameters, passParameters);
}
else
{
effectInstance.HasErrors = false;
// Do not update effect right away: passParameters might have changed since last compilation; just try to go through a CreateEffect that will properly update the effect synchronously
// TODO: This class (and maybe whole ParameterCollection system) need a complete rethink and rewrite with newest assumptions...
//UpdateEffect(effectInstance, currentlyCompilingEffect.Result, effectInstance.CurrentlyCompilingUsedParameters, passParameters);
}
effectChanged = true;
// Effect has been updated
effectInstance.CurrentlyCompilingEffect = null;
effectInstance.CurrentlyCompilingUsedParameters = null;
}
}
if (effectChanged || // Check again, in case effect was just finished async compilation
(effectInstance.Effect == null || !EffectSystem.IsValid(effectInstance.Effect) || HasCollectionChanged(effectInstance, passParameters) || effectInstance.HasErrors))
{
if (effectInstance.HasErrors)
{
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
var currentTime = DateTime.Now;
if (currentTime < effectInstance.LastErrorCheck + ErrorCheckTimeSpan)
{
// Wait a regular interval before retrying to compile effect (i.e. every second)
return false;
}
// Update last check time
effectInstance.LastErrorCheck = currentTime;
#else
// Other platforms: never try to recompile failed effects for now
return false;
#endif
}
CreateEffect(effectInstance, passParameters);
effectChanged = true;
}
return effectChanged;
}
/// <summary>
/// Loads the effect.
/// </summary>
/// <param name="effectName">Name of the effect.</param>
/// <param name="compilerParameters">The compiler parameters.</param>
/// <returns>A new instance of an effect.</returns>
/// <exception cref="System.InvalidOperationException">Could not compile shader. Need fallback.</exception>
public TaskOrResult<Effect> LoadEffect(string effectName, CompilerParameters compilerParameters)
{
ParameterCollection usedParameters;
return LoadEffect(effectName, compilerParameters, out usedParameters);
}
/// <summary>
/// Creates an effect.
/// </summary>
/// <param name="effectSystem">The effect system.</param>
/// <param name="effectName">Name of the effect.</param>
/// <returns>A new instance of an effect.</returns>
public static TaskOrResult<Effect> LoadEffect(this EffectSystem effectSystem, string effectName)
{
var compilerParameters = new CompilerParameters();
return effectSystem.LoadEffect(effectName, compilerParameters);
}
private void SwitchFallbackEffect(FallbackEffectType fallbackEffectType, DynamicEffectInstance effectInstance, ParameterCollection passParameters, CompilerParameters compilerParameters)
{
// Fallback for errors
effectInstance.HasErrors = true;
var fallbackEffect = ComputeFallbackEffect(this, fallbackEffectType, EffectName, compilerParameters);
UpdateEffect(effectInstance, fallbackEffect.Effect, fallbackEffect.UsedParameters, passParameters);
}