public EffectCompileCommand(AssetCompilerContext context, UDirectory baseUrl, string effectName, CompilerParameters compilerParameters, Package package)
{
this.context = context;
this.baseUrl = baseUrl;
this.effectName = effectName;
this.compilerParameters = compilerParameters;
this.package = package;
}
/// <summary>
/// Computes a hash <see cref="ObjectId"/> for the specified effect and compiler parameters.
/// </summary>
/// <param name="effectName">Name of the effect.</param>
/// <param name="compilerParameters">The compiler parameters.</param>
/// <returns>
/// EffectObjectIds.
/// </returns>
public static ObjectId Compute(string effectName, CompilerParameters compilerParameters)
{
lock (generatorLock)
{
if (generator == null)
{
generator = new ShaderMixinObjectId();
}
return generator.ComputeInternal(effectName, compilerParameters);
}
}
/// <summary>
/// Computes a hash <see cref="ObjectId"/> for the specified mixin.
/// </summary>
/// <param name="mixin">The mixin.</param>
/// <param name="mixinParameters">The mixin parameters.</param>
/// <returns>EffectObjectIds.</returns>
public static ObjectId Compute(ShaderMixinSource mixin, CompilerParameters compilerParameters)
{
lock (generatorLock)
{
if (generator == null)
{
generator = new ShaderMixinObjectId();
}
return generator.ComputeInternal(mixin, compilerParameters);
}
}
public CompilerResults Compile(ShaderSource shaderSource, CompilerParameters compilerParameters)
{
ShaderMixinSource mixinToCompile;
var shaderMixinGeneratorSource = shaderSource as ShaderMixinGeneratorSource;
if (shaderMixinGeneratorSource != null)
{
mixinToCompile = ShaderMixinManager.Generate(shaderMixinGeneratorSource.Name, compilerParameters);
}
else
{
mixinToCompile = shaderSource as ShaderMixinSource;
var shaderClassSource = shaderSource as ShaderClassSource;
if (shaderClassSource != null)
{
mixinToCompile = new ShaderMixinSource { Name = shaderClassSource.ClassName };
mixinToCompile.Mixins.Add(shaderClassSource);
}
if (mixinToCompile == null)
{
throw new ArgumentException("Unsupported ShaderSource type [{0}]. Supporting only ShaderMixinSource/xkfx, ShaderClassSource", "shaderSource");
}
if (string.IsNullOrEmpty(mixinToCompile.Name))
{
throw new ArgumentException("ShaderMixinSource must have a name", "shaderSource");
}
}
// Copy global parameters to used Parameters by default, as it is used by the compiler
mixinToCompile.UsedParameters.Set(CompilerParameters.GraphicsPlatformKey, compilerParameters.Platform);
mixinToCompile.UsedParameters.Set(CompilerParameters.GraphicsProfileKey, compilerParameters.Profile);
mixinToCompile.UsedParameters.Set(CompilerParameters.DebugKey, compilerParameters.Debug);
// Compile the whole mixin tree
var compilerResults = new CompilerResults { Module = string.Format("EffectCompile [{0}]", mixinToCompile.Name) };
var bytecode = Compile(mixinToCompile, compilerParameters);
// Since bytecode.Result is a struct, we check if any of its member has been set to know if it's valid
if (bytecode.Result.CompilationLog != null || bytecode.Task != null)
{
if (bytecode.Result.CompilationLog != null)
{
bytecode.Result.CompilationLog.CopyTo(compilerResults);
}
compilerResults.Bytecode = bytecode;
compilerResults.UsedParameters = mixinToCompile.UsedParameters;
}
return compilerResults;
}
public void Init()
{
// Create and mount database file system
var objDatabase = ObjectDatabase.CreateDefaultDatabase();
var databaseFileProvider = new DatabaseFileProvider(objDatabase);
ContentManager.GetFileProvider = () => databaseFileProvider;
Compiler = new EffectCompiler();
Compiler.SourceDirectories.Add("shaders");
MixinParameters = new CompilerParameters();
MixinParameters.EffectParameters.Platform = GraphicsPlatform.Direct3D11;
MixinParameters.EffectParameters.Profile = GraphicsProfile.Level_11_0;
ResultLogger = new LoggerResult();
}
public async Task<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
{
// Make sure we are connected
// TODO: Handle reconnections, etc...
var socketMessageLayer = await GetOrCreateConnection();
var shaderCompilerAnswer = (RemoteEffectCompilerEffectAnswer)await socketMessageLayer.SendReceiveAsync(new RemoteEffectCompilerEffectRequest
{
MixinTree = mixinTree,
UsedParameters = mixinTree.UsedParameters,
});
// TODO: Get LoggerResult as well
return new EffectBytecodeCompilerResult(shaderCompilerAnswer.EffectBytecode);
}
protected override void ChooseEffect(GraphicsDevice graphicsDevice)
{
// TODO: Free previous descriptor sets and layouts?
// Looks like the effect changed, it needs a recompilation
var compilerParameters = new CompilerParameters();
foreach (var effectParameterKey in Parameters.ParameterKeyInfos)
{
if (effectParameterKey.Key.Type == ParameterKeyType.Permutation)
{
// TODO GRAPHICS REFACTOR avoid direct access, esp. since permutation values might be separated from Objects at some point
compilerParameters.SetObject(effectParameterKey.Key, Parameters.ObjectValues[effectParameterKey.BindingSlot]);
}
}
effect = effectSystem.LoadEffect(effectName, compilerParameters).WaitForResult();
}
public void Compile()
{
Console.WriteLine(@"Inside Thread");
var parameters = new CompilerParameters();
parameters.EffectParameters.Platform = GraphicsPlatform.Direct3D11;
parameters.EffectParameters.Profile = GraphicsProfile.Level_11_0;
var mixinTree = new ShaderMixinSource() { Name = "TestParallelMix" };
var result = effectCompiler.Compile(mixinTree, parameters.EffectParameters, parameters).WaitForResult();
Assert.IsFalse(result.CompilationLog.HasErrors);
Assert.IsNotNull(result);
Console.WriteLine(@"Thread end");
}
private unsafe ObjectId ComputeInternal(ShaderMixinSource mixin, CompilerParameters compilerParameters)
{
// Write to memory stream
memStream.Position = 0;
writer.Write(EffectBytecode.MagicHeader); // Write the effect bytecode magic header
writer.Write(mixin);
writer.Write(compilerParameters.EffectParameters.Platform);
writer.Write(compilerParameters.EffectParameters.Profile);
writer.Write(compilerParameters.EffectParameters.Debug);
writer.Write(compilerParameters.EffectParameters.OptimizationLevel);
// Compute hash
objectIdBuilder.Reset();
objectIdBuilder.Write((byte*)buffer, (int)memStream.Position);
return objectIdBuilder.ComputeHash();
}
public void TestShaderCompilation()
{
var generics = new string[3];
generics[0] = "Texturing.Texture1";
generics[1] = "TEXCOORD0";
generics[2] = "float4(2.0,1,1,1)";
var compilerParameters = new CompilerParameters();
compilerParameters.Set(EffectSourceCodeKeys.Enable, true);
compilerParameters.EffectParameters.Profile = GraphicsProfile.Level_11_0;
var mixinSource = new ShaderMixinSource { Name = "TestShaderCompilationGenericClass" };
mixinSource.Mixins.Add(new ShaderClassSource("GenericClass2", generics));
var log = new CompilerResults();
var compiler = new EffectCompiler();
compiler.SourceDirectories.Add("shaders");
var effectByteCode = compiler.Compile(mixinSource, compilerParameters.EffectParameters, compilerParameters);
}
/// <summary>
/// Initializes a new instance of the <see cref="DynamicEffectCompiler" /> class.
/// </summary>
/// <param name="services">The services.</param>
/// <param name="effectName">Name of the effect.</param>
/// <param name="taskPriority">The task priority.</param>
/// <exception cref="System.ArgumentNullException">services
/// or
/// effectName</exception>
public DynamicEffectCompiler(IServiceRegistry services, string effectName, int taskPriority = 0)
{
if (services == null) throw new ArgumentNullException("services");
if (effectName == null) throw new ArgumentNullException("effectName");
Services = services;
this.effectName = effectName;
this.taskPriority = taskPriority;
EffectSystem = Services.GetSafeServiceAs<EffectSystem>();
GraphicsDevice = Services.GetSafeServiceAs<IGraphicsDeviceService>().GraphicsDevice;
parameterCollections = new FastListStruct<ParameterCollection>(8);
// Default behavior for fallback effect: load effect with same name but empty compiler parameters
ComputeFallbackEffect = (dynamicEffectCompiler, type, name, parameters) =>
{
ParameterCollection usedParameters;
var compilerParameters = new CompilerParameters { TaskPriority = -1 };
// We want high priority
var effect = dynamicEffectCompiler.EffectSystem.LoadEffect(effectName, compilerParameters, out usedParameters).WaitForResult();
return new ComputeFallbackEffectResult(effect, usedParameters);
};
}
private Effect GetOrCreateSelectedSpriteEffect(ref Effect effect, bool isSRgb)
{
if (effect == null)
{
var compilerParameters = new CompilerParameters { [SpriteBaseKeys.ColorIsSRgb] = isSRgb};
effect = EffectSystem.LoadEffect("SelectedSprite", compilerParameters).WaitForResult();
}
return effect;
}
/// <param name="context"></param>
/// <inheritdoc/>
public override void PrepareEffectPermutations(RenderDrawContext context)
{
base.PrepareEffectPermutations(context);
// TODO: Temporary until we have a better system for handling permutations
var renderEffects = RenderData.GetData(RenderEffectKey);
int effectSlotCount = EffectPermutationSlotCount;
Dispatcher.ForEach(RenderSystem.Views, view =>
{
var viewFeature = view.Features[Index];
Dispatcher.ForEach(viewFeature.RenderNodes, renderNodeReference =>
{
var renderNode = this.GetRenderNode(renderNodeReference);
var renderObject = renderNode.RenderObject;
// Get RenderEffect
var staticObjectNode = renderObject.StaticObjectNode;
var staticEffectObjectNode = staticObjectNode * effectSlotCount + effectSlots[renderNode.RenderStage.Index].Index;
var renderEffect = renderEffects[staticEffectObjectNode];
var effectSelector = renderObject.ActiveRenderStages[renderNode.RenderStage.Index].EffectSelector;
// Create it (first time) or regenerate it if effect changed
if (renderEffect == null || effectSelector != renderEffect.EffectSelector)
{
renderEffect = new RenderEffect(renderObject.ActiveRenderStages[renderNode.RenderStage.Index].EffectSelector);
renderEffects[staticEffectObjectNode] = renderEffect;
}
// Is it the first time this frame that we check this RenderEffect?
if (renderEffect.MarkAsUsed(RenderSystem))
{
renderEffect.EffectValidator.BeginEffectValidation();
}
});
});
// Step1: Perform permutations
PrepareEffectPermutationsImpl(context);
// Step2: Compile effects
Dispatcher.ForEach(RenderObjects, renderObject =>
{
//var renderObject = RenderObjects[index];
var staticObjectNode = renderObject.StaticObjectNode;
for (int i = 0; i < effectSlotCount; ++i)
{
var staticEffectObjectNode = staticObjectNode * effectSlotCount + i;
var renderEffect = renderEffects[staticEffectObjectNode];
// Skip if not used
if (renderEffect == null)
continue;
// Skip reflection update unless a state change requires it
renderEffect.IsReflectionUpdateRequired = false;
if (!renderEffect.IsUsedDuringThisFrame(RenderSystem))
continue;
// Skip if nothing changed
if (renderEffect.EffectValidator.ShouldSkip)
{
// Reset pending effect, as it is now obsolete anyway
renderEffect.Effect = null;
renderEffect.State = RenderEffectState.Skip;
}
else if (renderEffect.EffectValidator.EndEffectValidation() && (renderEffect.Effect == null || !renderEffect.Effect.SourceChanged))
{
InvalidateEffectPermutation(renderObject, renderEffect);
// Still, let's check if there is a pending effect compiling
var pendingEffect = renderEffect.PendingEffect;
if (pendingEffect == null || !pendingEffect.IsCompleted)
continue;
renderEffect.ClearFallbackParameters();
if (pendingEffect.IsFaulted)
{
renderEffect.State = RenderEffectState.Error;
renderEffect.Effect = ComputeFallbackEffect?.Invoke(renderObject, renderEffect, RenderEffectState.Error);
}
else
{
renderEffect.State = RenderEffectState.Normal;
renderEffect.Effect = pendingEffect.Result;
}
renderEffect.PendingEffect = null;
}
else
{
// Reset pending effect, as it is now obsolete anyway
renderEffect.PendingEffect = null;
renderEffect.State = RenderEffectState.Normal;
// CompilerParameters are ThreadStatic
if (staticCompilerParameters == null)
staticCompilerParameters = new CompilerParameters();
foreach (var effectValue in renderEffect.EffectValidator.EffectValues)
{
staticCompilerParameters.SetObject(effectValue.Key, effectValue.Value);
}
var asyncEffect = RenderSystem.EffectSystem.LoadEffect(renderEffect.EffectSelector.EffectName, staticCompilerParameters);
staticCompilerParameters.Clear();
renderEffect.Effect = asyncEffect.Result;
if (renderEffect.Effect == null)
{
// Effect still compiling, let's find if there is a fallback
renderEffect.ClearFallbackParameters();
renderEffect.Effect = ComputeFallbackEffect?.Invoke(renderObject, renderEffect, RenderEffectState.Compiling);
renderEffect.PendingEffect = asyncEffect.Task;
renderEffect.State = RenderEffectState.Compiling;
}
}
renderEffect.IsReflectionUpdateRequired = true;
}
});
// Step3: Uupdate reflection infos (offset, etc...)
foreach (var renderObject in RenderObjects)
{
var staticObjectNode = renderObject.StaticObjectNode;
for (int i = 0; i < effectSlotCount; ++i)
{
var staticEffectObjectNode = staticObjectNode * effectSlotCount + i;
var renderEffect = renderEffects[staticEffectObjectNode];
// Skip if not used
if (renderEffect == null || !renderEffect.IsReflectionUpdateRequired)
continue;
var effect = renderEffect.Effect;
if (effect == null && renderEffect.State == RenderEffectState.Compiling)
{
// Need to wait for completion
renderEffect.Effect = effect = renderEffect.PendingEffect.Result;
renderEffect.State = RenderEffectState.Normal;
}
var effectHashCode = effect != null ? (uint)effect.GetHashCode() : 0;
// Effect is last 16 bits
renderObject.StateSortKey = (renderObject.StateSortKey & 0xFFFF0000) | (effectHashCode & 0x0000FFFF);
if (effect != null)
{
RenderEffectReflection renderEffectReflection;
if (!InstantiatedEffects.TryGetValue(effect, out renderEffectReflection))
{
renderEffectReflection = new RenderEffectReflection();
// Build root signature automatically from reflection
renderEffectReflection.DescriptorReflection = EffectDescriptorSetReflection.New(RenderSystem.GraphicsDevice, effect.Bytecode, effectDescriptorSetSlots, "PerFrame");
renderEffectReflection.ResourceGroupDescriptions = new ResourceGroupDescription[renderEffectReflection.DescriptorReflection.Layouts.Count];
// Compute ResourceGroup hashes
for (int index = 0; index < renderEffectReflection.DescriptorReflection.Layouts.Count; index++)
{
var descriptorSet = renderEffectReflection.DescriptorReflection.Layouts[index];
if (descriptorSet.Layout == null)
continue;
var constantBufferReflection = effect.Bytecode.Reflection.ConstantBuffers.FirstOrDefault(x => x.Name == descriptorSet.Name);
renderEffectReflection.ResourceGroupDescriptions[index] = new ResourceGroupDescription(descriptorSet.Layout, constantBufferReflection);
}
renderEffectReflection.RootSignature = RootSignature.New(RenderSystem.GraphicsDevice, renderEffectReflection.DescriptorReflection);
renderEffectReflection.BufferUploader.Compile(RenderSystem.GraphicsDevice, renderEffectReflection.DescriptorReflection, effect.Bytecode);
// Prepare well-known descriptor set layouts
renderEffectReflection.PerDrawLayout = CreateDrawResourceGroupLayout(renderEffectReflection.ResourceGroupDescriptions[perDrawDescriptorSetSlot.Index], renderEffect.State);
renderEffectReflection.PerFrameLayout = CreateFrameResourceGroupLayout(renderEffectReflection.ResourceGroupDescriptions[perFrameDescriptorSetSlot.Index], renderEffect.State);
renderEffectReflection.PerViewLayout = CreateViewResourceGroupLayout(renderEffectReflection.ResourceGroupDescriptions[perViewDescriptorSetSlot.Index], renderEffect.State);
InstantiatedEffects.Add(effect, renderEffectReflection);
// Notify a new effect has been compiled
EffectCompiled?.Invoke(RenderSystem, effect, renderEffectReflection);
}
// Setup fallback parameters
if (renderEffect.State != RenderEffectState.Normal && renderEffectReflection.FallbackUpdaterLayout == null)
{
// Process all "non standard" layouts
var layoutMapping = new int[renderEffectReflection.DescriptorReflection.Layouts.Count - 3];
var layouts = new DescriptorSetLayoutBuilder[renderEffectReflection.DescriptorReflection.Layouts.Count - 3];
int layoutMappingIndex = 0;
for (int index = 0; index < renderEffectReflection.DescriptorReflection.Layouts.Count; index++)
{
var layout = renderEffectReflection.DescriptorReflection.Layouts[index];
// Skip well-known layouts (already handled)
if (layout.Name == "PerDraw" || layout.Name == "PerFrame" || layout.Name == "PerView")
continue;
layouts[layoutMappingIndex] = layout.Layout;
layoutMapping[layoutMappingIndex++] = index;
}
renderEffectReflection.FallbackUpdaterLayout = new EffectParameterUpdaterLayout(RenderSystem.GraphicsDevice, effect, layouts);
renderEffectReflection.FallbackResourceGroupMapping = layoutMapping;
}
// Update effect
renderEffect.Effect = effect;
renderEffect.Reflection = renderEffectReflection;
// Invalidate pipeline state (new effect)
renderEffect.PipelineState = null;
renderEffects[staticEffectObjectNode] = renderEffect;
}
else
{
renderEffect.Reflection = RenderEffectReflection.Empty;
renderEffect.PipelineState = null;
}
}
}
}
private CompilerResults GetCompilerResults(string effectName, CompilerParameters compilerParameters)
{
// Compile shader
var isXkfx = ShaderMixinManager.Contains(effectName);
// getting the effect from the used parameters only makes sense when the source files are the same
// TODO: improve this by updating earlyCompilerCache - cache can still be relevant
CompilerResults compilerResult = null;
if (isXkfx)
{
// perform an early test only based on the parameters
compilerResult = GetShaderFromParameters(effectName, compilerParameters);
}
if (compilerResult == null)
{
var source = isXkfx ? new ShaderMixinGeneratorSource(effectName) : (ShaderSource)new ShaderClassSource(effectName);
compilerResult = compiler.Compile(source, compilerParameters);
EffectUsed?.Invoke(new EffectCompileRequest(effectName, new CompilerParameters(compilerParameters)), compilerResult);
if (!compilerResult.HasErrors && isXkfx)
{
lock (earlyCompilerCache)
{
List<CompilerResults> effectCompilerResults;
if (!earlyCompilerCache.TryGetValue(effectName, out effectCompilerResults))
{
effectCompilerResults = new List<CompilerResults>();
earlyCompilerCache.Add(effectName, effectCompilerResults);
}
// Register bytecode used parameters so that they are checked when another effect is instanced
effectCompilerResults.Add(compilerResult);
}
}
}
foreach (var message in compilerResult.Messages)
{
Log.Log(message);
}
return compilerResult;
}
/// <summary>
/// Get the shader from the database based on the parameters used for its compilation.
/// </summary>
/// <param name="effectName">Name of the effect.</param>
/// <param name="parameters">The parameters.</param>
/// <returns>The EffectBytecode if found.</returns>
protected CompilerResults GetShaderFromParameters(string effectName, CompilerParameters parameters)
{
lock (earlyCompilerCache)
{
List<CompilerResults> compilerResultsList;
if (!earlyCompilerCache.TryGetValue(effectName, out compilerResultsList))
return null;
// Compiler Parameters are supposed to be created in the same order every time, so we just check if they were created in the same order (ParameterKeyInfos) with same values (ObjectValues)
// TODO GRAPHICS REFACTOR we could probably compute a hash for faster lookup
foreach (var compiledResults in compilerResultsList)
{
var compiledParameters = compiledResults.SourceParameters;
var compiledParameterKeyInfos = compiledParameters.ParameterKeyInfos;
var parameterKeyInfos = parameters.ParameterKeyInfos;
// Early check
if (parameterKeyInfos.Count != compiledParameterKeyInfos.Count)
continue;
for (int index = 0; index < parameterKeyInfos.Count; ++index)
{
var parameterKeyInfo = parameterKeyInfos[index];
var compiledParameterKeyInfo = compiledParameterKeyInfos[index];
if (parameterKeyInfo != compiledParameterKeyInfo)
goto different;
// Should not happen in practice (CompilerParameters should only consist of permutation values)
if (parameterKeyInfo.Key.Type != ParameterKeyType.Permutation)
continue;
for (int i = 0; i < parameterKeyInfo.Count; ++i)
{
var object1 = parameters.ObjectValues[parameterKeyInfo.BindingSlot + i];
var object2 = compiledParameters.ObjectValues[compiledParameterKeyInfo.BindingSlot + i];
if (object1 == null && object2 == null)
continue;
if ((object1 == null && object2 != null) || (object2 == null && object1 != null))
goto different;
if (!object1.Equals(object2))
goto different;
}
}
return compiledResults;
different:
;
}
}
return null;
}
private EffectBytecodeCompilerResult CompileBytecode(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters, ObjectId mixinObjectId, DatabaseFileProvider database, string compiledUrl)
{
// Open the database for writing
var log = new LoggerResult();
// Note: this compiler is expected to not be async and directly write stuff in localLogger
var compiledShader = base.Compile(mixinTree, effectParameters, compilerParameters).WaitForResult();
compiledShader.CompilationLog.CopyTo(log);
// If there are any errors, return immediately
if (log.HasErrors)
{
lock (compilingShaders)
{
compilingShaders.Remove(mixinObjectId);
}
return new EffectBytecodeCompilerResult(null, log);
}
// Compute the bytecodeId
var newBytecodeId = compiledShader.Bytecode.ComputeId();
// Check if we really need to store the bytecode
lock (bytecodes)
{
// Using custom serialization to the database to store an object with a custom id
// TODO: Check if we really need to write the bytecode everytime even if id is not changed
var memoryStream = new MemoryStream();
compiledShader.Bytecode.WriteTo(memoryStream);
memoryStream.Position = 0;
database.ObjectDatabase.Write(memoryStream, newBytecodeId, true);
database.AssetIndexMap[compiledUrl] = newBytecodeId;
// Save bytecode Id to the database cache as well
memoryStream.SetLength(0);
memoryStream.Write((byte[])newBytecodeId, 0, ObjectId.HashSize);
memoryStream.Position = 0;
database.ObjectDatabase.Write(memoryStream, mixinObjectId, true);
if (!bytecodes.ContainsKey(newBytecodeId))
{
log.Verbose("New effect compiled #{0} [{1}] (db: {2})\r\n{3}", effectCompileCount, mixinObjectId, newBytecodeId, compilerParameters?.ToStringPermutationsDetailed());
Interlocked.Increment(ref effectCompileCount);
// Replace or add new bytecode
bytecodes[newBytecodeId] = compiledShader.Bytecode;
}
}
lock (compilingShaders)
{
compilingShaders.Remove(mixinObjectId);
}
return compiledShader;
}
/// <summary>
/// Loads the effect.
/// </summary>
/// <param name="effectName">Name of the effect.</param>
/// <param name="compilerParameters">The compiler parameters.</param>
/// <param name="usedParameters">The used parameters.</param>
/// <returns>A new instance of an effect.</returns>
/// <exception cref="System.InvalidOperationException">Could not compile shader. Need fallback.</exception>
public TaskOrResult<Effect> LoadEffect(string effectName, CompilerParameters compilerParameters)
{
if (effectName == null) throw new ArgumentNullException("effectName");
if (compilerParameters == null) throw new ArgumentNullException("compilerParameters");
// Setup compilation parameters
// GraphicsDevice might have been not valid until this point, which is why we compute platform and profile only at this point
compilerParameters.EffectParameters.Platform = GraphicsDevice.Platform;
compilerParameters.EffectParameters.Profile = GraphicsDevice.ShaderProfile ?? GraphicsDevice.Features.RequestedProfile;
// Copy optimization/debug levels
compilerParameters.EffectParameters.OptimizationLevel = effectCompilerParameters.OptimizationLevel;
compilerParameters.EffectParameters.Debug = effectCompilerParameters.Debug;
// Get the compiled result
var compilerResult = GetCompilerResults(effectName, compilerParameters);
CheckResult(compilerResult);
// Only take the sub-effect
var bytecode = compilerResult.Bytecode;
if (bytecode.Task != null && !bytecode.Task.IsCompleted)
{
// Result was async, keep it async
// NOTE: There was some hangs when doing ContinueWith() (note: it might switch from EffectPriorityScheduler to TaskScheduler.Default, maybe something doesn't work well in this case?)
// it seems that TaskContinuationOptions.ExecuteSynchronously is helping in this case (also it will force continuation to execute right away on the thread pool, which is probably better)
// Not sure if the probably totally disappeared (esp. if something does a ContinueWith() externally on that) -- might need further investigation.
var result = bytecode.Task.ContinueWith(
x => CreateEffect(effectName, x.Result, compilerResult),
TaskContinuationOptions.ExecuteSynchronously);
return result;
}
else
{
return CreateEffect(effectName, bytecode.WaitForResult(), compilerResult);
}
}
public void TestGlslESCompiler()
{
VirtualFileSystem.RemountFileSystem("/shaders", "../../../../shaders");
VirtualFileSystem.RemountFileSystem("/baseShaders", "../../../../engine/SiliconStudio.Xenko.Graphics/Shaders");
VirtualFileSystem.RemountFileSystem("/compiler", "Compiler");
var compiler = new EffectCompiler();
compiler.SourceDirectories.Add("shaders");
compiler.SourceDirectories.Add("compiler");
compiler.SourceDirectories.Add("baseShaders");
var compilerParameters = new CompilerParameters { Platform = GraphicsPlatform.OpenGLES };
var results = compiler.Compile(new ShaderMixinGeneratorSource("ToGlslEffect"), compilerParameters);
}
private CompilerParameters BuildCompilerParameters(DynamicEffectInstance effectInstance, ParameterCollection passParameters)
{
var compilerParameters = new CompilerParameters();
parameterCollections.Clear();
if (passParameters != null)
{
parameterCollections.Add(passParameters);
}
effectInstance.FillParameterCollections(ref parameterCollections);
foreach (var parameterCollection in parameterCollections)
{
if (parameterCollection != null)
{
foreach (var parameter in parameterCollection.InternalValues)
{
compilerParameters.SetObject(parameter.Key, parameter.Value.Object);
}
}
}
compilerParameters.TaskPriority = taskPriority;
foreach (var parameter in GraphicsDevice.Parameters.InternalValues)
{
compilerParameters.SetObject(parameter.Key, parameter.Value.Object);
}
return compilerParameters;
}
public override TaskOrResult <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters = null)
{
throw new NotSupportedException("Shader Compilation is not allowed at run time on this platform.");
}
public void TestNoClean(out CompilerResults left, out CompilerResults right)
{
// Create and mount database file system
var objDatabase = ObjectDatabase.CreateDefaultDatabase();
using (var assetIndexMap = AssetIndexMap.Load(VirtualFileSystem.ApplicationDatabaseIndexPath))
{
var database = new DatabaseFileProvider(assetIndexMap, objDatabase);
AssetManager.GetFileProvider = () => database;
foreach (var shaderName in Directory.EnumerateFiles(@"..\..\sources\shaders", "*.xksl"))
CopyStream(database, shaderName);
foreach (var shaderName in Directory.EnumerateFiles(@"..\..\sources\engine\SiliconStudio.Xenko.Shaders.Tests\GameAssets\Compiler", "*.xksl"))
CopyStream(database, shaderName);
var compiler = new EffectCompiler();
compiler.SourceDirectories.Add("assets/shaders");
var compilerCache = new EffectCompilerCache(compiler);
var compilerParameters = new CompilerParameters {Platform = GraphicsPlatform.Direct3D11};
left = compilerCache.Compile(new ShaderMixinGeneratorSource("SimpleEffect"), compilerParameters);
right = compilerCache.Compile(new ShaderMixinGeneratorSource("SimpleEffect"), compilerParameters);
}
}
public override TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
{
throw new NotSupportedException("Shader Compilation is not allowed at run time on this platform.");
}
public override TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
{
var log = new LoggerResult();
// Load D3D compiler dll
// Note: No lock, it's probably fine if it gets called from multiple threads at the same time.
if (Platform.IsWindowsDesktop && !d3dCompilerLoaded)
{
NativeLibrary.PreloadLibrary("d3dcompiler_47.dll");
d3dCompilerLoaded = true;
}
var shaderMixinSource = mixinTree;
var fullEffectName = mixinTree.Name;
var usedParameters = mixinTree.UsedParameters;
// Make a copy of shaderMixinSource. Use deep clone since shaderMixinSource can be altered during compilation (e.g. macros)
var shaderMixinSourceCopy = new ShaderMixinSource();
shaderMixinSourceCopy.DeepCloneFrom(shaderMixinSource);
shaderMixinSource = shaderMixinSourceCopy;
// Generate platform-specific macros
var platform = usedParameters.Get(CompilerParameters.GraphicsPlatformKey);
switch (platform)
{
case GraphicsPlatform.Direct3D11:
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_DIRECT3D", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_DIRECT3D11", 1);
break;
case GraphicsPlatform.OpenGL:
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGL", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGLCORE", 1);
break;
case GraphicsPlatform.OpenGLES:
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGL", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGLES", 1);
break;
default:
throw new NotSupportedException();
}
// Generate profile-specific macros
var profile = usedParameters.Get(CompilerParameters.GraphicsProfileKey);
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_PROFILE", (int)profile);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_1", (int)GraphicsProfile.Level_9_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_2", (int)GraphicsProfile.Level_9_2);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_3", (int)GraphicsProfile.Level_9_3);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_0", (int)GraphicsProfile.Level_10_0);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_1", (int)GraphicsProfile.Level_10_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_0", (int)GraphicsProfile.Level_11_0);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_1", (int)GraphicsProfile.Level_11_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_2", (int)GraphicsProfile.Level_11_2);
var parsingResult = GetMixinParser().Parse(shaderMixinSource, shaderMixinSource.Macros.ToArray());
// Copy log from parser results to output
CopyLogs(parsingResult, log);
// Return directly if there are any errors
if (parsingResult.HasErrors)
{
return new EffectBytecodeCompilerResult(null, log);
}
// Convert the AST to HLSL
var writer = new SiliconStudio.Shaders.Writer.Hlsl.HlslWriter
{
EnablePreprocessorLine = true // Allow to output links to original xksl via #line pragmas
};
writer.Visit(parsingResult.Shader);
var shaderSourceText = writer.Text;
if (string.IsNullOrEmpty(shaderSourceText))
{
log.Error("No code generated for effect [{0}]", fullEffectName);
return new EffectBytecodeCompilerResult(null, log);
}
// -------------------------------------------------------
// Save shader log
// TODO: TEMP code to allow debugging generated shaders on Windows Desktop
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
var shaderId = ObjectId.FromBytes(Encoding.UTF8.GetBytes(shaderSourceText));
var logDir = Path.Combine(Directory.GetCurrentDirectory(), "log");
if (!Directory.Exists(logDir))
{
Directory.CreateDirectory(logDir);
}
var shaderSourceFilename = Path.Combine(logDir, "shader_" + fullEffectName.Replace('.', '_') + "_" + shaderId + ".hlsl");
lock (WriterLock) // protect write in case the same shader is created twice
{
// Write shader before generating to make sure that we are having a trace before compiling it (compiler may crash...etc.)
if (!File.Exists(shaderSourceFilename))
{
File.WriteAllText(shaderSourceFilename, shaderSourceText);
}
}
#else
string shaderSourceFilename = null;
#endif
// -------------------------------------------------------
var bytecode = new EffectBytecode { Reflection = parsingResult.Reflection, HashSources = parsingResult.HashSources };
// Select the correct backend compiler
IShaderCompiler compiler;
switch (platform)
{
#if SILICONSTUDIO_PLATFORM_WINDOWS
case GraphicsPlatform.Direct3D11:
compiler = new Direct3D.ShaderCompiler();
break;
#endif
case GraphicsPlatform.OpenGL:
case GraphicsPlatform.OpenGLES:
// get the number of render target outputs
var rtOutputs = 0;
var psOutput = parsingResult.Shader.Declarations.OfType<StructType>().FirstOrDefault(x => x.Name.Text == "PS_OUTPUT");
if (psOutput != null)
{
foreach (var rto in psOutput.Fields)
{
var sem = rto.Qualifiers.OfType<Semantic>().FirstOrDefault();
if (sem != null)
{
// special case SV_Target
if (rtOutputs == 0 && sem.Name.Text == "SV_Target")
{
rtOutputs = 1;
break;
}
for (var i = rtOutputs; i < 8; ++i)
{
if (sem.Name.Text == ("SV_Target" + i))
{
rtOutputs = i + 1;
break;
}
}
}
}
}
compiler = new OpenGL.ShaderCompiler(rtOutputs);
break;
default:
throw new NotSupportedException();
}
var shaderStageBytecodes = new List<ShaderBytecode>();
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
var stageStringBuilder = new StringBuilder();
#endif
// if the shader (non-compute) does not have a pixel shader, we should add it on OpenGL ES.
if (platform == GraphicsPlatform.OpenGLES && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Pixel) && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Compute))
{
parsingResult.EntryPoints.Add(ShaderStage.Pixel, null);
}
foreach (var stageBinding in parsingResult.EntryPoints)
{
// Compile
// TODO: We could compile stages in different threads to improve compiler throughput?
var result = compiler.Compile(shaderSourceText, stageBinding.Value, stageBinding.Key, usedParameters, bytecode.Reflection, shaderSourceFilename);
result.CopyTo(log);
if (result.HasErrors)
{
continue;
}
// -------------------------------------------------------
// Append bytecode id to shader log
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
stageStringBuilder.AppendLine("@G {0} => {1}".ToFormat(stageBinding.Key, result.Bytecode.Id));
#endif
// -------------------------------------------------------
shaderStageBytecodes.Add(result.Bytecode);
// When this is a compute shader, there is no need to scan other stages
if (stageBinding.Key == ShaderStage.Compute)
break;
}
// In case of Direct3D, we can safely remove reflection data as it is entirely resolved at compile time.
if (platform == GraphicsPlatform.Direct3D11)
{
CleanupReflection(bytecode.Reflection);
}
bytecode.Stages = shaderStageBytecodes.ToArray();
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
lock (WriterLock) // protect write in case the same shader is created twice
{
var builder = new StringBuilder();
builder.AppendLine("/**************************");
builder.AppendLine("***** Used Parameters *****");
builder.AppendLine("***************************");
builder.Append("@P EffectName: ");
builder.AppendLine(fullEffectName ?? "");
builder.Append(usedParameters.ToStringDetailed());
builder.AppendLine("***************************");
if (bytecode.Reflection.ConstantBuffers.Count > 0)
{
builder.AppendLine("**** ConstantBuffers ****");
builder.AppendLine("***************************");
foreach (var cBuffer in bytecode.Reflection.ConstantBuffers)
{
builder.AppendFormat("cbuffer {0} [Stage: {1}, Size: {2}]", cBuffer.Name, cBuffer.Stage, cBuffer.Size).AppendLine();
foreach (var parameter in cBuffer.Members)
{
builder.AppendFormat("@C {0} => {1}", parameter.Param.RawName, parameter.Param.KeyName).AppendLine();
}
}
builder.AppendLine("***************************");
}
if (bytecode.Reflection.ResourceBindings.Count > 0)
{
builder.AppendLine("****** Resources ******");
builder.AppendLine("***************************");
foreach (var resource in bytecode.Reflection.ResourceBindings)
{
var parameter = resource.Param;
builder.AppendFormat("@R {0} => {1} [Stage: {2}, Slot: ({3}-{4})]", parameter.RawName, parameter.KeyName, resource.Stage, resource.SlotStart, resource.SlotStart + resource.SlotCount - 1).AppendLine();
}
builder.AppendLine("***************************");
}
if (bytecode.HashSources.Count > 0)
{
builder.AppendLine("***** Sources *****");
builder.AppendLine("***************************");
foreach (var hashSource in bytecode.HashSources)
{
builder.AppendFormat("@S {0} => {1}", hashSource.Key, hashSource.Value).AppendLine();
}
builder.AppendLine("***************************");
}
if (bytecode.Stages.Length > 0)
{
builder.AppendLine("***** Stages *****");
builder.AppendLine("***************************");
builder.Append(stageStringBuilder);
builder.AppendLine("***************************");
}
builder.AppendLine("*************************/");
// Re-append the shader with all informations
builder.Append(shaderSourceText);
File.WriteAllText(shaderSourceFilename, builder.ToString());
}
#endif
return new EffectBytecodeCompilerResult(bytecode, log);
}
public override TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixin, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters = null)
{
var database = (FileProvider ?? ContentManager.FileProvider) as DatabaseFileProvider;
if (database == null)
{
throw new NotSupportedException("Using the cache requires to ContentManager.FileProvider to be valid.");
}
// Forward DatabaseFileProvider to actual compiler here
// Since we might be in a Task, it has to be forwarded manually (otherwise MicroThreadLocal ones wouldn't work during build)
// Note: this system might need an overhaul... (too many states?)
base.FileProvider = database;
var usedParameters = compilerParameters;
var mixinObjectId = ShaderMixinObjectId.Compute(mixin, usedParameters);
// Final url of the compiled bytecode
var compiledUrl = string.Format("{0}/{1}", CompiledShadersKey, mixinObjectId);
EffectBytecode bytecode = null;
lock (bytecodes)
{
// ------------------------------------------------------------------------------------------------------------
// 1) Try to load latest bytecode
// ------------------------------------------------------------------------------------------------------------
ObjectId bytecodeId;
if (database.AssetIndexMap.TryGetValue(compiledUrl, out bytecodeId))
{
bytecode = LoadEffectBytecode(database, bytecodeId);
}
// On non Windows platform, we are expecting to have the bytecode stored directly
if (Compiler is NullEffectCompiler && bytecode == null)
{
var stringBuilder = new StringBuilder();
stringBuilder.AppendFormat("Unable to find compiled shaders [{0}] for mixin [{1}] with parameters [{2}]", compiledUrl, mixin, usedParameters.ToStringPermutationsDetailed());
Log.Error(stringBuilder.ToString());
throw new InvalidOperationException(stringBuilder.ToString());
}
// ------------------------------------------------------------------------------------------------------------
// 2) Try to load from database cache
// ------------------------------------------------------------------------------------------------------------
if (bytecode == null && database.ObjectDatabase.Exists(mixinObjectId))
{
using (var stream = database.ObjectDatabase.OpenStream(mixinObjectId))
{
// We have an existing stream, make sure the shader is compiled
var objectIdBuffer = new byte[ObjectId.HashSize];
if (stream.Read(objectIdBuffer, 0, ObjectId.HashSize) == ObjectId.HashSize)
{
var newBytecodeId = new ObjectId(objectIdBuffer);
bytecode = LoadEffectBytecode(database, newBytecodeId);
if (bytecode != null)
{
// If we successfully retrieved it from cache, add it to index map so that it won't be collected and available for faster lookup
database.AssetIndexMap[compiledUrl] = newBytecodeId;
}
}
}
}
}
if (bytecode != null)
{
return new EffectBytecodeCompilerResult(bytecode);
}
// ------------------------------------------------------------------------------------------------------------
// 3) Compile the shader
// ------------------------------------------------------------------------------------------------------------
lock (compilingShaders)
{
Task<EffectBytecodeCompilerResult> compilingShaderTask;
if (compilingShaders.TryGetValue(mixinObjectId, out compilingShaderTask))
{
// Note: Task might still be compiling
return compilingShaderTask;
}
// Compile the mixin in a Task
if (CompileEffectAsynchronously)
{
var compilerParametersCopy = compilerParameters != null ? new CompilerParameters(compilerParameters) : null;
var resultTask = Task.Factory.StartNew(() => CompileBytecode(mixin, effectParameters, compilerParametersCopy, mixinObjectId, database, compiledUrl), CancellationToken.None, TaskCreationOptions.None, taskSchedulerSelector != null ? taskSchedulerSelector(mixin, compilerParametersCopy.EffectParameters) : TaskScheduler.Default);
compilingShaders.Add(mixinObjectId, resultTask);
return resultTask;
}
else
{
return CompileBytecode(mixin, effectParameters, compilerParameters, mixinObjectId, database, compiledUrl);
}
}
}
/// <summary>
/// Update a dynamic effect instance based on its parameters.
/// </summary>
/// <param name="effectInstance">A dynmaic effect instance</param>
/// <param name="passParameters">The pass parameters.</param>
/// <returns><c>true</c> if the effect was recomiled on the effect instance, <c>false</c> otherwise.</returns>
public bool Update(DynamicEffectInstance effectInstance, ParameterCollection passParameters)
{
bool effectChanged = false;
var currentlyCompilingEffect = effectInstance.CurrentlyCompilingEffect;
if (currentlyCompilingEffect != null)
{
if (currentlyCompilingEffect.IsCompleted)
{
if (currentlyCompilingEffect.IsFaulted)
{
var compilerParameters = new CompilerParameters();
effectInstance.CurrentlyCompilingUsedParameters.CopyTo(compilerParameters);
SwitchFallbackEffect(FallbackEffectType.Error, effectInstance, passParameters, compilerParameters);
}
else
{
effectInstance.HasErrors = false;
// Do not update effect right away: passParameters might have changed since last compilation; just try to go through a CreateEffect that will properly update the effect synchronously
// TODO: This class (and maybe whole ParameterCollection system) need a complete rethink and rewrite with newest assumptions...
//UpdateEffect(effectInstance, currentlyCompilingEffect.Result, effectInstance.CurrentlyCompilingUsedParameters, passParameters);
}
effectChanged = true;
// Effect has been updated
effectInstance.CurrentlyCompilingEffect = null;
effectInstance.CurrentlyCompilingUsedParameters = null;
}
}
if (effectChanged || // Check again, in case effect was just finished async compilation
(effectInstance.Effect == null || !EffectSystem.IsValid(effectInstance.Effect) || HasCollectionChanged(effectInstance, passParameters) || effectInstance.HasErrors))
{
if (effectInstance.HasErrors)
{
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
var currentTime = DateTime.Now;
if (currentTime < effectInstance.LastErrorCheck + ErrorCheckTimeSpan)
{
// Wait a regular interval before retrying to compile effect (i.e. every second)
return false;
}
// Update last check time
effectInstance.LastErrorCheck = currentTime;
#else
// Other platforms: never try to recompile failed effects for now
return false;
#endif
}
CreateEffect(effectInstance, passParameters);
effectChanged = true;
}
return effectChanged;
}
public void TestMaterial()
{
var compiler = new EffectCompiler { UseFileSystem = true };
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Xenko.Graphics\Shaders");
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Xenko.Engine\Shaders");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Core");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Lights");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Materials");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Shadows");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\ComputeColor");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Skinning");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Shading");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Transformation");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Utils");
var compilerParameters = new CompilerParameters { Platform = GraphicsPlatform.Direct3D11 };
var layers = new MaterialBlendLayers();
layers.Add(new MaterialBlendLayer
{
BlendMap = new ComputeFloat(0.5f),
Material = AttachedReferenceManager.CreateSerializableVersion<Material>(Guid.Empty, "fake")
});
var materialAsset = new MaterialAsset
{
Attributes = new MaterialAttributes()
{
Diffuse = new MaterialDiffuseMapFeature()
{
DiffuseMap = new ComputeColor(Color4.White)
},
DiffuseModel = new MaterialDiffuseLambertModelFeature()
},
Layers = layers
};
var fakeAsset = new MaterialAsset
{
Attributes = new MaterialAttributes()
{
Diffuse = new MaterialDiffuseMapFeature()
{
DiffuseMap = new ComputeColor(Color.Blue)
},
}
};
var context = new MaterialGeneratorContext { FindAsset = reference => fakeAsset };
var result = MaterialGenerator.Generate(new MaterialDescriptor { Attributes = materialAsset.Attributes, Layers = materialAsset.Layers }, context, "TestMaterial");
compilerParameters.Set(MaterialKeys.PixelStageSurfaceShaders, result.Material.Parameters.Get(MaterialKeys.PixelStageSurfaceShaders));
var directionalLightGroup = new ShaderClassSource("LightDirectionalGroup", 1);
compilerParameters.Set(LightingKeys.DirectLightGroups, new List<ShaderSource> { directionalLightGroup });
//compilerParameters.Set(LightingKeys.CastShadows, false);
//compilerParameters.Set(MaterialParameters.HasSkinningPosition, true);
//compilerParameters.Set(MaterialParameters.HasSkinningNormal, true);
compilerParameters.Set(MaterialKeys.HasNormalMap, true);
var results = compiler.Compile(new ShaderMixinGeneratorSource("XenkoEffectBase"), compilerParameters);
Assert.IsFalse(results.HasErrors);
}
private void SwitchFallbackEffect(FallbackEffectType fallbackEffectType, DynamicEffectInstance effectInstance, ParameterCollection passParameters, CompilerParameters compilerParameters)
{
// Fallback for errors
effectInstance.HasErrors = true;
var fallbackEffect = ComputeFallbackEffect(this, fallbackEffectType, EffectName, compilerParameters);
UpdateEffect(effectInstance, fallbackEffect.Effect, fallbackEffect.UsedParameters, passParameters);
}
public static void Write(string name, CompilerParameters parameters, EffectBytecode effectData, TextWriter writer)
{
const string codeTemplate = @"//------------------------------------------------------------------------------
// <auto-generated>
// Xenko Effect Compiler File Generated:
{0}//
// Command Line: {7}
// Changes to this file may cause incorrect behavior and will be lost if
// the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------
namespace {1}
{{
{2} class {3}
{{
{4} static readonly byte[] {5} = new byte[] {{
{6}
}};
}}
}}
";
var effectToGenerateText = new StringBuilder();
effectToGenerateText.AppendFormat("// Effect [{0}]\r\n", name);
var buffer = new MemoryStream();
effectData.WriteTo(buffer);
var bufferAsText = new StringBuilder();
var bufferArray = buffer.ToArray();
for (int i = 0; i < bufferArray.Length; i++)
{
bufferAsText.Append(bufferArray[i]).Append(", ");
if (i > 0 && (i % 64) == 0)
{
bufferAsText.AppendLine();
}
}
var classDeclaration = parameters.Get(EffectSourceCodeKeys.ClassDeclaration);
var fieldDeclaration = parameters.Get(EffectSourceCodeKeys.FieldDeclaration);
var nameSpace = parameters.Get(EffectSourceCodeKeys.Namespace);
var className = parameters.Get(EffectSourceCodeKeys.ClassName) ?? name;
var fieldName = parameters.Get(EffectSourceCodeKeys.FieldName);
var commandLine = string.Join(" ", Environment.GetCommandLineArgs());
var graphicsPlatform = parameters.EffectParameters.Platform;
string xenkoDefine = "undefined";
switch (graphicsPlatform)
{
case GraphicsPlatform.Direct3D11:
case GraphicsPlatform.Direct3D12:
xenkoDefine = "SILICONSTUDIO_XENKO_GRAPHICS_API_DIRECT3D";
break;
case GraphicsPlatform.OpenGL:
xenkoDefine = "SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGLCORE";
break;
case GraphicsPlatform.OpenGLES:
xenkoDefine = "SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGLES";
break;
case GraphicsPlatform.Vulkan:
xenkoDefine = "SILICONSTUDIO_XENKO_GRAPHICS_API_VULKAN";
break;
}
writer.WriteLine("#if {0}", xenkoDefine);
writer.Write(codeTemplate,
effectToGenerateText, // {0}
nameSpace, // {1}
classDeclaration, // {2}
className, // {3}
fieldDeclaration, // {4}
fieldName, // {5}
bufferAsText, // {6}
commandLine); // {7}
writer.WriteLine("#endif");
writer.Flush();
}
public void TestStream()
{
var compiler = new EffectCompiler { UseFileSystem = true };
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Xenko.Shaders.Tests\GameAssets\Compiler");
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Xenko.Graphics\Shaders");
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Xenko.Engine\Shaders");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Core");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Lights");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Materials");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Shadows");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\ComputeColor");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Skinning");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Shading");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Transformation");
compiler.SourceDirectories.Add(@"..\..\sources\shaders\Utils");
var compilerParameters = new CompilerParameters { Platform = GraphicsPlatform.Direct3D11 };
var results = compiler.Compile(new ShaderClassSource("TestStream"), compilerParameters);
Assert.IsFalse(results.HasErrors);
}
public override TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
{
return compiler.Compile(mixinTree, compilerParameters);
}
public void TestMixinAndComposeKeys()
{
var compiler = new EffectCompiler { UseFileSystem = true };
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Xenko.Graphics\Shaders");
compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Xenko.Shaders.Tests\GameAssets\Mixins");
var compilerParameters = new CompilerParameters {Platform = GraphicsPlatform.Direct3D11};
var subCompute1Key = TestABC.TestParameters.UseComputeColor2.ComposeWith("SubCompute1");
var subCompute2Key = TestABC.TestParameters.UseComputeColor2.ComposeWith("SubCompute2");
var subComputesKey = TestABC.TestParameters.UseComputeColorRedirect.ComposeWith("SubComputes[0]");
compilerParameters.Set(subCompute1Key, true);
compilerParameters.Set(subComputesKey, true);
var results = compiler.Compile(new ShaderMixinGeneratorSource("test_mixin_compose_keys"), compilerParameters);
Assert.IsFalse(results.HasErrors);
var mainBytecode = results.Bytecode.WaitForResult();
Assert.IsFalse(mainBytecode.CompilationLog.HasErrors);
Assert.NotNull(mainBytecode.Bytecode.Reflection.ConstantBuffers);
Assert.AreEqual(1, mainBytecode.Bytecode.Reflection.ConstantBuffers.Count);
var cbuffer = mainBytecode.Bytecode.Reflection.ConstantBuffers[0];
Assert.NotNull(cbuffer.Members);
Assert.AreEqual(2, cbuffer.Members.Length);
// Check that ComputeColor2.Color is correctly composed for variables
var computeColorSubCompute2 = ComputeColor2Keys.Color.ComposeWith("SubCompute1");
var computeColorSubComputes = ComputeColor2Keys.Color.ComposeWith("ColorRedirect.SubComputes[0]");
var members = cbuffer.Members.Select(member => member.Param.KeyName).ToList();
Assert.IsTrue(members.Contains(computeColorSubCompute2.Name));
Assert.IsTrue(members.Contains(computeColorSubComputes.Name));
}
public override TaskOrResult <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters)
{
var log = new LoggerResult();
// Load D3D compiler dll
// Note: No lock, it's probably fine if it gets called from multiple threads at the same time.
if (Platform.IsWindowsDesktop && !d3dCompilerLoaded)
{
NativeLibrary.PreloadLibrary("d3dcompiler_47.dll");
d3dCompilerLoaded = true;
}
var shaderMixinSource = mixinTree;
var fullEffectName = mixinTree.Name;
// Make a copy of shaderMixinSource. Use deep clone since shaderMixinSource can be altered during compilation (e.g. macros)
var shaderMixinSourceCopy = new ShaderMixinSource();
shaderMixinSourceCopy.DeepCloneFrom(shaderMixinSource);
shaderMixinSource = shaderMixinSourceCopy;
// Generate platform-specific macros
switch (effectParameters.Platform)
{
case GraphicsPlatform.Direct3D11:
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_DIRECT3D", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_DIRECT3D11", 1);
break;
case GraphicsPlatform.Direct3D12:
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_DIRECT3D", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_DIRECT3D12", 1);
break;
case GraphicsPlatform.OpenGL:
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGL", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGLCORE", 1);
break;
case GraphicsPlatform.OpenGLES:
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGL", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGLES", 1);
break;
default:
throw new NotSupportedException();
}
// Generate profile-specific macros
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_PROFILE", (int)effectParameters.Profile);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_1", (int)GraphicsProfile.Level_9_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_2", (int)GraphicsProfile.Level_9_2);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_3", (int)GraphicsProfile.Level_9_3);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_0", (int)GraphicsProfile.Level_10_0);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_1", (int)GraphicsProfile.Level_10_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_0", (int)GraphicsProfile.Level_11_0);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_1", (int)GraphicsProfile.Level_11_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_2", (int)GraphicsProfile.Level_11_2);
var parsingResult = GetMixinParser().Parse(shaderMixinSource, shaderMixinSource.Macros.ToArray());
// Copy log from parser results to output
CopyLogs(parsingResult, log);
// Return directly if there are any errors
if (parsingResult.HasErrors)
{
return(new EffectBytecodeCompilerResult(null, log));
}
// Convert the AST to HLSL
var writer = new SiliconStudio.Shaders.Writer.Hlsl.HlslWriter
{
EnablePreprocessorLine = false // Allow to output links to original pdxsl via #line pragmas
};
writer.Visit(parsingResult.Shader);
var shaderSourceText = writer.Text;
if (string.IsNullOrEmpty(shaderSourceText))
{
log.Error("No code generated for effect [{0}]", fullEffectName);
return(new EffectBytecodeCompilerResult(null, log));
}
// -------------------------------------------------------
// Save shader log
// TODO: TEMP code to allow debugging generated shaders on Windows Desktop
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
var shaderId = ObjectId.FromBytes(Encoding.UTF8.GetBytes(shaderSourceText));
var logDir = Path.Combine(Directory.GetCurrentDirectory(), "log");
if (!Directory.Exists(logDir))
{
Directory.CreateDirectory(logDir);
}
var shaderSourceFilename = Path.Combine(logDir, "shader_" + fullEffectName.Replace('.', '_') + "_" + shaderId + ".hlsl");
lock (WriterLock) // protect write in case the same shader is created twice
{
// Write shader before generating to make sure that we are having a trace before compiling it (compiler may crash...etc.)
if (!File.Exists(shaderSourceFilename))
{
File.WriteAllText(shaderSourceFilename, shaderSourceText);
}
}
#else
string shaderSourceFilename = null;
#endif
// -------------------------------------------------------
var bytecode = new EffectBytecode {
Reflection = parsingResult.Reflection, HashSources = parsingResult.HashSources
};
// Select the correct backend compiler
IShaderCompiler compiler;
switch (effectParameters.Platform)
{
#if SILICONSTUDIO_PLATFORM_WINDOWS
case GraphicsPlatform.Direct3D11:
case GraphicsPlatform.Direct3D12:
compiler = new Direct3D.ShaderCompiler();
break;
#endif
case GraphicsPlatform.OpenGL:
case GraphicsPlatform.OpenGLES:
// get the number of render target outputs
var rtOutputs = 0;
var psOutput = parsingResult.Shader.Declarations.OfType <StructType>().FirstOrDefault(x => x.Name.Text == "PS_OUTPUT");
if (psOutput != null)
{
foreach (var rto in psOutput.Fields)
{
var sem = rto.Qualifiers.OfType <Semantic>().FirstOrDefault();
if (sem != null)
{
// special case SV_Target
if (rtOutputs == 0 && sem.Name.Text == "SV_Target")
{
rtOutputs = 1;
break;
}
for (var i = rtOutputs; i < 8; ++i)
{
if (sem.Name.Text == ("SV_Target" + i))
{
rtOutputs = i + 1;
break;
}
}
}
}
}
compiler = new OpenGL.ShaderCompiler(rtOutputs);
break;
default:
throw new NotSupportedException();
}
var shaderStageBytecodes = new List <ShaderBytecode>();
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
var stageStringBuilder = new StringBuilder();
#endif
// if the shader (non-compute) does not have a pixel shader, we should add it on OpenGL ES.
if (effectParameters.Platform == GraphicsPlatform.OpenGLES && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Pixel) && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Compute))
{
parsingResult.EntryPoints.Add(ShaderStage.Pixel, null);
}
foreach (var stageBinding in parsingResult.EntryPoints)
{
// Compile
// TODO: We could compile stages in different threads to improve compiler throughput?
var result = compiler.Compile(shaderSourceText, stageBinding.Value, stageBinding.Key, effectParameters, bytecode.Reflection, shaderSourceFilename);
result.CopyTo(log);
if (result.HasErrors)
{
continue;
}
// -------------------------------------------------------
// Append bytecode id to shader log
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
stageStringBuilder.AppendLine("@G {0} => {1}".ToFormat(stageBinding.Key, result.Bytecode.Id));
if (result.DisassembleText != null)
{
stageStringBuilder.Append(result.DisassembleText);
}
#endif
// -------------------------------------------------------
shaderStageBytecodes.Add(result.Bytecode);
// When this is a compute shader, there is no need to scan other stages
if (stageBinding.Key == ShaderStage.Compute)
{
break;
}
}
// Remove unused reflection data, as it is entirely resolved at compile time.
CleanupReflection(bytecode.Reflection);
bytecode.Stages = shaderStageBytecodes.ToArray();
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
lock (WriterLock) // protect write in case the same shader is created twice
{
var builder = new StringBuilder();
builder.AppendLine("/**************************");
builder.AppendLine("***** Compiler Parameters *****");
builder.AppendLine("***************************");
builder.Append("@P EffectName: ");
builder.AppendLine(fullEffectName ?? "");
builder.Append(compilerParameters?.ToStringPermutationsDetailed());
builder.AppendLine("***************************");
if (bytecode.Reflection.ConstantBuffers.Count > 0)
{
builder.AppendLine("**** ConstantBuffers ****");
builder.AppendLine("***************************");
foreach (var cBuffer in bytecode.Reflection.ConstantBuffers)
{
builder.AppendFormat("cbuffer {0} [Size: {1}]", cBuffer.Name, cBuffer.Size).AppendLine();
foreach (var parameter in cBuffer.Members)
{
builder.AppendFormat("@C {0} => {1}", parameter.RawName, parameter.KeyInfo.KeyName).AppendLine();
}
}
builder.AppendLine("***************************");
}
if (bytecode.Reflection.ResourceBindings.Count > 0)
{
builder.AppendLine("****** Resources ******");
builder.AppendLine("***************************");
foreach (var resource in bytecode.Reflection.ResourceBindings)
{
builder.AppendFormat("@R {0} => {1} [Stage: {2}, Slot: ({3}-{4})]", resource.RawName, resource.KeyInfo.KeyName, resource.Stage, resource.SlotStart, resource.SlotStart + resource.SlotCount - 1).AppendLine();
}
builder.AppendLine("***************************");
}
if (bytecode.HashSources.Count > 0)
{
builder.AppendLine("***** Sources *****");
builder.AppendLine("***************************");
foreach (var hashSource in bytecode.HashSources)
{
builder.AppendFormat("@S {0} => {1}", hashSource.Key, hashSource.Value).AppendLine();
}
builder.AppendLine("***************************");
}
if (bytecode.Stages.Length > 0)
{
builder.AppendLine("***** Stages *****");
builder.AppendLine("***************************");
builder.Append(stageStringBuilder);
builder.AppendLine("***************************");
}
builder.AppendLine("*************************/");
// Re-append the shader with all informations
builder.Append(shaderSourceText);
File.WriteAllText(shaderSourceFilename, builder.ToString());
}
#endif
return(new EffectBytecodeCompilerResult(bytecode, log));
}
