public EffectCompileCommand(AssetCompilerContext context, UDirectory baseUrl, string effectName, CompilerParameters compilerParameters)
 {
     this.context = context;
     this.baseUrl = baseUrl;
     this.effectName = effectName;
     this.compilerParameters = compilerParameters;
 }
        /// <summary>
        /// Initializes a new instance of the <see cref="DynamicEffectCompiler" /> class.
        /// </summary>
        /// <param name="services">The services.</param>
        /// <param name="effectName">Name of the effect.</param>
        /// <param name="asyncDynamicEffectCompiler">if set to <c>true</c> it can compile effect asynchronously.</param>
        /// <exception cref="System.ArgumentNullException">services
        /// or
        /// effectName</exception>
        public DynamicEffectCompiler(IServiceRegistry services, string effectName, int taskPriority = 0)
        {
            if (services == null) throw new ArgumentNullException("services");
            if (effectName == null) throw new ArgumentNullException("effectName");

            Services = services;
            this.effectName = effectName;
            this.taskPriority = taskPriority;
            EffectSystem = Services.GetSafeServiceAs<EffectSystem>();
            GraphicsDevice = Services.GetSafeServiceAs<IGraphicsDeviceService>().GraphicsDevice;
            parameterCollections = new FastList<ParameterCollection>();

            // Default behavior for fallback effect: load effect with same name but empty compiler parameters
            ComputeFallbackEffect = (dynamicEffectCompiler, type, name, parameters) =>
            {
                ParameterCollection usedParameters;
                var compilerParameters = new CompilerParameters();

                // We want high priority
                compilerParameters.TaskPriority = -1;

                var effect = dynamicEffectCompiler.EffectSystem.LoadEffect(effectName, compilerParameters, out usedParameters).WaitForResult();
                return new ComputeFallbackEffectResult(effect, usedParameters);
            };
        }
        private async Task<EffectBytecodeCompilerResult> CompileAsync(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
        {
            // Make sure we are connected
            // TODO: Handle reconnections, etc...
            await shaderCompilerConnected;

            return await shaderCompilerTarget.Compile(mixinTree, compilerParameters);
        }
        public CompilerResults Compile(ShaderSource shaderSource, CompilerParameters compilerParameters)
        {
            ShaderMixinSource mixinToCompile;
            var shaderMixinGeneratorSource = shaderSource as ShaderMixinGeneratorSource;

            if (shaderMixinGeneratorSource != null)
            {
                mixinToCompile = ShaderMixinManager.Generate(shaderMixinGeneratorSource.Name, compilerParameters);
            }
            else
            {
                mixinToCompile = shaderSource as ShaderMixinSource;
                var shaderClassSource = shaderSource as ShaderClassSource;

                if (shaderClassSource != null)
                {
                    mixinToCompile = new ShaderMixinSource { Name = shaderClassSource.ClassName };
                    mixinToCompile.Mixins.Add(shaderClassSource);
                }

                if (mixinToCompile == null)
                {
                    throw new ArgumentException("Unsupported ShaderSource type [{0}]. Supporting only ShaderMixinSource/pdxfx, ShaderClassSource", "shaderSource");
                }
                if (string.IsNullOrEmpty(mixinToCompile.Name))
                {
                    throw new ArgumentException("ShaderMixinSource must have a name", "shaderSource");
                }
            }

            // Copy global parameters to used Parameters by default, as it is used by the compiler
            mixinToCompile.UsedParameters.Set(CompilerParameters.GraphicsPlatformKey, compilerParameters.Platform);
            mixinToCompile.UsedParameters.Set(CompilerParameters.GraphicsProfileKey, compilerParameters.Profile);
            mixinToCompile.UsedParameters.Set(CompilerParameters.DebugKey, compilerParameters.Debug);

            // Compile the whole mixin tree
            var compilerResults = new CompilerResults { Module = string.Format("EffectCompile [{0}]", mixinToCompile.Name) };
            var bytecode = Compile(mixinToCompile, compilerParameters);

            // Since bytecode.Result is a struct, we check if any of its member has been set to know if it's valid
            if (bytecode.Result.CompilationLog != null || bytecode.Task != null)
            {
                if (bytecode.Result.CompilationLog != null)
                {
                    bytecode.Result.CompilationLog.CopyTo(compilerResults);
                }
                compilerResults.Bytecode = bytecode;
                compilerResults.UsedParameters = mixinToCompile.UsedParameters;
            }
            return compilerResults;
        }
        public async Task<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
        {
            // Make sure we are connected
            // TODO: Handle reconnections, etc...
            var socketMessageLayer = await GetOrCreateConnection();

            var shaderCompilerAnswer = (RemoteEffectCompilerEffectAnswer)await socketMessageLayer.SendReceiveAsync(new RemoteEffectCompilerEffectRequest
            {
                MixinTree = mixinTree,
                UsedParameters = mixinTree.UsedParameters,
            });

            // TODO: Get LoggerResult as well
            return new EffectBytecodeCompilerResult(shaderCompilerAnswer.EffectBytecode);
        }
 public override TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
 {
     throw new NotSupportedException("Shader Compilation is not allowed at run time on this platform.");
 }
Beispiel #7
0
        /// <summary>
        /// Loads the effect.
        /// </summary>
        /// <param name="effectName">Name of the effect.</param>
        /// <param name="compilerParameters">The compiler parameters.</param>
        /// <param name="usedParameters">The used parameters.</param>
        /// <returns>A new instance of an effect.</returns>
        /// <exception cref="System.InvalidOperationException">Could not compile shader. Need fallback.</exception>
        public TaskOrResult<Effect> LoadEffect(string effectName, CompilerParameters compilerParameters, out ParameterCollection usedParameters)
        {
            if (effectName == null) throw new ArgumentNullException("effectName");
            if (compilerParameters == null) throw new ArgumentNullException("compilerParameters");

            // Get the compiled result
            var compilerResult = GetCompilerResults(effectName, compilerParameters);
            CheckResult(compilerResult);

            // Only take the sub-effect
            var bytecode = compilerResult.Bytecode;
            usedParameters = compilerResult.UsedParameters;

            if (bytecode.Task != null && !bytecode.Task.IsCompleted)
            {
                // Result was async, keep it async
                return bytecode.Task.ContinueWith(x => CreateEffect(effectName, x.Result, compilerResult));
            }
            else
            {
                return CreateEffect(effectName, bytecode.WaitForResult(), compilerResult);
            }
        }
Beispiel #8
0
        public override TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
        {
            var log = new LoggerResult();

            // Load D3D compiler dll
            // Note: No lock, it's probably fine if it gets called from multiple threads at the same time.
            if (Platform.IsWindowsDesktop && !d3dCompilerLoaded)
            {
                NativeLibrary.PreloadLibrary("d3dcompiler_47.dll");
                d3dCompilerLoaded = true;
            }

            var shaderMixinSource = mixinTree;
            var fullEffectName = mixinTree.Name;
            var usedParameters = mixinTree.UsedParameters;

            // Make a copy of shaderMixinSource. Use deep clone since shaderMixinSource can be altered during compilation (e.g. macros)
            var shaderMixinSourceCopy = new ShaderMixinSource();
            shaderMixinSourceCopy.DeepCloneFrom(shaderMixinSource);
            shaderMixinSource = shaderMixinSourceCopy;

            // Generate platform-specific macros
            var platform = usedParameters.Get(CompilerParameters.GraphicsPlatformKey);
            switch (platform)
            {
                case GraphicsPlatform.Direct3D11:
                    shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_API_DIRECT3D", 1);
                    shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_API_DIRECT3D11", 1);
                    break;
                case GraphicsPlatform.OpenGL:
                    shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGL", 1);
                    shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGLCORE", 1);
                    break;
                case GraphicsPlatform.OpenGLES:
                    shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGL", 1);
                    shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGLES", 1);
                    break;
                default:
                    throw new NotSupportedException();
            }

            // Generate profile-specific macros
            var profile = usedParameters.Get(CompilerParameters.GraphicsProfileKey);
            shaderMixinSource.AddMacro("SILICONSTUDIO_PARADOX_GRAPHICS_PROFILE", (int)profile);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_1", (int)GraphicsProfile.Level_9_1);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_2", (int)GraphicsProfile.Level_9_2);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_3", (int)GraphicsProfile.Level_9_3);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_0", (int)GraphicsProfile.Level_10_0);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_1", (int)GraphicsProfile.Level_10_1);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_0", (int)GraphicsProfile.Level_11_0);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_1", (int)GraphicsProfile.Level_11_1);
            shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_2", (int)GraphicsProfile.Level_11_2);

            var parsingResult = GetMixinParser().Parse(shaderMixinSource, shaderMixinSource.Macros.ToArray());

            // Copy log from parser results to output
            CopyLogs(parsingResult, log);

            // Return directly if there are any errors
            if (parsingResult.HasErrors)
            {
                return new EffectBytecodeCompilerResult(null, log);
            }

            // Convert the AST to HLSL
            var writer = new SiliconStudio.Shaders.Writer.Hlsl.HlslWriter
            {
                EnablePreprocessorLine = true // Allow to output links to original pdxsl via #line pragmas
            };
            writer.Visit(parsingResult.Shader);
            var shaderSourceText = writer.Text;

            if (string.IsNullOrEmpty(shaderSourceText))
            {
                log.Error("No code generated for effect [{0}]", fullEffectName);
                return new EffectBytecodeCompilerResult(null, log);
            }

            // -------------------------------------------------------
            // Save shader log
            // TODO: TEMP code to allow debugging generated shaders on Windows Desktop
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
            var shaderId = ObjectId.FromBytes(Encoding.UTF8.GetBytes(shaderSourceText));

            var logDir = "log";
            if (!Directory.Exists(logDir))
            {
                Directory.CreateDirectory(logDir);
            }
            var shaderSourceFilename = Path.Combine(logDir, "shader_" +  fullEffectName.Replace('.', '_') + "_" + shaderId + ".hlsl");
            lock (WriterLock) // protect write in case the same shader is created twice
            {
                // Write shader before generating to make sure that we are having a trace before compiling it (compiler may crash...etc.)
                if (!File.Exists(shaderSourceFilename))
                {
                    File.WriteAllText(shaderSourceFilename, shaderSourceText);
                }
            }
#else
            string shaderSourceFilename = null;
#endif
            // -------------------------------------------------------

            var bytecode = new EffectBytecode { Reflection = parsingResult.Reflection, HashSources = parsingResult.HashSources };

            // Select the correct backend compiler
            IShaderCompiler compiler;
            switch (platform)
            {
#if SILICONSTUDIO_PLATFORM_WINDOWS
                case GraphicsPlatform.Direct3D11:
                    compiler = new Direct3D.ShaderCompiler();
                    break;
#endif
                case GraphicsPlatform.OpenGL:
                case GraphicsPlatform.OpenGLES:
                    // get the number of render target outputs
                    var rtOutputs = 0;
                    var psOutput = parsingResult.Shader.Declarations.OfType<StructType>().FirstOrDefault(x => x.Name.Text == "PS_OUTPUT");
                    if (psOutput != null)
                    {
                        foreach (var rto in psOutput.Fields)
                        {
                            var sem = rto.Qualifiers.OfType<Semantic>().FirstOrDefault();
                            if (sem != null)
                            {
                                // special case SV_Target
                                if (rtOutputs == 0 && sem.Name.Text == "SV_Target")
                                {
                                    rtOutputs = 1;
                                    break;
                                }
                                for (var i = rtOutputs; i < 8; ++i)
                                {
                                    if (sem.Name.Text == ("SV_Target" + i))
                                    {
                                        rtOutputs = i + 1;
                                        break;
                                    }
                                }
                            }
                        }
                    }
                    compiler = new OpenGL.ShaderCompiler(rtOutputs);
                    break;
                default:
                    throw new NotSupportedException();
            }

            var shaderStageBytecodes = new List<ShaderBytecode>();

#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
            var stageStringBuilder = new StringBuilder();
#endif
            // if the shader (non-compute) does not have a pixel shader, we should add it on OpenGL ES.
            if (platform == GraphicsPlatform.OpenGLES && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Pixel) && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Compute))
            {
                parsingResult.EntryPoints.Add(ShaderStage.Pixel, null);
            }

            foreach (var stageBinding in parsingResult.EntryPoints)
            {
                // Compile
                // TODO: We could compile stages in different threads to improve compiler throughput?
                var result = compiler.Compile(shaderSourceText, stageBinding.Value, stageBinding.Key, usedParameters, bytecode.Reflection, shaderSourceFilename);
                result.CopyTo(log);

                if (result.HasErrors)
                {
                    continue;
                }

                // -------------------------------------------------------
                // Append bytecode id to shader log
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
                stageStringBuilder.AppendLine("@G    {0} => {1}".ToFormat(stageBinding.Key, result.Bytecode.Id));
#endif
                // -------------------------------------------------------

                shaderStageBytecodes.Add(result.Bytecode);

                // When this is a compute shader, there is no need to scan other stages
                if (stageBinding.Key == ShaderStage.Compute)
                    break;
            }

            // In case of Direct3D, we can safely remove reflection data as it is entirely resolved at compile time.
            if (platform == GraphicsPlatform.Direct3D11)
            {
                CleanupReflection(bytecode.Reflection);
            }
            bytecode.Stages = shaderStageBytecodes.ToArray();

#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
            lock (WriterLock) // protect write in case the same shader is created twice
            {
                var builder = new StringBuilder();
                builder.AppendLine("/**************************");
                builder.AppendLine("***** Used Parameters *****");
                builder.AppendLine("***************************");
                builder.Append("@P EffectName: ");
                builder.AppendLine(fullEffectName ?? "");
                builder.Append(usedParameters.ToStringDetailed());
                builder.AppendLine("***************************");

                if (bytecode.Reflection.ConstantBuffers.Count > 0)
                {
                    builder.AppendLine("****  ConstantBuffers  ****");
                    builder.AppendLine("***************************");
                    foreach (var cBuffer in bytecode.Reflection.ConstantBuffers)
                    {
                        builder.AppendFormat("cbuffer {0} [Stage: {1}, Size: {2}]", cBuffer.Name, cBuffer.Stage, cBuffer.Size).AppendLine();
                        foreach (var parameter in cBuffer.Members)
                        {
                            builder.AppendFormat("@C    {0} => {1}", parameter.Param.RawName, parameter.Param.KeyName).AppendLine();
                        }
                    }
                    builder.AppendLine("***************************");
                }

                if (bytecode.Reflection.ResourceBindings.Count > 0)
                {
                    builder.AppendLine("******  Resources    ******");
                    builder.AppendLine("***************************");
                    foreach (var resource in bytecode.Reflection.ResourceBindings)
                    {
                        var parameter = resource.Param;
                        builder.AppendFormat("@R    {0} => {1} [Stage: {2}, Slot: ({3}-{4})]", parameter.RawName, parameter.KeyName, resource.Stage, resource.SlotStart, resource.SlotStart + resource.SlotCount - 1).AppendLine();
                    }
                    builder.AppendLine("***************************");
                }

                if (bytecode.HashSources.Count > 0)
                {
                    builder.AppendLine("*****     Sources     *****");
                    builder.AppendLine("***************************");
                    foreach (var hashSource in bytecode.HashSources)
                    {
                        builder.AppendFormat("@S    {0} => {1}", hashSource.Key, hashSource.Value).AppendLine();
                    }
                    builder.AppendLine("***************************");
                }

                if (bytecode.Stages.Length > 0)
                {
                    builder.AppendLine("*****     Stages      *****");
                    builder.AppendLine("***************************");
                    builder.Append(stageStringBuilder);
                    builder.AppendLine("***************************");
                }
                builder.AppendLine("*************************/");

                // Re-append the shader with all informations
                builder.Append(shaderSourceText);

                File.WriteAllText(shaderSourceFilename, builder.ToString());
            }
#endif

            return new EffectBytecodeCompilerResult(bytecode, log);
        }
Beispiel #9
0
        public void TestMaterial()
        {
            var compiler = new EffectCompiler { UseFileSystem = true };
            compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Graphics\Shaders");
            compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Engine\Shaders");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Core");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Lights");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Materials");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Shadows");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\ComputeColor");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Skinning");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Shading");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Transformation");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Utils");
            var compilerParameters = new CompilerParameters { Platform = GraphicsPlatform.Direct3D11 };

            var layers = new MaterialBlendLayers();
            layers.Add(new MaterialBlendLayer
            {
                BlendMap = new ComputeFloat(0.5f),
                Material =  AttachedReferenceManager.CreateSerializableVersion<Material>(Guid.Empty, "fake")
            });

            var materialAsset = new MaterialAsset
            {
                Attributes = new MaterialAttributes()
                {
                    Diffuse = new MaterialDiffuseMapFeature()
                    {
                        DiffuseMap = new ComputeColor(Color4.White)
                    },
                    DiffuseModel = new MaterialDiffuseLambertModelFeature()
                },
                Layers = layers
            };

            var fakeAsset = new MaterialAsset
            {
                Attributes = new MaterialAttributes()
                {
                    Diffuse = new MaterialDiffuseMapFeature()
                    {
                        DiffuseMap = new ComputeColor(Color.Blue)
                    },
                }
            };

            var context = new MaterialGeneratorContext { FindAsset = reference => fakeAsset };
            var result = MaterialGenerator.Generate(new MaterialDescriptor { Attributes = materialAsset.Attributes, Layers = materialAsset.Layers }, context, "TestMaterial");

            compilerParameters.Set(MaterialKeys.PixelStageSurfaceShaders, result.Material.Parameters.Get(MaterialKeys.PixelStageSurfaceShaders));
            var directionalLightGroup = new ShaderClassSource("LightDirectionalGroup", 1);
            compilerParameters.Set(LightingKeys.DirectLightGroups, new List<ShaderSource> { directionalLightGroup });
            //compilerParameters.Set(LightingKeys.CastShadows, false);
            //compilerParameters.Set(MaterialParameters.HasSkinningPosition, true);
            //compilerParameters.Set(MaterialParameters.HasSkinningNormal, true);
            compilerParameters.Set(MaterialKeys.HasNormalMap, true);

            var results = compiler.Compile(new ShaderMixinGeneratorSource("ParadoxEffectBase"), compilerParameters);

            Assert.IsFalse(results.HasErrors);
        }
Beispiel #10
0
        public void TestNoClean(out CompilerResults left, out CompilerResults right)
        {
            // Create and mount database file system
            var objDatabase = ObjectDatabase.CreateDefaultDatabase();
            using (var assetIndexMap = AssetIndexMap.Load(VirtualFileSystem.ApplicationDatabaseIndexPath))
            {
                var database = new DatabaseFileProvider(assetIndexMap, objDatabase);
                AssetManager.GetFileProvider = () => database;

                foreach (var shaderName in Directory.EnumerateFiles(@"..\..\sources\shaders", "*.pdxsl"))
                    CopyStream(database, shaderName);

                foreach (var shaderName in Directory.EnumerateFiles(@"..\..\sources\engine\SiliconStudio.Paradox.Shaders.Tests\GameAssets\Compiler", "*.pdxsl"))
                    CopyStream(database, shaderName);

                var compiler = new EffectCompiler();
                compiler.SourceDirectories.Add("assets/shaders");
                var compilerCache = new EffectCompilerCache(compiler);

                var compilerParameters = new CompilerParameters {Platform = GraphicsPlatform.Direct3D11};

                left = compilerCache.Compile(new ShaderMixinGeneratorSource("SimpleEffect"), compilerParameters);
                right = compilerCache.Compile(new ShaderMixinGeneratorSource("SimpleEffect"), compilerParameters);
            }
        }
        public override TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixin, CompilerParameters compilerParameters)
        {
            var database = (FileProvider ?? AssetManager.FileProvider) as DatabaseFileProvider;
            if (database == null)
            {
                throw new NotSupportedException("Using the cache requires to AssetManager.FileProvider to be valid.");
            }

            // Forward DatabaseFileProvider to actual compiler here
            // Since we might be in a Task, it has to be forwarded manually (otherwise MicroThreadLocal ones wouldn't work during build)
            // Note: this system might need an overhaul... (too many states?)
            base.FileProvider = database;

            var usedParameters = mixin.UsedParameters;
            var mixinObjectId = ShaderMixinObjectId.Compute(mixin, usedParameters);

            // Final url of the compiled bytecode
            var compiledUrl = string.Format("{0}/{1}", CompiledShadersKey, mixinObjectId);

            EffectBytecode bytecode = null;
            lock (bytecodes)
            {                
                // ------------------------------------------------------------------------------------------------------------
                // 1) Try to load latest bytecode
                // ------------------------------------------------------------------------------------------------------------
                ObjectId bytecodeId;
                if (database.AssetIndexMap.TryGetValue(compiledUrl, out bytecodeId))
                {
                    bytecode = LoadEffectBytecode(database, bytecodeId);
                }

                // On non Windows platform, we are expecting to have the bytecode stored directly
                if (Compiler is NullEffectCompiler && bytecode == null)
                {
                    var stringBuilder = new StringBuilder();
                    stringBuilder.AppendFormat("Unable to find compiled shaders [{0}] for mixin [{1}] with parameters [{2}]", compiledUrl, mixin, usedParameters.ToStringDetailed());
                    Log.Error(stringBuilder.ToString());
                    throw new InvalidOperationException(stringBuilder.ToString());
                }

                // ------------------------------------------------------------------------------------------------------------
                // 2) Try to load from database cache
                // ------------------------------------------------------------------------------------------------------------
                if (bytecode == null && database.ObjectDatabase.Exists(mixinObjectId))
                {
                    using (var stream = database.ObjectDatabase.OpenStream(mixinObjectId))
                    {
                        // We have an existing stream, make sure the shader is compiled
                        var objectIdBuffer = new byte[ObjectId.HashSize];
                        if (stream.Read(objectIdBuffer, 0, ObjectId.HashSize) == ObjectId.HashSize)
                        {
                            var newBytecodeId = new ObjectId(objectIdBuffer);
                            bytecode = LoadEffectBytecode(database, newBytecodeId);

                            if (bytecode != null)
                            {
                                // If we successfully retrieved it from cache, add it to index map so that it won't be collected and available for faster lookup 
                                database.AssetIndexMap[compiledUrl] = newBytecodeId;
                            }
                        }
                    }
                }
            }

            if (bytecode != null)
            {
                return new EffectBytecodeCompilerResult(bytecode);
            }

            // ------------------------------------------------------------------------------------------------------------
            // 3) Compile the shader
            // ------------------------------------------------------------------------------------------------------------
            lock (compilingShaders)
            {
                Task<EffectBytecodeCompilerResult> compilingShaderTask;
                if (compilingShaders.TryGetValue(mixinObjectId, out compilingShaderTask))
                {
                    // Note: Task might still be compiling
                    return compilingShaderTask;
                }

                // Compile the mixin in a Task
                if (CompileEffectAsynchronously)
                {
                    var resultTask = Task.Factory.StartNew(() => CompileBytecode(mixin, compilerParameters, mixinObjectId, database, compiledUrl, usedParameters), CancellationToken.None, TaskCreationOptions.None, taskSchedulerSelector != null ? taskSchedulerSelector(mixin, compilerParameters) : TaskScheduler.Default);

                    compilingShaders.Add(mixinObjectId, resultTask);

                    return resultTask;
                }
                else
                {
                    return CompileBytecode(mixin, compilerParameters, mixinObjectId, database, compiledUrl, usedParameters);
                }
            }
        }
        private EffectBytecodeCompilerResult CompileBytecode(ShaderMixinSource mixinTree, CompilerParameters compilerParameters, ObjectId mixinObjectId, DatabaseFileProvider database, string compiledUrl, ShaderMixinParameters usedParameters)
        {
            // Open the database for writing
            var log = new LoggerResult();

            // Note: this compiler is expected to not be async and directly write stuff in localLogger
            var compiledShader = base.Compile(mixinTree, compilerParameters).WaitForResult();
            compiledShader.CompilationLog.CopyTo(log);
            
            // If there are any errors, return immediately
            if (log.HasErrors)
            {
                lock (compilingShaders)
                {
                    compilingShaders.Remove(mixinObjectId);
                }

                return new EffectBytecodeCompilerResult(null, log);
            }

            // Compute the bytecodeId
            var newBytecodeId = compiledShader.Bytecode.ComputeId();

            // Check if we really need to store the bytecode
            lock (bytecodes)
            {
                // Using custom serialization to the database to store an object with a custom id
                // TODO: Check if we really need to write the bytecode everytime even if id is not changed
                var memoryStream = new MemoryStream();
                compiledShader.Bytecode.WriteTo(memoryStream);
                memoryStream.Position = 0;
                database.ObjectDatabase.Write(memoryStream, newBytecodeId, true);
                database.AssetIndexMap[compiledUrl] = newBytecodeId;

                // Save bytecode Id to the database cache as well
                memoryStream.SetLength(0);
                memoryStream.Write((byte[])newBytecodeId, 0, ObjectId.HashSize);
                memoryStream.Position = 0;
                database.ObjectDatabase.Write(memoryStream, mixinObjectId, true);

                if (!bytecodes.ContainsKey(newBytecodeId))
                {
                    log.Info("New effect compiled #{0} [{1}] (db: {2})\r\n{3}", effectCompileCount, mixinObjectId, newBytecodeId, usedParameters.ToStringDetailed());
                    Interlocked.Increment(ref effectCompileCount);

                    // Replace or add new bytecode
                    bytecodes[newBytecodeId] = compiledShader.Bytecode;
                }
            }

            lock (compilingShaders)
            {
                compilingShaders.Remove(mixinObjectId);
            }

            return compiledShader;
        }
 private async Task<EffectBytecodeCompilerResult> CompileAsync(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
 {
     return await remoteEffectCompilerClient.Compile(mixinTree, compilerParameters);
 }
        private Effect GetOrCreateSelectedSpriteEffect(ref Effect effect, bool isSRgb)
        {
            if (effect == null)
            {
                var compilerParameters = new CompilerParameters { [SpriteBaseKeys.ColorIsSRgb] = isSRgb};
                effect = EffectSystem.LoadEffect("SelectedSprite", compilerParameters).WaitForResult();
            }

            return effect;
        }
Beispiel #15
0
        /// <summary>
        /// Get the shader from the database based on the parameters used for its compilation.
        /// </summary>
        /// <param name="effectName">Name of the effect.</param>
        /// <param name="parameters">The parameters.</param>
        /// <returns>The EffectBytecode if found.</returns>
        protected CompilerResults GetShaderFromParameters(string effectName, CompilerParameters parameters)
        {
            lock (earlyCompilerCache)
            {
                List<CompilerResults> compilerResultsList;
                if (!earlyCompilerCache.TryGetValue(effectName, out compilerResultsList))
                    return null;

                // TODO: Optimize it so that search is not linear?
                // Probably not trivial for subset testing
                foreach (var compiledResults in compilerResultsList)
                {
                    if (parameters.Contains(compiledResults.UsedParameters))
                    {
                        return compiledResults;
                    }
                }
            }

            return null;
        }
Beispiel #16
0
        private CompilerResults GetCompilerResults(string effectName, CompilerParameters compilerParameters)
        {
            compilerParameters.Profile = GraphicsDevice.ShaderProfile.HasValue ? GraphicsDevice.ShaderProfile.Value : GraphicsDevice.Features.Profile;
#if SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGLCORE
            compilerParameters.Platform = GraphicsPlatform.OpenGL;
#endif
#if SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGLES 
            compilerParameters.Platform = GraphicsPlatform.OpenGLES;
#endif

            // Compile shader
            var isPdxfx = ShaderMixinManager.Contains(effectName);

            // getting the effect from the used parameters only makes sense when the source files are the same
            // TODO: improve this by updating earlyCompilerCache - cache can still be relevant

            CompilerResults compilerResult = null;

            if (isPdxfx)
            {
                // perform an early test only based on the parameters
                compilerResult = GetShaderFromParameters(effectName, compilerParameters);
            }

            if (compilerResult == null)
            {
                var source = isPdxfx ? new ShaderMixinGeneratorSource(effectName) : (ShaderSource)new ShaderClassSource(effectName);
                compilerResult = compiler.Compile(source, compilerParameters);

                var effectRequested = EffectUsed;
                if (effectRequested != null)
                {
                    effectRequested(new EffectCompileRequest(effectName, compilerResult.UsedParameters));
                }
                
                if (!compilerResult.HasErrors && isPdxfx)
                {
                    lock (earlyCompilerCache)
                    {
                        List<CompilerResults> effectCompilerResults;
                        if (!earlyCompilerCache.TryGetValue(effectName, out effectCompilerResults))
                        {
                            effectCompilerResults = new List<CompilerResults>();
                            earlyCompilerCache.Add(effectName, effectCompilerResults);
                        }

                        // Register bytecode used parameters so that they are checked when another effect is instanced
                        effectCompilerResults.Add(compilerResult);
                    }
                }
            }

            foreach (var message in compilerResult.Messages)
            {
                Log.Log(message);
            }

            return compilerResult;
        }
Beispiel #17
0
        public void TestStream()
        {
            var compiler = new EffectCompiler { UseFileSystem = true };
            compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Shaders.Tests\GameAssets\Compiler");
            compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Graphics\Shaders");
            compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Engine\Shaders");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Core");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Lights");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Materials");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Shadows");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\ComputeColor");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Skinning");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Shading");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Transformation");
            compiler.SourceDirectories.Add(@"..\..\sources\shaders\Utils");
            var compilerParameters = new CompilerParameters { Platform = GraphicsPlatform.Direct3D11 };
            var results = compiler.Compile(new ShaderClassSource("TestStream"), compilerParameters);

            Assert.IsFalse(results.HasErrors);
        }
Beispiel #18
0
        public void TestMixinAndComposeKeys()
        {
            var compiler = new EffectCompiler { UseFileSystem = true };
            compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Graphics\Shaders");
            compiler.SourceDirectories.Add(@"..\..\sources\engine\SiliconStudio.Paradox.Shaders.Tests\GameAssets\Mixins");

            var compilerParameters = new CompilerParameters {Platform = GraphicsPlatform.Direct3D11};

            var subCompute1Key = TestABC.TestParameters.UseComputeColor2.ComposeWith("SubCompute1");
            var subCompute2Key = TestABC.TestParameters.UseComputeColor2.ComposeWith("SubCompute2");
            var subComputesKey = TestABC.TestParameters.UseComputeColorRedirect.ComposeWith("SubComputes[0]");

            compilerParameters.Set(subCompute1Key, true);
            compilerParameters.Set(subComputesKey, true);

            var results = compiler.Compile(new ShaderMixinGeneratorSource("test_mixin_compose_keys"), compilerParameters);

            Assert.IsFalse(results.HasErrors);

            var mainBytecode = results.Bytecode.WaitForResult();
            Assert.IsFalse(mainBytecode.CompilationLog.HasErrors);

            Assert.NotNull(mainBytecode.Bytecode.Reflection.ConstantBuffers);
            Assert.AreEqual(1, mainBytecode.Bytecode.Reflection.ConstantBuffers.Count);
            var cbuffer = mainBytecode.Bytecode.Reflection.ConstantBuffers[0];

            Assert.NotNull(cbuffer.Members);
            Assert.AreEqual(2, cbuffer.Members.Length);


            // Check that ComputeColor2.Color is correctly composed for variables
            var computeColorSubCompute2 = ComputeColor2Keys.Color.ComposeWith("SubCompute1");
            var computeColorSubComputes = ComputeColor2Keys.Color.ComposeWith("ColorRedirect.SubComputes[0]");

            var members = cbuffer.Members.Select(member => member.Param.KeyName).ToList();
            Assert.IsTrue(members.Contains(computeColorSubCompute2.Name));
            Assert.IsTrue(members.Contains(computeColorSubComputes.Name));
        }
        public static void Write(string name, CompilerParameters parameters, EffectBytecode effectData, TextWriter writer)
        {
            const string codeTemplate = @"//------------------------------------------------------------------------------
// <auto-generated>
//     Paradox Effect Compiler File Generated:
{0}//
//     Command Line: {7}
//     Changes to this file may cause incorrect behavior and will be lost if
//     the code is regenerated.
// </auto-generated>
//------------------------------------------------------------------------------

namespace {1} 
{{
    {2} class {3}
    {{
        {4} static readonly byte[] {5} = new byte[] {{
{6}
        }};
    }}
}}
";
            var effectToGenerateText = new StringBuilder();
            effectToGenerateText.AppendFormat("//     Effect [{0}]\r\n", name);

            var buffer = new MemoryStream();
            effectData.WriteTo(buffer);

            var bufferAsText = new StringBuilder();
            var bufferArray = buffer.ToArray();
            for (int i = 0; i < bufferArray.Length; i++)
            {
                bufferAsText.Append(bufferArray[i]).Append(", ");
                if (i > 0 && (i % 64) == 0)
                {
                    bufferAsText.AppendLine();
                }
            }

            var classDeclaration = parameters.Get(EffectSourceCodeKeys.ClassDeclaration);
            var fieldDeclaration = parameters.Get(EffectSourceCodeKeys.FieldDeclaration);
            var nameSpace = parameters.Get(EffectSourceCodeKeys.Namespace);
            var className = parameters.Get(EffectSourceCodeKeys.ClassName) ?? name;
            var fieldName = parameters.Get(EffectSourceCodeKeys.FieldName);

            var commandLine = string.Join(" ", Environment.GetCommandLineArgs());

            var graphicsPlatform = parameters.Get(CompilerParameters.GraphicsPlatformKey);
            string paradoxDefine = "undefined";
            switch (graphicsPlatform)
            {
                case GraphicsPlatform.Direct3D11:
                    paradoxDefine = "SILICONSTUDIO_PARADOX_GRAPHICS_API_DIRECT3D11";
                    break;
                case GraphicsPlatform.OpenGL:
                    paradoxDefine = "SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGLCORE";
                    break;
                case GraphicsPlatform.OpenGLES:
                    paradoxDefine = "SILICONSTUDIO_PARADOX_GRAPHICS_API_OPENGLES";
                    break;
            }

            writer.WriteLine("#if {0}", paradoxDefine);
            writer.Write(codeTemplate,
                         effectToGenerateText, // {0} 
                         nameSpace,            // {1} 
                         classDeclaration,     // {2} 
                         className,            // {3} 
                         fieldDeclaration,     // {4} 
                         fieldName,            // {5} 
                         bufferAsText,         // {6}
                         commandLine);         // {7}

            writer.WriteLine("#endif");

            writer.Flush();
        }
Beispiel #20
0
        public void TestGlslESCompiler()
        {
            VirtualFileSystem.RemountFileSystem("/shaders", "../../../../shaders");
            VirtualFileSystem.RemountFileSystem("/baseShaders", "../../../../engine/SiliconStudio.Paradox.Graphics/Shaders");
            VirtualFileSystem.RemountFileSystem("/compiler", "Compiler");

            var compiler = new EffectCompiler();

            compiler.SourceDirectories.Add("shaders");
            compiler.SourceDirectories.Add("compiler");
            compiler.SourceDirectories.Add("baseShaders");

            var compilerParameters = new CompilerParameters { Platform = GraphicsPlatform.OpenGLES };

            var results = compiler.Compile(new ShaderMixinGeneratorSource("ToGlslEffect"), compilerParameters);
        }
 /// <summary>
 /// Compiles the ShaderMixinSource into a platform bytecode.
 /// </summary>
 /// <param name="mixinTree">The mixin tree.</param>
 /// <param name="compilerParameters">The compiler parameters.</param>
 /// <returns>The platform-dependent bytecode.</returns>
 public abstract TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters);
 public override TaskOrResult<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
 {
     return CompileAsync(mixinTree, compilerParameters);
 }
        private void CreateEffect(DynamicEffectInstance effectInstance, ParameterCollection passParameters)
        {
            var compilerParameters = new CompilerParameters();
            parameterCollections.Clear(true);
            if (passParameters != null)
            {
                parameterCollections.Add(passParameters);
            }
            effectInstance.FillParameterCollections(parameterCollections);

            foreach (var parameterCollection in parameterCollections)
            {
                if (parameterCollection != null)
                {
                    foreach (var parameter in parameterCollection.InternalValues)
                    {
                        compilerParameters.SetObject(parameter.Key, parameter.Value.Object);
                    }
                }
            }

            compilerParameters.TaskPriority = taskPriority;

            foreach (var parameter in GraphicsDevice.Parameters.InternalValues)
            {
                compilerParameters.SetObject(parameter.Key, parameter.Value.Object);
            }

            // Compile shader
            // possible exception in LoadEffect
            TaskOrResult<Effect> effect;
            ParameterCollection usedParameters;
            try
            {
                effect = EffectSystem.LoadEffect(EffectName, compilerParameters, out usedParameters);
            }
            catch (Exception)
            {
                effectInstance.HasErrors = true;

                // Fallback to error effect
                var fallbackEffect = ComputeFallbackEffect(this, FallbackEffectType.Error, EffectName, compilerParameters);
                UpdateEffect(effectInstance, fallbackEffect.Effect, fallbackEffect.UsedParameters, passParameters);

                return;
            }

            // Do we have an async compilation?
            if (asyncEffectCompiler && effect.Task != null)
            {
                effectInstance.CurrentlyCompilingEffect = effect.Task;
                effectInstance.CurrentlyCompilingUsedParameters = usedParameters;

                if (!effectInstance.HasErrors) // If there was an error, stay in that state (we don't want to switch between reloading and error states)
                {
                    // Fallback to default effect
                    var fallbackEffect = ComputeFallbackEffect(this, FallbackEffectType.Compiling, EffectName, compilerParameters);
                    UpdateEffect(effectInstance, fallbackEffect.Effect, fallbackEffect.UsedParameters, passParameters);
                }
                return;
            }

            var compiledEffect = effect.WaitForResult();

            UpdateEffect(effectInstance, compiledEffect, usedParameters, passParameters);

            // Effect has been updated
            effectInstance.CurrentlyCompilingEffect = null;
            effectInstance.CurrentlyCompilingUsedParameters = null;
        }
        public async Task<EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, CompilerParameters compilerParameters)
        {
            var socketContextClient = await socketContextClientTCS.Task;

            var shaderCompilerAnswer = (ShaderCompilerAnswer)await socketContextClient.SendReceiveAsync(new ShaderCompilerRequest
            {
                MixinTree = mixinTree,
            });

            // TODO: Get LoggerResult as well
            return new EffectBytecodeCompilerResult(shaderCompilerAnswer.EffectBytecode);
        }
Beispiel #25
0
        private CompilerParameters BuildCompilerParameters(DynamicEffectInstance effectInstance, ParameterCollection passParameters)
        {
            var compilerParameters = new CompilerParameters();
            parameterCollections.Clear();
            if (passParameters != null)
            {
                parameterCollections.Add(passParameters);
            }
            effectInstance.FillParameterCollections(ref parameterCollections);

            foreach (var parameterCollection in parameterCollections)
            {
                if (parameterCollection != null)
                {
                    foreach (var parameter in parameterCollection.InternalValues)
                    {
                        compilerParameters.SetObject(parameter.Key, parameter.Value.Object);
                    }
                }
            }

            compilerParameters.TaskPriority = taskPriority;

            foreach (var parameter in GraphicsDevice.Parameters.InternalValues)
            {
                compilerParameters.SetObject(parameter.Key, parameter.Value.Object);
            }
            return compilerParameters;
        }
        /// <summary>
        /// Update a dynamic effect instance based on its parameters.
        /// </summary>
        /// <param name="effectInstance">A dynmaic effect instance</param>
        /// <param name="passParameters">The pass parameters.</param>
        /// <returns><c>true</c> if the effect was recomiled on the effect instance, <c>false</c> otherwise.</returns>
        public bool Update(DynamicEffectInstance effectInstance, ParameterCollection passParameters)
        {
            bool effectChanged = false;

            var currentlyCompilingEffect = effectInstance.CurrentlyCompilingEffect;
            if (currentlyCompilingEffect != null)
            {
                if (currentlyCompilingEffect.IsCompleted)
                {
                    if (currentlyCompilingEffect.IsFaulted)
                    {
                        var compilerParameters = new CompilerParameters();
                        effectInstance.CurrentlyCompilingUsedParameters.CopyTo(compilerParameters);

                        effectInstance.HasErrors = true;

                        // Fallback for errors
                        var fallbackEffect = ComputeFallbackEffect(this, FallbackEffectType.Error, EffectName, compilerParameters);
                        UpdateEffect(effectInstance, fallbackEffect.Effect, fallbackEffect.UsedParameters, passParameters);
                    }
                    else
                    {
                        effectInstance.HasErrors = false;
                        // Do not update effect right away: passParameters might have changed since last compilation; just try to go through a CreateEffect that will properly update the effect synchronously
                        // TODO: This class (and maybe whole ParameterCollection system) need a complete rethink and rewrite with newest assumptions...
                        //UpdateEffect(effectInstance, currentlyCompilingEffect.Result, effectInstance.CurrentlyCompilingUsedParameters, passParameters);
                    }

                    effectChanged = true;

                    // Effect has been updated
                    effectInstance.CurrentlyCompilingEffect = null;
                    effectInstance.CurrentlyCompilingUsedParameters = null;
                }
            }

            if (effectChanged || // Check again, in case effect was just finished async compilation
                (effectInstance.Effect == null || !EffectSystem.IsValid(effectInstance.Effect) || HasCollectionChanged(effectInstance, passParameters) || effectInstance.HasErrors))
            {
                if (effectInstance.HasErrors)
                {
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
                    var currentTime = DateTime.Now;
                    if (currentTime < effectInstance.LastErrorCheck + ErrorCheckTimeSpan)
                    {
                        // Wait a regular interval before retrying to compile effect (i.e. every second)
                        return false;
                    }

                    // Update last check time
                    effectInstance.LastErrorCheck = currentTime;
#else
                    // Other platforms: never try to recompile failed effects for now
                    return false;
#endif
                }

                CreateEffect(effectInstance, passParameters);
                effectChanged = true;
            }

            return effectChanged;
        }
Beispiel #27
0
 /// <summary>
 /// Loads the effect.
 /// </summary>
 /// <param name="effectName">Name of the effect.</param>
 /// <param name="compilerParameters">The compiler parameters.</param>
 /// <returns>A new instance of an effect.</returns>
 /// <exception cref="System.InvalidOperationException">Could not compile shader. Need fallback.</exception>
 public TaskOrResult<Effect> LoadEffect(string effectName, CompilerParameters compilerParameters)
 {
     ParameterCollection usedParameters;
     return LoadEffect(effectName, compilerParameters, out usedParameters);
 }
 /// <summary>
 /// Creates an effect.
 /// </summary>
 /// <param name="effectSystem">The effect system.</param>
 /// <param name="effectName">Name of the effect.</param>
 /// <returns>A new instance of an effect.</returns>
 public static TaskOrResult<Effect> LoadEffect(this EffectSystem effectSystem, string effectName)
 {
     var compilerParameters = new CompilerParameters();
     return effectSystem.LoadEffect(effectName, compilerParameters);
 }
Beispiel #29
0
 private void SwitchFallbackEffect(FallbackEffectType fallbackEffectType, DynamicEffectInstance effectInstance, ParameterCollection passParameters, CompilerParameters compilerParameters)
 {
     // Fallback for errors
     effectInstance.HasErrors = true;
     var fallbackEffect = ComputeFallbackEffect(this, fallbackEffectType, EffectName, compilerParameters);
     UpdateEffect(effectInstance, fallbackEffect.Effect, fallbackEffect.UsedParameters, passParameters);
 }