private ObjectId GetHashId()
{
// This methods use the stream that is already filled-up by the standard binary serialization of the object
// Here we add ids and overrides metadata informations to the stream in order to calculate an accurate id
var stream = streamOrValueType as MemoryStream;
if (stream != null)
{
// ------------------------------------------------------
// Un-comment the following code to debug the ObjectId of the serialized version without taking into account overrides
// ------------------------------------------------------
//var savedPosition = stream.Position;
//stream.Position = 0;
//var intermediateHashId = ObjectId.FromBytes(stream.ToArray());
//stream.Position = savedPosition;
var writer = new BinarySerializationWriter(stream);
// Write invariant objects
foreach (var invarialtObject in invariantObjects)
{
writer.SerializeExtended(invarialtObject, ArchiveMode.Serialize);
}
writer.Flush();
stream.Position = 0;
return(ObjectId.FromBytes(stream.ToArray()));
}
return(ObjectId.Empty);
}
/// <summary>
/// Generates a unique identifier from location.
/// </summary>
/// <param name="location">The location.</param>
/// <returns>Guid.</returns>
public static AssetId GenerateIdFromLocation(string packageName, string location)
{
if (location == null)
{
throw new ArgumentNullException(nameof(location));
}
return((AssetId)ObjectId.Combine(ObjectId.FromBytes(Encoding.UTF8.GetBytes(packageName)), ObjectId.FromBytes(Encoding.UTF8.GetBytes(location))).ToGuid());
}
/// <summary>
/// Generates a unique identifier from location.
/// </summary>
/// <param name="location">The location.</param>
/// <returns>Guid.</returns>
public static Guid GenerateGuidFromLocation(string location)
{
if (location == null)
{
throw new ArgumentNullException("location");
}
return(ObjectId.FromBytes(Encoding.UTF8.GetBytes(location)).ToGuid());
}
public unsafe static string ConvertTypeId(this TypeReference type)
{
var typeName = type.ConvertCSharp(false);
var typeId = ObjectId.FromBytes(Encoding.UTF8.GetBytes(typeName));
var typeIdHash = (uint *)&typeId;
return(string.Format("new {0}(0x{1:x8}, 0x{2:x8}, 0x{3:x8}, 0x{4:x8})", typeof(ObjectId).FullName, typeIdHash[0], typeIdHash[1], typeIdHash[2], typeIdHash[3]));
}
private static ModelMaterial AttachId(ModelMaterial modelMaterial)
{
// Compute an id for the list item based on the name of the material
var materialNameKey = modelMaterial.Name;
var modelMaterialId = ObjectId.FromBytes(Encoding.UTF8.GetBytes(materialNameKey)).ToGuid();
IdentifiableHelper.SetId(modelMaterial, modelMaterialId);
return(modelMaterial);
}
private static void EnsureSerializationTypeId([NotNull] DataSerializer dataSerializer)
{
// Ensure a serialization type ID has been generated (otherwise do so now)
if (dataSerializer.SerializationTypeId == ObjectId.Empty)
{
// Need to generate serialization type id
var typeName = dataSerializer.SerializationType.FullName;
dataSerializer.SerializationTypeId = ObjectId.FromBytes(System.Text.Encoding.UTF8.GetBytes(typeName));
}
}
/// <summary>
/// Converts the hlsl code into glsl and stores the result as plain text
/// </summary>
/// <param name="shaderSource">the hlsl shader</param>
/// <param name="entryPoint">the entrypoint function name</param>
/// <param name="stage">the shader pipeline stage</param>
/// <param name="compilerParameters"></param>
/// <param name="reflection">the reflection gathered from the hlsl analysis</param>
/// <param name="sourceFilename">the name of the source file</param>
/// <returns></returns>
public ShaderBytecodeResult Compile(string shaderSource, string entryPoint, ShaderStage stage, CompilerParameters compilerParameters, EffectReflection reflection, string sourceFilename = null)
{
var isOpenGLES = compilerParameters.EffectParameters.Platform == GraphicsPlatform.OpenGLES;
var isOpenGLES3 = compilerParameters.EffectParameters.Profile >= GraphicsProfile.Level_10_0;
var shaderBytecodeResult = new ShaderBytecodeResult();
byte[] rawData;
var shader = Compile(shaderSource, entryPoint, stage, isOpenGLES, isOpenGLES3, shaderBytecodeResult, reflection, sourceFilename);
if (shader == null)
{
return(shaderBytecodeResult);
}
if (isOpenGLES)
{
// store both ES 2 and ES 3 on OpenGL ES platforms
var shaderBytecodes = new ShaderLevelBytecode();
if (isOpenGLES3)
{
shaderBytecodes.DataES3 = shader;
shaderBytecodes.DataES2 = null;
}
else
{
shaderBytecodes.DataES2 = shader;
shaderBytecodes.DataES3 = Compile(shaderSource, entryPoint, stage, true, true, shaderBytecodeResult, reflection, sourceFilename);
}
using (var stream = new MemoryStream())
{
BinarySerialization.Write(stream, shaderBytecodes);
#if !SILICONSTUDIO_RUNTIME_CORECLR
rawData = stream.GetBuffer();
#else
// FIXME: Manu: The call to "ToArray()" might be slower than "GetBuffer()"
rawData = stream.ToArray();
#endif
}
}
else
{
// store string on OpenGL platforms
rawData = Encoding.ASCII.GetBytes(shader);
}
var bytecodeId = ObjectId.FromBytes(rawData);
var bytecode = new ShaderBytecode(bytecodeId, rawData);
bytecode.Stage = stage;
shaderBytecodeResult.Bytecode = bytecode;
return(shaderBytecodeResult);
}
/// <summary>
/// Adds the shader source registered manually.
/// </summary>
/// <param name="type">The type.</param>
/// <param name="sourceCode">The source code.</param>
/// <param name="sourcePath">The source path.</param>
public void AddShaderSource(string type, string sourceCode, string sourcePath)
{
lock (locker)
{
var shaderSource = new ShaderSourceWithHash()
{
Source = sourceCode, Path = sourcePath
};
shaderSource.Hash = ObjectId.FromBytes(Encoding.UTF8.GetBytes(shaderSource.Source));
loadedShaderSources[type] = shaderSource;
classNameToPath[type] = sourcePath;
}
}
public void TestImportWithDuplicateWithinSameSession()
{
var name = "TestAssetImport";
var file = Path.Combine(Path.GetTempPath(), name + ".tmp");
const string fileContent = "This is the file content";
File.WriteAllText(file, fileContent);
var fileHash = ObjectId.FromBytes(Encoding.UTF8.GetBytes(fileContent));
// Create a project with an asset reference a raw file
var project = new Package();
using (var session = new PackageSession(project))
{
var importSession = new AssetImportSession(session);
// ------------------------------------------------------------------
// Step 1: Add files to session
// ------------------------------------------------------------------
importSession.AddFile(file, project, UDirectory.Empty);
// Simulate using another importer to duplicate the assets
importSession.AddFile(file, new CustomImporter(), project, UDirectory.Empty);
Assert.AreEqual(2, importSession.Imports[0].ByImporters.Count);
// ------------------------------------------------------------------
// Step 2: Stage assets
// ------------------------------------------------------------------
var stageResult = importSession.Stage();
//importSession.Imports[0].ByImporters[1].Items[0].SelectedItem = importSession.Imports[0].ByImporters[1].Items[0].Merges[0].PreviousItem;
// Merge 0 into 1
importSession.Imports[0].ByImporters[0].Items[0].SelectedItem = importSession.Imports[0].ByImporters[0].Items[0].Merges[0].PreviousItem;
// ------------------------------------------------------------------
// Step 4: Merge the asset specified by the previous step
// ------------------------------------------------------------------
importSession.Merge();
Assert.IsTrue(stageResult);
// ------------------------------------------------------------------
// Step 3: Import merged asset
// ------------------------------------------------------------------
var result = importSession.Import();
Assert.AreEqual(3, project.Assets.Count);
// TODO Add more tests
}
}
/// <summary>
/// Build the ModuleMixinInfo class
/// </summary>
/// <param name="shaderSource">the ShaderSource to load</param>
/// <param name="macros">the macros applied on the source</param>
/// <returns>the ModuleMixinInfo</returns>
private ModuleMixinInfo BuildMixinInfo(ShaderSource shaderSource, Stride.Core.Shaders.Parser.ShaderMacro[] macros)
{
ModuleMixinInfo mixinInfo = null;
if (shaderSource is ShaderClassCode)
{
var shaderClassSource = shaderSource as ShaderClassCode;
mixinInfo = new ModuleMixinInfo
{
ShaderSource = shaderClassSource,
Macros = macros,
ReferencedShaders = { shaderClassSource.ClassName }
};
LoadMixinFromClassSource(mixinInfo);
}
else if (shaderSource is ShaderMixinSource)
{
var shaderMixinSource = shaderSource as ShaderMixinSource;
var shaderName = "Mix" + lastMixIndex;
++lastMixIndex;
var fakeAst = new ShaderClassType(shaderName);
foreach (var classSource in shaderMixinSource.Mixins)
{
Identifier name;
if (classSource.GenericArguments != null && classSource.GenericArguments.Length > 0)
{
name = new IdentifierGeneric(classSource.ClassName, classSource.GenericArguments.Select(x => new Identifier(x.ToString())).ToArray());
}
else
{
name = new Identifier(classSource.ClassName);
}
fakeAst.BaseClasses.Add(new TypeName(name));
}
mixinInfo = new ModuleMixinInfo
{
MixinGenericName = shaderName,
Macros = macros,
MixinAst = fakeAst,
ShaderSource = shaderSource,
SourceHash = ObjectId.FromBytes(Encoding.UTF8.GetBytes(shaderName)),
Instanciated = true
};
}
return(mixinInfo);
}
/// <summary>
/// Converts the hlsl code into glsl and stores the result as plain text
/// </summary>
/// <param name="shaderSource">the hlsl shader</param>
/// <param name="entryPoint">the entrypoint function name</param>
/// <param name="stage">the shader pipeline stage</param>
/// <param name="compilerParameters"></param>
/// <param name="reflection">the reflection gathered from the hlsl analysis</param>
/// <param name="sourceFilename">the name of the source file</param>
/// <returns></returns>
public ShaderBytecodeResult Compile(string shaderSource, string entryPoint, ShaderStage stage, ShaderMixinParameters compilerParameters, EffectReflection reflection, string sourceFilename = null)
{
var isOpenGLES = compilerParameters.Get(CompilerParameters.GraphicsPlatformKey) == GraphicsPlatform.OpenGLES;
var isOpenGLES3 = compilerParameters.Get(CompilerParameters.GraphicsProfileKey) >= GraphicsProfile.Level_10_0;
var shaderBytecodeResult = new ShaderBytecodeResult();
byte[] rawData;
var shader = Compile(shaderSource, entryPoint, stage, isOpenGLES, isOpenGLES3, shaderBytecodeResult, sourceFilename);
if (shader == null)
{
return(shaderBytecodeResult);
}
if (isOpenGLES)
{
// store both ES 2 and ES 3 on OpenGL ES platforms
var shaderBytecodes = new ShaderLevelBytecode();
if (isOpenGLES3)
{
shaderBytecodes.DataES3 = shader;
shaderBytecodes.DataES2 = null;
}
else
{
shaderBytecodes.DataES2 = shader;
shaderBytecodes.DataES3 = Compile(shaderSource, entryPoint, stage, true, true, shaderBytecodeResult, sourceFilename);
}
using (var stream = new MemoryStream())
{
BinarySerialization.Write(stream, shaderBytecodes);
rawData = stream.GetBuffer();
}
}
else
{
// store string on OpenGL platforms
rawData = Encoding.ASCII.GetBytes(shader);
}
var bytecodeId = ObjectId.FromBytes(rawData);
var bytecode = new ShaderBytecode(bytecodeId, rawData);
bytecode.Stage = stage;
shaderBytecodeResult.Bytecode = bytecode;
return(shaderBytecodeResult);
}
private void PrepareSerializer(DataSerializer dataSerializer)
{
// Ensure a serialization type ID has been generated (otherwise do so now)
if (dataSerializer.SerializationTypeId == ObjectId.Empty)
{
// Need to generate serialization type id
var typeName = dataSerializer.SerializationType.FullName;
dataSerializer.SerializationTypeId = ObjectId.FromBytes(System.Text.Encoding.UTF8.GetBytes(typeName));
}
if (dataSerializer is IDataSerializerInitializer)
{
((IDataSerializerInitializer)dataSerializer).Initialize(this);
}
}
public ShaderBytecodeResult Compile(string shaderSource, string entryPoint, ShaderStage stage, ShaderMixinParameters compilerParameters, EffectReflection reflection, string sourceFilename = null)
{
var isDebug = compilerParameters.Get(CompilerParameters.DebugKey);
var profile = compilerParameters.Get(CompilerParameters.GraphicsProfileKey);
var shaderModel = ShaderStageToString(stage) + "_" + ShaderProfileFromGraphicsProfile(profile);
var shaderFlags = ShaderFlags.None;
if (isDebug)
{
shaderFlags = ShaderFlags.OptimizationLevel0 | ShaderFlags.Debug;
}
SharpDX.Configuration.ThrowOnShaderCompileError = false;
// Compile using D3DCompiler
var compilationResult = SharpDX.D3DCompiler.ShaderBytecode.Compile(shaderSource, entryPoint, shaderModel, shaderFlags, EffectFlags.None, null, null, sourceFilename);
var byteCodeResult = new ShaderBytecodeResult();
if (compilationResult.HasErrors || compilationResult.Bytecode == null)
{
// Log compilation errors
byteCodeResult.Error(compilationResult.Message);
}
else
{
// As effect bytecode binary can changed when having debug infos (with d3dcompiler_47), we are calculating a bytecodeId on the stripped version
var rawData = compilationResult.Bytecode.Strip(StripFlags.CompilerStripDebugInformation | StripFlags.CompilerStripReflectionData);
var bytecodeId = ObjectId.FromBytes(rawData);
byteCodeResult.Bytecode = new ShaderBytecode(bytecodeId, compilationResult.Bytecode.Data)
{
Stage = stage
};
// If compilation succeed, then we can update reflection.
UpdateReflection(byteCodeResult.Bytecode, reflection, byteCodeResult);
if (!string.IsNullOrEmpty(compilationResult.Message))
{
byteCodeResult.Warning(compilationResult.Message);
}
}
return(byteCodeResult);
}
private static AssetItem ImportSkeleton(List <AssetItem> assetReferences, UFile assetSource, UFile localPath, EntityInfo entityInfo)
{
var asset = new SkeletonAsset {
Source = assetSource
};
if (entityInfo.Nodes != null)
{
for (int i = 0; i < entityInfo.Nodes.Count; i++)
{
var node = entityInfo.Nodes[i];
var nodeInfo = new NodeInformation(node.Name, node.Depth, node.Preserve);
// Try to keep identifier id consistent
// TODO: We might remove this as we don't expect Skeleton asset to be inherited, but they could
int sameNameAndDepthCount = 0;
for (int j = 0; j < i; j++)
{
var againstNode = entityInfo.Nodes[i];
// If we found a node with the same name and depth, we use a increment a counter
if (againstNode.Name == node.Name && againstNode.Depth == node.Depth)
{
sameNameAndDepthCount++;
}
}
var nodeNameKey = nodeInfo.Name + nodeInfo.Depth + ((sameNameAndDepthCount > 0) ? "_" + sameNameAndDepthCount : string.Empty);
var nodeId = ObjectId.FromBytes(Encoding.UTF8.GetBytes(nodeNameKey)).ToGuid();
IdentifiableHelper.SetId(nodeInfo, nodeId);
asset.Nodes.Add(nodeInfo);
}
}
if (entityInfo.AnimationNodes != null && entityInfo.AnimationNodes.Count > 0)
{
asset.PreserveNodes(entityInfo.AnimationNodes);
}
var skeletonUrl = new UFile(localPath.GetFileName() + " Skeleton");
var assetItem = new AssetItem(skeletonUrl, asset);
assetReferences.Add(assetItem);
return(assetItem);
}
/// <summary>
/// Computes the hash from an assembly based on its AssemblyFileVersion. Recurse to all assembly dependencies.
/// </summary>
/// <param name="assembly">The assembly.</param>
/// <returns>A full hash of this assembly, including all its dependencies.</returns>
public static string ComputeAssemblyHash(Assembly assembly)
{
string hash;
lock (assemblyToHash)
{
if (!assemblyToHash.TryGetValue(assembly, out hash))
{
var assemblies = new HashSet <Assembly>();
var text = new StringBuilder();
ComputeAssemblyHash(assembly, assemblies, text);
hash = ObjectId.FromBytes(Encoding.UTF8.GetBytes(text.ToString())).ToString();
assemblyToHash.Add(assembly, hash);
Log.Debug($"Assembly Hash [{assembly.GetName().Name}] => [{hash}]");
}
}
return(hash);
}
private LoadedShaderClassType LoadShaderClass(ShaderClassSource classSource, string generics, LoggerResult log, SiliconStudio.Shaders.Parser.ShaderMacro[] macros = null)
{
var type = classSource.ClassName;
if (type == null)
{
throw new ArgumentNullException("type");
}
var shaderSourceKey = new ShaderSourceKey(type, generics, macros);
lock (loadedShaders)
{
// Already instantiated
LoadedShaderClassType shaderClass;
if (loadedShaders.TryGetValue(shaderSourceKey, out shaderClass))
{
return(shaderClass);
}
// Load file
var shaderSource = SourceManager.LoadShaderSource(type);
string preprocessedSource;
try
{
preprocessedSource = PreProcessor.Run(shaderSource.Source, shaderSource.Path, macros);
}
catch (Exception ex)
{
log.Error(MessageCode.ErrorUnexpectedException, new SourceSpan(new SourceLocation(shaderSource.Path, 0, 1, 1), 1), ex);
return(null);
}
byte[] byteArray = Encoding.UTF8.GetBytes(preprocessedSource);
var hashPreprocessSource = ObjectId.FromBytes(byteArray);
// Compile
var parsingResult = XenkoShaderParser.TryParse(preprocessedSource, shaderSource.Path);
parsingResult.CopyTo(log);
if (parsingResult.HasErrors)
{
return(null);
}
var shader = parsingResult.Shader;
// As shaders can be embedded in namespaces, get only the shader class and make sure there is only one in a xksl.
var shaderClassTypes = XenkoShaderParser.GetShaderClassTypes(shader.Declarations).ToList();
if (shaderClassTypes.Count != 1)
{
var sourceSpan = new SourceSpan(new SourceLocation(shaderSource.Path, 0, 0, 0), 1);
if (shaderClassTypes.Count > 1)
{
sourceSpan = shaderClassTypes[1].Span;
}
log.Error(XenkoMessageCode.ShaderMustContainSingleClassDeclaration, sourceSpan, type);
return(null);
}
shaderClass = new LoadedShaderClassType();
shaderClass.Type = shaderClassTypes.First();
shaderClass.SourcePath = shaderSource.Path;
shaderClass.SourceHash = shaderSource.Hash;
shaderClass.PreprocessedSourceHash = hashPreprocessSource;
shaderClass.IsInstanciated = false;
// TODO: We should not use Console. Change the way we log things here
// Console.WriteLine("Loading Shader {0}{1}", type, macros != null && macros.Length > 0 ? String.Format("<{0}>", string.Join(", ", macros)) : string.Empty);
// If the file name is not matching the class name, provide an error
if (shaderClass.Type.Name.Text != type)
{
log.Error(XenkoMessageCode.FileNameNotMatchingClassName, shaderClass.Type.Name.Span, type, shaderClass.Type.Name.Text);
return(null);
}
loadedShaders.Add(shaderSourceKey, shaderClass);
return(shaderClass);
}
}
public ShaderBytecodeResult Compile(string shaderSource, string entryPoint, ShaderStage stage, EffectCompilerParameters effectParameters, EffectReflection reflection, string sourceFilename = null)
{
var isDebug = effectParameters.Debug;
var optimLevel = effectParameters.OptimizationLevel;
var profile = effectParameters.Profile;
var shaderModel = ShaderStageToString(stage) + "_" + ShaderProfileFromGraphicsProfile(profile);
var shaderFlags = ShaderFlags.None;
if (isDebug)
{
shaderFlags = ShaderFlags.Debug;
}
switch (optimLevel)
{
case 0:
shaderFlags |= ShaderFlags.OptimizationLevel0;
break;
case 1:
shaderFlags |= ShaderFlags.OptimizationLevel1;
break;
case 2:
shaderFlags |= ShaderFlags.OptimizationLevel2;
break;
case 3:
shaderFlags |= ShaderFlags.OptimizationLevel3;
break;
}
SharpDX.Configuration.ThrowOnShaderCompileError = false;
// Compile using D3DCompiler
var compilationResult = SharpDX.D3DCompiler.ShaderBytecode.Compile(shaderSource, entryPoint, shaderModel, shaderFlags, EffectFlags.None, null, null, sourceFilename);
var byteCodeResult = new ShaderBytecodeResult();
if (compilationResult.HasErrors || compilationResult.Bytecode == null)
{
// Log compilation errors
byteCodeResult.Error(compilationResult.Message);
}
else
{
// TODO: Make this optional
try
{
byteCodeResult.DisassembleText = compilationResult.Bytecode.Disassemble();
}
catch (SharpDXException)
{
}
// As effect bytecode binary can changed when having debug infos (with d3dcompiler_47), we are calculating a bytecodeId on the stripped version
var rawData = compilationResult.Bytecode.Strip(StripFlags.CompilerStripDebugInformation | StripFlags.CompilerStripReflectionData);
var bytecodeId = ObjectId.FromBytes(rawData);
byteCodeResult.Bytecode = new ShaderBytecode(bytecodeId, compilationResult.Bytecode.Data)
{
Stage = stage
};
// If compilation succeed, then we can update reflection.
UpdateReflection(byteCodeResult.Bytecode, reflection, byteCodeResult);
if (!string.IsNullOrEmpty(compilationResult.Message))
{
byteCodeResult.Warning(compilationResult.Message);
}
}
return(byteCodeResult);
}
private ObjectId GetHashId()
{
// This methods use the stream that is already filled-up by the standard binary serialization of the object
// Here we add ids and overrides metadata informations to the stream in order to calculate an accurate id
var stream = streamOrValueType as MemoryStream;
if (stream != null)
{
// ------------------------------------------------------
// Un-comment the following code to debug the ObjectId of the serialized version without taking into account overrides
// ------------------------------------------------------
//var savedPosition = stream.Position;
//stream.Position = 0;
//var intermediateHashId = ObjectId.FromBytes(stream.ToArray());
//stream.Position = savedPosition;
var writer = new BinarySerializationWriter(stream);
Dictionary <string, OverrideType> overrides = null;
List <string> orderedNames = null;
foreach (var objectRef in objectReferences)
{
//// If the object is actually a reference to another asset, we can skip it as their won't be any overrides
//if (AttachedReferenceManager.GetAttachedReference(objectRef) != null)
//{
// continue;
//}
// Else gets the id if there are any (including shadows that are not part of the standard serialization)
var shadowObject = ShadowObject.GetOrCreate(objectRef);
if (shadowObject.IsIdentifiable)
{
// Get the shadow id (may be a non-shadow, so we may duplicate it in the stream (e.g Entity)
// but it should not be a big deal
var id = shadowObject.GetId(objectRef);
writer.Write(id);
}
// Dump all members with overrides informations
foreach (var item in shadowObject)
{
if (item.Key.Item2 == Override.OverrideKey)
{
// Use the member name to ensure a stable id
var memberName = ((IMemberDescriptor)item.Key.Item1).Name;
// Only creates the overrides dictionary if needed
if (overrides == null)
{
overrides = new Dictionary <string, OverrideType>();
}
overrides.Add(memberName, (OverrideType)item.Value);
}
}
// Write any overrides information to the stream
if (overrides != null)
{
// Collect names and order them by alphabetical order in order to make sure that we will get a stable id
// (Dictionary doesn't ensure order)
if (orderedNames == null)
{
orderedNames = new List <string>();
}
orderedNames.Clear();
foreach (var entry in overrides)
{
orderedNames.Add(entry.Key);
}
orderedNames.Sort();
// Write all overrides for the current object reference
foreach (var name in orderedNames)
{
writer.Write(name);
// Write the override as an int
writer.Write((int)overrides[name]);
}
// Clear overrides for next entry
overrides.Clear();
}
}
writer.Flush();
stream.Position = 0;
return(ObjectId.FromBytes(stream.ToArray()));
}
return(ObjectId.Empty);
}
/// <summary>
/// Generates serializer code using Cecil.
/// </summary>
/// <param name="registry"></param>
private static void GenerateSerializerCode(ComplexSerializerRegistry registry, out ObjectId serializationHash)
{
var hash = new ObjectIdBuilder();
// First, hash global binary format version, in case it gets bumped
hash.Write(DataSerializer.BinaryFormatVersion);
var assembly = registry.Assembly;
var strideCoreModule = assembly.GetStrideCoreModule();
var dataSerializerTypeRef = assembly.MainModule.ImportReference(strideCoreModule.GetType("Stride.Core.Serialization.DataSerializer`1"));
var serializerSelectorType = strideCoreModule.GetType("Stride.Core.Serialization.SerializerSelector");
var serializerSelectorTypeRef = assembly.MainModule.ImportReference(serializerSelectorType);
var serializerSelectorGetSerializerRef = assembly.MainModule.ImportReference(serializerSelectorType.Methods.Single(x => x.Name == "GetSerializer" && x.Parameters.Count == 0 && x.GenericParameters.Count == 1));
var memberSerializerCreateRef = assembly.MainModule.ImportReference(strideCoreModule.GetType("Stride.Core.Serialization.MemberSerializer`1").Methods.Single(x => x.Name == "Create"));
var dataSerializerSerializeMethod = dataSerializerTypeRef.Resolve().Methods.Single(x => x.Name == "Serialize" && (x.Attributes & MethodAttributes.Abstract) != 0);
var dataSerializerSerializeMethodRef = assembly.MainModule.ImportReference(dataSerializerSerializeMethod);
// Generate serializer code for each type (we generate code similar to ComplexClassSerializerGenerator.tt, see this file for reference)
foreach (var complexType in registry.Context.ComplexTypes)
{
var type = complexType.Key;
var serializerType = (TypeDefinition)complexType.Value.SerializerType;
var genericParameters = serializerType.GenericParameters.ToArray <TypeReference>();
var typeWithGenerics = type.MakeGenericType(genericParameters);
// Hash
hash.Write(typeWithGenerics.FullName);
TypeReference parentType = null;
FieldDefinition parentSerializerField = null;
if (complexType.Value.IsComplexSerializerProcessParentType)
{
parentType = ResolveGenericsVisitor.Process(serializerType, type.BaseType);
serializerType.Fields.Add(parentSerializerField = new FieldDefinition("parentSerializer", Mono.Cecil.FieldAttributes.Private, dataSerializerTypeRef.MakeGenericType(parentType)));
hash.Write("parent");
}
var serializableItems = ComplexSerializerRegistry.GetSerializableItems(type).ToArray();
var serializableItemInfos = new Dictionary <TypeReference, (FieldDefinition SerializerField, TypeReference Type)>(TypeReferenceEqualityComparer.Default);
var localsByTypes = new Dictionary <TypeReference, VariableDefinition>(TypeReferenceEqualityComparer.Default);
ResolveGenericsVisitor genericResolver = null;
if (type.HasGenericParameters)
{
var genericMapping = new Dictionary <TypeReference, TypeReference>();
for (int i = 0; i < type.GenericParameters.Count; i++)
{
genericMapping[type.GenericParameters[i]] = serializerType.GenericParameters[i];
}
genericResolver = new ResolveGenericsVisitor(genericMapping);
}
foreach (var serializableItem in serializableItems)
{
if (serializableItemInfos.ContainsKey(serializableItem.Type))
{
continue;
}
var serializableItemType = serializableItem.Type;
if (genericResolver != null)
{
serializableItemType = genericResolver.VisitDynamic(serializableItemType);
}
var fieldDefinition = new FieldDefinition($"{Utilities.BuildValidClassName(serializableItemType.FullName)}Serializer", Mono.Cecil.FieldAttributes.Private, dataSerializerTypeRef.MakeGenericType(serializableItemType));
serializableItemInfos.Add(serializableItem.Type, (fieldDefinition, serializableItemType));
serializerType.Fields.Add(fieldDefinition);
hash.Write(serializableItem.Type.FullName);
hash.Write(serializableItem.Name);
hash.Write(serializableItem.AssignBack);
}
// Add constructor (call parent constructor)
var ctor = new MethodDefinition(".ctor",
MethodAttributes.RTSpecialName | MethodAttributes.SpecialName | MethodAttributes.HideBySig |
MethodAttributes.Public, assembly.MainModule.TypeSystem.Void);
ctor.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_0));
ctor.Body.Instructions.Add(Instruction.Create(OpCodes.Call, assembly.MainModule.ImportReference(serializerType.BaseType.Resolve().GetEmptyConstructor(true)).MakeGeneric(typeWithGenerics)));
ctor.Body.Instructions.Add(Instruction.Create(OpCodes.Ret));
serializerType.Methods.Add(ctor);
// Add Initialize method
var initialize = new MethodDefinition("Initialize", MethodAttributes.Public | MethodAttributes.HideBySig | MethodAttributes.Virtual, assembly.MainModule.TypeSystem.Void);
initialize.Parameters.Add(new ParameterDefinition("serializerSelector", ParameterAttributes.None, serializerSelectorTypeRef));
if (complexType.Value.IsComplexSerializerProcessParentType)
{
initialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_0));
initialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_1));
initialize.Body.Instructions.Add(Instruction.Create(OpCodes.Callvirt, serializerSelectorGetSerializerRef.MakeGenericMethod(parentType)));
initialize.Body.Instructions.Add(Instruction.Create(OpCodes.Stfld, parentSerializerField.MakeGeneric(genericParameters)));
}
foreach (var serializableItem in serializableItemInfos)
{
initialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_0));
initialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_1));
initialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldc_I4_1));
initialize.Body.Instructions.Add(Instruction.Create(OpCodes.Call, memberSerializerCreateRef.MakeGeneric(serializableItem.Value.Type)));
initialize.Body.Instructions.Add(Instruction.Create(OpCodes.Stfld, serializableItem.Value.SerializerField.MakeGeneric(genericParameters)));
}
initialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ret));
serializerType.Methods.Add(initialize);
// Add Serialize method
var serialize = new MethodDefinition("Serialize", MethodAttributes.Public | MethodAttributes.HideBySig | MethodAttributes.Virtual, assembly.MainModule.TypeSystem.Void);
serialize.Parameters.Add(new ParameterDefinition("obj", ParameterAttributes.None, typeWithGenerics.MakeByReferenceType()));
// Copy other parameters from parent method
for (int i = 1; i < dataSerializerSerializeMethod.Parameters.Count; ++i)
{
var parentParameter = dataSerializerSerializeMethod.Parameters[i];
serialize.Parameters.Add(new ParameterDefinition(parentParameter.Name, ParameterAttributes.None, assembly.MainModule.ImportReference(parentParameter.ParameterType)));
}
if (complexType.Value.IsComplexSerializerProcessParentType)
{
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_0));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldfld, parentSerializerField.MakeGeneric(genericParameters)));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_1));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_2));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_3));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Callvirt, dataSerializerSerializeMethodRef.MakeGeneric(parentType)));
}
if (serializableItems.Length > 0)
{
var blockStartInstructions = new[] { Instruction.Create(OpCodes.Nop), Instruction.Create(OpCodes.Nop) };
// Iterate over ArchiveMode
for (int i = 0; i < 2; ++i)
{
var archiveMode = i == 0 ? ArchiveMode.Serialize : ArchiveMode.Deserialize;
// Check mode
if (archiveMode == ArchiveMode.Serialize)
{
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_2));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldc_I4, (int)archiveMode));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ceq));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Brfalse, blockStartInstructions[0]));
}
else
{
serialize.Body.Instructions.Add(blockStartInstructions[0]);
}
foreach (var serializableItem in serializableItems)
{
if (serializableItem.HasFixedAttribute)
{
throw new NotImplementedException("FixedBuffer attribute is not supported.");
}
var memberAssignBack = serializableItem.AssignBack;
var memberVariableName = (serializableItem.MemberInfo is PropertyDefinition || !memberAssignBack) ? ComplexSerializerRegistry.CreateMemberVariableName(serializableItem.MemberInfo) : null;
var serializableItemInfo = serializableItemInfos[serializableItem.Type];
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_0));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldfld, serializableItemInfo.SerializerField.MakeGeneric(genericParameters)));
var fieldReference = serializableItem.MemberInfo is FieldReference?assembly.MainModule.ImportReference((FieldReference)serializableItem.MemberInfo).MakeGeneric(genericParameters) : null;
if (memberVariableName != null)
{
// Use a temporary variable
if (!localsByTypes.TryGetValue(serializableItemInfo.Type, out var tempLocal))
{
tempLocal = new VariableDefinition(serializableItemInfo.Type);
localsByTypes.Add(serializableItemInfo.Type, tempLocal);
serialize.Body.Variables.Add(tempLocal);
serialize.Body.InitLocals = true;
}
if (!(archiveMode == ArchiveMode.Deserialize && memberAssignBack))
{
// obj.Member
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_1));
if (!type.IsValueType)
{
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldind_Ref));
}
if (serializableItem.MemberInfo is PropertyDefinition property)
{
var getMethod = property.Resolve().GetMethod;
serialize.Body.Instructions.Add(Instruction.Create(getMethod.IsVirtual ? OpCodes.Callvirt : OpCodes.Call, assembly.MainModule.ImportReference(getMethod).MakeGeneric(genericParameters)));
}
else if (serializableItem.MemberInfo is FieldDefinition)
{
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldfld, fieldReference));
}
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Stloc, tempLocal));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldloca, tempLocal));
}
else
{
// default(T)
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldloca, tempLocal));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Dup));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Initobj, serializableItemInfo.Type));
}
}
else
{
// Use object directly
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_1));
if (!type.IsValueType)
{
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldind_Ref));
}
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldflda, fieldReference));
}
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_2));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_3));
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Callvirt, dataSerializerSerializeMethodRef.MakeGeneric(serializableItemInfo.Type)));
if (archiveMode == ArchiveMode.Deserialize && memberVariableName != null && memberAssignBack)
{
// Need to copy back to object
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldarg_1));
if (!type.IsValueType)
{
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldind_Ref));
}
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ldloc, localsByTypes[serializableItemInfo.Type]));
if (serializableItem.MemberInfo is PropertyDefinition property)
{
var setMethod = property.Resolve().SetMethod;
serialize.Body.Instructions.Add(Instruction.Create(setMethod.IsVirtual ? OpCodes.Callvirt : OpCodes.Call, assembly.MainModule.ImportReference(setMethod).MakeGeneric(genericParameters)));
}
else if (serializableItem.MemberInfo is FieldDefinition)
{
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Stfld, fieldReference));
}
}
}
if (archiveMode == ArchiveMode.Serialize)
{
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Br, blockStartInstructions[1]));
}
}
serialize.Body.Instructions.Add(blockStartInstructions[1]);
}
serialize.Body.Instructions.Add(Instruction.Create(OpCodes.Ret));
serializerType.Methods.Add(serialize);
//assembly.MainModule.Types.Add(serializerType);
}
var mscorlibAssembly = CecilExtensions.FindCorlibAssembly(assembly);
var reflectionAssembly = CecilExtensions.FindReflectionAssembly(assembly);
// String
var stringType = mscorlibAssembly.MainModule.GetTypeResolved(typeof(string).FullName);
var stringTypeRef = assembly.MainModule.ImportReference(stringType);
// Type
var typeType = mscorlibAssembly.MainModule.GetTypeResolved(typeof(Type).FullName);
var typeTypeRef = assembly.MainModule.ImportReference(typeType);
var getTypeFromHandleMethod = typeType.Methods.First(x => x.Name == nameof(Type.GetTypeFromHandle));
var getTokenInfoExMethod = reflectionAssembly.MainModule.GetTypeResolved("System.Reflection.IntrospectionExtensions").Resolve().Methods.First(x => x.Name == nameof(IntrospectionExtensions.GetTypeInfo));
var typeInfoType = reflectionAssembly.MainModule.GetTypeResolved(typeof(TypeInfo).FullName);
// NOTE: TypeInfo.Assembly/Module could be on the type itself or on its parent MemberInfo depending on runtime
var getTypeInfoAssembly = typeInfoType.Properties.Concat(typeInfoType.BaseType.Resolve().Properties).First(x => x.Name == nameof(TypeInfo.Assembly)).GetMethod;
var getTypeInfoModule = typeInfoType.Properties.Concat(typeInfoType.BaseType.Resolve().Properties).First(x => x.Name == nameof(TypeInfo.Module)).GetMethod;
var typeHandleProperty = typeType.Properties.First(x => x.Name == nameof(Type.TypeHandle));
var getTypeHandleMethodRef = assembly.MainModule.ImportReference(typeHandleProperty.GetMethod);
// Generate code
var serializerFactoryType = new TypeDefinition("Stride.Core.DataSerializers",
Utilities.BuildValidClassName(assembly.Name.Name) + "SerializerFactory",
TypeAttributes.BeforeFieldInit | TypeAttributes.AnsiClass | TypeAttributes.AutoClass |
TypeAttributes.Public | TypeAttributes.Sealed | TypeAttributes.Abstract,
assembly.MainModule.TypeSystem.Object);
assembly.MainModule.Types.Add(serializerFactoryType);
var dataSerializerModeTypeRef = assembly.MainModule.ImportReference(strideCoreModule.GetType("Stride.Core.Serialization.DataSerializerGenericMode"));
var dataSerializerGlobalAttribute = strideCoreModule.GetType("Stride.Core.Serialization.DataSerializerGlobalAttribute");
var dataSerializerGlobalCtorRef = assembly.MainModule.ImportReference(dataSerializerGlobalAttribute.GetConstructors().Single(x => !x.IsStatic && x.Parameters.Count == 5));
foreach (var profile in registry.Context.SerializableTypesProfiles)
{
foreach (var type in profile.Value.SerializableTypes.Where(x => x.Value.IsLocal))
{
// Generating: [DataSerializerGlobalAttribute(<#= type.Value.SerializerType != null ? $"typeof({type.Value.SerializerType.ConvertCSharp(false)})" : "null" #>, typeof(<#= type.Key.ConvertCSharp(false) #>), DataSerializerGenericMode.<#= type.Value.Mode.ToString() #>, <#=type.Value.Inherited ? "true" : "false"#>, <#=type.Value.ComplexSerializer ? "true" : "false"#>, Profile = "<#=profile.Key#>")]
serializerFactoryType.CustomAttributes.Add(new CustomAttribute(dataSerializerGlobalCtorRef)
{
ConstructorArguments =
{
new CustomAttributeArgument(typeTypeRef, type.Value.SerializerType != null ? assembly.MainModule.ImportReference(type.Value.SerializerType) : null),
new CustomAttributeArgument(typeTypeRef, assembly.MainModule.ImportReference(type.Key)),
new CustomAttributeArgument(dataSerializerModeTypeRef, type.Value.GenericsMode),
new CustomAttributeArgument(assembly.MainModule.TypeSystem.Boolean, type.Value.IsInherited),
new CustomAttributeArgument(assembly.MainModule.TypeSystem.Boolean, type.Value.IsComplexSerializer),
},
Properties =
{
new CustomAttributeNamedArgument("Profile", new CustomAttributeArgument(assembly.MainModule.TypeSystem.String, profile.Key))
},
});
}
foreach (var type in profile.Value.GenericSerializableTypes.Where(x => x.Value.IsLocal))
{
// Generating: [DataSerializerGlobalAttribute(<#= type.Value.SerializerType != null ? $"typeof({type.Value.SerializerType.ConvertCSharp(true)})" : "null" #>, typeof(<#= type.Key.ConvertCSharp(true) #>), DataSerializerGenericMode.<#= type.Value.Mode.ToString() #>, <#=type.Value.Inherited ? "true" : "false"#>, <#=type.Value.ComplexSerializer ? "true" : "false"#>, Profile = "<#=profile.Key#>")]
serializerFactoryType.CustomAttributes.Add(new CustomAttribute(dataSerializerGlobalCtorRef)
{
ConstructorArguments =
{
new CustomAttributeArgument(typeTypeRef, type.Value.SerializerType != null ? assembly.MainModule.ImportReference(type.Value.SerializerType) : null),
new CustomAttributeArgument(typeTypeRef, assembly.MainModule.ImportReference(type.Key)),
new CustomAttributeArgument(dataSerializerModeTypeRef, type.Value.GenericsMode),
new CustomAttributeArgument(assembly.MainModule.TypeSystem.Boolean, type.Value.IsInherited),
new CustomAttributeArgument(assembly.MainModule.TypeSystem.Boolean, type.Value.IsComplexSerializer),
},
Properties =
{
new CustomAttributeNamedArgument("Profile", new CustomAttributeArgument(assembly.MainModule.TypeSystem.String, profile.Key))
},
});
}
}
// Create Initialize method
var initializeMethod = new MethodDefinition("Initialize",
MethodAttributes.Assembly | MethodAttributes.HideBySig | MethodAttributes.Static,
assembly.MainModule.TypeSystem.Void);
serializerFactoryType.Methods.Add(initializeMethod);
// Make sure it is called at module startup
initializeMethod.AddModuleInitializer(-1000);
var initializeMethodIL = initializeMethod.Body.GetILProcessor();
// Generating: var assemblySerializers = new AssemblySerializers(typeof(<#=registry.ClassName#>).GetTypeInfo().Assembly);
var assemblySerializersType = strideCoreModule.GetType("Stride.Core.Serialization.AssemblySerializers");
var assemblySerializersGetDataContractAliasesRef = assembly.MainModule.ImportReference(assemblySerializersType.Properties.First(x => x.Name == "DataContractAliases").GetMethod);
var assemblySerializersGetDataContractAliasesAdd = assemblySerializersGetDataContractAliasesRef.ReturnType.Resolve().Methods.First(x => x.Name == "Add");
var dataContractAliasTypeRef = ((GenericInstanceType)assemblySerializersGetDataContractAliasesRef.ReturnType).GenericArguments[0];
var dataContractAliasTypeCtorRef = assembly.MainModule.ImportReference(dataContractAliasTypeRef.Resolve().GetConstructors().Single());
var assemblySerializersGetDataContractAliasesAddRef = assembly.MainModule.ImportReference(assemblySerializersGetDataContractAliasesAdd).MakeGeneric(dataContractAliasTypeRef);
initializeMethodIL.Emit(OpCodes.Ldtoken, serializerFactoryType);
initializeMethodIL.Emit(OpCodes.Call, assembly.MainModule.ImportReference(getTypeFromHandleMethod));
initializeMethodIL.Emit(OpCodes.Call, assembly.MainModule.ImportReference(getTokenInfoExMethod));
initializeMethodIL.Emit(OpCodes.Callvirt, assembly.MainModule.ImportReference(getTypeInfoAssembly));
initializeMethodIL.Emit(OpCodes.Newobj, assembly.MainModule.ImportReference(assemblySerializersType.Methods.Single(x => x.IsConstructor && x.Parameters.Count == 1)));
foreach (var alias in registry.Context.DataContractAliases)
{
initializeMethodIL.Emit(OpCodes.Dup);
// Generating: assemblySerializers.DataContractAliases.Add(new AssemblySerializers.DataContractAlias(@"<#= alias.Item1 #>", typeof(<#= alias.Item2.ConvertCSharp(true) #>), <#=alias.Item3 ? "true" : "false"#>));
initializeMethodIL.Emit(OpCodes.Call, assemblySerializersGetDataContractAliasesRef);
initializeMethodIL.Emit(OpCodes.Ldstr, alias.Item1);
initializeMethodIL.Emit(OpCodes.Ldtoken, assembly.MainModule.ImportReference(alias.Item2));
initializeMethodIL.Emit(OpCodes.Call, assembly.MainModule.ImportReference(getTypeFromHandleMethod));
initializeMethodIL.Emit(alias.Item3 ? OpCodes.Ldc_I4_1 : OpCodes.Ldc_I4_0);
initializeMethodIL.Emit(OpCodes.Newobj, dataContractAliasTypeCtorRef);
initializeMethodIL.Emit(OpCodes.Call, assemblySerializersGetDataContractAliasesAddRef);
}
var assemblySerializersGetModulesRef = assembly.MainModule.ImportReference(assemblySerializersType.Properties.First(x => x.Name == "Modules").GetMethod);
var assemblySerializersGetModulesAdd = assemblySerializersGetModulesRef.ReturnType.Resolve().Methods.First(x => x.Name == "Add");
var moduleRef = ((GenericInstanceType)assemblySerializersGetModulesRef.ReturnType).GenericArguments[0];
var assemblySerializersGetModulesAddRef = assembly.MainModule.ImportReference(assemblySerializersGetModulesAdd).MakeGeneric(moduleRef);
foreach (var referencedAssemblySerializerFactoryType in registry.ReferencedAssemblySerializerFactoryTypes)
{
initializeMethodIL.Emit(OpCodes.Dup);
// Generating: assemblySerializers.Modules.Add(typeof(<#=referencedAssemblySerializerFactoryType.ConvertCSharp()#>).GetTypeInfo().Module);
initializeMethodIL.Emit(OpCodes.Call, assemblySerializersGetModulesRef);
initializeMethodIL.Emit(OpCodes.Ldtoken, assembly.MainModule.ImportReference(referencedAssemblySerializerFactoryType));
initializeMethodIL.Emit(OpCodes.Call, assembly.MainModule.ImportReference(getTypeFromHandleMethod));
initializeMethodIL.Emit(OpCodes.Call, assembly.MainModule.ImportReference(getTokenInfoExMethod));
initializeMethodIL.Emit(OpCodes.Callvirt, assembly.MainModule.ImportReference(getTypeInfoModule));
initializeMethodIL.Emit(OpCodes.Call, assemblySerializersGetModulesAddRef);
}
var objectIdCtorRef = assembly.MainModule.ImportReference(strideCoreModule.GetType("Stride.Core.Storage.ObjectId").GetConstructors().Single(x => x.Parameters.Count == 4));
var serializerEntryTypeCtorRef = assembly.MainModule.ImportReference(strideCoreModule.GetType("Stride.Core.Serialization.AssemblySerializerEntry").GetConstructors().Single());
var assemblySerializersPerProfileType = strideCoreModule.GetType("Stride.Core.Serialization.AssemblySerializersPerProfile");
var assemblySerializersPerProfileTypeAddRef = assembly.MainModule.ImportReference(assemblySerializersPerProfileType.BaseType.Resolve().Methods.First(x => x.Name == "Add")).MakeGeneric(serializerEntryTypeCtorRef.DeclaringType);
var assemblySerializersPerProfileTypeCtorRef = assembly.MainModule.ImportReference(assemblySerializersPerProfileType.GetEmptyConstructor());
var assemblySerializersGetProfilesRef = assembly.MainModule.ImportReference(assemblySerializersType.Properties.First(x => x.Name == "Profiles").GetMethod);
var assemblySerializersGetProfilesSetItemRef = assembly.MainModule.ImportReference(assemblySerializersGetProfilesRef.ReturnType.Resolve().Methods.First(x => x.Name == "set_Item"))
.MakeGeneric(((GenericInstanceType)assemblySerializersGetProfilesRef.ReturnType).GenericArguments.ToArray());
var runtimeHelpersType = mscorlibAssembly.MainModule.GetTypeResolved(typeof(RuntimeHelpers).FullName);
var runClassConstructorMethod = assembly.MainModule.ImportReference(runtimeHelpersType.Methods.Single(x => x.IsPublic && x.Name == "RunClassConstructor" && x.Parameters.Count == 1 && x.Parameters[0].ParameterType.FullName == typeof(RuntimeTypeHandle).FullName));
foreach (var profile in registry.Context.SerializableTypesProfiles)
{
initializeMethodIL.Emit(OpCodes.Dup);
// Generating: var assemblySerializersProfile = new AssemblySerializersPerProfile();
// Generating: assemblySerializers.Profiles["<#=profile.Key#>"] = assemblySerializersProfile;
initializeMethodIL.Emit(OpCodes.Callvirt, assemblySerializersGetProfilesRef);
initializeMethodIL.Emit(OpCodes.Ldstr, profile.Key);
initializeMethodIL.Emit(OpCodes.Newobj, assemblySerializersPerProfileTypeCtorRef);
foreach (var type in profile.Value.SerializableTypes.Where(x => x.Value.IsLocal))
{
// Generating: assemblySerializersProfile.Add(new AssemblySerializerEntry(<#=type.Key.ConvertTypeId()#>, typeof(<#= type.Key.ConvertCSharp() #>), <# if (type.Value.SerializerType != null) { #>typeof(<#= type.Value.SerializerType.ConvertCSharp() #>)<# } else { #>null<# } #>));
initializeMethodIL.Emit(OpCodes.Dup);
var typeName = type.Key.ConvertToValidCSharp(false);
var typeId = ObjectId.FromBytes(Encoding.UTF8.GetBytes(typeName));
unsafe
{
var typeIdHash = (int *)&typeId;
for (int i = 0; i < ObjectId.HashSize / 4; ++i)
{
initializeMethodIL.Emit(OpCodes.Ldc_I4, typeIdHash[i]);
}
}
initializeMethodIL.Emit(OpCodes.Newobj, objectIdCtorRef);
initializeMethodIL.Emit(OpCodes.Ldtoken, assembly.MainModule.ImportReference(type.Key));
initializeMethodIL.Emit(OpCodes.Call, assembly.MainModule.ImportReference(getTypeFromHandleMethod));
if (type.Value.SerializerType != null)
{
initializeMethodIL.Emit(OpCodes.Ldtoken, assembly.MainModule.ImportReference(type.Value.SerializerType));
initializeMethodIL.Emit(OpCodes.Call, assembly.MainModule.ImportReference(getTypeFromHandleMethod));
}
else
{
initializeMethodIL.Emit(OpCodes.Ldnull);
}
initializeMethodIL.Emit(OpCodes.Newobj, serializerEntryTypeCtorRef);
initializeMethodIL.Emit(OpCodes.Callvirt, assemblySerializersPerProfileTypeAddRef);
if (type.Value.SerializerType?.Resolve()?.Methods.Any(x => x.IsConstructor && x.IsStatic) == true)
{
// Generating: System.Runtime.CompilerServices.RuntimeHelpers.RunClassConstructor(typeof(<#=type.Value.SerializerType.ConvertCSharp()#>).TypeHandle);
initializeMethodIL.Append(Instruction.Create(OpCodes.Ldtoken, type.Value.SerializerType));
initializeMethodIL.Append(Instruction.Create(OpCodes.Call, assembly.MainModule.ImportReference(getTypeFromHandleMethod)));
initializeMethodIL.Append(Instruction.Create(OpCodes.Callvirt, getTypeHandleMethodRef));
initializeMethodIL.Append(Instruction.Create(OpCodes.Call, runClassConstructorMethod));
}
}
initializeMethodIL.Emit(OpCodes.Callvirt, assemblySerializersGetProfilesSetItemRef);
}
// Generating: DataSerializerFactory.RegisterSerializationAssembly(assemblySerializers);
var dataSerializerFactoryRegisterSerializationAssemblyMethodRef = assembly.MainModule.ImportReference(strideCoreModule.GetType("Stride.Core.Serialization.DataSerializerFactory").Methods.Single(x => x.Name == "RegisterSerializationAssembly" && x.Parameters[0].ParameterType.FullName == assemblySerializersType.FullName));
initializeMethodIL.Emit(OpCodes.Call, dataSerializerFactoryRegisterSerializationAssemblyMethodRef);
// Generating: AssemblyRegistry.Register(typeof(<#=registry.ClassName#>).GetTypeInfo().Assembly, AssemblyCommonCategories.Engine);
initializeMethodIL.Emit(OpCodes.Ldtoken, serializerFactoryType);
initializeMethodIL.Emit(OpCodes.Call, assembly.MainModule.ImportReference(getTypeFromHandleMethod));
initializeMethodIL.Emit(OpCodes.Call, assembly.MainModule.ImportReference(getTokenInfoExMethod));
initializeMethodIL.Emit(OpCodes.Callvirt, assembly.MainModule.ImportReference(getTypeInfoAssembly));
// create new[] { AssemblyCommonCategories.Engine }
initializeMethodIL.Emit(OpCodes.Ldc_I4_1);
initializeMethodIL.Emit(OpCodes.Newarr, assembly.MainModule.TypeSystem.String);
initializeMethodIL.Emit(OpCodes.Dup);
initializeMethodIL.Emit(OpCodes.Ldc_I4_0);
initializeMethodIL.Emit(OpCodes.Ldstr, Core.Reflection.AssemblyCommonCategories.Engine);
initializeMethodIL.Emit(OpCodes.Stelem_Ref);
var assemblyRegistryRegisterMethodRef = assembly.MainModule.ImportReference(strideCoreModule.GetType("Stride.Core.Reflection.AssemblyRegistry").Methods.Single(x => x.Name == "Register" && x.Parameters[1].ParameterType.IsArray));
initializeMethodIL.Emit(OpCodes.Call, assemblyRegistryRegisterMethodRef);
initializeMethodIL.Emit(OpCodes.Ret);
// Add AssemblySerializerFactoryAttribute
var assemblySerializerFactoryAttribute = strideCoreModule.GetType("Stride.Core.Serialization.AssemblySerializerFactoryAttribute");
assembly.CustomAttributes.Add(new CustomAttribute(assembly.MainModule.ImportReference(assemblySerializerFactoryAttribute.GetEmptyConstructor()))
{
Fields =
{
new CustomAttributeNamedArgument("Type", new CustomAttributeArgument(typeTypeRef, serializerFactoryType)),
}
});
serializationHash = hash.ComputeHash();
}
private void UpdateDataSerializers()
{
lock (Lock)
{
if (invalidated)
{
var newDataSerializersByType = new Dictionary <Type, DataSerializer>();
var newDataSerializersByTypeId = new Dictionary <ObjectId, DataSerializer>();
referenceTypes = new HashSet <Type>();
// Create list of combined serializers
var combinedSerializers = new Dictionary <Type, AssemblySerializerEntry>();
lock (DataSerializerFactory.Lock)
{
foreach (var profile in profiles)
{
Dictionary <Type, AssemblySerializerEntry> serializersPerProfile;
if (DataSerializerFactory.DataSerializersPerProfile.TryGetValue(profile, out serializersPerProfile))
{
foreach (var serializer in serializersPerProfile)
{
combinedSerializers[serializer.Key] = serializer.Value;
}
}
}
}
// Create new list of serializers (it will create new ones, and remove unused ones)
foreach (var serializer in combinedSerializers)
{
DataSerializer dataSerializer;
if (!dataSerializersByType.TryGetValue(serializer.Key, out dataSerializer))
{
if (serializer.Value.SerializerType != null)
{
// New serializer, let's create it
dataSerializer = (DataSerializer)Activator.CreateInstance(serializer.Value.SerializerType);
dataSerializer.SerializationTypeId = serializer.Value.Id;
// Ensure a serialization type ID has been generated (otherwise do so now)
if (dataSerializer.SerializationTypeId == ObjectId.Empty)
{
// Need to generate serialization type id
var typeName = dataSerializer.SerializationType.FullName;
dataSerializer.SerializationTypeId = ObjectId.FromBytes(System.Text.Encoding.UTF8.GetBytes(typeName));
}
}
}
newDataSerializersByType[serializer.Key] = dataSerializer;
newDataSerializersByTypeId[serializer.Value.Id] = dataSerializer;
}
dataSerializersByType = newDataSerializersByType;
dataSerializersByTypeId = newDataSerializersByTypeId;
invalidated = false;
}
}
}
private static ObjectId CalculateHashFromSource(string source)
{
return(ObjectId.FromBytes(Encoding.UTF8.GetBytes(source)));
}
/// <summary>
/// Loads the shader source with the specified type name.
/// </summary>
/// <param name="type">The typeName.</param>
/// <returns>ShaderSourceWithHash.</returns>
/// <exception cref="System.IO.FileNotFoundException">If the file was not found</exception>
public ShaderSourceWithHash LoadShaderSource(string type)
{
lock (locker)
{
// Load file
ShaderSourceWithHash shaderSource;
if (!loadedShaderSources.TryGetValue(type, out shaderSource))
{
var sourceUrl = FindFilePath(type);
if (sourceUrl != null)
{
shaderSource = new ShaderSourceWithHash();
if (!UrlToFilePath.TryGetValue(sourceUrl, out shaderSource.Path))
{
shaderSource.Path = sourceUrl;
}
// On Windows, Always try to load first from the original URL in order to get the latest version
if (Platform.IsWindowsDesktop)
{
// TODO: the "/path" is hardcoded, used in ImportStreamCommand and EffectSystem. Find a place to share this correctly.
var pathUrl = sourceUrl + "/path";
if (FileExists(pathUrl))
{
using (var fileStream = OpenStream(pathUrl))
{
string shaderSourcePath;
using (var sr = new StreamReader(fileStream, Encoding.UTF8))
shaderSourcePath = sr.ReadToEnd();
if (File.Exists(shaderSourcePath))
{
// Replace path with a local path
shaderSource.Path = Path.Combine(Environment.CurrentDirectory, shaderSourcePath);
// Optimization: It currently reads the source file twice
shaderSource.Hash = ObjectId.FromBytes(File.ReadAllBytes(shaderSourcePath));
using (var sourceStream = File.Open(shaderSourcePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
using (var sr = new StreamReader(sourceStream))
shaderSource.Source = sr.ReadToEnd();
}
}
}
}
}
if (shaderSource.Source == null)
{
using (var sourceStream = OpenStream(sourceUrl))
{
var databaseStream = sourceStream as IDatabaseStream;
var fileStream = sourceStream as FileStream;
if (databaseStream != null || fileStream != null)
{
using (var sr = new StreamReader(sourceStream))
shaderSource.Source = sr.ReadToEnd();
if (databaseStream != null)
{
shaderSource.Hash = databaseStream.ObjectId;
}
else
{
shaderSource.Hash = ObjectId.FromBytes(File.ReadAllBytes(sourceUrl));
}
}
else
{
throw new Exception(string.Format("Unsupported Stream type to load shader [{0}.pdxsl]", type));
}
}
}
loadedShaderSources[type] = shaderSource;
}
else
{
throw new FileNotFoundException(string.Format("Unable to find shader [{0}]", type), string.Format("{0}.pdxsl", type));
}
}
return(shaderSource);
}
}
public override TaskOrResult <EffectBytecodeCompilerResult> Compile(ShaderMixinSource mixinTree, EffectCompilerParameters effectParameters, CompilerParameters compilerParameters)
{
var log = new LoggerResult();
// Load D3D compiler dll
// Note: No lock, it's probably fine if it gets called from multiple threads at the same time.
if (Platform.IsWindowsDesktop && !d3dCompilerLoaded)
{
NativeLibrary.PreloadLibrary("d3dcompiler_47.dll");
d3dCompilerLoaded = true;
}
var shaderMixinSource = mixinTree;
var fullEffectName = mixinTree.Name;
// Make a copy of shaderMixinSource. Use deep clone since shaderMixinSource can be altered during compilation (e.g. macros)
var shaderMixinSourceCopy = new ShaderMixinSource();
shaderMixinSourceCopy.DeepCloneFrom(shaderMixinSource);
shaderMixinSource = shaderMixinSourceCopy;
// Generate platform-specific macros
switch (effectParameters.Platform)
{
case GraphicsPlatform.Direct3D11:
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_DIRECT3D", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_DIRECT3D11", 1);
break;
case GraphicsPlatform.Direct3D12:
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_DIRECT3D", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_DIRECT3D12", 1);
break;
case GraphicsPlatform.OpenGL:
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGL", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGLCORE", 1);
break;
case GraphicsPlatform.OpenGLES:
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGL", 1);
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_OPENGLES", 1);
break;
case GraphicsPlatform.Vulkan:
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_API_VULKAN", 1);
break;
default:
throw new NotSupportedException();
}
// Generate profile-specific macros
shaderMixinSource.AddMacro("SILICONSTUDIO_XENKO_GRAPHICS_PROFILE", (int)effectParameters.Profile);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_1", (int)GraphicsProfile.Level_9_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_2", (int)GraphicsProfile.Level_9_2);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_9_3", (int)GraphicsProfile.Level_9_3);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_0", (int)GraphicsProfile.Level_10_0);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_10_1", (int)GraphicsProfile.Level_10_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_0", (int)GraphicsProfile.Level_11_0);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_1", (int)GraphicsProfile.Level_11_1);
shaderMixinSource.AddMacro("GRAPHICS_PROFILE_LEVEL_11_2", (int)GraphicsProfile.Level_11_2);
// In .xksl, class has been renamed to shader to avoid ambiguities with HLSL
shaderMixinSource.AddMacro("class", "shader");
var parsingResult = GetMixinParser().Parse(shaderMixinSource, shaderMixinSource.Macros.ToArray());
// Copy log from parser results to output
CopyLogs(parsingResult, log);
// Return directly if there are any errors
if (parsingResult.HasErrors)
{
return(new EffectBytecodeCompilerResult(null, log));
}
// Convert the AST to HLSL
var writer = new SiliconStudio.Shaders.Writer.Hlsl.HlslWriter
{
EnablePreprocessorLine = false // Allow to output links to original pdxsl via #line pragmas
};
writer.Visit(parsingResult.Shader);
var shaderSourceText = writer.Text;
if (string.IsNullOrEmpty(shaderSourceText))
{
log.Error($"No code generated for effect [{fullEffectName}]");
return(new EffectBytecodeCompilerResult(null, log));
}
// -------------------------------------------------------
// Save shader log
// TODO: TEMP code to allow debugging generated shaders on Windows Desktop
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
var shaderId = ObjectId.FromBytes(Encoding.UTF8.GetBytes(shaderSourceText));
var logDir = Path.Combine(Directory.GetCurrentDirectory(), "log");
if (!Directory.Exists(logDir))
{
Directory.CreateDirectory(logDir);
}
var shaderSourceFilename = Path.Combine(logDir, "shader_" + fullEffectName.Replace('.', '_') + "_" + shaderId + ".hlsl");
lock (WriterLock) // protect write in case the same shader is created twice
{
// Write shader before generating to make sure that we are having a trace before compiling it (compiler may crash...etc.)
if (!File.Exists(shaderSourceFilename))
{
File.WriteAllText(shaderSourceFilename, shaderSourceText);
}
}
#else
string shaderSourceFilename = null;
#endif
// -------------------------------------------------------
var bytecode = new EffectBytecode {
Reflection = parsingResult.Reflection, HashSources = parsingResult.HashSources
};
// Select the correct backend compiler
IShaderCompiler compiler;
switch (effectParameters.Platform)
{
#if SILICONSTUDIO_PLATFORM_WINDOWS
case GraphicsPlatform.Direct3D11:
case GraphicsPlatform.Direct3D12:
compiler = new Direct3D.ShaderCompiler();
break;
#endif
case GraphicsPlatform.OpenGL:
case GraphicsPlatform.OpenGLES:
case GraphicsPlatform.Vulkan:
// get the number of render target outputs
var rtOutputs = 0;
var psOutput = parsingResult.Shader.Declarations.OfType <StructType>().FirstOrDefault(x => x.Name.Text == "PS_OUTPUT");
if (psOutput != null)
{
foreach (var rto in psOutput.Fields)
{
var sem = rto.Qualifiers.OfType <Semantic>().FirstOrDefault();
if (sem != null)
{
// special case SV_Target
if (rtOutputs == 0 && sem.Name.Text == "SV_Target")
{
rtOutputs = 1;
break;
}
for (var i = rtOutputs; i < 8; ++i)
{
if (sem.Name.Text == ("SV_Target" + i))
{
rtOutputs = i + 1;
break;
}
}
}
}
}
compiler = new OpenGL.ShaderCompiler(rtOutputs);
break;
default:
throw new NotSupportedException();
}
var shaderStageBytecodes = new List <ShaderBytecode>();
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
var stageStringBuilder = new StringBuilder();
#endif
// if the shader (non-compute) does not have a pixel shader, we should add it for OpenGL and OpenGL ES.
if ((effectParameters.Platform == GraphicsPlatform.OpenGL || effectParameters.Platform == GraphicsPlatform.OpenGLES) && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Pixel) && !parsingResult.EntryPoints.ContainsKey(ShaderStage.Compute))
{
parsingResult.EntryPoints.Add(ShaderStage.Pixel, null);
}
foreach (var stageBinding in parsingResult.EntryPoints)
{
// Compile
// TODO: We could compile stages in different threads to improve compiler throughput?
var result = compiler.Compile(shaderSourceText, stageBinding.Value, stageBinding.Key, effectParameters, bytecode.Reflection, shaderSourceFilename);
result.CopyTo(log);
if (result.HasErrors)
{
continue;
}
// -------------------------------------------------------
// Append bytecode id to shader log
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
stageStringBuilder.AppendLine("@G {0} => {1}".ToFormat(stageBinding.Key, result.Bytecode.Id));
if (result.DisassembleText != null)
{
stageStringBuilder.Append(result.DisassembleText);
}
#endif
// -------------------------------------------------------
shaderStageBytecodes.Add(result.Bytecode);
// When this is a compute shader, there is no need to scan other stages
if (stageBinding.Key == ShaderStage.Compute)
{
break;
}
}
// Remove unused reflection data, as it is entirely resolved at compile time.
CleanupReflection(bytecode.Reflection);
bytecode.Stages = shaderStageBytecodes.ToArray();
#if SILICONSTUDIO_PLATFORM_WINDOWS_DESKTOP
lock (WriterLock) // protect write in case the same shader is created twice
{
var builder = new StringBuilder();
builder.AppendLine("/**************************");
builder.AppendLine("***** Compiler Parameters *****");
builder.AppendLine("***************************");
builder.Append("@P EffectName: ");
builder.AppendLine(fullEffectName ?? "");
builder.Append(compilerParameters?.ToStringPermutationsDetailed());
builder.AppendLine("***************************");
if (bytecode.Reflection.ConstantBuffers.Count > 0)
{
builder.AppendLine("**** ConstantBuffers ****");
builder.AppendLine("***************************");
foreach (var cBuffer in bytecode.Reflection.ConstantBuffers)
{
builder.AppendFormat("cbuffer {0} [Size: {1}]", cBuffer.Name, cBuffer.Size).AppendLine();
foreach (var parameter in cBuffer.Members)
{
builder.AppendFormat("@C {0} => {1}", parameter.RawName, parameter.KeyInfo.KeyName).AppendLine();
}
}
builder.AppendLine("***************************");
}
if (bytecode.Reflection.ResourceBindings.Count > 0)
{
builder.AppendLine("****** Resources ******");
builder.AppendLine("***************************");
foreach (var resource in bytecode.Reflection.ResourceBindings)
{
builder.AppendFormat("@R {0} => {1} [Stage: {2}, Slot: ({3}-{4})]", resource.RawName, resource.KeyInfo.KeyName, resource.Stage, resource.SlotStart, resource.SlotStart + resource.SlotCount - 1).AppendLine();
}
builder.AppendLine("***************************");
}
if (bytecode.HashSources.Count > 0)
{
builder.AppendLine("***** Sources *****");
builder.AppendLine("***************************");
foreach (var hashSource in bytecode.HashSources)
{
builder.AppendFormat("@S {0} => {1}", hashSource.Key, hashSource.Value).AppendLine();
}
builder.AppendLine("***************************");
}
if (bytecode.Stages.Length > 0)
{
builder.AppendLine("***** Stages *****");
builder.AppendLine("***************************");
builder.Append(stageStringBuilder);
builder.AppendLine("***************************");
}
builder.AppendLine("*************************/");
// Re-append the shader with all informations
builder.Append(shaderSourceText);
File.WriteAllText(shaderSourceFilename, builder.ToString());
}
#endif
return(new EffectBytecodeCompilerResult(bytecode, log));
}
private ShaderClassType LoadShaderClass(string type, string generics, LoggerResult log, SiliconStudio.Shaders.Parser.ShaderMacro[] macros = null, HashSet <string> modifiedShaders = null)
{
if (type == null)
{
throw new ArgumentNullException("type");
}
var shaderSourceKey = new ShaderSourceKey(type, generics, macros);
lock (loadedShaders)
{
// Already instantiated
ShaderClassType shaderClass;
if (loadedShaders.TryGetValue(shaderSourceKey, out shaderClass))
{
return(shaderClass);
}
// Load file
var shaderSource = SourceManager.LoadShaderSource(type, modifiedShaders);
// TODO USE ORIGINAL SOURCE PATH and not to object database path
var preprocessedSource = PreProcessor.Run(shaderSource.Source, shaderSource.Path, macros);
byte[] byteArray = Encoding.ASCII.GetBytes(preprocessedSource);
var hashPreprocessSource = ObjectId.FromBytes(byteArray);
// Compile
var parsingResult = ParadoxShaderParser.TryParse(preprocessedSource, shaderSource.Path);
parsingResult.CopyTo(log);
if (parsingResult.HasErrors)
{
return(null);
}
var shader = parsingResult.Shader;
// As shaders can be embedded in namespaces, get only the shader class and make sure there is only one in a pdxsl.
var shaderClassTypes = GetShaderClassTypes(shader.Declarations).ToList();
if (shaderClassTypes.Count != 1)
{
throw new InvalidOperationException(string.Format("Shader [{0}] must contain only a single Shader class type intead of [{1}]", type, shaderClassTypes.Count));
}
shaderClass = shaderClassTypes.First();
shaderClass.SourcePath = shaderSource.Path;
shaderClass.SourceHash = shaderSource.Hash;
shaderClass.PreprocessedSourceHash = hashPreprocessSource;
shaderClass.IsInstanciated = false;
// TODO: We should not use Console. Change the way we log things here
Console.WriteLine("Loading Shader {0}{1}", type, macros != null && macros.Length > 0 ? String.Format("<{0}>", string.Join(", ", macros)) : string.Empty);
if (shaderClass.Name.Text != type)
{
throw new InvalidOperationException(string.Format("Unable to load shader [{0}] not maching class name [{1}]", type, shaderClass.Name.Text));
}
// Only full version are stored
loadedShaders.Add(shaderSourceKey, shaderClass);
return(shaderClass);
}
}
/// <summary>
/// Loads the shader source with the specified type name.
/// </summary>
/// <param name="type">The typeName.</param>
/// <param name="modifiedShaders">The list of modified shaders.</param>
/// <returns>ShaderSourceWithHash.</returns>
/// <exception cref="System.IO.FileNotFoundException">If the file was not found</exception>
public ShaderSourceWithHash LoadShaderSource(string type, HashSet <string> modifiedShaders = null)
{
lock (locker)
{
// Load file
ShaderSourceWithHash shaderSource;
if (!loadedShaderSources.TryGetValue(type, out shaderSource))
{
var sourceUrl = FindFilePath(type);
if (sourceUrl != null)
{
shaderSource = new ShaderSourceWithHash();
if (modifiedShaders != null && modifiedShaders.Contains(sourceUrl))
{
using (var fileStream = AssetManager.FileProvider.OpenStream(sourceUrl + "/path", VirtualFileMode.Open, VirtualFileAccess.Read, VirtualFileShare.Read))
{
string shaderSourcePath;
using (var sr = new StreamReader(fileStream, Encoding.UTF8))
shaderSourcePath = sr.ReadToEnd();
try
{
using (var sourceStream = File.Open(shaderSourcePath, FileMode.Open, FileAccess.Read))
{
using (var sr = new StreamReader(sourceStream))
shaderSource.Source = sr.ReadToEnd();
}
}
catch (FileNotFoundException)
{
throw new FileNotFoundException(string.Format("Unable to find shader [{0}] on disk", type), string.Format("{0}.pdxsl", type));
}
}
}
else
{
using (var fileStream = AssetManager.FileProvider.OpenStream(sourceUrl, VirtualFileMode.Open, VirtualFileAccess.Read, VirtualFileShare.Read))
{
using (var sr = new StreamReader(fileStream))
shaderSource.Source = sr.ReadToEnd();
var databaseStream = fileStream as IDatabaseStream;
if (databaseStream != null)
{
shaderSource.Hash = databaseStream.ObjectId;
}
}
}
// If the file was loaded from the database, use the ObjectId returned by the database, otherwise compute it directly
if (shaderSource.Hash == ObjectId.Empty)
{
shaderSource.Hash = ObjectId.FromBytes(Encoding.UTF8.GetBytes(shaderSource.Source));
}
// Convert URL to absolute file path
// TODO can we handle path differently? Current code is just a hack
UrlToFilePath.TryGetValue(sourceUrl, out shaderSource.Path);
// If Path is null, set it to type at least to be able to have more information
if (shaderSource.Path == null)
{
shaderSource.Path = type;
}
loadedShaderSources[type] = shaderSource;
}
else
{
throw new FileNotFoundException(string.Format("Unable to find shader [{0}]", type), string.Format("{0}.pdxsl", type));
}
}
return(shaderSource);
}
}
/// <summary>
/// Loads the shader source with the specified type name.
/// </summary>
/// <param name="type">The typeName.</param>
/// <returns>ShaderSourceWithHash.</returns>
/// <exception cref="System.IO.FileNotFoundException">If the file was not found</exception>
public ShaderSourceWithHash LoadShaderSource(string type)
{
lock (locker)
{
// Load file
ShaderSourceWithHash shaderSource;
if (!loadedShaderSources.TryGetValue(type, out shaderSource))
{
var sourceUrl = FindFilePath(type);
if (sourceUrl != null)
{
shaderSource = new ShaderSourceWithHash();
if (!UrlToFilePath.TryGetValue(sourceUrl, out shaderSource.Path))
{
shaderSource.Path = sourceUrl;
}
// On Windows, Always try to load first from the original URL in order to get the latest version
if (Platform.IsWindowsDesktop)
{
// TODO: the "/path" is hardcoded, used in ImportStreamCommand and EffectSystem. Find a place to share this correctly.
var pathUrl = sourceUrl + "/path";
if (FileExists(pathUrl))
{
using (var fileStream = OpenStream(pathUrl))
{
string shaderSourcePath;
using (var sr = new StreamReader(fileStream, Encoding.UTF8))
shaderSourcePath = sr.ReadToEnd();
if (File.Exists(shaderSourcePath))
{
byte[] fileData = null;
for (int tries = 10; tries >= 0; --tries)
{
try
{
fileData = File.ReadAllBytes(shaderSourcePath);
break;
}
catch (IOException)
{
// Try again
}
}
if (fileData != null)
{
// Replace path with a local path
shaderSource.Path = Path.Combine(PlatformFolders.ApplicationBinaryDirectory, shaderSourcePath);
shaderSource.Hash = ObjectId.FromBytes(fileData);
// Note: we can't use Encoding.UTF8.GetString directly because there might be the UTF8 BOM at the beginning of the file
using (StreamReader reader = new StreamReader(new MemoryStream(fileData), Encoding.UTF8))
shaderSource.Source = reader.ReadToEnd();
}
}
}
}
}
if (shaderSource.Source == null)
{
using (var sourceStream = OpenStream(sourceUrl))
{
var databaseStream = sourceStream as IDatabaseStream;
using (var sr = new StreamReader(sourceStream))
{
shaderSource.Source = sr.ReadToEnd();
if (databaseStream == null)
{
sourceStream.Position = 0;
var data = new byte[sourceStream.Length];
sourceStream.Read(data, 0, (int)sourceStream.Length);
shaderSource.Hash = ObjectId.FromBytes(data);
}
else
{
shaderSource.Hash = databaseStream.ObjectId;
}
}
}
}
loadedShaderSources[type] = shaderSource;
}
else
{
throw new FileNotFoundException($"Unable to find shader [{type}]", $"{type}.xksl");
}
}
return(shaderSource);
}
}
public void TestImportSessionSimple()
{
var name = "TestAssetImport";
var file = Path.Combine(Path.GetTempPath(), name + ".tmp");
const string fileContent = "This is the file content";
File.WriteAllText(file, fileContent);
var fileHash = ObjectId.FromBytes(Encoding.UTF8.GetBytes(fileContent));
// Create a project with an asset reference a raw file
var project = new Package();
using (var session = new PackageSession(project))
{
var importSession = new AssetImportSession(session);
// ------------------------------------------------------------------
// Step 1: Add files to session
// ------------------------------------------------------------------
importSession.AddFile(file, project, UDirectory.Empty);
// ------------------------------------------------------------------
// Step 2: Stage assets
// ------------------------------------------------------------------
var stageResult = importSession.Stage();
Assert.IsTrue(stageResult);
Assert.AreEqual(0, project.Assets.Count);
// ------------------------------------------------------------------
// Step 3: Import asset directly (we don't try to merge)
// ------------------------------------------------------------------
importSession.Import();
Assert.AreEqual(2, project.Assets.Count);
var assetItem = project.Assets.FirstOrDefault(item => item.Asset is AssetImport);
Assert.NotNull(assetItem);
var importedAsset = (AssetImport)assetItem.Asset;
Assert.IsInstanceOf <AssetImport>(importedAsset.Base.Asset);
Assert.AreEqual((string)AssetBase.DefaultImportBase, importedAsset.Base.Location);
// ------------------------------------------------------------------
// Reset the import session
// ------------------------------------------------------------------
importSession.Reset();
// Get the list of asset already in the project
var ids = project.Assets.Select(item => item.Id).ToList();
ids.Sort();
// ------------------------------------------------------------------
// Step 1: Add same file to the session
// ------------------------------------------------------------------
importSession.AddFile(file, project, UDirectory.Empty);
// ------------------------------------------------------------------
// Step 2: Stage assets
// ------------------------------------------------------------------
stageResult = importSession.Stage();
// Select the previous item so that we will perform a merge
foreach (var fileToImport in importSession.Imports)
{
foreach (var import in fileToImport.ByImporters)
{
foreach (var assetImportMergeGroup in import.Items)
{
Assert.AreEqual(1, assetImportMergeGroup.Merges.Count);
// Check that we are matching correctly a previously imported asset
var previousItem = assetImportMergeGroup.Merges[0].PreviousItem;
Assert.IsTrue(project.Assets.ContainsById(previousItem.Id));
// Select the previous asset
assetImportMergeGroup.SelectedItem = previousItem;
}
}
}
// ------------------------------------------------------------------
// Step 3: Merge the asset specified by the previous step
// ------------------------------------------------------------------
importSession.Merge();
Assert.IsTrue(stageResult);
Assert.AreEqual(2, project.Assets.Count);
// ------------------------------------------------------------------
// Step 4: Import merged asset
// ------------------------------------------------------------------
importSession.Import();
Assert.AreEqual(2, project.Assets.Count);
// Get the list of asset already in the project
var newIds = project.Assets.Select(item => item.Id).ToList();
newIds.Sort();
// Check that we have exactly the same number of assets
Assert.AreEqual(ids, newIds);
// Check that new AssetObjectTestRaw.Value is setup to the new value (1, was previously 0)
var assetRaw = project.Assets.Select(item => item.Asset).OfType <AssetObjectTestSub>().FirstOrDefault();
Assert.IsNotNull(assetRaw);
Assert.IsNotNull(assetRaw.Base);
Assert.IsInstanceOf <AssetObjectTestSub>(assetRaw.Base.Asset);
var assetRawBase = (AssetObjectTestSub)assetRaw.Base.Asset;
Assert.AreEqual(1, assetRaw.Value);
Assert.AreEqual(1, assetRawBase.Value);
}
}
/// <summary>
/// Converts the hlsl code into glsl and stores the result as plain text
/// </summary>
/// <param name="shaderSource">the hlsl shader</param>
/// <param name="entryPoint">the entrypoint function name</param>
/// <param name="stage">the shader pipeline stage</param>
/// <param name="effectParameters"></param>
/// <param name="reflection">the reflection gathered from the hlsl analysis</param>
/// <param name="sourceFilename">the name of the source file</param>
/// <returns></returns>
public ShaderBytecodeResult Compile(string shaderSource, string entryPoint, ShaderStage stage, EffectCompilerParameters effectParameters, EffectReflection reflection, string sourceFilename = null)
{
var shaderBytecodeResult = new ShaderBytecodeResult();
byte[] rawData;
var inputAttributeNames = new Dictionary <int, string>();
var resourceBindings = new Dictionary <string, int>();
GlslShaderPlatform shaderPlatform;
int shaderVersion;
switch (effectParameters.Platform)
{
case GraphicsPlatform.OpenGL:
shaderPlatform = GlslShaderPlatform.OpenGL;
shaderVersion = 410;
break;
case GraphicsPlatform.OpenGLES:
shaderPlatform = GlslShaderPlatform.OpenGLES;
shaderVersion = effectParameters.Profile >= GraphicsProfile.Level_10_0 ? 300 : 100;
break;
case GraphicsPlatform.Vulkan:
shaderPlatform = GlslShaderPlatform.Vulkan;
shaderVersion = 450;
break;
default:
throw new ArgumentOutOfRangeException("effectParameters.Platform");
}
var shader = Compile(shaderSource, entryPoint, stage, shaderPlatform, shaderVersion, shaderBytecodeResult, reflection, inputAttributeNames, resourceBindings, sourceFilename);
if (shader == null)
{
return(shaderBytecodeResult);
}
if (effectParameters.Platform == GraphicsPlatform.OpenGLES) // TODO: Add check to run on android only. The current version breaks OpenGL ES on windows.
{
//TODO: Remove this ugly hack!
if (shaderSource.Contains($"Texture2D XenkoInternal_TextureExt0") && shader.Contains("uniform sampler2D"))
{
if (shaderPlatform != GlslShaderPlatform.OpenGLES || shaderVersion != 300)
{
throw new Exception("Invalid GLES platform or version: require OpenGLES 300");
}
shader = shader.Replace("uniform sampler2D", "uniform samplerExternalOES");
shader = shader.Replace("#version 300 es", "#version 300 es\n#extension GL_OES_EGL_image_external_essl3 : require");
}
}
if (effectParameters.Platform == GraphicsPlatform.OpenGLES)
{
// store both ES 2 and ES 3 on OpenGL ES platforms
var shaderBytecodes = new ShaderLevelBytecode();
if (effectParameters.Profile >= GraphicsProfile.Level_10_0)
{
shaderBytecodes.DataES3 = shader;
shaderBytecodes.DataES2 = null;
}
else
{
shaderBytecodes.DataES2 = shader;
shaderBytecodes.DataES3 = Compile(shaderSource, entryPoint, stage, GlslShaderPlatform.OpenGLES, 300, shaderBytecodeResult, reflection, inputAttributeNames, resourceBindings, sourceFilename);
}
using (var stream = new MemoryStream())
{
BinarySerialization.Write(stream, shaderBytecodes);
rawData = stream.GetBuffer();
}
}
else if (effectParameters.Platform == GraphicsPlatform.Vulkan)
{
string inputFileExtension;
switch (stage)
{
case ShaderStage.Vertex: inputFileExtension = ".vert"; break;
case ShaderStage.Pixel: inputFileExtension = ".frag"; break;
case ShaderStage.Geometry: inputFileExtension = ".geom"; break;
case ShaderStage.Domain: inputFileExtension = ".tese"; break;
case ShaderStage.Hull: inputFileExtension = ".tesc"; break;
case ShaderStage.Compute: inputFileExtension = ".comp"; break;
default:
shaderBytecodeResult.Error("Unknown shader profile");
return(shaderBytecodeResult);
}
var inputFileName = Path.ChangeExtension(Path.GetTempFileName(), inputFileExtension);
var outputFileName = Path.ChangeExtension(inputFileName, ".spv");
// Write shader source to disk
File.WriteAllBytes(inputFileName, Encoding.ASCII.GetBytes(shader));
// Run shader compiler
var filename = Platform.Type == PlatformType.Windows ? "glslangValidator.exe" : "glslangValidator";
ShellHelper.RunProcessAndRedirectToLogger(filename, $"-V -o {outputFileName} {inputFileName}", null, shaderBytecodeResult);
if (!File.Exists(outputFileName))
{
shaderBytecodeResult.Error("Failed to generate SPIR-V from GLSL");
return(shaderBytecodeResult);
}
// Read compiled shader
var shaderBytecodes = new ShaderInputBytecode
{
InputAttributeNames = inputAttributeNames,
ResourceBindings = resourceBindings,
Data = File.ReadAllBytes(outputFileName),
};
using (var stream = new MemoryStream())
{
BinarySerialization.Write(stream, shaderBytecodes);
rawData = stream.ToArray();
}
// Cleanup temp files
File.Delete(inputFileName);
File.Delete(outputFileName);
}
else
{
// store string on OpenGL platforms
rawData = Encoding.UTF8.GetBytes(shader);
}
var bytecodeId = ObjectId.FromBytes(rawData);
var bytecode = new ShaderBytecode(bytecodeId, rawData);
bytecode.Stage = stage;
shaderBytecodeResult.Bytecode = bytecode;
return(shaderBytecodeResult);
}
/// <summary>
/// Generates a unique identifier from location.
/// </summary>
/// <param name="location">The location.</param>
/// <returns>Guid.</returns>
public static AssetId GenerateIdFromLocation(string location)
{
if (location == null) throw new ArgumentNullException(nameof(location));
return (AssetId)ObjectId.FromBytes(Encoding.UTF8.GetBytes(location)).ToGuid();
}
/// <summary>
/// Converts the hlsl code into glsl and stores the result as plain text
/// </summary>
/// <param name="shaderSource">the hlsl shader</param>
/// <param name="entryPoint">the entrypoint function name</param>
/// <param name="stage">the shader pipeline stage</param>
/// <param name="effectParameters"></param>
/// <param name="reflection">the reflection gathered from the hlsl analysis</param>
/// <param name="sourceFilename">the name of the source file</param>
/// <returns></returns>
public ShaderBytecodeResult Compile(string shaderSource, string entryPoint, ShaderStage stage, EffectCompilerParameters effectParameters, EffectReflection reflection, string sourceFilename = null)
{
var shaderBytecodeResult = new ShaderBytecodeResult();
byte[] rawData;
GlslShaderPlatform shaderPlatform;
int shaderVersion;
switch (effectParameters.Platform)
{
case GraphicsPlatform.OpenGL:
shaderPlatform = GlslShaderPlatform.OpenGL;
shaderVersion = 420;
break;
case GraphicsPlatform.OpenGLES:
shaderPlatform = GlslShaderPlatform.OpenGLES;
shaderVersion = effectParameters.Profile >= GraphicsProfile.Level_10_0 ? 300 : 100;
break;
default:
throw new ArgumentOutOfRangeException("effectParameters.Platform");
}
var shader = Compile(shaderSource, entryPoint, stage, shaderPlatform, shaderVersion, shaderBytecodeResult, reflection, sourceFilename);
if (shader == null)
{
return(shaderBytecodeResult);
}
if (effectParameters.Platform == GraphicsPlatform.OpenGLES)
{
// store both ES 2 and ES 3 on OpenGL ES platforms
var shaderBytecodes = new ShaderLevelBytecode();
if (effectParameters.Profile >= GraphicsProfile.Level_10_0)
{
shaderBytecodes.DataES3 = shader;
shaderBytecodes.DataES2 = null;
}
else
{
shaderBytecodes.DataES2 = shader;
shaderBytecodes.DataES3 = Compile(shaderSource, entryPoint, stage, GlslShaderPlatform.OpenGLES, 300, shaderBytecodeResult, reflection, sourceFilename);
}
using (var stream = new MemoryStream())
{
BinarySerialization.Write(stream, shaderBytecodes);
#if !SILICONSTUDIO_RUNTIME_CORECLR
rawData = stream.GetBuffer();
#else
// FIXME: Manu: The call to "ToArray()" might be slower than "GetBuffer()"
rawData = stream.ToArray();
#endif
}
}
else
{
// store string on OpenGL platforms
rawData = Encoding.ASCII.GetBytes(shader);
}
var bytecodeId = ObjectId.FromBytes(rawData);
var bytecode = new ShaderBytecode(bytecodeId, rawData);
bytecode.Stage = stage;
shaderBytecodeResult.Bytecode = bytecode;
return(shaderBytecodeResult);
}
