internal void Initialize(TranslationRunner runner, ShaderLanguage language, List <string> preprocessorSymbols, TranslationFlags flags)
{
Flags = flags;
Runner = runner;
Language = language;
ParseOptions = new CSharpParseOptions(LanguageVersion.CSharp7_3, DocumentationMode.Parse, SourceCodeKind.Regular, preprocessorSymbols);
}
public static string Translate(byte[] data)
{
TranslationFlags flags = TranslationFlags.DebugMode;
return(Translator.CreateContext(0,
new GpuAccessor(data), flags).Translate(out _).Code);
}
private static Block[][] DecodeShader(
ulong address,
IGpuAccessor gpuAccessor,
TranslationFlags flags,
TranslationCounts counts,
out ShaderConfig config)
{
Block[][] cfg;
ulong maxEndAddress = 0;
bool hasBindless;
if ((flags & TranslationFlags.Compute) != 0)
{
config = new ShaderConfig(gpuAccessor, flags, counts);
cfg = Decoder.Decode(gpuAccessor, address, out hasBindless);
}
else
{
config = new ShaderConfig(new ShaderHeader(gpuAccessor, address), gpuAccessor, flags, counts);
cfg = Decoder.Decode(gpuAccessor, address + HeaderSize, out hasBindless);
}
if (hasBindless)
{
config.SetUsedFeature(FeatureFlags.Bindless);
}
for (int funcIndex = 0; funcIndex < cfg.Length; funcIndex++)
{
for (int blkIndex = 0; blkIndex < cfg[funcIndex].Length; blkIndex++)
{
Block block = cfg[funcIndex][blkIndex];
if (maxEndAddress < block.EndAddress)
{
maxEndAddress = block.EndAddress;
}
if (!hasBindless)
{
for (int index = 0; index < block.OpCodes.Count; index++)
{
if (block.OpCodes[index] is OpCodeTextureBase texture)
{
config.TextureHandlesForCache.Add(texture.HandleOffset);
}
}
}
}
}
config.SizeAdd((int)maxEndAddress + (flags.HasFlag(TranslationFlags.Compute) ? 0 : HeaderSize));
return(cfg);
}
private TranslationResult BuildResult(TranslationContext context, TranslationFlags flags)
{
var msg = new List <TranslationMessage>(context.Messages);
var output = new Dictionary <string, ShaderTranslationResult>();
foreach (ShaderTranslationContext sc in context.Shaders)
{
var cBuffers = new List <ConstantBufferInfo>();
var includes = new Dictionary <string, ShaderTranslationResult>();
var variables = new List <ShaderMember>();
// Constant Buffers
foreach (KeyValuePair <string, MappedConstantBuffer> p in sc.ConstantBuffers)
{
List <ShaderMember> cBufferVariables = new List <ShaderMember>();
foreach (MappedField mField in p.Value.Variables)
{
cBufferVariables.Add(PopulateMember(mField));
}
cBuffers.Add(new ConstantBufferInfo(p.Key, cBufferVariables, p.Value.BindSlots));
}
// Fields
foreach (MappedField mField in sc.MappedFields)
{
variables.Add(PopulateMember(mField));
}
Dictionary <string, EntryPointInfo> epDict = new Dictionary <string, EntryPointInfo>();
foreach (KeyValuePair <string, MappedEntryPoint> pair in sc.EntryPointsByName)
{
EntryPointInfo info = pair.Value.Info;
epDict.Add(pair.Key, new EntryPointInfo()
{
StartIndex = info.StartIndex,
EndIndex = info.EndIndex,
GeometryMaxVertexOut = info.GeometryMaxVertexOut,
GeometryType = info.GeometryType,
HullControlPoints = info.HullControlPoints,
HullPartitioning = info.HullPartitioning,
HullPatchConstantCallback = info.HullPatchConstantCallback,
HullTopology = info.HullTopology,
Name = info.Name,
PatchType = info.PatchType,
ThreadGroupSize = info.ThreadGroupSize,
Type = info.Type,
});
}
ShaderTranslationResult shader = new ShaderTranslationResult(sc.Name, sc.FinalSource, epDict, includes, cBuffers, variables);
output.Add(sc.Name, shader);
}
return(new TranslationResult(msg, output));
}
public static ShaderProgram Translate(
ulong address,
IGpuAccessor gpuAccessor,
TranslationFlags flags,
TranslationCounts counts = null)
{
counts ??= new TranslationCounts();
return(Translate(DecodeShader(address, gpuAccessor, flags, counts, out ShaderConfig config), config));
}
public ShaderConfig(IGpuAccessor gpuAccessor, TranslationFlags flags, TranslationCounts counts)
{
Stage = ShaderStage.Compute;
GpuAccessor = gpuAccessor;
Flags = flags;
_counts = counts;
TextureHandlesForCache = new HashSet <int>();
_usedTextures = new Dictionary <TextureInfo, TextureMeta>();
_usedImages = new Dictionary <TextureInfo, TextureMeta>();
}
/// <summary>
/// Strips and re-adds the correct amount of indentation to a code string.
/// </summary>
/// <param name="target"></param>
/// <param name="minIndent">The minimum level of indentation.</param>
/// <returns></returns>
internal static void CorrectIndents(ref string target, TranslationFlags flags)
{
string[] lines = target.Trim().Split(_delimiters, StringSplitOptions.None);
int indent = 0;
bool prevLineEmpty = false;
int newLineCount = lines.Length;
for (int i = 0; i < newLineCount; i++)
{
lines[i] = lines[i].Trim().Replace(" ", " ");
// Remove consecutive empty lines.
bool curLineEmpty = string.IsNullOrWhiteSpace(lines[i]);
int cPos = i + 1;
if (prevLineEmpty && curLineEmpty && cPos < lines.Length)
{
Array.Copy(lines, cPos, lines, i, lines.Length - cPos);
i--;
newLineCount--;
continue;
}
// TODO improve this later. It's fugly!
bool blockStarted = lines[i].StartsWith(BlockOpen) || lines[i].EndsWith(BlockOpen);
bool blockEnded = lines[i].EndsWith(BlockClosed) ||
lines[i].EndsWith(BlockClosed + ";") ||
lines[i].StartsWith(BlockClosed + "//") ||
lines[i].StartsWith(BlockClosed + " //");
if (lines[i].StartsWith(BlockClosed) && blockEnded)
{
indent = Math.Max(indent - 1, 0);
}
lines[i] = new string('\t', indent) + lines[i];
if (lines[i].StartsWith("//"))
{
if ((flags & TranslationFlags.StripComments) == TranslationFlags.StripComments)
{
lines[i] = "";
}
}
if (blockStarted && !blockEnded)
{
indent++;
}
prevLineEmpty = curLineEmpty;
}
target = string.Join(Environment.NewLine, lines, 0, newLineCount);
}
public ShaderConfig(TranslationFlags flags, TranslatorCallbacks callbacks)
{
Stage = ShaderStage.Compute;
OutputTopology = OutputTopology.PointList;
MaxOutputVertices = 0;
OmapTargets = null;
OmapSampleMask = false;
OmapDepth = false;
Flags = flags;
_callbacks = callbacks;
}
public ShaderConfig(ShaderHeader header, TranslationFlags flags, TranslatorCallbacks callbacks)
{
Stage = header.Stage;
OutputTopology = header.OutputTopology;
MaxOutputVertices = header.MaxOutputVertexCount;
OmapTargets = header.OmapTargets;
OmapSampleMask = header.OmapSampleMask;
OmapDepth = header.OmapDepth;
Flags = flags;
_callbacks = callbacks;
}
public ShaderConfig(ShaderHeader header, IGpuAccessor gpuAccessor, TranslationFlags flags, TranslationCounts counts) : this(gpuAccessor, flags, counts)
{
Stage = header.Stage;
GpPassthrough = header.Stage == ShaderStage.Geometry && header.GpPassthrough;
OutputTopology = header.OutputTopology;
MaxOutputVertices = header.MaxOutputVertexCount;
LocalMemorySize = header.ShaderLocalMemoryLowSize + header.ShaderLocalMemoryHighSize;
ImapTypes = header.ImapTypes;
OmapTargets = header.OmapTargets;
OmapSampleMask = header.OmapSampleMask;
OmapDepth = header.OmapDepth;
}
static void Main(string[] args)
{
if (args.Length == 1 || args.Length == 2)
{
TranslationFlags flags = TranslationFlags.DebugMode;
if (args.Length == 2 && args[0] == "--compute")
{
flags |= TranslationFlags.Compute;
}
byte[] data = File.ReadAllBytes(args[^ 1]);
public static TranslatorContext CreateContext(
ulong address,
IGpuAccessor gpuAccessor,
TranslationFlags flags,
TranslationCounts counts = null)
{
counts ??= new TranslationCounts();
Block[][] cfg = DecodeShader(address, gpuAccessor, flags, counts, out ShaderConfig config);
return(new TranslatorContext(address, cfg, config));
}
/// <summary>
/// Translates the binary Maxwell shader code to something that the host API accepts.
/// </summary>
/// <remarks>
/// This will combine the "Vertex A" and "Vertex B" shader stages, if specified, into one shader.
/// </remarks>
/// <param name="state">Current GPU state</param>
/// <param name="counts">Cumulative shader resource counts</param>
/// <param name="flags">Flags that controls shader translation</param>
/// <param name="stage">Shader stage</param>
/// <param name="gpuVa">GPU virtual address of the shader code</param>
/// <param name="gpuVaA">Optional GPU virtual address of the "Vertex A" shader code</param>
/// <returns>Compiled graphics shader code</returns>
private ShaderCodeHolder TranslateGraphicsShader(
GpuState state,
TranslationCounts counts,
TranslationFlags flags,
ShaderStage stage,
ulong gpuVa,
ulong gpuVaA = 0)
{
if (gpuVa == 0)
{
return(null);
}
GpuAccessor gpuAccessor = new GpuAccessor(_context, state, (int)stage - 1);
if (gpuVaA != 0)
{
ShaderProgram program = Translator.Translate(gpuVaA, gpuVa, gpuAccessor, flags, counts);
byte[] codeA = _context.MemoryManager.GetSpan(gpuVaA, program.SizeA).ToArray();
byte[] codeB = _context.MemoryManager.GetSpan(gpuVa, program.Size).ToArray();
_dumper.Dump(codeA, compute: false, out string fullPathA, out string codePathA);
_dumper.Dump(codeB, compute: false, out string fullPathB, out string codePathB);
if (fullPathA != null && fullPathB != null && codePathA != null && codePathB != null)
{
program.Prepend("// " + codePathB);
program.Prepend("// " + fullPathB);
program.Prepend("// " + codePathA);
program.Prepend("// " + fullPathA);
}
return(new ShaderCodeHolder(program, codeB, codeA));
}
else
{
ShaderProgram program = Translator.Translate(gpuVa, gpuAccessor, flags, counts);
byte[] code = _context.MemoryManager.GetSpan(gpuVa, program.Size).ToArray();
_dumper.Dump(code, compute: false, out string fullPath, out string codePath);
if (fullPath != null && codePath != null)
{
program.Prepend("// " + codePath);
program.Prepend("// " + fullPath);
}
return(new ShaderCodeHolder(program, code));
}
}
public ShaderConfig(ShaderHeader header, TranslationFlags flags, TranslatorCallbacks callbacks)
{
Stage = header.Stage;
OutputTopology = header.OutputTopology;
MaxOutputVertices = header.MaxOutputVertexCount;
LocalMemorySize = header.ShaderLocalMemoryLowSize + header.ShaderLocalMemoryHighSize;
ImapTypes = header.ImapTypes;
OmapTargets = header.OmapTargets;
OmapSampleMask = header.OmapSampleMask;
OmapDepth = header.OmapDepth;
Flags = flags;
_callbacks = callbacks;
}
public ShaderConfig(IGpuAccessor gpuAccessor, TranslationFlags flags)
{
Stage = ShaderStage.Compute;
OutputTopology = OutputTopology.PointList;
MaxOutputVertices = 0;
LocalMemorySize = 0;
ImapTypes = null;
OmapTargets = null;
OmapSampleMask = false;
OmapDepth = false;
GpuAccessor = gpuAccessor;
Flags = flags;
}
public static TranslatorContext CreateContext(
ulong addressA,
ulong addressB,
IGpuAccessor gpuAccessor,
TranslationFlags flags,
TranslationCounts counts = null)
{
counts ??= new TranslationCounts();
Block[][] cfgA = DecodeShader(addressA, gpuAccessor, flags | TranslationFlags.VertexA, counts, out ShaderConfig configA);
Block[][] cfgB = DecodeShader(addressB, gpuAccessor, flags, counts, out ShaderConfig configB);
return(new TranslatorContext(addressA, addressB, cfgA, cfgB, configA, configB));
}
public ShaderConfig(ShaderHeader header, IGpuAccessor gpuAccessor, TranslationFlags flags)
{
Stage = header.Stage;
OutputTopology = header.OutputTopology;
MaxOutputVertices = header.MaxOutputVertexCount;
LocalMemorySize = header.ShaderLocalMemoryLowSize + header.ShaderLocalMemoryHighSize;
ImapTypes = header.ImapTypes;
OmapTargets = header.OmapTargets;
OmapSampleMask = header.OmapSampleMask;
OmapDepth = header.OmapDepth;
GpuAccessor = gpuAccessor;
Flags = flags;
UsedFeatures = FeatureFlags.None;
}
internal void Initialize(ShaderTranslationContext sc, TranslationFlags flags)
{
_flags = flags;
_language = sc.Language;
_currentScope = Pooling.Scopes.Get();
_currentScope.Type = ScopeType.Class;
_currentScope.TypeInfo = new ShaderType(_language, sc.ShaderType.Name, sc.ShaderType);
_currentScope.Namespace = $"{sc.ShaderType.Namespace}.{sc.ShaderType.Name}";
_rootScope = _currentScope;
_firstSegment = Pooling.SourceSegments.Get();
_firstSegment.Value = "";
_curSegment = _firstSegment;
_lastSegment = _curSegment;
}
/// <summary>
/// Decode the binary Maxwell shader code to a translator context.
/// </summary>
/// <remarks>
/// This will combine the "Vertex A" and "Vertex B" shader stages, if specified, into one shader.
/// </remarks>
/// <param name="state">Current GPU state</param>
/// <param name="counts">Cumulative shader resource counts</param>
/// <param name="flags">Flags that controls shader translation</param>
/// <param name="stage">Shader stage</param>
/// <param name="gpuVa">GPU virtual address of the shader code</param>
/// <returns>The generated translator context</returns>
private TranslatorContext DecodeGraphicsShader(
GpuState state,
TranslationCounts counts,
TranslationFlags flags,
ShaderStage stage,
ulong gpuVa)
{
if (gpuVa == 0)
{
return(null);
}
GpuAccessor gpuAccessor = new GpuAccessor(_context, state, (int)stage - 1);
return(Translator.CreateContext(gpuVa, gpuAccessor, flags, counts));
}
public static ShaderProgram Translate(
ulong addressA,
ulong addressB,
IGpuAccessor gpuAccessor,
TranslationFlags flags,
TranslationCounts counts = null)
{
counts ??= new TranslationCounts();
FunctionCode[] funcA = DecodeShader(addressA, gpuAccessor, flags | TranslationFlags.VertexA, counts, out ShaderConfig configA);
FunctionCode[] funcB = DecodeShader(addressB, gpuAccessor, flags, counts, out ShaderConfig config);
config.SetUsedFeature(configA.UsedFeatures);
return(Translate(Combine(funcA, funcB), config, configA.Size));
}
public ShaderConfig(ShaderHeader header, IGpuAccessor gpuAccessor, TranslationFlags flags, TranslationCounts counts)
{
Stage = header.Stage;
OutputTopology = header.OutputTopology;
MaxOutputVertices = header.MaxOutputVertexCount;
LocalMemorySize = header.ShaderLocalMemoryLowSize + header.ShaderLocalMemoryHighSize;
ImapTypes = header.ImapTypes;
OmapTargets = header.OmapTargets;
OmapSampleMask = header.OmapSampleMask;
OmapDepth = header.OmapDepth;
GpuAccessor = gpuAccessor;
Flags = flags;
Size = 0;
UsedFeatures = FeatureFlags.None;
Counts = counts;
TextureHandlesForCache = new HashSet <int>();
}
public ShaderConfig(IGpuAccessor gpuAccessor, TranslationFlags flags, TranslationCounts counts)
{
Stage = ShaderStage.Compute;
OutputTopology = OutputTopology.PointList;
MaxOutputVertices = 0;
LocalMemorySize = 0;
ImapTypes = null;
OmapTargets = null;
OmapSampleMask = false;
OmapDepth = false;
GpuAccessor = gpuAccessor;
Flags = flags;
Size = 0;
UsedFeatures = FeatureFlags.None;
Counts = counts;
TextureHandlesForCache = new HashSet <int>();
}
/// <summary>
/// Decode the binary Maxwell shader code to a translator context.
/// </summary>
/// <remarks>
/// This will combine the "Vertex A" and "Vertex B" shader stages, if specified, into one shader.
/// </remarks>
/// <param name="channel">GPU channel</param>
/// <param name="gas">GPU accessor state</param>
/// <param name="counts">Cumulative shader resource counts</param>
/// <param name="flags">Flags that controls shader translation</param>
/// <param name="stage">Shader stage</param>
/// <param name="gpuVa">GPU virtual address of the shader code</param>
/// <returns>The generated translator context</returns>
private TranslatorContext DecodeGraphicsShader(
GpuChannel channel,
GpuAccessorState gas,
TranslationCounts counts,
TranslationFlags flags,
ShaderStage stage,
ulong gpuVa)
{
if (gpuVa == 0)
{
return(null);
}
GpuAccessor gpuAccessor = new GpuAccessor(_context, channel, gas, (int)stage - 1);
var options = new TranslationOptions(TargetLanguage.Glsl, TargetApi.OpenGL, flags);
return(Translator.CreateContext(gpuVa, gpuAccessor, options, counts));
}
/// <summary>
/// Decode the binary Maxwell shader code to a translator context.
/// </summary>
/// <remarks>
/// This will combine the "Vertex A" and "Vertex B" shader stages, if specified, into one shader.
/// </remarks>
/// <param name="channel">GPU channel</param>
/// <param name="gas">GPU accessor state</param>
/// <param name="tfd">Transform feedback descriptors</param>
/// <param name="flags">Flags that controls shader translation</param>
/// <param name="stage">Shader stage</param>
/// <param name="gpuVa">GPU virtual address of the shader code</param>
/// <returns>The generated translator context</returns>
private TranslatorContext DecodeGraphicsShader(
GpuChannel channel,
GpuAccessorState gas,
TransformFeedbackDescriptor[] tfd,
TranslationFlags flags,
ShaderStage stage,
ulong gpuVa)
{
if (gpuVa == 0)
{
return(null);
}
GpuAccessor gpuAccessor = new GpuAccessor(_context, channel, gas, tfd, (int)stage - 1);
var options = CreateTranslationOptions(flags);
return(Translator.CreateContext(gpuVa, gpuAccessor, options));
}
static void HandleArguments(Options options)
{
TranslationFlags flags = TranslationFlags.DebugMode;
if (options.Compute)
{
flags |= TranslationFlags.Compute;
}
byte[] data = File.ReadAllBytes(options.InputPath);
TranslationOptions translationOptions = new TranslationOptions(options.TargetLanguage, options.TargetApi, flags);
ShaderProgram program = Translator.CreateContext(0, new GpuAccessor(data), translationOptions).Translate(out _);
if (options.OutputPath == null)
{
if (program.BinaryCode != null)
{
using Stream outputStream = Console.OpenStandardOutput();
outputStream.Write(program.BinaryCode);
}
else
{
Console.WriteLine(program.Code);
}
}
else
{
if (program.BinaryCode != null)
{
File.WriteAllBytes(options.OutputPath, program.BinaryCode);
}
else
{
File.WriteAllText(options.OutputPath, program.Code);
}
}
}
internal static void RemoveWhitespace(ref string target, TranslationFlags flags)
{
string[] lines = target.Trim().Split(_delimiters, StringSplitOptions.None);
for (int i = 0; i < lines.Length; i++)
{
// TODO check if the output language supports comment blocks. If not, add a line break to the end of the comment.
lines[i] = lines[i].Trim();
if (lines[i].StartsWith("//"))
{
if ((flags & TranslationFlags.StripComments) == TranslationFlags.StripComments)
{
lines[i] = "";
}
else
{
lines[i] = $"/* {lines[i]} */";
}
}
}
target = string.Join("", lines);
}
static Regex csharpFormat = new Regex(@"(^|[^\{])\{\d\}"); // matches "{0}" unless the first bracket is escaped, e.g. "{{0}"
public ResourceItem(ResXDataNode data)
{
if (data == null)
{
throw new ArgumentNullException();
}
if (data != null)
{
_name = data.Name;
_value = data.GetValue(_assemblyname) as String;
_metadata_comment = data.Comment;
if (!String.IsNullOrEmpty(Value))
{
// Since this is being read from .resx, assume it's a .Net/C# style string
if (csharpFormat.IsMatch(Value))
{
_metadata_flags |= TranslationFlags.csharpFormatString;
}
}
}
}
/// <summary>
/// Decode the binary Maxwell shader code to a translator context.
/// </summary>
/// <remarks>
/// This will combine the "Vertex A" and "Vertex B" shader stages, if specified, into one shader.
/// </remarks>
/// <param name="state">Current GPU state</param>
/// <param name="counts">Cumulative shader resource counts</param>
/// <param name="flags">Flags that controls shader translation</param>
/// <param name="stage">Shader stage</param>
/// <param name="gpuVa">GPU virtual address of the shader code</param>
/// <returns>The generated translator context</returns>
private TranslatorContext DecodeGraphicsShader(
GpuState state,
TranslationCounts counts,
TranslationFlags flags,
ShaderStage stage,
ulong gpuVa)
{
if (gpuVa == 0)
{
return(null);
}
GpuAccessorState gas = new GpuAccessorState(
state.Get <PoolState>(MethodOffset.TexturePoolState).Address.Pack(),
state.Get <PoolState>(MethodOffset.TexturePoolState).MaximumId,
state.Get <int>(MethodOffset.TextureBufferIndex),
state.Get <Boolean32>(MethodOffset.EarlyZForce));
GpuAccessor gpuAccessor = new GpuAccessor(_context, state.Channel, gas, (int)stage - 1);
var options = new TranslationOptions(TargetLanguage.Glsl, TargetApi.OpenGL, flags);
return(Translator.CreateContext(gpuVa, gpuAccessor, options, counts));
}
/// <summary>
/// Converts the provided C# source code to the specified shader language.
/// </summary>
/// <param name="cSharpSources">A dictionary containing source code by file or friendly name. The name is used to identify the source code in message logs.</param>
/// <param name="outputLanguage">The language that the input source code should be translated to.</param>
/// <param name="flags">A set of flags to change the default behaviour of the converter.</param>
/// <param name="preprocessorSymbols">A list of defined preprocessor symbols.</param>
/// <returns></returns>
public TranslationResult Translate(Dictionary <string, string> cSharpSources,
OutputLanguage outputLanguage,
TranslationFlags flags = TranslationFlags.None,
List <string> preprocessorSymbols = null)
{
if (_disposed)
{
throw new ObjectDisposedException("Translator instance has been disposed.");
}
TranslationArgs tArgs = new TranslationArgs()
{
CSharpSources = new Dictionary <string, string>(cSharpSources),
Flags = flags,
Language = outputLanguage,
PreprocessorSymbols = preprocessorSymbols ?? new List <string>(),
};
TranslationContext context = _runner.Run(tArgs);
TranslationResult result = BuildResult(context, flags);
context.Recycle();
return(result);
}
static Regex csharpFormat = new Regex(@"(^|[^\{])\{\d\}"); // matches "{0}" unless the first bracket is escaped, e.g. "{{0}"
public ResourceItem(ResXDataNode data)
{
if (data == null) throw new ArgumentNullException();
if (data != null) {
_name = data.Name;
_value = data.GetValue(_assemblyname) as String;
_metadata_comment = data.Comment;
if (!String.IsNullOrEmpty(Value)) {
// Since this is being read from .resx, assume it's a .Net/C# style string
if (csharpFormat.IsMatch(Value)) _metadata_flags |= TranslationFlags.csharpFormatString;
}
}
}
/// <summary>
/// Initialize the cache.
/// </summary>
internal void Initialize()
{
if (GraphicsConfig.EnableShaderCache && GraphicsConfig.TitleId != null)
{
_cacheManager = new CacheManager(CacheGraphicsApi.OpenGL, CacheHashType.XxHash128, "glsl", GraphicsConfig.TitleId, ShaderCodeGenVersion);
bool isReadOnly = _cacheManager.IsReadOnly;
HashSet <Hash128> invalidEntries = null;
if (isReadOnly)
{
Logger.Warning?.Print(LogClass.Gpu, "Loading shader cache in read-only mode (cache in use by another program!)");
}
else
{
invalidEntries = new HashSet <Hash128>();
}
ReadOnlySpan <Hash128> guestProgramList = _cacheManager.GetGuestProgramList();
using AutoResetEvent progressReportEvent = new AutoResetEvent(false);
_shaderCount = 0;
_totalShaderCount = guestProgramList.Length;
ShaderCacheStateChanged?.Invoke(ShaderCacheState.Start, _shaderCount, _totalShaderCount);
Thread progressReportThread = null;
if (guestProgramList.Length > 0)
{
progressReportThread = new Thread(ReportProgress)
{
Name = "ShaderCache.ProgressReporter",
Priority = ThreadPriority.Lowest,
IsBackground = true
};
progressReportThread.Start(progressReportEvent);
}
// Make sure these are initialized before doing compilation.
Capabilities caps = _context.Capabilities;
int maxTaskCount = Math.Min(Environment.ProcessorCount, 8);
int programIndex = 0;
List <ShaderCompileTask> activeTasks = new List <ShaderCompileTask>();
AutoResetEvent taskDoneEvent = new AutoResetEvent(false);
// This thread dispatches tasks to do shader translation, and creates programs that OpenGL will link in the background.
// The program link status is checked in a non-blocking manner so that multiple shaders can be compiled at once.
while (programIndex < guestProgramList.Length || activeTasks.Count > 0)
{
if (activeTasks.Count < maxTaskCount && programIndex < guestProgramList.Length)
{
// Begin a new shader compilation.
Hash128 key = guestProgramList[programIndex];
byte[] hostProgramBinary = _cacheManager.GetHostProgramByHash(ref key);
bool hasHostCache = hostProgramBinary != null;
IProgram hostProgram = null;
// If the program sources aren't in the cache, compile from saved guest program.
byte[] guestProgram = _cacheManager.GetGuestProgramByHash(ref key);
if (guestProgram == null)
{
Logger.Error?.Print(LogClass.Gpu, $"Ignoring orphan shader hash {key} in cache (is the cache incomplete?)");
// Should not happen, but if someone messed with the cache it's better to catch it.
invalidEntries?.Add(key);
_shaderCount = ++programIndex;
continue;
}
ReadOnlySpan <byte> guestProgramReadOnlySpan = guestProgram;
ReadOnlySpan <GuestShaderCacheEntry> cachedShaderEntries = GuestShaderCacheEntry.Parse(ref guestProgramReadOnlySpan, out GuestShaderCacheHeader fileHeader);
if (cachedShaderEntries[0].Header.Stage == ShaderStage.Compute)
{
Debug.Assert(cachedShaderEntries.Length == 1);
GuestShaderCacheEntry entry = cachedShaderEntries[0];
HostShaderCacheEntry[] hostShaderEntries = null;
// Try loading host shader binary.
if (hasHostCache)
{
hostShaderEntries = HostShaderCacheEntry.Parse(hostProgramBinary, out ReadOnlySpan <byte> hostProgramBinarySpan);
hostProgramBinary = hostProgramBinarySpan.ToArray();
hostProgram = _context.Renderer.LoadProgramBinary(hostProgramBinary);
}
ShaderCompileTask task = new ShaderCompileTask(taskDoneEvent);
activeTasks.Add(task);
task.OnCompiled(hostProgram, (bool isHostProgramValid, ShaderCompileTask task) =>
{
ShaderProgram program = null;
ShaderProgramInfo shaderProgramInfo = null;
if (isHostProgramValid)
{
// Reconstruct code holder.
program = new ShaderProgram(entry.Header.Stage, "");
shaderProgramInfo = hostShaderEntries[0].ToShaderProgramInfo();
ShaderCodeHolder shader = new ShaderCodeHolder(program, shaderProgramInfo, entry.Code);
_cpProgramsDiskCache.Add(key, new ShaderBundle(hostProgram, shader));
return(true);
}
else
{
// If the host program was rejected by the gpu driver or isn't in cache, try to build from program sources again.
Task compileTask = Task.Run(() =>
{
IGpuAccessor gpuAccessor = new CachedGpuAccessor(_context, entry.Code, entry.Header.GpuAccessorHeader, entry.TextureDescriptors);
var options = new TranslationOptions(TargetLanguage.Glsl, TargetApi.OpenGL, DefaultFlags | TranslationFlags.Compute);
program = Translator.CreateContext(0, gpuAccessor, options).Translate(out shaderProgramInfo);
});
task.OnTask(compileTask, (bool _, ShaderCompileTask task) =>
{
ShaderCodeHolder shader = new ShaderCodeHolder(program, shaderProgramInfo, entry.Code);
Logger.Info?.Print(LogClass.Gpu, $"Host shader {key} got invalidated, rebuilding from guest...");
// Compile shader and create program as the shader program binary got invalidated.
shader.HostShader = _context.Renderer.CompileShader(ShaderStage.Compute, shader.Program.Code);
hostProgram = _context.Renderer.CreateProgram(new IShader[] { shader.HostShader }, null);
task.OnCompiled(hostProgram, (bool isNewProgramValid, ShaderCompileTask task) =>
{
// As the host program was invalidated, save the new entry in the cache.
hostProgramBinary = HostShaderCacheEntry.Create(hostProgram.GetBinary(), new ShaderCodeHolder[] { shader });
if (!isReadOnly)
{
if (hasHostCache)
{
_cacheManager.ReplaceHostProgram(ref key, hostProgramBinary);
}
else
{
Logger.Warning?.Print(LogClass.Gpu, $"Add missing host shader {key} in cache (is the cache incomplete?)");
_cacheManager.AddHostProgram(ref key, hostProgramBinary);
}
}
_cpProgramsDiskCache.Add(key, new ShaderBundle(hostProgram, shader));
return(true);
});
return(false); // Not finished: still need to compile the host program.
});
return(false); // Not finished: translating the program.
}
});
}
else
{
Debug.Assert(cachedShaderEntries.Length == Constants.ShaderStages);
ShaderCodeHolder[] shaders = new ShaderCodeHolder[cachedShaderEntries.Length];
List <ShaderProgram> shaderPrograms = new List <ShaderProgram>();
TransformFeedbackDescriptor[] tfd = CacheHelper.ReadTransformFeedbackInformation(ref guestProgramReadOnlySpan, fileHeader);
TranslationFlags flags = DefaultFlags;
if (tfd != null)
{
flags |= TranslationFlags.Feedback;
}
TranslationCounts counts = new TranslationCounts();
HostShaderCacheEntry[] hostShaderEntries = null;
// Try loading host shader binary.
if (hasHostCache)
{
hostShaderEntries = HostShaderCacheEntry.Parse(hostProgramBinary, out ReadOnlySpan <byte> hostProgramBinarySpan);
hostProgramBinary = hostProgramBinarySpan.ToArray();
hostProgram = _context.Renderer.LoadProgramBinary(hostProgramBinary);
}
ShaderCompileTask task = new ShaderCompileTask(taskDoneEvent);
activeTasks.Add(task);
GuestShaderCacheEntry[] entries = cachedShaderEntries.ToArray();
task.OnCompiled(hostProgram, (bool isHostProgramValid, ShaderCompileTask task) =>
{
Task compileTask = Task.Run(() =>
{
TranslatorContext[] shaderContexts = null;
if (!isHostProgramValid)
{
shaderContexts = new TranslatorContext[1 + entries.Length];
for (int i = 0; i < entries.Length; i++)
{
GuestShaderCacheEntry entry = entries[i];
if (entry == null)
{
continue;
}
IGpuAccessor gpuAccessor = new CachedGpuAccessor(_context, entry.Code, entry.Header.GpuAccessorHeader, entry.TextureDescriptors);
var options = new TranslationOptions(TargetLanguage.Glsl, TargetApi.OpenGL, flags);
shaderContexts[i + 1] = Translator.CreateContext(0, gpuAccessor, options, counts);
if (entry.Header.SizeA != 0)
{
var options2 = new TranslationOptions(TargetLanguage.Glsl, TargetApi.OpenGL, flags | TranslationFlags.VertexA);
shaderContexts[0] = Translator.CreateContext((ulong)entry.Header.Size, gpuAccessor, options2, counts);
}
}
}
// Reconstruct code holder.
for (int i = 0; i < entries.Length; i++)
{
GuestShaderCacheEntry entry = entries[i];
if (entry == null)
{
continue;
}
ShaderProgram program;
ShaderProgramInfo shaderProgramInfo;
if (isHostProgramValid)
{
program = new ShaderProgram(entry.Header.Stage, "");
shaderProgramInfo = hostShaderEntries[i].ToShaderProgramInfo();
}
else
{
IGpuAccessor gpuAccessor = new CachedGpuAccessor(_context, entry.Code, entry.Header.GpuAccessorHeader, entry.TextureDescriptors);
int stageIndex = i + 1;
TranslatorContext currentStage = shaderContexts[stageIndex];
TranslatorContext nextStage = GetNextStageContext(shaderContexts, stageIndex);
TranslatorContext vertexA = stageIndex == 1 ? shaderContexts[0] : null;
program = currentStage.Translate(out shaderProgramInfo, nextStage, vertexA);
}
// NOTE: Vertex B comes first in the shader cache.
byte[] code = entry.Code.AsSpan().Slice(0, entry.Header.Size).ToArray();
byte[] code2 = entry.Header.SizeA != 0 ? entry.Code.AsSpan().Slice(entry.Header.Size, entry.Header.SizeA).ToArray() : null;
shaders[i] = new ShaderCodeHolder(program, shaderProgramInfo, code, code2);
shaderPrograms.Add(program);
}
});
task.OnTask(compileTask, (bool _, ShaderCompileTask task) =>
{
// If the host program was rejected by the gpu driver or isn't in cache, try to build from program sources again.
if (!isHostProgramValid)
{
Logger.Info?.Print(LogClass.Gpu, $"Host shader {key} got invalidated, rebuilding from guest...");
List <IShader> hostShaders = new List <IShader>();
// Compile shaders and create program as the shader program binary got invalidated.
for (int stage = 0; stage < Constants.ShaderStages; stage++)
{
ShaderProgram program = shaders[stage]?.Program;
if (program == null)
{
continue;
}
IShader hostShader = _context.Renderer.CompileShader(program.Stage, program.Code);
shaders[stage].HostShader = hostShader;
hostShaders.Add(hostShader);
}
hostProgram = _context.Renderer.CreateProgram(hostShaders.ToArray(), tfd);
task.OnCompiled(hostProgram, (bool isNewProgramValid, ShaderCompileTask task) =>
{
// As the host program was invalidated, save the new entry in the cache.
hostProgramBinary = HostShaderCacheEntry.Create(hostProgram.GetBinary(), shaders);
if (!isReadOnly)
{
if (hasHostCache)
{
_cacheManager.ReplaceHostProgram(ref key, hostProgramBinary);
}
else
{
Logger.Warning?.Print(LogClass.Gpu, $"Add missing host shader {key} in cache (is the cache incomplete?)");
_cacheManager.AddHostProgram(ref key, hostProgramBinary);
}
}
_gpProgramsDiskCache.Add(key, new ShaderBundle(hostProgram, shaders));
return(true);
});
return(false); // Not finished: still need to compile the host program.
}
else
{
_gpProgramsDiskCache.Add(key, new ShaderBundle(hostProgram, shaders));
return(true);
}
});
return(false); // Not finished: translating the program.
});
}
_shaderCount = ++programIndex;
}
// Process the queue.
for (int i = 0; i < activeTasks.Count; i++)
{
ShaderCompileTask task = activeTasks[i];
if (task.IsDone())
{
activeTasks.RemoveAt(i--);
}
}
if (activeTasks.Count == maxTaskCount)
{
// Wait for a task to be done, or for 1ms.
// Host shader compilation cannot signal when it is done,
// so the 1ms timeout is required to poll status.
taskDoneEvent.WaitOne(1);
}
}
if (!isReadOnly)
{
// Remove entries that are broken in the cache
_cacheManager.RemoveManifestEntries(invalidEntries);
_cacheManager.FlushToArchive();
_cacheManager.Synchronize();
}
progressReportEvent.Set();
progressReportThread?.Join();
ShaderCacheStateChanged?.Invoke(ShaderCacheState.Loaded, _shaderCount, _totalShaderCount);
Logger.Info?.Print(LogClass.Gpu, $"Shader cache loaded {_shaderCount} entries.");
}
}
