//FORMAT OF SHADER CONTENT ARCHIVE:
//int32 holding shader flags
//Count of path-shaders stored in int32
//{Contiguous list of path-shaders}
//
//Where each element is represented as:
//Path name
//Shader bytecode size in bytes (int64)
//Shader bytecode
//
//Count of path-timestamps stored in int32
//{Contiguous list of path-timestamps}
//
//Where each element is represented as:
//Path name
//timestamp
//
//Count of path-dependencies stored in int32
//{Contiguous list of path-dependencies}
//
//Where each element is represented as:
//Path name
//number of dependencies
//[list of dependency paths]
//
//Where each Unique element name is represented by a UTF-16 string, stored as:
//Character count (int32)
//{Characters}
public static void Save(ShaderCompilationCache cache, string path)
{
using (var archiveFileStream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None))
{
Save(cache, archiveFileStream);
}
}
public void CopyFrom(string source, ShaderCompilationCache loadedCache)
{
lock (CompiledShaders)
{
//Just try every stage. Could be smarter about this, but nah.
foreach (var stage in MetadataParsing.Stages)
{
//The old cached data is still valid. Copy it over into the new cache and skip.
//Note that there may be multiple compiled shaders due to shader permutations.
string sourceWithExtension = source + stage.Extension;
var existingShaders = from pair in loadedCache.CompiledShaders
where pair.Key.Name == sourceWithExtension
select pair;
foreach (var entry in existingShaders)
{
CompiledShaders.Add(entry.Key, entry.Value);
}
}
TimeStamps[source] = loadedCache.TimeStamps[source];
var dependencies = new HashSet <string>(loadedCache.Dependencies[source]);
foreach (var dependency in dependencies)
{
TimeStamps[dependency] = loadedCache.TimeStamps[dependency];
}
Dependencies[source] = dependencies;
}
}
public static bool TryLoad(string path, out ShaderCompilationCache cache)
{
if (!File.Exists(path))
{
cache = null;
return(false);
}
using (var stream = File.OpenRead(path))
{
return(TryLoad(stream, out cache));
}
}
public static void Save(ShaderCompilationCache cache, Stream outputStream)
{
//Save the number of shaders.
using (var writer = new BinaryWriter(outputStream))
{
writer.Write((int)cache.ShaderFlags);
writer.Write(cache.CompiledShaders.Count);
//Save every path-shader in sequence.
foreach (var element in cache.CompiledShaders)
{
//Write the element's name.
writer.Write(element.Key.Name);
writer.Write(element.Key.Defines.Length);
for (int i = 0; i < element.Key.Defines.Length; ++i)
{
writer.Write(element.Key.Defines[i]);
}
//Write the size of the shader bytecode.
writer.Write(element.Value.Data.Length);
//Write the bytecode itself.
writer.Write(element.Value.Data, 0, element.Value.Data.Length);
}
//Save the number of timestamped references.
//Note that timestamps and dependencies do not have define permutations. Timestamps and dependencies do not change with respect to permutations.
writer.Write(cache.TimeStamps.Count);
foreach (var element in cache.TimeStamps)
{
writer.Write(element.Key);
//Write the current time stamp onto the element.
writer.Write(element.Value);
}
writer.Write(cache.Dependencies.Count);
foreach (var pathDependenciesPair in cache.Dependencies)
{
writer.Write(pathDependenciesPair.Key);
writer.Write(pathDependenciesPair.Value.Count);
foreach (var dependency in pathDependenciesPair.Value)
{
writer.Write(dependency);
}
}
}
}
public static bool TryLoad(Stream stream, out ShaderCompilationCache cache)
{
using (var reader = new BinaryReader(stream))
{
try
{
cache = new ShaderCompilationCache((ShaderFlags)reader.ReadInt32());
byte[] data = new byte[16384];
var shaderDataCount = reader.ReadInt32();
for (int i = 0; i < shaderDataCount; ++i)
{
var shaderSource = SourceShader.Read(reader);
//Read the size in bytes of the content element data itself.
int sizeInBytes = reader.ReadInt32();
if (data.Length < sizeInBytes)
{
data = new byte[sizeInBytes];
}
reader.Read(data, 0, sizeInBytes);
ShaderBytecode bytecode;
unsafe
{
fixed(byte *buffer = data)
{
bytecode = new ShaderBytecode(new IntPtr(buffer), sizeInBytes);
}
}
cache.CompiledShaders.Add(shaderSource, bytecode);
}
var timeStampCount = reader.ReadInt32();
for (int i = 0; i < timeStampCount; ++i)
{
var shaderSource = reader.ReadString();
//Read the time stamp.
long timeStamp = reader.ReadInt64();
cache.TimeStamps.Add(shaderSource, timeStamp);
}
var dependenciesCount = reader.ReadInt32();
for (int i = 0; i < dependenciesCount; ++i)
{
var shaderSource = reader.ReadString();
var pathDependenciesCount = reader.ReadInt32();
var dependencies = new HashSet <string>();
for (int j = 0; j < pathDependenciesCount; ++j)
{
dependencies.Add(reader.ReadString());
}
cache.Dependencies.Add(shaderSource, dependencies);
}
}
catch
{
cache = null;
return(false);
}
return(true);
}
}
private static void CollectCompilationTargets(string source, string workingPath,
ShaderFileCache shaderFileCache, ShaderCompilationCache loadedCache, ShaderCompilationCache cache,
List <ShaderCompilationResult> errors, List <ShaderCompilationTarget> compilationTargets)
{
if (!shaderFileCache.TryLoad(source, out string shaderCode))
{
lock (errors)
{
errors.Add(new ShaderCompilationResult("Shader", "", "", source, 0, 0, 0, 0, "Could not find file " + source));
}
return;
}
//Only compile this shader if the shader has been modified since the previous compile.
var currentTimeStamp = File.GetLastWriteTime(source).Ticks;
if (loadedCache.ShaderFlags == cache.ShaderFlags && loadedCache.TimeStamps.TryGetValue(source, out long previousTimeStamp)) //If this file was contained before, we MIGHT be able to skip its compilation.
{
if (currentTimeStamp <= previousTimeStamp) //If the file hasn't been updated, we MIGHT be able to skip.
{
//Check the timestamps associated with the dependencies of this file.
//If any are newer than the previous snapshot, skip.
bool allowSkip = true;
if (loadedCache.Dependencies.TryGetValue(source, out HashSet <string> loadedDependencyPaths))
{
foreach (var dependency in loadedDependencyPaths)
{
//The loadedCache is guaranteed to contain a time stamp for any dependency referenced,
//because the only time any dependency is added to the list the dependency is ALSO put into the timestamps list.
var previousDependencyTimeStamp = loadedCache.TimeStamps[dependency];
var currentDependencyTimeStamp = File.GetLastWriteTime(dependency).Ticks;
if (currentDependencyTimeStamp > previousDependencyTimeStamp)
{
//One of the dependencies has been updated. This shader must be compiled.
allowSkip = false;
break;
}
}
}
else
{
//One of the dependencies could not be found in the old cache. Implies a new dependency was added; must compile.
allowSkip = false;
}
if (allowSkip)
{
Console.WriteLine($"Shader up to date: {Path.GetFileName(source)}");
cache.CopyFrom(source, loadedCache);
return;
}
}
}
var localWorkingPath = Path.GetDirectoryName(source);
var include = new IncludeHandler(shaderFileCache, workingPath, localWorkingPath);
var stages = new List <ShaderStage>();
var macroGroups = new List <MacroGroup>();
var metadataParsingErrors = new List <MetadataParsingError>();
MetadataParsing.Parse(source, shaderCode, stages, macroGroups, metadataParsingErrors);
//Prepass to collect include metadata. Seems hacky, oh well.
try
{
ShaderBytecode.Preprocess(shaderCode, null, include);
}
catch (CompilationException e)
{
if (ParseCompilerResult(e.Message, out string filePath, out int lineBegin, out int columnBegin, out int lineEnd, out int columnEnd, out string description))
{
var errorChecker = new Regex("error X(?<errorCode>[0-9]{4}): ");
var errorCode = errorChecker.Match(e.Message).Groups["errorCode"].Value;
lock (errors)
{
errors.Add(new ShaderCompilationResult("Shader", errorCode, "", source, lineBegin, columnBegin, lineEnd, columnEnd, description));
}
}