/// <summary>
/// Construct a new cache manifest header.
/// </summary>
/// <param name="version">The version of the cache</param>
/// <param name="graphicsApi">The graphics api used for this cache</param>
/// <param name="hashType">The hash type used for this cache</param>
public CacheManifestHeader(ulong version, CacheGraphicsApi graphicsApi, CacheHashType hashType)
{
Version = version;
GraphicsApi = graphicsApi;
HashType = hashType;
TableChecksum = 0;
}
/// <summary>
/// Create a new cache manager instance
/// </summary>
/// <param name="graphicsApi">The graphics api in use</param>
/// <param name="hashType">The hash type in use for the cache</param>
/// <param name="shaderProvider">The name of the codegen provider</param>
/// <param name="titleId">The guest application title ID</param>
/// <param name="shaderCodeGenVersion">Version of the codegen</param>
public CacheManager(CacheGraphicsApi graphicsApi, CacheHashType hashType, string shaderProvider, string titleId, ulong shaderCodeGenVersion)
{
_graphicsApi = graphicsApi;
_hashType = hashType;
_shaderProvider = shaderProvider;
string baseCacheDirectory = Path.Combine(AppDataManager.GamesDirPath, titleId, "cache", "shader");
_guestProgramCache = new CacheCollection(baseCacheDirectory, _hashType, CacheGraphicsApi.Guest, "", "program", GuestCacheVersion);
_hostProgramCache = new CacheCollection(baseCacheDirectory, _hashType, _graphicsApi, _shaderProvider, "host", shaderCodeGenVersion);
}
/// <summary>
/// Create a new cache manager instance
/// </summary>
/// <param name="graphicsApi">The graphics api in use</param>
/// <param name="hashType">The hash type in use for the cache</param>
/// <param name="shaderProvider">The name of the codegen provider</param>
/// <param name="titleId">The guest application title ID</param>
/// <param name="shaderCodeGenVersion">Version of the codegen</param>
public CacheManager(CacheGraphicsApi graphicsApi, CacheHashType hashType, string shaderProvider, string titleId, ulong shaderCodeGenVersion)
{
_graphicsApi = graphicsApi;
_hashType = hashType;
_shaderProvider = shaderProvider;
string baseCacheDirectory = CacheHelper.GetBaseCacheDirectory(titleId);
_guestProgramCache = new CacheCollection(baseCacheDirectory, _hashType, CacheGraphicsApi.Guest, "", "program", GuestCacheVersion);
_hostProgramCache = new CacheCollection(baseCacheDirectory, _hashType, _graphicsApi, _shaderProvider, "host", shaderCodeGenVersion);
}
/// <summary>
/// Create a new cache collection.
/// </summary>
/// <param name="baseCacheDirectory">The directory of the shader cache</param>
/// <param name="hashType">The hash type of the shader cache</param>
/// <param name="graphicsApi">The graphics api of the shader cache</param>
/// <param name="shaderProvider">The shader provider name of the shader cache</param>
/// <param name="cacheName">The name of the cache</param>
/// <param name="version">The version of the cache</param>
public CacheCollection(string baseCacheDirectory, CacheHashType hashType, CacheGraphicsApi graphicsApi, string shaderProvider, string cacheName, ulong version)
{
if (hashType != CacheHashType.XxHash128)
{
throw new NotImplementedException($"{hashType}");
}
_cacheDirectory = GenerateCachePath(baseCacheDirectory, graphicsApi, shaderProvider, cacheName);
_graphicsApi = graphicsApi;
_hashType = hashType;
_version = version;
_hashTable = new HashSet <Hash128>();
Load();
_fileWriterWorkerQueue = new AsyncWorkQueue <CacheFileOperationTask>(HandleCacheTask, $"CacheCollection.Worker.{cacheName}");
}
/// <summary>
/// Recompute all the hashes of a given cache.
/// </summary>
/// <param name="guestBaseCacheDirectory">The guest cache directory path</param>
/// <param name="hostBaseCacheDirectory">The host cache directory path</param>
/// <param name="graphicsApi">The graphics api in use</param>
/// <param name="hashType">The hash type in use</param>
/// <param name="newVersion">The version to write in the host and guest manifest after migration</param>
private static void RecomputeHashes(string guestBaseCacheDirectory, string hostBaseCacheDirectory, CacheGraphicsApi graphicsApi, CacheHashType hashType, ulong newVersion)
{
string guestManifestPath = CacheHelper.GetManifestPath(guestBaseCacheDirectory);
string hostManifestPath = CacheHelper.GetManifestPath(hostBaseCacheDirectory);
if (CacheHelper.TryReadManifestFile(guestManifestPath, CacheGraphicsApi.Guest, hashType, out _, out HashSet <Hash128> guestEntries))
{
CacheHelper.TryReadManifestFile(hostManifestPath, graphicsApi, hashType, out _, out HashSet <Hash128> hostEntries);
Logger.Info?.Print(LogClass.Gpu, "Shader cache hashes need to be recomputed, performing migration...");
string guestArchivePath = CacheHelper.GetArchivePath(guestBaseCacheDirectory);
string hostArchivePath = CacheHelper.GetArchivePath(hostBaseCacheDirectory);
ZipArchive guestArchive = ZipFile.Open(guestArchivePath, ZipArchiveMode.Update);
ZipArchive hostArchive = ZipFile.Open(hostArchivePath, ZipArchiveMode.Update);
CacheHelper.EnsureArchiveUpToDate(guestBaseCacheDirectory, guestArchive, guestEntries);
CacheHelper.EnsureArchiveUpToDate(hostBaseCacheDirectory, hostArchive, hostEntries);
int programIndex = 0;
HashSet <Hash128> newEntries = new HashSet <Hash128>();
foreach (Hash128 oldHash in guestEntries)
{
byte[] guestProgram = CacheHelper.ReadFromArchive(guestArchive, oldHash);
Logger.Info?.Print(LogClass.Gpu, $"Migrating shader {oldHash} ({programIndex + 1} / {guestEntries.Count})");
if (guestProgram != null)
{
ReadOnlySpan <byte> guestProgramReadOnlySpan = guestProgram;
ReadOnlySpan <GuestShaderCacheEntry> cachedShaderEntries = GuestShaderCacheEntry.Parse(ref guestProgramReadOnlySpan, out GuestShaderCacheHeader fileHeader);
TransformFeedbackDescriptor[] tfd = CacheHelper.ReadTransformationFeedbackInformations(ref guestProgramReadOnlySpan, fileHeader);
Hash128 newHash = CacheHelper.ComputeGuestHashFromCache(cachedShaderEntries, tfd);
if (newHash != oldHash)
{
MoveEntry(guestArchive, oldHash, newHash);
MoveEntry(hostArchive, oldHash, newHash);
}
else
{
Logger.Warning?.Print(LogClass.Gpu, $"Same hashes for shader {oldHash}");
}
newEntries.Add(newHash);
}
programIndex++;
}
byte[] newGuestManifestContent = CacheHelper.ComputeManifest(newVersion, CacheGraphicsApi.Guest, hashType, newEntries);
byte[] newHostManifestContent = CacheHelper.ComputeManifest(newVersion, graphicsApi, hashType, newEntries);
File.WriteAllBytes(guestManifestPath, newGuestManifestContent);
File.WriteAllBytes(hostManifestPath, newHostManifestContent);
guestArchive.Dispose();
hostArchive.Dispose();
}
}
/// <summary>
/// Check and run cache migration if needed.
/// </summary>
/// <param name="baseCacheDirectory">The base path of the cache</param>
/// <param name="graphicsApi">The graphics api in use</param>
/// <param name="hashType">The hash type in use</param>
/// <param name="shaderProvider">The shader provider name of the cache</param>
public static void Run(string baseCacheDirectory, CacheGraphicsApi graphicsApi, CacheHashType hashType, string shaderProvider)
{
string guestBaseCacheDirectory = CacheHelper.GenerateCachePath(baseCacheDirectory, CacheGraphicsApi.Guest, "", "program");
string hostBaseCacheDirectory = CacheHelper.GenerateCachePath(baseCacheDirectory, graphicsApi, shaderProvider, "host");
if (CacheHelper.TryReadManifestHeader(CacheHelper.GetManifestPath(guestBaseCacheDirectory), out CacheManifestHeader header))
{
if (NeedHashRecompute(header.Version, out ulong newVersion))
{
RecomputeHashes(guestBaseCacheDirectory, hostBaseCacheDirectory, graphicsApi, hashType, newVersion);
}
}
}
/// <summary>
/// Compute a cache manifest from runtime data.
/// </summary>
/// <param name="version">The version of the cache</param>
/// <param name="graphicsApi">The graphics api used by the cache</param>
/// <param name="hashType">The hash type of the cache</param>
/// <param name="entries">The entries in the cache</param>
/// <returns>The cache manifest from runtime data</returns>
public static byte[] ComputeManifest(ulong version, CacheGraphicsApi graphicsApi, CacheHashType hashType, HashSet <Hash128> entries)
{
if (hashType != CacheHashType.XxHash128)
{
throw new NotImplementedException($"{hashType}");
}
CacheManifestHeader manifestHeader = new CacheManifestHeader(version, graphicsApi, hashType);
byte[] data = new byte[Unsafe.SizeOf <CacheManifestHeader>() + entries.Count * Unsafe.SizeOf <Hash128>()];
// CacheManifestHeader has the same size as a Hash128.
Span <Hash128> dataSpan = MemoryMarshal.Cast <byte, Hash128>(data.AsSpan()).Slice(1);
int i = 0;
foreach (Hash128 hash in entries)
{
dataSpan[i++] = hash;
}
manifestHeader.UpdateChecksum(data.AsSpan(Unsafe.SizeOf <CacheManifestHeader>()));
MemoryMarshal.Write(data, ref manifestHeader);
return(data);
}
/// <summary>
/// Try to read the manifest from a given file path.
/// </summary>
/// <param name="manifestPath">The path to the manifest file</param>
/// <param name="graphicsApi">The graphics api used by the cache</param>
/// <param name="hashType">The hash type of the cache</param>
/// <param name="header">The manifest header read</param>
/// <param name="entries">The entries read from the cache manifest</param>
/// <returns>Return true if the manifest was read</returns>
public static bool TryReadManifestFile(string manifestPath, CacheGraphicsApi graphicsApi, CacheHashType hashType, out CacheManifestHeader header, out HashSet <Hash128> entries)
{
header = default;
entries = new HashSet <Hash128>();
if (File.Exists(manifestPath))
{
Memory <byte> rawManifest = File.ReadAllBytes(manifestPath);
if (MemoryMarshal.TryRead(rawManifest.Span, out header))
{
Memory <byte> hashTableRaw = rawManifest.Slice(Unsafe.SizeOf <CacheManifestHeader>());
bool isValid = header.IsValid(graphicsApi, hashType, hashTableRaw.Span);
if (isValid)
{
ReadOnlySpan <Hash128> hashTable = MemoryMarshal.Cast <byte, Hash128>(hashTableRaw.Span);
foreach (Hash128 hash in hashTable)
{
entries.Add(hash);
}
}
return(isValid);
}
}
return(false);
}
/// <summary>
/// Check the validity of the header.
/// </summary>
/// <param name="version">The target version in use</param>
/// <param name="graphicsApi">The target graphics api in use</param>
/// <param name="hashType">The target hash type in use</param>
/// <param name="data">The data after this header</param>
/// <returns>True if the header is valid</returns>
public bool IsValid(ulong version, CacheGraphicsApi graphicsApi, CacheHashType hashType, ReadOnlySpan <byte> data)
{
return(Version == version && GraphicsApi == graphicsApi && HashType == hashType && TableChecksum == CalculateCrc16(data));
}
