/// <summary> /// Construct a new cache manifest header. /// </summary> /// <param name="version">The version of the cache</param> /// <param name="graphicsApi">The graphics api used for this cache</param> /// <param name="hashType">The hash type used for this cache</param> public CacheManifestHeader(ulong version, CacheGraphicsApi graphicsApi, CacheHashType hashType) { Version = version; GraphicsApi = graphicsApi; HashType = hashType; TableChecksum = 0; }
/// <summary> /// Create a new cache manager instance /// </summary> /// <param name="graphicsApi">The graphics api in use</param> /// <param name="hashType">The hash type in use for the cache</param> /// <param name="shaderProvider">The name of the codegen provider</param> /// <param name="titleId">The guest application title ID</param> /// <param name="shaderCodeGenVersion">Version of the codegen</param> public CacheManager(CacheGraphicsApi graphicsApi, CacheHashType hashType, string shaderProvider, string titleId, ulong shaderCodeGenVersion) { _graphicsApi = graphicsApi; _hashType = hashType; _shaderProvider = shaderProvider; string baseCacheDirectory = Path.Combine(AppDataManager.GamesDirPath, titleId, "cache", "shader"); _guestProgramCache = new CacheCollection(baseCacheDirectory, _hashType, CacheGraphicsApi.Guest, "", "program", GuestCacheVersion); _hostProgramCache = new CacheCollection(baseCacheDirectory, _hashType, _graphicsApi, _shaderProvider, "host", shaderCodeGenVersion); }
/// <summary> /// Create a new cache manager instance /// </summary> /// <param name="graphicsApi">The graphics api in use</param> /// <param name="hashType">The hash type in use for the cache</param> /// <param name="shaderProvider">The name of the codegen provider</param> /// <param name="titleId">The guest application title ID</param> /// <param name="shaderCodeGenVersion">Version of the codegen</param> public CacheManager(CacheGraphicsApi graphicsApi, CacheHashType hashType, string shaderProvider, string titleId, ulong shaderCodeGenVersion) { _graphicsApi = graphicsApi; _hashType = hashType; _shaderProvider = shaderProvider; string baseCacheDirectory = CacheHelper.GetBaseCacheDirectory(titleId); _guestProgramCache = new CacheCollection(baseCacheDirectory, _hashType, CacheGraphicsApi.Guest, "", "program", GuestCacheVersion); _hostProgramCache = new CacheCollection(baseCacheDirectory, _hashType, _graphicsApi, _shaderProvider, "host", shaderCodeGenVersion); }
/// <summary> /// Create a new cache collection. /// </summary> /// <param name="baseCacheDirectory">The directory of the shader cache</param> /// <param name="hashType">The hash type of the shader cache</param> /// <param name="graphicsApi">The graphics api of the shader cache</param> /// <param name="shaderProvider">The shader provider name of the shader cache</param> /// <param name="cacheName">The name of the cache</param> /// <param name="version">The version of the cache</param> public CacheCollection(string baseCacheDirectory, CacheHashType hashType, CacheGraphicsApi graphicsApi, string shaderProvider, string cacheName, ulong version) { if (hashType != CacheHashType.XxHash128) { throw new NotImplementedException($"{hashType}"); } _cacheDirectory = GenerateCachePath(baseCacheDirectory, graphicsApi, shaderProvider, cacheName); _graphicsApi = graphicsApi; _hashType = hashType; _version = version; _hashTable = new HashSet <Hash128>(); Load(); _fileWriterWorkerQueue = new AsyncWorkQueue <CacheFileOperationTask>(HandleCacheTask, $"CacheCollection.Worker.{cacheName}"); }
/// <summary> /// Recompute all the hashes of a given cache. /// </summary> /// <param name="guestBaseCacheDirectory">The guest cache directory path</param> /// <param name="hostBaseCacheDirectory">The host cache directory path</param> /// <param name="graphicsApi">The graphics api in use</param> /// <param name="hashType">The hash type in use</param> /// <param name="newVersion">The version to write in the host and guest manifest after migration</param> private static void RecomputeHashes(string guestBaseCacheDirectory, string hostBaseCacheDirectory, CacheGraphicsApi graphicsApi, CacheHashType hashType, ulong newVersion) { string guestManifestPath = CacheHelper.GetManifestPath(guestBaseCacheDirectory); string hostManifestPath = CacheHelper.GetManifestPath(hostBaseCacheDirectory); if (CacheHelper.TryReadManifestFile(guestManifestPath, CacheGraphicsApi.Guest, hashType, out _, out HashSet <Hash128> guestEntries)) { CacheHelper.TryReadManifestFile(hostManifestPath, graphicsApi, hashType, out _, out HashSet <Hash128> hostEntries); Logger.Info?.Print(LogClass.Gpu, "Shader cache hashes need to be recomputed, performing migration..."); string guestArchivePath = CacheHelper.GetArchivePath(guestBaseCacheDirectory); string hostArchivePath = CacheHelper.GetArchivePath(hostBaseCacheDirectory); ZipArchive guestArchive = ZipFile.Open(guestArchivePath, ZipArchiveMode.Update); ZipArchive hostArchive = ZipFile.Open(hostArchivePath, ZipArchiveMode.Update); CacheHelper.EnsureArchiveUpToDate(guestBaseCacheDirectory, guestArchive, guestEntries); CacheHelper.EnsureArchiveUpToDate(hostBaseCacheDirectory, hostArchive, hostEntries); int programIndex = 0; HashSet <Hash128> newEntries = new HashSet <Hash128>(); foreach (Hash128 oldHash in guestEntries) { byte[] guestProgram = CacheHelper.ReadFromArchive(guestArchive, oldHash); Logger.Info?.Print(LogClass.Gpu, $"Migrating shader {oldHash} ({programIndex + 1} / {guestEntries.Count})"); if (guestProgram != null) { ReadOnlySpan <byte> guestProgramReadOnlySpan = guestProgram; ReadOnlySpan <GuestShaderCacheEntry> cachedShaderEntries = GuestShaderCacheEntry.Parse(ref guestProgramReadOnlySpan, out GuestShaderCacheHeader fileHeader); TransformFeedbackDescriptor[] tfd = CacheHelper.ReadTransformationFeedbackInformations(ref guestProgramReadOnlySpan, fileHeader); Hash128 newHash = CacheHelper.ComputeGuestHashFromCache(cachedShaderEntries, tfd); if (newHash != oldHash) { MoveEntry(guestArchive, oldHash, newHash); MoveEntry(hostArchive, oldHash, newHash); } else { Logger.Warning?.Print(LogClass.Gpu, $"Same hashes for shader {oldHash}"); } newEntries.Add(newHash); } programIndex++; } byte[] newGuestManifestContent = CacheHelper.ComputeManifest(newVersion, CacheGraphicsApi.Guest, hashType, newEntries); byte[] newHostManifestContent = CacheHelper.ComputeManifest(newVersion, graphicsApi, hashType, newEntries); File.WriteAllBytes(guestManifestPath, newGuestManifestContent); File.WriteAllBytes(hostManifestPath, newHostManifestContent); guestArchive.Dispose(); hostArchive.Dispose(); } }
/// <summary> /// Check and run cache migration if needed. /// </summary> /// <param name="baseCacheDirectory">The base path of the cache</param> /// <param name="graphicsApi">The graphics api in use</param> /// <param name="hashType">The hash type in use</param> /// <param name="shaderProvider">The shader provider name of the cache</param> public static void Run(string baseCacheDirectory, CacheGraphicsApi graphicsApi, CacheHashType hashType, string shaderProvider) { string guestBaseCacheDirectory = CacheHelper.GenerateCachePath(baseCacheDirectory, CacheGraphicsApi.Guest, "", "program"); string hostBaseCacheDirectory = CacheHelper.GenerateCachePath(baseCacheDirectory, graphicsApi, shaderProvider, "host"); if (CacheHelper.TryReadManifestHeader(CacheHelper.GetManifestPath(guestBaseCacheDirectory), out CacheManifestHeader header)) { if (NeedHashRecompute(header.Version, out ulong newVersion)) { RecomputeHashes(guestBaseCacheDirectory, hostBaseCacheDirectory, graphicsApi, hashType, newVersion); } } }
/// <summary> /// Compute a cache manifest from runtime data. /// </summary> /// <param name="version">The version of the cache</param> /// <param name="graphicsApi">The graphics api used by the cache</param> /// <param name="hashType">The hash type of the cache</param> /// <param name="entries">The entries in the cache</param> /// <returns>The cache manifest from runtime data</returns> public static byte[] ComputeManifest(ulong version, CacheGraphicsApi graphicsApi, CacheHashType hashType, HashSet <Hash128> entries) { if (hashType != CacheHashType.XxHash128) { throw new NotImplementedException($"{hashType}"); } CacheManifestHeader manifestHeader = new CacheManifestHeader(version, graphicsApi, hashType); byte[] data = new byte[Unsafe.SizeOf <CacheManifestHeader>() + entries.Count * Unsafe.SizeOf <Hash128>()]; // CacheManifestHeader has the same size as a Hash128. Span <Hash128> dataSpan = MemoryMarshal.Cast <byte, Hash128>(data.AsSpan()).Slice(1); int i = 0; foreach (Hash128 hash in entries) { dataSpan[i++] = hash; } manifestHeader.UpdateChecksum(data.AsSpan(Unsafe.SizeOf <CacheManifestHeader>())); MemoryMarshal.Write(data, ref manifestHeader); return(data); }
/// <summary> /// Try to read the manifest from a given file path. /// </summary> /// <param name="manifestPath">The path to the manifest file</param> /// <param name="graphicsApi">The graphics api used by the cache</param> /// <param name="hashType">The hash type of the cache</param> /// <param name="header">The manifest header read</param> /// <param name="entries">The entries read from the cache manifest</param> /// <returns>Return true if the manifest was read</returns> public static bool TryReadManifestFile(string manifestPath, CacheGraphicsApi graphicsApi, CacheHashType hashType, out CacheManifestHeader header, out HashSet <Hash128> entries) { header = default; entries = new HashSet <Hash128>(); if (File.Exists(manifestPath)) { Memory <byte> rawManifest = File.ReadAllBytes(manifestPath); if (MemoryMarshal.TryRead(rawManifest.Span, out header)) { Memory <byte> hashTableRaw = rawManifest.Slice(Unsafe.SizeOf <CacheManifestHeader>()); bool isValid = header.IsValid(graphicsApi, hashType, hashTableRaw.Span); if (isValid) { ReadOnlySpan <Hash128> hashTable = MemoryMarshal.Cast <byte, Hash128>(hashTableRaw.Span); foreach (Hash128 hash in hashTable) { entries.Add(hash); } } return(isValid); } } return(false); }
/// <summary> /// Check the validity of the header. /// </summary> /// <param name="version">The target version in use</param> /// <param name="graphicsApi">The target graphics api in use</param> /// <param name="hashType">The target hash type in use</param> /// <param name="data">The data after this header</param> /// <returns>True if the header is valid</returns> public bool IsValid(ulong version, CacheGraphicsApi graphicsApi, CacheHashType hashType, ReadOnlySpan <byte> data) { return(Version == version && GraphicsApi == graphicsApi && HashType == hashType && TableChecksum == CalculateCrc16(data)); }