/// <summary> /// Reads the host code for a given shader, if existent. /// </summary> /// <param name="context">GPU context</param> /// <param name="tocFileStream">Host TOC file stream, intialized if needed</param> /// <param name="dataFileStream">Host data file stream, initialized if needed</param> /// <param name="guestShaders">Guest shader code for each active stage</param> /// <param name="programIndex">Index of the program on the cache</param> /// <param name="expectedTimestamp">Timestamp of the shared cache file. The host file must be newer than it</param> /// <returns>Host binary code, or null if not found</returns> private (byte[], CachedShaderStage[]) ReadHostCode( GpuContext context, ref Stream tocFileStream, ref Stream dataFileStream, GuestCodeAndCbData?[] guestShaders, int programIndex, ulong expectedTimestamp) { if (tocFileStream == null && dataFileStream == null) { string tocFilePath = Path.Combine(_basePath, GetHostTocFileName(context)); string dataFilePath = Path.Combine(_basePath, GetHostDataFileName(context)); if (!File.Exists(tocFilePath) || !File.Exists(dataFilePath)) { return(null, null); } tocFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: false); dataFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: false); BinarySerializer tempTocReader = new BinarySerializer(tocFileStream); TocHeader header = new TocHeader(); tempTocReader.Read(ref header); if (header.Timestamp < expectedTimestamp) { return(null, null); } } int offset = Unsafe.SizeOf <TocHeader>() + programIndex * Unsafe.SizeOf <OffsetAndSize>(); if (offset + Unsafe.SizeOf <OffsetAndSize>() > tocFileStream.Length) { return(null, null); } if ((ulong)offset >= (ulong)dataFileStream.Length) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } tocFileStream.Seek(offset, SeekOrigin.Begin); BinarySerializer tocReader = new BinarySerializer(tocFileStream); OffsetAndSize offsetAndSize = new OffsetAndSize(); tocReader.Read(ref offsetAndSize); if (offsetAndSize.Offset >= (ulong)dataFileStream.Length) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } dataFileStream.Seek((long)offsetAndSize.Offset, SeekOrigin.Begin); byte[] hostCode = new byte[offsetAndSize.UncompressedSize]; BinarySerializer.ReadCompressed(dataFileStream, hostCode); CachedShaderStage[] shaders = new CachedShaderStage[guestShaders.Length]; BinarySerializer dataReader = new BinarySerializer(dataFileStream); dataFileStream.Seek((long)(offsetAndSize.Offset + offsetAndSize.CompressedSize), SeekOrigin.Begin); dataReader.BeginCompression(); for (int index = 0; index < guestShaders.Length; index++) { if (!guestShaders[index].HasValue) { continue; } GuestCodeAndCbData guestShader = guestShaders[index].Value; ShaderProgramInfo info = index != 0 || guestShaders.Length == 1 ? ReadShaderProgramInfo(ref dataReader) : null; shaders[index] = new CachedShaderStage(info, guestShader.Code, guestShader.Cb1Data); } dataReader.EndCompression(); return(hostCode, shaders); }
/// <summary> /// Adds a shader to the cache. /// </summary> /// <param name="context">GPU context</param> /// <param name="program">Cached program</param> /// <param name="hostCode">Optional host binary code</param> /// <param name="streams">Output streams to use</param> public void AddShader(GpuContext context, CachedShaderProgram program, ReadOnlySpan <byte> hostCode, DiskCacheOutputStreams streams = null) { uint stagesBitMask = 0; for (int index = 0; index < program.Shaders.Length; index++) { var shader = program.Shaders[index]; if (shader == null || (shader.Info != null && shader.Info.Stage == ShaderStage.Compute)) { continue; } stagesBitMask |= 1u << index; } var tocFileStream = streams != null ? streams.TocFileStream : DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: true); var dataFileStream = streams != null ? streams.DataFileStream : DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: true); ulong timestamp = (ulong)DateTime.UtcNow.Subtract(DateTime.UnixEpoch).TotalSeconds; if (tocFileStream.Length == 0) { TocHeader header = new TocHeader(); CreateToc(tocFileStream, ref header, TocsMagic, CodeGenVersion, timestamp); } tocFileStream.Seek(0, SeekOrigin.End); dataFileStream.Seek(0, SeekOrigin.End); BinarySerializer tocWriter = new BinarySerializer(tocFileStream); BinarySerializer dataWriter = new BinarySerializer(dataFileStream); ulong dataOffset = (ulong)dataFileStream.Position; tocWriter.Write(ref dataOffset); DataEntry entry = new DataEntry(); entry.StagesBitMask = stagesBitMask; dataWriter.BeginCompression(DiskCacheCommon.GetCompressionAlgorithm()); dataWriter.Write(ref entry); DataEntryPerStage stageEntry = new DataEntryPerStage(); for (int index = 0; index < program.Shaders.Length; index++) { var shader = program.Shaders[index]; if (shader == null) { continue; } stageEntry.GuestCodeIndex = _guestStorage.AddShader(shader.Code, shader.Cb1Data); dataWriter.Write(ref stageEntry); } program.SpecializationState.Write(ref dataWriter); dataWriter.EndCompression(); if (streams == null) { tocFileStream.Dispose(); dataFileStream.Dispose(); } if (hostCode.IsEmpty) { return; } WriteHostCode(context, hostCode, program.Shaders, streams, timestamp); }
/// <summary> /// Loads all shaders from the cache. /// </summary> /// <param name="context">GPU context</param> /// <param name="loader">Parallel disk cache loader</param> public void LoadShaders(GpuContext context, ParallelDiskCacheLoader loader) { if (!CacheExists()) { return; } Stream hostTocFileStream = null; Stream hostDataFileStream = null; try { using var tocFileStream = DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: false); using var dataFileStream = DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: false); using var guestTocFileStream = _guestStorage.OpenTocFileStream(); using var guestDataFileStream = _guestStorage.OpenDataFileStream(); BinarySerializer tocReader = new BinarySerializer(tocFileStream); BinarySerializer dataReader = new BinarySerializer(dataFileStream); TocHeader header = new TocHeader(); if (!tocReader.TryRead(ref header) || header.Magic != TocsMagic) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } if (header.FormatVersion != FileFormatVersionPacked) { throw new DiskCacheLoadException(DiskCacheLoadResult.IncompatibleVersion); } bool loadHostCache = header.CodeGenVersion == CodeGenVersion; int programIndex = 0; DataEntry entry = new DataEntry(); while (tocFileStream.Position < tocFileStream.Length && loader.Active) { ulong dataOffset = 0; tocReader.Read(ref dataOffset); if ((ulong)dataOffset >= (ulong)dataFileStream.Length) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } dataFileStream.Seek((long)dataOffset, SeekOrigin.Begin); dataReader.BeginCompression(); dataReader.Read(ref entry); uint stagesBitMask = entry.StagesBitMask; if ((stagesBitMask & ~0x3fu) != 0) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } bool isCompute = stagesBitMask == 0; if (isCompute) { stagesBitMask = 1; } GuestCodeAndCbData?[] guestShaders = new GuestCodeAndCbData?[isCompute ? 1: Constants.ShaderStages + 1]; DataEntryPerStage stageEntry = new DataEntryPerStage(); while (stagesBitMask != 0) { int stageIndex = BitOperations.TrailingZeroCount(stagesBitMask); dataReader.Read(ref stageEntry); guestShaders[stageIndex] = _guestStorage.LoadShader( guestTocFileStream, guestDataFileStream, stageEntry.GuestCodeIndex); stagesBitMask &= ~(1u << stageIndex); } ShaderSpecializationState specState = ShaderSpecializationState.Read(ref dataReader); dataReader.EndCompression(); if (loadHostCache) { (byte[] hostCode, CachedShaderStage[] shaders) = ReadHostCode( context, ref hostTocFileStream, ref hostDataFileStream, guestShaders, programIndex, header.Timestamp); if (hostCode != null) { bool hasFragmentShader = shaders.Length > 5 && shaders[5] != null; int fragmentOutputMap = hasFragmentShader ? shaders[5].Info.FragmentOutputMap : -1; ShaderInfo shaderInfo = specState.PipelineState.HasValue ? new ShaderInfo(fragmentOutputMap, specState.PipelineState.Value, fromCache: true) : new ShaderInfo(fragmentOutputMap, fromCache: true); IProgram hostProgram; if (context.Capabilities.Api == TargetApi.Vulkan) { ShaderSource[] shaderSources = ShaderBinarySerializer.Unpack(shaders, hostCode, isCompute); hostProgram = context.Renderer.CreateProgram(shaderSources, shaderInfo); } else { hostProgram = context.Renderer.LoadProgramBinary(hostCode, hasFragmentShader, shaderInfo); } CachedShaderProgram program = new CachedShaderProgram(hostProgram, specState, shaders); loader.QueueHostProgram(program, hostCode, programIndex, isCompute); } else { loadHostCache = false; } } if (!loadHostCache) { loader.QueueGuestProgram(guestShaders, specState, programIndex, isCompute); } loader.CheckCompilation(); programIndex++; } } finally { _guestStorage.ClearMemoryCache(); hostTocFileStream?.Dispose(); hostDataFileStream?.Dispose(); } }