/// <summary> /// Writes the host binary code on the host cache. /// </summary> /// <param name="context">GPU context</param> /// <param name="hostCode">Host binary code</param> /// <param name="shaders">Shader stages to be added to the host cache</param> /// <param name="streams">Output streams to use</param> /// <param name="timestamp">File creation timestamp</param> private void WriteHostCode( GpuContext context, ReadOnlySpan <byte> hostCode, CachedShaderStage[] shaders, DiskCacheOutputStreams streams, ulong timestamp) { var tocFileStream = streams != null ? streams.HostTocFileStream : DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: true); var dataFileStream = streams != null ? streams.HostDataFileStream : DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: true); if (tocFileStream.Length == 0) { TocHeader header = new TocHeader(); CreateToc(tocFileStream, ref header, TochMagic, 0, timestamp); } tocFileStream.Seek(0, SeekOrigin.End); dataFileStream.Seek(0, SeekOrigin.End); BinarySerializer tocWriter = new BinarySerializer(tocFileStream); BinarySerializer dataWriter = new BinarySerializer(dataFileStream); OffsetAndSize offsetAndSize = new OffsetAndSize(); offsetAndSize.Offset = (ulong)dataFileStream.Position; offsetAndSize.UncompressedSize = (uint)hostCode.Length; long dataStartPosition = dataFileStream.Position; BinarySerializer.WriteCompressed(dataFileStream, hostCode, DiskCacheCommon.GetCompressionAlgorithm()); offsetAndSize.CompressedSize = (uint)(dataFileStream.Position - dataStartPosition); tocWriter.Write(ref offsetAndSize); dataWriter.BeginCompression(DiskCacheCommon.GetCompressionAlgorithm()); for (int index = 0; index < shaders.Length; index++) { if (shaders[index] != null) { WriteShaderProgramInfo(ref dataWriter, shaders[index].Info); } } dataWriter.EndCompression(); if (streams == null) { tocFileStream.Dispose(); dataFileStream.Dispose(); } }
/// <summary> /// Adds a shader to the cache. /// </summary> /// <param name="context">GPU context</param> /// <param name="program">Cached program</param> /// <param name="hostCode">Optional host binary code</param> /// <param name="streams">Output streams to use</param> public void AddShader(GpuContext context, CachedShaderProgram program, ReadOnlySpan <byte> hostCode, DiskCacheOutputStreams streams = null) { uint stagesBitMask = 0; for (int index = 0; index < program.Shaders.Length; index++) { var shader = program.Shaders[index]; if (shader == null || (shader.Info != null && shader.Info.Stage == ShaderStage.Compute)) { continue; } stagesBitMask |= 1u << index; } var tocFileStream = streams != null ? streams.TocFileStream : DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: true); var dataFileStream = streams != null ? streams.DataFileStream : DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: true); if (tocFileStream.Length == 0) { TocHeader header = new TocHeader(); CreateToc(tocFileStream, ref header, TocsMagic, CodeGenVersion); } tocFileStream.Seek(0, SeekOrigin.End); dataFileStream.Seek(0, SeekOrigin.End); BinarySerializer tocWriter = new BinarySerializer(tocFileStream); BinarySerializer dataWriter = new BinarySerializer(dataFileStream); ulong dataOffset = (ulong)dataFileStream.Position; tocWriter.Write(ref dataOffset); DataEntry entry = new DataEntry(); entry.StagesBitMask = stagesBitMask; dataWriter.BeginCompression(DiskCacheCommon.GetCompressionAlgorithm()); dataWriter.Write(ref entry); DataEntryPerStage stageEntry = new DataEntryPerStage(); for (int index = 0; index < program.Shaders.Length; index++) { var shader = program.Shaders[index]; if (shader == null) { continue; } stageEntry.GuestCodeIndex = _guestStorage.AddShader(shader.Code, shader.Cb1Data); dataWriter.Write(ref stageEntry); WriteShaderProgramInfo(ref dataWriter, shader.Info); } program.SpecializationState.Write(ref dataWriter); dataWriter.EndCompression(); if (streams == null) { tocFileStream.Dispose(); dataFileStream.Dispose(); } if (hostCode.IsEmpty) { return; } WriteHostCode(context, hostCode, -1, streams); }
/// <summary> /// Loads all shaders from the cache. /// </summary> /// <param name="context">GPU context</param> /// <param name="loader">Parallel disk cache loader</param> public void LoadShaders(GpuContext context, ParallelDiskCacheLoader loader) { if (!CacheExists()) { return; } Stream hostTocFileStream = null; Stream hostDataFileStream = null; try { using var tocFileStream = DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: false); using var dataFileStream = DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: false); using var guestTocFileStream = _guestStorage.OpenTocFileStream(); using var guestDataFileStream = _guestStorage.OpenDataFileStream(); BinarySerializer tocReader = new BinarySerializer(tocFileStream); BinarySerializer dataReader = new BinarySerializer(dataFileStream); TocHeader header = new TocHeader(); if (!tocReader.TryRead(ref header) || header.Magic != TocsMagic) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } if (header.FormatVersion != FileFormatVersionPacked) { throw new DiskCacheLoadException(DiskCacheLoadResult.IncompatibleVersion); } bool loadHostCache = header.CodeGenVersion == CodeGenVersion; int programIndex = 0; DataEntry entry = new DataEntry(); while (tocFileStream.Position < tocFileStream.Length && loader.Active) { ulong dataOffset = 0; tocReader.Read(ref dataOffset); if ((ulong)dataOffset >= (ulong)dataFileStream.Length) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } dataFileStream.Seek((long)dataOffset, SeekOrigin.Begin); dataReader.BeginCompression(); dataReader.Read(ref entry); uint stagesBitMask = entry.StagesBitMask; if ((stagesBitMask & ~0x3fu) != 0) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } bool isCompute = stagesBitMask == 0; if (isCompute) { stagesBitMask = 1; } CachedShaderStage[] shaders = new CachedShaderStage[isCompute ? 1 : Constants.ShaderStages + 1]; DataEntryPerStage stageEntry = new DataEntryPerStage(); while (stagesBitMask != 0) { int stageIndex = BitOperations.TrailingZeroCount(stagesBitMask); dataReader.Read(ref stageEntry); ShaderProgramInfo info = stageIndex != 0 || isCompute?ReadShaderProgramInfo(ref dataReader) : null; (byte[] guestCode, byte[] cb1Data) = _guestStorage.LoadShader( guestTocFileStream, guestDataFileStream, stageEntry.GuestCodeIndex); shaders[stageIndex] = new CachedShaderStage(info, guestCode, cb1Data); stagesBitMask &= ~(1u << stageIndex); } ShaderSpecializationState specState = ShaderSpecializationState.Read(ref dataReader); dataReader.EndCompression(); if (loadHostCache) { byte[] hostCode = ReadHostCode(context, ref hostTocFileStream, ref hostDataFileStream, programIndex); if (hostCode != null) { bool hasFragmentShader = shaders.Length > 5 && shaders[5] != null; int fragmentOutputMap = hasFragmentShader ? shaders[5].Info.FragmentOutputMap : -1; IProgram hostProgram = context.Renderer.LoadProgramBinary(hostCode, hasFragmentShader, new ShaderInfo(fragmentOutputMap)); CachedShaderProgram program = new CachedShaderProgram(hostProgram, specState, shaders); loader.QueueHostProgram(program, hostProgram, programIndex, isCompute); } else { loadHostCache = false; } } if (!loadHostCache) { loader.QueueGuestProgram(shaders, specState, programIndex, isCompute); } loader.CheckCompilation(); programIndex++; } } finally { _guestStorage.ClearMemoryCache(); hostTocFileStream?.Dispose(); hostDataFileStream?.Dispose(); } }
/// <summary> /// Reads the host code for a given shader, if existent. /// </summary> /// <param name="context">GPU context</param> /// <param name="tocFileStream">Host TOC file stream, intialized if needed</param> /// <param name="dataFileStream">Host data file stream, initialized if needed</param> /// <param name="guestShaders">Guest shader code for each active stage</param> /// <param name="programIndex">Index of the program on the cache</param> /// <param name="expectedTimestamp">Timestamp of the shared cache file. The host file must be newer than it</param> /// <returns>Host binary code, or null if not found</returns> private (byte[], CachedShaderStage[]) ReadHostCode( GpuContext context, ref Stream tocFileStream, ref Stream dataFileStream, GuestCodeAndCbData?[] guestShaders, int programIndex, ulong expectedTimestamp) { if (tocFileStream == null && dataFileStream == null) { string tocFilePath = Path.Combine(_basePath, GetHostTocFileName(context)); string dataFilePath = Path.Combine(_basePath, GetHostDataFileName(context)); if (!File.Exists(tocFilePath) || !File.Exists(dataFilePath)) { return(null, null); } tocFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: false); dataFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: false); BinarySerializer tempTocReader = new BinarySerializer(tocFileStream); TocHeader header = new TocHeader(); tempTocReader.Read(ref header); if (header.Timestamp < expectedTimestamp) { return(null, null); } } int offset = Unsafe.SizeOf <TocHeader>() + programIndex * Unsafe.SizeOf <OffsetAndSize>(); if (offset + Unsafe.SizeOf <OffsetAndSize>() > tocFileStream.Length) { return(null, null); } if ((ulong)offset >= (ulong)dataFileStream.Length) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } tocFileStream.Seek(offset, SeekOrigin.Begin); BinarySerializer tocReader = new BinarySerializer(tocFileStream); OffsetAndSize offsetAndSize = new OffsetAndSize(); tocReader.Read(ref offsetAndSize); if (offsetAndSize.Offset >= (ulong)dataFileStream.Length) { throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric); } dataFileStream.Seek((long)offsetAndSize.Offset, SeekOrigin.Begin); byte[] hostCode = new byte[offsetAndSize.UncompressedSize]; BinarySerializer.ReadCompressed(dataFileStream, hostCode); CachedShaderStage[] shaders = new CachedShaderStage[guestShaders.Length]; BinarySerializer dataReader = new BinarySerializer(dataFileStream); dataFileStream.Seek((long)(offsetAndSize.Offset + offsetAndSize.CompressedSize), SeekOrigin.Begin); dataReader.BeginCompression(); for (int index = 0; index < guestShaders.Length; index++) { if (!guestShaders[index].HasValue) { continue; } GuestCodeAndCbData guestShader = guestShaders[index].Value; ShaderProgramInfo info = index != 0 || guestShaders.Length == 1 ? ReadShaderProgramInfo(ref dataReader) : null; shaders[index] = new CachedShaderStage(info, guestShader.Code, guestShader.Cb1Data); } dataReader.EndCompression(); return(hostCode, shaders); }