Esempio n. 1
0
        /// <summary>
        /// Writes the host binary code on the host cache.
        /// </summary>
        /// <param name="context">GPU context</param>
        /// <param name="hostCode">Host binary code</param>
        /// <param name="shaders">Shader stages to be added to the host cache</param>
        /// <param name="streams">Output streams to use</param>
        /// <param name="timestamp">File creation timestamp</param>
        private void WriteHostCode(
            GpuContext context,
            ReadOnlySpan <byte> hostCode,
            CachedShaderStage[] shaders,
            DiskCacheOutputStreams streams,
            ulong timestamp)
        {
            var tocFileStream  = streams != null ? streams.HostTocFileStream : DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: true);
            var dataFileStream = streams != null ? streams.HostDataFileStream : DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: true);

            if (tocFileStream.Length == 0)
            {
                TocHeader header = new TocHeader();
                CreateToc(tocFileStream, ref header, TochMagic, 0, timestamp);
            }

            tocFileStream.Seek(0, SeekOrigin.End);
            dataFileStream.Seek(0, SeekOrigin.End);

            BinarySerializer tocWriter  = new BinarySerializer(tocFileStream);
            BinarySerializer dataWriter = new BinarySerializer(dataFileStream);

            OffsetAndSize offsetAndSize = new OffsetAndSize();

            offsetAndSize.Offset           = (ulong)dataFileStream.Position;
            offsetAndSize.UncompressedSize = (uint)hostCode.Length;

            long dataStartPosition = dataFileStream.Position;

            BinarySerializer.WriteCompressed(dataFileStream, hostCode, DiskCacheCommon.GetCompressionAlgorithm());

            offsetAndSize.CompressedSize = (uint)(dataFileStream.Position - dataStartPosition);

            tocWriter.Write(ref offsetAndSize);

            dataWriter.BeginCompression(DiskCacheCommon.GetCompressionAlgorithm());

            for (int index = 0; index < shaders.Length; index++)
            {
                if (shaders[index] != null)
                {
                    WriteShaderProgramInfo(ref dataWriter, shaders[index].Info);
                }
            }

            dataWriter.EndCompression();

            if (streams == null)
            {
                tocFileStream.Dispose();
                dataFileStream.Dispose();
            }
        }
Esempio n. 2
0
        /// <summary>
        /// Adds a shader to the cache.
        /// </summary>
        /// <param name="context">GPU context</param>
        /// <param name="program">Cached program</param>
        /// <param name="hostCode">Optional host binary code</param>
        /// <param name="streams">Output streams to use</param>
        public void AddShader(GpuContext context, CachedShaderProgram program, ReadOnlySpan <byte> hostCode, DiskCacheOutputStreams streams = null)
        {
            uint stagesBitMask = 0;

            for (int index = 0; index < program.Shaders.Length; index++)
            {
                var shader = program.Shaders[index];
                if (shader == null || (shader.Info != null && shader.Info.Stage == ShaderStage.Compute))
                {
                    continue;
                }

                stagesBitMask |= 1u << index;
            }

            var tocFileStream  = streams != null ? streams.TocFileStream : DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: true);
            var dataFileStream = streams != null ? streams.DataFileStream : DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: true);

            if (tocFileStream.Length == 0)
            {
                TocHeader header = new TocHeader();
                CreateToc(tocFileStream, ref header, TocsMagic, CodeGenVersion);
            }

            tocFileStream.Seek(0, SeekOrigin.End);
            dataFileStream.Seek(0, SeekOrigin.End);

            BinarySerializer tocWriter  = new BinarySerializer(tocFileStream);
            BinarySerializer dataWriter = new BinarySerializer(dataFileStream);

            ulong dataOffset = (ulong)dataFileStream.Position;

            tocWriter.Write(ref dataOffset);

            DataEntry entry = new DataEntry();

            entry.StagesBitMask = stagesBitMask;

            dataWriter.BeginCompression(DiskCacheCommon.GetCompressionAlgorithm());
            dataWriter.Write(ref entry);

            DataEntryPerStage stageEntry = new DataEntryPerStage();

            for (int index = 0; index < program.Shaders.Length; index++)
            {
                var shader = program.Shaders[index];
                if (shader == null)
                {
                    continue;
                }

                stageEntry.GuestCodeIndex = _guestStorage.AddShader(shader.Code, shader.Cb1Data);

                dataWriter.Write(ref stageEntry);

                WriteShaderProgramInfo(ref dataWriter, shader.Info);
            }

            program.SpecializationState.Write(ref dataWriter);
            dataWriter.EndCompression();

            if (streams == null)
            {
                tocFileStream.Dispose();
                dataFileStream.Dispose();
            }

            if (hostCode.IsEmpty)
            {
                return;
            }

            WriteHostCode(context, hostCode, -1, streams);
        }
Esempio n. 3
0
        /// <summary>
        /// Loads all shaders from the cache.
        /// </summary>
        /// <param name="context">GPU context</param>
        /// <param name="loader">Parallel disk cache loader</param>
        public void LoadShaders(GpuContext context, ParallelDiskCacheLoader loader)
        {
            if (!CacheExists())
            {
                return;
            }

            Stream hostTocFileStream  = null;
            Stream hostDataFileStream = null;

            try
            {
                using var tocFileStream  = DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: false);
                using var dataFileStream = DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: false);

                using var guestTocFileStream  = _guestStorage.OpenTocFileStream();
                using var guestDataFileStream = _guestStorage.OpenDataFileStream();

                BinarySerializer tocReader  = new BinarySerializer(tocFileStream);
                BinarySerializer dataReader = new BinarySerializer(dataFileStream);

                TocHeader header = new TocHeader();

                if (!tocReader.TryRead(ref header) || header.Magic != TocsMagic)
                {
                    throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric);
                }

                if (header.FormatVersion != FileFormatVersionPacked)
                {
                    throw new DiskCacheLoadException(DiskCacheLoadResult.IncompatibleVersion);
                }

                bool loadHostCache = header.CodeGenVersion == CodeGenVersion;

                int programIndex = 0;

                DataEntry entry = new DataEntry();

                while (tocFileStream.Position < tocFileStream.Length && loader.Active)
                {
                    ulong dataOffset = 0;
                    tocReader.Read(ref dataOffset);

                    if ((ulong)dataOffset >= (ulong)dataFileStream.Length)
                    {
                        throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric);
                    }

                    dataFileStream.Seek((long)dataOffset, SeekOrigin.Begin);

                    dataReader.BeginCompression();
                    dataReader.Read(ref entry);
                    uint stagesBitMask = entry.StagesBitMask;

                    if ((stagesBitMask & ~0x3fu) != 0)
                    {
                        throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric);
                    }

                    bool isCompute = stagesBitMask == 0;
                    if (isCompute)
                    {
                        stagesBitMask = 1;
                    }

                    CachedShaderStage[] shaders = new CachedShaderStage[isCompute ? 1 : Constants.ShaderStages + 1];

                    DataEntryPerStage stageEntry = new DataEntryPerStage();

                    while (stagesBitMask != 0)
                    {
                        int stageIndex = BitOperations.TrailingZeroCount(stagesBitMask);

                        dataReader.Read(ref stageEntry);

                        ShaderProgramInfo info = stageIndex != 0 || isCompute?ReadShaderProgramInfo(ref dataReader) : null;

                        (byte[] guestCode, byte[] cb1Data) = _guestStorage.LoadShader(
                            guestTocFileStream,
                            guestDataFileStream,
                            stageEntry.GuestCodeIndex);

                        shaders[stageIndex] = new CachedShaderStage(info, guestCode, cb1Data);

                        stagesBitMask &= ~(1u << stageIndex);
                    }

                    ShaderSpecializationState specState = ShaderSpecializationState.Read(ref dataReader);
                    dataReader.EndCompression();

                    if (loadHostCache)
                    {
                        byte[] hostCode = ReadHostCode(context, ref hostTocFileStream, ref hostDataFileStream, programIndex);

                        if (hostCode != null)
                        {
                            bool     hasFragmentShader = shaders.Length > 5 && shaders[5] != null;
                            int      fragmentOutputMap = hasFragmentShader ? shaders[5].Info.FragmentOutputMap : -1;
                            IProgram hostProgram       = context.Renderer.LoadProgramBinary(hostCode, hasFragmentShader, new ShaderInfo(fragmentOutputMap));

                            CachedShaderProgram program = new CachedShaderProgram(hostProgram, specState, shaders);

                            loader.QueueHostProgram(program, hostProgram, programIndex, isCompute);
                        }
                        else
                        {
                            loadHostCache = false;
                        }
                    }

                    if (!loadHostCache)
                    {
                        loader.QueueGuestProgram(shaders, specState, programIndex, isCompute);
                    }

                    loader.CheckCompilation();
                    programIndex++;
                }
            }
            finally
            {
                _guestStorage.ClearMemoryCache();

                hostTocFileStream?.Dispose();
                hostDataFileStream?.Dispose();
            }
        }
Esempio n. 4
0
        /// <summary>
        /// Reads the host code for a given shader, if existent.
        /// </summary>
        /// <param name="context">GPU context</param>
        /// <param name="tocFileStream">Host TOC file stream, intialized if needed</param>
        /// <param name="dataFileStream">Host data file stream, initialized if needed</param>
        /// <param name="guestShaders">Guest shader code for each active stage</param>
        /// <param name="programIndex">Index of the program on the cache</param>
        /// <param name="expectedTimestamp">Timestamp of the shared cache file. The host file must be newer than it</param>
        /// <returns>Host binary code, or null if not found</returns>
        private (byte[], CachedShaderStage[]) ReadHostCode(
            GpuContext context,
            ref Stream tocFileStream,
            ref Stream dataFileStream,
            GuestCodeAndCbData?[] guestShaders,
            int programIndex,
            ulong expectedTimestamp)
        {
            if (tocFileStream == null && dataFileStream == null)
            {
                string tocFilePath  = Path.Combine(_basePath, GetHostTocFileName(context));
                string dataFilePath = Path.Combine(_basePath, GetHostDataFileName(context));

                if (!File.Exists(tocFilePath) || !File.Exists(dataFilePath))
                {
                    return(null, null);
                }

                tocFileStream  = DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: false);
                dataFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: false);

                BinarySerializer tempTocReader = new BinarySerializer(tocFileStream);

                TocHeader header = new TocHeader();

                tempTocReader.Read(ref header);

                if (header.Timestamp < expectedTimestamp)
                {
                    return(null, null);
                }
            }

            int offset = Unsafe.SizeOf <TocHeader>() + programIndex * Unsafe.SizeOf <OffsetAndSize>();

            if (offset + Unsafe.SizeOf <OffsetAndSize>() > tocFileStream.Length)
            {
                return(null, null);
            }

            if ((ulong)offset >= (ulong)dataFileStream.Length)
            {
                throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric);
            }

            tocFileStream.Seek(offset, SeekOrigin.Begin);

            BinarySerializer tocReader = new BinarySerializer(tocFileStream);

            OffsetAndSize offsetAndSize = new OffsetAndSize();

            tocReader.Read(ref offsetAndSize);

            if (offsetAndSize.Offset >= (ulong)dataFileStream.Length)
            {
                throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric);
            }

            dataFileStream.Seek((long)offsetAndSize.Offset, SeekOrigin.Begin);

            byte[] hostCode = new byte[offsetAndSize.UncompressedSize];

            BinarySerializer.ReadCompressed(dataFileStream, hostCode);

            CachedShaderStage[] shaders    = new CachedShaderStage[guestShaders.Length];
            BinarySerializer    dataReader = new BinarySerializer(dataFileStream);

            dataFileStream.Seek((long)(offsetAndSize.Offset + offsetAndSize.CompressedSize), SeekOrigin.Begin);

            dataReader.BeginCompression();

            for (int index = 0; index < guestShaders.Length; index++)
            {
                if (!guestShaders[index].HasValue)
                {
                    continue;
                }

                GuestCodeAndCbData guestShader = guestShaders[index].Value;
                ShaderProgramInfo  info        = index != 0 || guestShaders.Length == 1 ? ReadShaderProgramInfo(ref dataReader) : null;

                shaders[index] = new CachedShaderStage(info, guestShader.Code, guestShader.Cb1Data);
            }

            dataReader.EndCompression();

            return(hostCode, shaders);
        }