static Ptc() { _infosStream = new MemoryStream(); _codesStream = new MemoryStream(); _relocsStream = new MemoryStream(); _unwindInfosStream = new MemoryStream(); _infosWriter = new BinaryWriter(_infosStream, EncodingCache.UTF8NoBOM, true); _binaryFormatter = new BinaryFormatter(); _waitEvent = new ManualResetEvent(true); _loggerEvent = new AutoResetEvent(false); _basePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), BaseDir); _lock = new object(); _disposed = false; PtcJumpTable = new PtcJumpTable(); TitleIdText = TitleIdTextDefault; DisplayVersion = DisplayVersionDefault; CachePathActual = string.Empty; CachePathBackup = string.Empty; Disable(); AppDomain.CurrentDomain.UnhandledException += CurrentDomain_UnhandledException; AppDomain.CurrentDomain.ProcessExit += CurrentDomain_ProcessExit; }
static Ptc() { _infosStream = new MemoryStream(); _codesStream = new MemoryStream(); _relocsStream = new MemoryStream(); _unwindInfosStream = new MemoryStream(); _infosWriter = new BinaryWriter(_infosStream, EncodingCache.UTF8NoBOM, true); _waitEvent = new ManualResetEvent(true); _loggerEvent = new AutoResetEvent(false); _lock = new object(); _disposed = false; PtcJumpTable = new PtcJumpTable(); TitleIdText = TitleIdTextDefault; DisplayVersion = DisplayVersionDefault; CachePathActual = string.Empty; CachePathBackup = string.Empty; Disable(); }
private static void PreSave() { _waitEvent.Reset(); try { string fileNameActual = string.Concat(CachePathActual, ".cache"); string fileNameBackup = string.Concat(CachePathBackup, ".cache"); FileInfo fileInfoActual = new FileInfo(fileNameActual); if (fileInfoActual.Exists && fileInfoActual.Length != 0L) { File.Copy(fileNameActual, fileNameBackup, true); } Save(fileNameActual); } finally { ResetCarriersIfNeeded(); PtcJumpTable.ClearIfNeeded(); GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; } _waitEvent.Set(); }
public static void Initialize(string titleIdText, string displayVersion, bool enabled) { Wait(); ClearMemoryStreams(); PtcJumpTable.Clear(); PtcProfiler.Stop(); PtcProfiler.Wait(); PtcProfiler.ClearEntries(); if (String.IsNullOrEmpty(titleIdText) || titleIdText == TitleIdTextDefault) { TitleIdText = TitleIdTextDefault; DisplayVersion = DisplayVersionDefault; CachePathActual = string.Empty; CachePathBackup = string.Empty; Disable(); return; } Logger.PrintInfo(LogClass.Ptc, $"Initializing Profiled Persistent Translation Cache (enabled: {enabled})."); TitleIdText = titleIdText; DisplayVersion = !String.IsNullOrEmpty(displayVersion) ? displayVersion : DisplayVersionDefault; if (enabled) { string workPathActual = Path.Combine(_basePath, "games", TitleIdText, "cache", "cpu", ActualDir); string workPathBackup = Path.Combine(_basePath, "games", TitleIdText, "cache", "cpu", BackupDir); if (!Directory.Exists(workPathActual)) { Directory.CreateDirectory(workPathActual); } if (!Directory.Exists(workPathBackup)) { Directory.CreateDirectory(workPathBackup); } CachePathActual = Path.Combine(workPathActual, DisplayVersion); CachePathBackup = Path.Combine(workPathBackup, DisplayVersion); Enable(); PreLoad(); PtcProfiler.PreLoad(); } else { CachePathActual = string.Empty; CachePathBackup = string.Empty; Disable(); } }
internal static void LoadTranslations(ConcurrentDictionary <ulong, TranslatedFunction> funcs, IntPtr pageTablePointer, JumpTable jumpTable) { if ((int)_infosStream.Length == 0 || (int)_codesStream.Length == 0 || (int)_relocsStream.Length == 0 || (int)_unwindInfosStream.Length == 0) { return; } Debug.Assert(funcs.Count == 0); _infosStream.Seek(0L, SeekOrigin.Begin); _codesStream.Seek(0L, SeekOrigin.Begin); _relocsStream.Seek(0L, SeekOrigin.Begin); _unwindInfosStream.Seek(0L, SeekOrigin.Begin); using (BinaryReader infosReader = new BinaryReader(_infosStream, EncodingCache.UTF8NoBOM, true)) using (BinaryReader codesReader = new BinaryReader(_codesStream, EncodingCache.UTF8NoBOM, true)) using (BinaryReader relocsReader = new BinaryReader(_relocsStream, EncodingCache.UTF8NoBOM, true)) using (BinaryReader unwindInfosReader = new BinaryReader(_unwindInfosStream, EncodingCache.UTF8NoBOM, true)) { int infosEntriesCount = (int)_infosStream.Length / InfoEntry.Stride; for (int i = 0; i < infosEntriesCount; i++) { InfoEntry infoEntry = ReadInfo(infosReader); byte[] code = ReadCode(codesReader, infoEntry.CodeLen); if (infoEntry.RelocEntriesCount != 0) { RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount); PatchCode(code, relocEntries, pageTablePointer, jumpTable); } UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader); TranslatedFunction func = FastTranslate(code, unwindInfo, infoEntry.HighCq); funcs.AddOrUpdate((ulong)infoEntry.Address, func, (key, oldFunc) => func.HighCq && !oldFunc.HighCq ? func : oldFunc); } } if (_infosStream.Position < _infosStream.Length || _codesStream.Position < _codesStream.Length || _relocsStream.Position < _relocsStream.Length || _unwindInfosStream.Position < _unwindInfosStream.Length) { throw new Exception("Could not reach the end of one or more memory streams."); } jumpTable.Initialize(PtcJumpTable, funcs); PtcJumpTable.WriteJumpTable(jumpTable, funcs); PtcJumpTable.WriteDynamicTable(jumpTable); }
public static void Serialize(MemoryStream stream, PtcJumpTable ptcJumpTable) { using (BinaryWriter writer = new BinaryWriter(stream, EncodingCache.UTF8NoBOM, true)) { writer.Write((int)ptcJumpTable._jumpTable.Count); foreach (var tableEntry in ptcJumpTable._jumpTable) { writer.Write((int)tableEntry.EntryIndex); writer.Write((long)tableEntry.GuestAddress); writer.Write((int)tableEntry.HostAddress); } writer.Write((int)ptcJumpTable._dynamicTable.Count); foreach (var tableEntry in ptcJumpTable._dynamicTable) { writer.Write((int)tableEntry.EntryIndex); writer.Write((long)tableEntry.GuestAddress); writer.Write((int)tableEntry.HostAddress); } writer.Write((int)ptcJumpTable.Targets.Count); foreach (ulong address in ptcJumpTable.Targets) { writer.Write((ulong)address); } writer.Write((int)ptcJumpTable.Dependants.Count); foreach (var kv in ptcJumpTable.Dependants) { writer.Write((ulong)kv.Key); // address writer.Write((int)kv.Value.Count); foreach (int entry in kv.Value) { writer.Write((int)entry); } } writer.Write((int)ptcJumpTable.Owners.Count); foreach (var kv in ptcJumpTable.Owners) { writer.Write((ulong)kv.Key); // address writer.Write((int)kv.Value.Count); foreach (int entry in kv.Value) { writer.Write((int)entry); } } } }
public static void Serialize(Stream stream, PtcJumpTable ptcJumpTable) { SerializeList(stream, ptcJumpTable._jumpTable); SerializeList(stream, ptcJumpTable._dynamicTable); SerializeList(stream, ptcJumpTable.Targets); SerializeDictionary(stream, ptcJumpTable.Dependants, (stream, list) => SerializeList(stream, list)); SerializeDictionary(stream, ptcJumpTable.Owners, (stream, list) => SerializeList(stream, list)); }
private static void Save(string fileName) { int translatedFuncsCount; using (MemoryStream stream = new MemoryStream()) using (MD5 md5 = MD5.Create()) { int hashSize = md5.HashSize / 8; stream.Seek((long)hashSize, SeekOrigin.Begin); WriteHeader(stream); _infosStream.WriteTo(stream); _codesStream.WriteTo(stream); _relocsStream.WriteTo(stream); _unwindInfosStream.WriteTo(stream); PtcJumpTable.Serialize(stream, PtcJumpTable); translatedFuncsCount = GetInfosEntriesCount(); ClearMemoryStreams(); PtcJumpTable.Clear(); stream.Seek((long)hashSize, SeekOrigin.Begin); byte[] hash = md5.ComputeHash(stream); stream.Seek(0L, SeekOrigin.Begin); stream.Write(hash, 0, hashSize); using (FileStream compressedStream = new FileStream(fileName, FileMode.OpenOrCreate)) using (DeflateStream deflateStream = new DeflateStream(compressedStream, SaveCompressionLevel, true)) { try { stream.WriteTo(deflateStream); } catch { compressedStream.Position = 0L; } if (compressedStream.Position < compressedStream.Length) { compressedStream.SetLength(compressedStream.Position); } } } long fileSize = new FileInfo(fileName).Length; Logger.Info?.Print(LogClass.Ptc, $"Saved Translation Cache (size: {fileSize} bytes, translated functions: {translatedFuncsCount})."); }
public static int GetSerializeSize(PtcJumpTable ptcJumpTable) { int size = 0; size += GetSerializeSizeList(ptcJumpTable._jumpTable); size += GetSerializeSizeList(ptcJumpTable._dynamicTable); size += GetSerializeSizeList(ptcJumpTable.Targets); size += GetSerializeSizeDictionary(ptcJumpTable.Dependants, (list) => GetSerializeSizeList(list)); size += GetSerializeSizeDictionary(ptcJumpTable.Owners, (list) => GetSerializeSizeList(list)); return(size); }
internal static void MakeAndSaveTranslations(ConcurrentDictionary <ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable) { var profiledFuncsToTranslate = PtcProfiler.GetProfiledFuncsToTranslate(funcs); if (profiledFuncsToTranslate.Count == 0) { return; } _translateCount = 0; ThreadPool.QueueUserWorkItem(TranslationLogger, profiledFuncsToTranslate.Count); int maxDegreeOfParallelism = (Environment.ProcessorCount * 3) / 4; Parallel.ForEach(profiledFuncsToTranslate, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, (item, state) => { ulong address = item.Key; Debug.Assert(PtcProfiler.IsAddressInStaticCodeRange(address)); TranslatedFunction func = Translator.Translate(memory, jumpTable, address, item.Value.mode, item.Value.highCq); bool isAddressUnique = funcs.TryAdd(address, func); Debug.Assert(isAddressUnique, $"The address 0x{address:X16} is not unique."); if (func.HighCq) { jumpTable.RegisterFunction(address, func); } Interlocked.Increment(ref _translateCount); if (State != PtcState.Enabled) { state.Stop(); } }); _loggerEvent.Set(); PtcJumpTable.Initialize(jumpTable); PtcJumpTable.ReadJumpTable(jumpTable); PtcJumpTable.ReadDynamicTable(jumpTable); ThreadPool.QueueUserWorkItem(PreSave); }
static Ptc() { InitializeCarriers(); _headerMagic = BinaryPrimitives.ReadUInt64LittleEndian(EncodingCache.UTF8NoBOM.GetBytes(HeaderMagicString).AsSpan()); _waitEvent = new ManualResetEvent(true); _lock = new object(); _disposed = false; PtcJumpTable = new PtcJumpTable(); TitleIdText = TitleIdTextDefault; DisplayVersion = DisplayVersionDefault; CachePathActual = string.Empty; CachePathBackup = string.Empty; Disable(); }
private static unsafe void Save(string fileName) { int translatedFuncsCount; InnerHeader innerHeader = new InnerHeader(); innerHeader.Magic = _innerHeaderMagic; innerHeader.InfosLength = (int)_infosStream.Length; innerHeader.CodesLength = _codesList.Length(); innerHeader.RelocsLength = (int)_relocsStream.Length; innerHeader.UnwindInfosLength = (int)_unwindInfosStream.Length; innerHeader.PtcJumpTableLength = PtcJumpTable.GetSerializeSize(PtcJumpTable); OuterHeader outerHeader = new OuterHeader(); outerHeader.Magic = _outerHeaderMagic; outerHeader.CacheFileVersion = InternalVersion; outerHeader.Endianness = GetEndianness(); outerHeader.FeatureInfo = GetFeatureInfo(); outerHeader.MemoryManagerMode = GetMemoryManagerMode(); outerHeader.OSPlatform = GetOSPlatform(); outerHeader.UncompressedStreamSize = (long)Unsafe.SizeOf <InnerHeader>() + innerHeader.InfosLength + innerHeader.CodesLength + innerHeader.RelocsLength + innerHeader.UnwindInfosLength + innerHeader.PtcJumpTableLength; outerHeader.SetHeaderHash(); IntPtr intPtr = IntPtr.Zero; try { intPtr = Marshal.AllocHGlobal(new IntPtr(outerHeader.UncompressedStreamSize)); using (UnmanagedMemoryStream stream = new((byte *)intPtr.ToPointer(), outerHeader.UncompressedStreamSize, outerHeader.UncompressedStreamSize, FileAccess.ReadWrite)) { stream.Seek((long)Unsafe.SizeOf <InnerHeader>(), SeekOrigin.Begin); ReadOnlySpan <byte> infosBytes = new(stream.PositionPointer, innerHeader.InfosLength); _infosStream.WriteTo(stream); ReadOnlySpan <byte> codesBytes = (int)innerHeader.CodesLength > 0 ? new(stream.PositionPointer, (int)innerHeader.CodesLength) : ReadOnlySpan <byte> .Empty; _codesList.WriteTo(stream); ReadOnlySpan <byte> relocsBytes = new(stream.PositionPointer, innerHeader.RelocsLength); _relocsStream.WriteTo(stream); ReadOnlySpan <byte> unwindInfosBytes = new(stream.PositionPointer, innerHeader.UnwindInfosLength); _unwindInfosStream.WriteTo(stream); ReadOnlySpan <byte> ptcJumpTableBytes = new(stream.PositionPointer, innerHeader.PtcJumpTableLength); PtcJumpTable.Serialize(stream, PtcJumpTable); Debug.Assert(stream.Position == stream.Length); innerHeader.InfosHash = XXHash128.ComputeHash(infosBytes); innerHeader.CodesHash = XXHash128.ComputeHash(codesBytes); innerHeader.RelocsHash = XXHash128.ComputeHash(relocsBytes); innerHeader.UnwindInfosHash = XXHash128.ComputeHash(unwindInfosBytes); innerHeader.PtcJumpTableHash = XXHash128.ComputeHash(ptcJumpTableBytes); innerHeader.SetHeaderHash(); stream.Seek(0L, SeekOrigin.Begin); SerializeStructure(stream, innerHeader); translatedFuncsCount = GetEntriesCount(); ResetCarriersIfNeeded(); PtcJumpTable.ClearIfNeeded(); using (FileStream compressedStream = new(fileName, FileMode.OpenOrCreate)) using (DeflateStream deflateStream = new(compressedStream, SaveCompressionLevel, true)) { try { SerializeStructure(compressedStream, outerHeader); stream.Seek(0L, SeekOrigin.Begin); stream.CopyTo(deflateStream); } catch { compressedStream.Position = 0L; } if (compressedStream.Position < compressedStream.Length) { compressedStream.SetLength(compressedStream.Position); } } } } finally { if (intPtr != IntPtr.Zero) { Marshal.FreeHGlobal(intPtr); } } long fileSize = new FileInfo(fileName).Length; if (fileSize != 0L) { Logger.Info?.Print(LogClass.Ptc, $"Saved Translation Cache (size: {fileSize} bytes, translated functions: {translatedFuncsCount})."); } }
private static unsafe bool Load(string fileName, bool isBackup) { using (FileStream compressedStream = new(fileName, FileMode.Open)) using (DeflateStream deflateStream = new(compressedStream, CompressionMode.Decompress, true)) { OuterHeader outerHeader = DeserializeStructure <OuterHeader>(compressedStream); if (!outerHeader.IsHeaderValid()) { InvalidateCompressedStream(compressedStream); return(false); } if (outerHeader.Magic != _outerHeaderMagic) { InvalidateCompressedStream(compressedStream); return(false); } if (outerHeader.CacheFileVersion != InternalVersion) { InvalidateCompressedStream(compressedStream); return(false); } if (outerHeader.Endianness != GetEndianness()) { InvalidateCompressedStream(compressedStream); return(false); } if (outerHeader.FeatureInfo != GetFeatureInfo()) { InvalidateCompressedStream(compressedStream); return(false); } if (outerHeader.MemoryManagerMode != GetMemoryManagerMode()) { InvalidateCompressedStream(compressedStream); return(false); } if (outerHeader.OSPlatform != GetOSPlatform()) { InvalidateCompressedStream(compressedStream); return(false); } IntPtr intPtr = IntPtr.Zero; try { intPtr = Marshal.AllocHGlobal(new IntPtr(outerHeader.UncompressedStreamSize)); using (UnmanagedMemoryStream stream = new((byte *)intPtr.ToPointer(), outerHeader.UncompressedStreamSize, outerHeader.UncompressedStreamSize, FileAccess.ReadWrite)) { try { deflateStream.CopyTo(stream); } catch { InvalidateCompressedStream(compressedStream); return(false); } Debug.Assert(stream.Position == stream.Length); stream.Seek(0L, SeekOrigin.Begin); InnerHeader innerHeader = DeserializeStructure <InnerHeader>(stream); if (!innerHeader.IsHeaderValid()) { InvalidateCompressedStream(compressedStream); return(false); } if (innerHeader.Magic != _innerHeaderMagic) { InvalidateCompressedStream(compressedStream); return(false); } ReadOnlySpan <byte> infosBytes = new(stream.PositionPointer, innerHeader.InfosLength); stream.Seek(innerHeader.InfosLength, SeekOrigin.Current); Hash128 infosHash = XXHash128.ComputeHash(infosBytes); if (innerHeader.InfosHash != infosHash) { InvalidateCompressedStream(compressedStream); return(false); } ReadOnlySpan <byte> codesBytes = (int)innerHeader.CodesLength > 0 ? new(stream.PositionPointer, (int)innerHeader.CodesLength) : ReadOnlySpan <byte> .Empty; stream.Seek(innerHeader.CodesLength, SeekOrigin.Current); Hash128 codesHash = XXHash128.ComputeHash(codesBytes); if (innerHeader.CodesHash != codesHash) { InvalidateCompressedStream(compressedStream); return(false); } ReadOnlySpan <byte> relocsBytes = new(stream.PositionPointer, innerHeader.RelocsLength); stream.Seek(innerHeader.RelocsLength, SeekOrigin.Current); Hash128 relocsHash = XXHash128.ComputeHash(relocsBytes); if (innerHeader.RelocsHash != relocsHash) { InvalidateCompressedStream(compressedStream); return(false); } ReadOnlySpan <byte> unwindInfosBytes = new(stream.PositionPointer, innerHeader.UnwindInfosLength); stream.Seek(innerHeader.UnwindInfosLength, SeekOrigin.Current); Hash128 unwindInfosHash = XXHash128.ComputeHash(unwindInfosBytes); if (innerHeader.UnwindInfosHash != unwindInfosHash) { InvalidateCompressedStream(compressedStream); return(false); } ReadOnlySpan <byte> ptcJumpTableBytes = new(stream.PositionPointer, innerHeader.PtcJumpTableLength); stream.Seek(innerHeader.PtcJumpTableLength, SeekOrigin.Current); Debug.Assert(stream.Position == stream.Length); Hash128 ptcJumpTableHash = XXHash128.ComputeHash(ptcJumpTableBytes); if (innerHeader.PtcJumpTableHash != ptcJumpTableHash) { InvalidateCompressedStream(compressedStream); return(false); } stream.Seek((long)Unsafe.SizeOf <InnerHeader>(), SeekOrigin.Begin); _infosStream.Write(infosBytes); stream.Seek(innerHeader.InfosLength, SeekOrigin.Current); _codesList.ReadFrom(stream); _relocsStream.Write(relocsBytes); stream.Seek(innerHeader.RelocsLength, SeekOrigin.Current); _unwindInfosStream.Write(unwindInfosBytes); stream.Seek(innerHeader.UnwindInfosLength, SeekOrigin.Current); PtcJumpTable = PtcJumpTable.Deserialize(stream); Debug.Assert(stream.Position == stream.Length); } } finally { if (intPtr != IntPtr.Zero) { Marshal.FreeHGlobal(intPtr); } } } long fileSize = new FileInfo(fileName).Length; Logger.Info?.Print(LogClass.Ptc, $"{(isBackup ? "Loaded Backup Translation Cache" : "Loaded Translation Cache")} (size: {fileSize} bytes, translated functions: {GetEntriesCount()})."); return(true); }
internal static void MakeAndSaveTranslations(ConcurrentDictionary <ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable) { var profiledFuncsToTranslate = PtcProfiler.GetProfiledFuncsToTranslate(funcs); if (profiledFuncsToTranslate.Count == 0) { return; } _translateCount = 0; ThreadPool.QueueUserWorkItem(TranslationLogger, profiledFuncsToTranslate.Count); void TranslateFuncs() { while (profiledFuncsToTranslate.TryDequeue(out var item)) { ulong address = item.address; Debug.Assert(PtcProfiler.IsAddressInStaticCodeRange(address)); TranslatedFunction func = Translator.Translate(memory, jumpTable, address, item.mode, item.highCq); bool isAddressUnique = funcs.TryAdd(address, func); Debug.Assert(isAddressUnique, $"The address 0x{address:X16} is not unique."); if (func.HighCq) { jumpTable.RegisterFunction(address, func); } Interlocked.Increment(ref _translateCount); if (State != PtcState.Enabled) { break; } } } int maxDegreeOfParallelism = (Environment.ProcessorCount * 3) / 4; List <Thread> threads = new List <Thread>(); for (int i = 0; i < maxDegreeOfParallelism; i++) { Thread thread = new Thread(TranslateFuncs); thread.IsBackground = true; threads.Add(thread); } threads.ForEach((thread) => thread.Start()); threads.ForEach((thread) => thread.Join()); threads.Clear(); Translator.ResetPools(); _loggerEvent.Set(); PtcJumpTable.Initialize(jumpTable); PtcJumpTable.ReadJumpTable(jumpTable); PtcJumpTable.ReadDynamicTable(jumpTable); ThreadPool.QueueUserWorkItem(PreSave); }
internal static void LoadTranslations(ConcurrentDictionary <ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable) { if ((int)_infosStream.Length == 0 || (int)_codesStream.Length == 0 || (int)_relocsStream.Length == 0 || (int)_unwindInfosStream.Length == 0) { return; } Debug.Assert(funcs.Count == 0); _infosStream.Seek(0L, SeekOrigin.Begin); _codesStream.Seek(0L, SeekOrigin.Begin); _relocsStream.Seek(0L, SeekOrigin.Begin); _unwindInfosStream.Seek(0L, SeekOrigin.Begin); using (BinaryReader infosReader = new BinaryReader(_infosStream, EncodingCache.UTF8NoBOM, true)) using (BinaryReader codesReader = new BinaryReader(_codesStream, EncodingCache.UTF8NoBOM, true)) using (BinaryReader relocsReader = new BinaryReader(_relocsStream, EncodingCache.UTF8NoBOM, true)) using (BinaryReader unwindInfosReader = new BinaryReader(_unwindInfosStream, EncodingCache.UTF8NoBOM, true)) { for (int i = 0; i < GetInfosEntriesCount(); i++) { InfoEntry infoEntry = ReadInfo(infosReader); if (infoEntry.Stubbed) { SkipCode(infoEntry.CodeLen); SkipReloc(infoEntry.RelocEntriesCount); SkipUnwindInfo(unwindInfosReader); } else if (infoEntry.HighCq || !PtcProfiler.ProfiledFuncs.TryGetValue(infoEntry.Address, out var value) || !value.highCq) { byte[] code = ReadCode(codesReader, infoEntry.CodeLen); if (infoEntry.RelocEntriesCount != 0) { RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount); PatchCode(code, relocEntries, memory.PageTablePointer, jumpTable); } UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader); TranslatedFunction func = FastTranslate(code, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq); bool isAddressUnique = funcs.TryAdd(infoEntry.Address, func); Debug.Assert(isAddressUnique, $"The address 0x{infoEntry.Address:X16} is not unique."); } else { infoEntry.Stubbed = true; UpdateInfo(infoEntry); StubCode(infoEntry.CodeLen); StubReloc(infoEntry.RelocEntriesCount); StubUnwindInfo(unwindInfosReader); } } } if (_infosStream.Position < _infosStream.Length || _codesStream.Position < _codesStream.Length || _relocsStream.Position < _relocsStream.Length || _unwindInfosStream.Position < _unwindInfosStream.Length) { throw new Exception("Could not reach the end of one or more memory streams."); } jumpTable.Initialize(PtcJumpTable, funcs); PtcJumpTable.WriteJumpTable(jumpTable, funcs); PtcJumpTable.WriteDynamicTable(jumpTable); Logger.Info?.Print(LogClass.Ptc, $"{funcs.Count} translated functions loaded"); }
private static bool Load(string fileName, bool isBackup) { using (FileStream compressedStream = new FileStream(fileName, FileMode.Open)) using (DeflateStream deflateStream = new DeflateStream(compressedStream, CompressionMode.Decompress, true)) using (MemoryStream stream = new MemoryStream()) using (MD5 md5 = MD5.Create()) { int hashSize = md5.HashSize / 8; try { deflateStream.CopyTo(stream); } catch { InvalidateCompressedStream(compressedStream); return(false); } stream.Seek(0L, SeekOrigin.Begin); byte[] currentHash = new byte[hashSize]; stream.Read(currentHash, 0, hashSize); byte[] expectedHash = md5.ComputeHash(stream); if (!CompareHash(currentHash, expectedHash)) { InvalidateCompressedStream(compressedStream); return(false); } stream.Seek((long)hashSize, SeekOrigin.Begin); Header header = ReadHeader(stream); if (header.Magic != HeaderMagic) { InvalidateCompressedStream(compressedStream); return(false); } if (header.CacheFileVersion != InternalVersion) { InvalidateCompressedStream(compressedStream); return(false); } if (header.FeatureInfo != GetFeatureInfo()) { InvalidateCompressedStream(compressedStream); return(false); } if (header.OSPlatform != GetOSPlatform()) { InvalidateCompressedStream(compressedStream); return(false); } if (header.InfosLen % InfoEntry.Stride != 0) { InvalidateCompressedStream(compressedStream); return(false); } byte[] infosBuf = new byte[header.InfosLen]; byte[] codesBuf = new byte[header.CodesLen]; byte[] relocsBuf = new byte[header.RelocsLen]; byte[] unwindInfosBuf = new byte[header.UnwindInfosLen]; stream.Read(infosBuf, 0, header.InfosLen); stream.Read(codesBuf, 0, header.CodesLen); stream.Read(relocsBuf, 0, header.RelocsLen); stream.Read(unwindInfosBuf, 0, header.UnwindInfosLen); try { PtcJumpTable = PtcJumpTable.Deserialize(stream); } catch { PtcJumpTable = new PtcJumpTable(); InvalidateCompressedStream(compressedStream); return(false); } _infosStream.Write(infosBuf, 0, header.InfosLen); _codesStream.Write(codesBuf, 0, header.CodesLen); _relocsStream.Write(relocsBuf, 0, header.RelocsLen); _unwindInfosStream.Write(unwindInfosBuf, 0, header.UnwindInfosLen); } long fileSize = new FileInfo(fileName).Length; Logger.Info?.Print(LogClass.Ptc, $"{(isBackup ? "Loaded Backup Translation Cache" : "Loaded Translation Cache")} (size: {fileSize} bytes, translated functions: {GetInfosEntriesCount()})."); return(true); }
private static unsafe bool Load(string fileName, bool isBackup) { using (FileStream compressedStream = new(fileName, FileMode.Open)) using (DeflateStream deflateStream = new(compressedStream, CompressionMode.Decompress, true)) { Hash128 currentSizeHash = DeserializeStructure <Hash128>(compressedStream); Span <byte> sizeBytes = new byte[sizeof(int)]; compressedStream.Read(sizeBytes); Hash128 expectedSizeHash = XXHash128.ComputeHash(sizeBytes); if (currentSizeHash != expectedSizeHash) { InvalidateCompressedStream(compressedStream); return(false); } int size = BinaryPrimitives.ReadInt32LittleEndian(sizeBytes); IntPtr intPtr = IntPtr.Zero; try { intPtr = Marshal.AllocHGlobal(size); using (UnmanagedMemoryStream stream = new((byte *)intPtr.ToPointer(), size, size, FileAccess.ReadWrite)) { try { deflateStream.CopyTo(stream); } catch { InvalidateCompressedStream(compressedStream); return(false); } int hashSize = Unsafe.SizeOf <Hash128>(); stream.Seek(0L, SeekOrigin.Begin); Hash128 currentHash = DeserializeStructure <Hash128>(stream); ReadOnlySpan <byte> streamBytes = new(stream.PositionPointer, (int)(stream.Length - stream.Position)); Hash128 expectedHash = XXHash128.ComputeHash(streamBytes); if (currentHash != expectedHash) { InvalidateCompressedStream(compressedStream); return(false); } stream.Seek((long)hashSize, SeekOrigin.Begin); Header header = ReadHeader(stream); if (header.Magic != _headerMagic) { InvalidateCompressedStream(compressedStream); return(false); } if (header.CacheFileVersion != InternalVersion) { InvalidateCompressedStream(compressedStream); return(false); } if (header.Endianness != GetEndianness()) { InvalidateCompressedStream(compressedStream); return(false); } if (header.FeatureInfo != GetFeatureInfo()) { InvalidateCompressedStream(compressedStream); return(false); } if (header.OSPlatform != GetOSPlatform()) { InvalidateCompressedStream(compressedStream); return(false); } if (header.InfosLen % InfoEntry.Stride != 0) { InvalidateCompressedStream(compressedStream); return(false); } ReadOnlySpan <byte> infosBuf = new(stream.PositionPointer, header.InfosLen); stream.Seek(header.InfosLen, SeekOrigin.Current); ReadOnlySpan <byte> codesBuf = new(stream.PositionPointer, header.CodesLen); stream.Seek(header.CodesLen, SeekOrigin.Current); ReadOnlySpan <byte> relocsBuf = new(stream.PositionPointer, header.RelocsLen); stream.Seek(header.RelocsLen, SeekOrigin.Current); ReadOnlySpan <byte> unwindInfosBuf = new(stream.PositionPointer, header.UnwindInfosLen); stream.Seek(header.UnwindInfosLen, SeekOrigin.Current); try { PtcJumpTable = PtcJumpTable.Deserialize(stream); } catch { PtcJumpTable = new PtcJumpTable(); InvalidateCompressedStream(compressedStream); return(false); } _infosStream.Write(infosBuf); _codesStream.Write(codesBuf); _relocsStream.Write(relocsBuf); _unwindInfosStream.Write(unwindInfosBuf); } } finally { if (intPtr != IntPtr.Zero) { Marshal.FreeHGlobal(intPtr); } } } long fileSize = new FileInfo(fileName).Length; Logger.Info?.Print(LogClass.Ptc, $"{(isBackup ? "Loaded Backup Translation Cache" : "Loaded Translation Cache")} (size: {fileSize} bytes, translated functions: {GetInfosEntriesCount()})."); return(true); }
internal static void LoadTranslations( ConcurrentDictionary <ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable, EntryTable <uint> countTable) { if (AreCarriersEmpty()) { return; } long infosStreamLength = _infosStream.Length; long relocsStreamLength = _relocsStream.Length; long unwindInfosStreamLength = _unwindInfosStream.Length; _infosStream.Seek(0L, SeekOrigin.Begin); _relocsStream.Seek(0L, SeekOrigin.Begin); _unwindInfosStream.Seek(0L, SeekOrigin.Begin); using (BinaryReader relocsReader = new(_relocsStream, EncodingCache.UTF8NoBOM, true)) using (BinaryReader unwindInfosReader = new(_unwindInfosStream, EncodingCache.UTF8NoBOM, true)) { for (int index = 0; index < GetEntriesCount(); index++) { InfoEntry infoEntry = DeserializeStructure <InfoEntry>(_infosStream); if (infoEntry.Stubbed) { SkipCode(index, infoEntry.CodeLength); SkipReloc(infoEntry.RelocEntriesCount); SkipUnwindInfo(unwindInfosReader); continue; } bool isEntryChanged = infoEntry.Hash != ComputeHash(memory, infoEntry.Address, infoEntry.GuestSize); if (isEntryChanged || (!infoEntry.HighCq && PtcProfiler.ProfiledFuncs.TryGetValue(infoEntry.Address, out var value) && value.HighCq)) { infoEntry.Stubbed = true; infoEntry.CodeLength = 0; UpdateInfo(infoEntry); StubCode(index); StubReloc(infoEntry.RelocEntriesCount); StubUnwindInfo(unwindInfosReader); if (isEntryChanged) { PtcJumpTable.Clean(infoEntry.Address); Logger.Info?.Print(LogClass.Ptc, $"Invalidated translated function (address: 0x{infoEntry.Address:X16})"); } continue; } byte[] code = ReadCode(index, infoEntry.CodeLength); Counter <uint> callCounter = null; if (infoEntry.RelocEntriesCount != 0) { RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount); PatchCode(code, relocEntries, memory.PageTablePointer, jumpTable, countTable, out callCounter); } UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader); TranslatedFunction func = FastTranslate(code, callCounter, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq); bool isAddressUnique = funcs.TryAdd(infoEntry.Address, func); Debug.Assert(isAddressUnique, $"The address 0x{infoEntry.Address:X16} is not unique."); } } if (_infosStream.Length != infosStreamLength || _infosStream.Position != infosStreamLength || _relocsStream.Length != relocsStreamLength || _relocsStream.Position != relocsStreamLength || _unwindInfosStream.Length != unwindInfosStreamLength || _unwindInfosStream.Position != unwindInfosStreamLength) { throw new Exception("The length of a memory stream has changed, or its position has not reached or has exceeded its end."); } jumpTable.Initialize(PtcJumpTable, funcs); PtcJumpTable.WriteJumpTable(jumpTable, funcs); PtcJumpTable.WriteDynamicTable(jumpTable); Logger.Info?.Print(LogClass.Ptc, $"{funcs.Count} translated functions loaded"); }
private static bool Load(string fileName) { using (FileStream compressedStream = new FileStream(fileName, FileMode.Open)) using (DeflateStream deflateStream = new DeflateStream(compressedStream, CompressionMode.Decompress, true)) using (MemoryStream stream = new MemoryStream()) using (System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create()) { int hashSize = md5.HashSize / 8; deflateStream.CopyTo(stream); stream.Seek(0L, SeekOrigin.Begin); byte[] currentHash = new byte[hashSize]; stream.Read(currentHash, 0, hashSize); byte[] expectedHash = md5.ComputeHash(stream); if (!CompareHash(currentHash, expectedHash)) { InvalidateCompressedStream(compressedStream); return(false); } stream.Seek((long)hashSize, SeekOrigin.Begin); Header header = ReadHeader(stream); if (header.Magic != HeaderMagic) { InvalidateCompressedStream(compressedStream); return(false); } if (header.CacheFileVersion != InternalVersion) { InvalidateCompressedStream(compressedStream); return(false); } if (header.FeatureInfo != GetFeatureInfo()) { InvalidateCompressedStream(compressedStream); return(false); } if (header.InfosLen % InfoEntry.Stride != 0) { InvalidateCompressedStream(compressedStream); return(false); } byte[] infosBuf = new byte[header.InfosLen]; byte[] codesBuf = new byte[header.CodesLen]; byte[] relocsBuf = new byte[header.RelocsLen]; byte[] unwindInfosBuf = new byte[header.UnwindInfosLen]; stream.Read(infosBuf, 0, header.InfosLen); stream.Read(codesBuf, 0, header.CodesLen); stream.Read(relocsBuf, 0, header.RelocsLen); stream.Read(unwindInfosBuf, 0, header.UnwindInfosLen); try { PtcJumpTable = (PtcJumpTable)_binaryFormatter.Deserialize(stream); } catch { PtcJumpTable = new PtcJumpTable(); InvalidateCompressedStream(compressedStream); return(false); } _infosStream.Write(infosBuf, 0, header.InfosLen); _codesStream.Write(codesBuf, 0, header.CodesLen); _relocsStream.Write(relocsBuf, 0, header.RelocsLen); _unwindInfosStream.Write(unwindInfosBuf, 0, header.UnwindInfosLen); return(true); } }
private static unsafe void Save(string fileName) { int translatedFuncsCount; int hashSize = Unsafe.SizeOf <Hash128>(); int size = hashSize + Header.Size + GetMemoryStreamsLength() + PtcJumpTable.GetSerializeSize(PtcJumpTable); Span <byte> sizeBytes = new byte[sizeof(int)]; BinaryPrimitives.WriteInt32LittleEndian(sizeBytes, size); Hash128 sizeHash = XXHash128.ComputeHash(sizeBytes); Span <byte> sizeHashBytes = new byte[hashSize]; MemoryMarshal.Write <Hash128>(sizeHashBytes, ref sizeHash); IntPtr intPtr = IntPtr.Zero; try { intPtr = Marshal.AllocHGlobal(size); using (UnmanagedMemoryStream stream = new((byte *)intPtr.ToPointer(), size, size, FileAccess.ReadWrite)) { stream.Seek((long)hashSize, SeekOrigin.Begin); WriteHeader(stream); _infosStream.WriteTo(stream); _codesStream.WriteTo(stream); _relocsStream.WriteTo(stream); _unwindInfosStream.WriteTo(stream); PtcJumpTable.Serialize(stream, PtcJumpTable); stream.Seek((long)hashSize, SeekOrigin.Begin); ReadOnlySpan <byte> streamBytes = new(stream.PositionPointer, (int)(stream.Length - stream.Position)); Hash128 hash = XXHash128.ComputeHash(streamBytes); stream.Seek(0L, SeekOrigin.Begin); SerializeStructure(stream, hash); translatedFuncsCount = GetInfosEntriesCount(); ResetMemoryStreamsIfNeeded(); PtcJumpTable.ClearIfNeeded(); using (FileStream compressedStream = new(fileName, FileMode.OpenOrCreate)) using (DeflateStream deflateStream = new(compressedStream, SaveCompressionLevel, true)) { try { compressedStream.Write(sizeHashBytes); compressedStream.Write(sizeBytes); stream.Seek(0L, SeekOrigin.Begin); stream.CopyTo(deflateStream); } catch { compressedStream.Position = 0L; } if (compressedStream.Position < compressedStream.Length) { compressedStream.SetLength(compressedStream.Position); } } } } finally { if (intPtr != IntPtr.Zero) { Marshal.FreeHGlobal(intPtr); } } long fileSize = new FileInfo(fileName).Length; if (fileSize != 0L) { Logger.Info?.Print(LogClass.Ptc, $"Saved Translation Cache (size: {fileSize} bytes, translated functions: {translatedFuncsCount})."); } }
private static unsafe void Save(string fileName) { int translatedFuncsCount; int headerSize = Unsafe.SizeOf <Header>(); Header header = new Header() { Magic = _headerMagic, CacheFileVersion = InternalVersion, Endianness = GetEndianness(), FeatureInfo = GetFeatureInfo(), OSPlatform = GetOSPlatform(), InfosLength = (int)_infosStream.Length, CodesLength = _codesList.Length(), RelocsLength = (int)_relocsStream.Length, UnwindInfosLength = (int)_unwindInfosStream.Length, PtcJumpTableLength = PtcJumpTable.GetSerializeSize(PtcJumpTable) }; long size = (long)headerSize + header.InfosLength + header.CodesLength + header.RelocsLength + header.UnwindInfosLength + header.PtcJumpTableLength; Span <byte> sizeBytes = new byte[sizeof(long)]; BinaryPrimitives.WriteInt64LittleEndian(sizeBytes, size); Hash128 sizeHash = XXHash128.ComputeHash(sizeBytes); Span <byte> sizeHashBytes = new byte[Unsafe.SizeOf <Hash128>()]; MemoryMarshal.Write <Hash128>(sizeHashBytes, ref sizeHash); IntPtr intPtr = IntPtr.Zero; try { intPtr = Marshal.AllocHGlobal(new IntPtr(size)); using (UnmanagedMemoryStream stream = new((byte *)intPtr.ToPointer(), size, size, FileAccess.ReadWrite)) { stream.Seek((long)headerSize, SeekOrigin.Begin); ReadOnlySpan <byte> infosBytes = new(stream.PositionPointer, header.InfosLength); _infosStream.WriteTo(stream); ReadOnlySpan <byte> codesBytes = (int)header.CodesLength > 0 ? new(stream.PositionPointer, (int)header.CodesLength) : ReadOnlySpan <byte> .Empty; _codesList.WriteTo(stream); ReadOnlySpan <byte> relocsBytes = new(stream.PositionPointer, header.RelocsLength); _relocsStream.WriteTo(stream); ReadOnlySpan <byte> unwindInfosBytes = new(stream.PositionPointer, header.UnwindInfosLength); _unwindInfosStream.WriteTo(stream); ReadOnlySpan <byte> ptcJumpTableBytes = new(stream.PositionPointer, header.PtcJumpTableLength); PtcJumpTable.Serialize(stream, PtcJumpTable); header.InfosHash = XXHash128.ComputeHash(infosBytes); header.CodesHash = XXHash128.ComputeHash(codesBytes); header.RelocsHash = XXHash128.ComputeHash(relocsBytes); header.UnwindInfosHash = XXHash128.ComputeHash(unwindInfosBytes); header.PtcJumpTableHash = XXHash128.ComputeHash(ptcJumpTableBytes); Debug.Assert(stream.Position == stream.Length); stream.Seek(0L, SeekOrigin.Begin); SerializeStructure(stream, header); translatedFuncsCount = GetEntriesCount(); ResetCarriersIfNeeded(); PtcJumpTable.ClearIfNeeded(); using (FileStream compressedStream = new(fileName, FileMode.OpenOrCreate)) using (DeflateStream deflateStream = new(compressedStream, SaveCompressionLevel, true)) { try { compressedStream.Write(sizeHashBytes); compressedStream.Write(sizeBytes); stream.Seek(0L, SeekOrigin.Begin); stream.CopyTo(deflateStream); } catch { compressedStream.Position = 0L; } if (compressedStream.Position < compressedStream.Length) { compressedStream.SetLength(compressedStream.Position); } } } } finally { if (intPtr != IntPtr.Zero) { Marshal.FreeHGlobal(intPtr); } } long fileSize = new FileInfo(fileName).Length; if (fileSize != 0L) { Logger.Info?.Print(LogClass.Ptc, $"Saved Translation Cache (size: {fileSize} bytes, translated functions: {translatedFuncsCount})."); } }
internal static void MakeAndSaveTranslations( ConcurrentDictionary <ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable, EntryTable <uint> countTable) { var profiledFuncsToTranslate = PtcProfiler.GetProfiledFuncsToTranslate(funcs); _translateCount = 0; _translateTotalCount = profiledFuncsToTranslate.Count; int degreeOfParallelism = new DegreeOfParallelism(4d, 75d, 12.5d).GetDegreeOfParallelism(0, 32); if (_translateTotalCount == 0 || degreeOfParallelism == 0) { ResetCarriersIfNeeded(); PtcJumpTable.ClearIfNeeded(); GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; return; } Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism}"); PtcStateChanged?.Invoke(PtcLoadingState.Start, _translateCount, _translateTotalCount); using AutoResetEvent progressReportEvent = new AutoResetEvent(false); Thread progressReportThread = new Thread(ReportProgress) { Name = "Ptc.ProgressReporter", Priority = ThreadPriority.Lowest, IsBackground = true }; progressReportThread.Start(progressReportEvent); void TranslateFuncs() { while (profiledFuncsToTranslate.TryDequeue(out var item)) { ulong address = item.address; Debug.Assert(PtcProfiler.IsAddressInStaticCodeRange(address)); TranslatedFunction func = Translator.Translate(memory, jumpTable, countTable, address, item.funcProfile.Mode, item.funcProfile.HighCq); bool isAddressUnique = funcs.TryAdd(address, func); Debug.Assert(isAddressUnique, $"The address 0x{address:X16} is not unique."); if (func.HighCq) { jumpTable.RegisterFunction(address, func); } Interlocked.Increment(ref _translateCount); if (State != PtcState.Enabled) { break; } } Translator.DisposePools(); } List <Thread> threads = new List <Thread>(); for (int i = 0; i < degreeOfParallelism; i++) { Thread thread = new Thread(TranslateFuncs); thread.IsBackground = true; threads.Add(thread); } threads.ForEach((thread) => thread.Start()); threads.ForEach((thread) => thread.Join()); threads.Clear(); progressReportEvent.Set(); progressReportThread.Join(); PtcStateChanged?.Invoke(PtcLoadingState.Loaded, _translateCount, _translateTotalCount); Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism}"); PtcJumpTable.Initialize(jumpTable); PtcJumpTable.ReadJumpTable(jumpTable); PtcJumpTable.ReadDynamicTable(jumpTable); Thread preSaveThread = new Thread(PreSave); preSaveThread.IsBackground = true; preSaveThread.Start(); }
internal static void MakeAndSaveTranslations(ConcurrentDictionary <ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable) { if (PtcProfiler.ProfiledFuncs.Count == 0) { return; } _translateCount = 0; _rejitCount = 0; ThreadPool.QueueUserWorkItem(TranslationLogger, (funcs.Count, PtcProfiler.ProfiledFuncs.Count)); int maxDegreeOfParallelism = (Environment.ProcessorCount * 3) / 4; Parallel.ForEach(PtcProfiler.ProfiledFuncs, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, (item, state) => { ulong address = item.Key; Debug.Assert(PtcProfiler.IsAddressInStaticCodeRange(address)); if (!funcs.ContainsKey(address)) { TranslatedFunction func = Translator.Translate(memory, jumpTable, address, item.Value.mode, item.Value.highCq); funcs.TryAdd(address, func); if (func.HighCq) { jumpTable.RegisterFunction(address, func); } Interlocked.Increment(ref _translateCount); } else if (item.Value.highCq && !funcs[address].HighCq) { TranslatedFunction func = Translator.Translate(memory, jumpTable, address, item.Value.mode, highCq: true); funcs[address] = func; jumpTable.RegisterFunction(address, func); Interlocked.Increment(ref _rejitCount); } if (State != PtcState.Enabled) { state.Stop(); } }); _loggerEvent.Set(); if (_translateCount != 0 || _rejitCount != 0) { PtcJumpTable.Initialize(jumpTable); PtcJumpTable.ReadJumpTable(jumpTable); PtcJumpTable.ReadDynamicTable(jumpTable); ThreadPool.QueueUserWorkItem(PreSave); } }