/// <summary> /// HashDB manager /// MUST BE A POWER OF 2 /// </summary> /// <param name="minBlockSize">POWER of 2</param> /// <param name="DB">Primary DB</param> /// <param name="relocFolder">Relocation data</param> /// <param name="Size">POWER OF 2!</param> public HashDB(int minBlockSize, string DB, string relocFolder, long Size = 0) { HashDBFile = DB; HashDBBitMap = DB + ".bin"; if (Size != 0) { DBSize = (long)FractHashTree.RoundUpPow2(Size); } if (!File.Exists(HashDBFile)) { if (!FractHashTree.IsPow2(DBSize)) { throw new InternalBufferOverflowException($"DB SIZE not a power of 2!"); } using (var fileStream = new FileStream(HashDBFile, FileMode.Create, FileAccess.Write, FileShare.None)) fileStream.SetLength(DBSize + (DB_READ_SIZE)); } else { DBSize = (long)FractHashTree.RoundDownPow2(new FileInfo(HashDBFile).Length); } // Divide by HASH size DBEntries = (ulong)DBSize >> HASH_SHIFT; DBEntriesMask = (ulong)DBEntries - 1; MinBlockSize = minBlockSize; ReRe = new ReReDB(relocFolder); HDBBitMap = new UnsafeHelp(HashDBBitMap, (long)DBEntries); }
static IEnumerable <long> MapScanFile(String File, long From, int ScanData, int Count) { List <long> rv = new List <long>(); // TODO: These streams should be persistent across these calls right? // TODO: This path is only 1 time and pretty infrequent so far though using (var fs = new FileStream(File, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { var mapName = Path.GetFileNameWithoutExtension(File) + From.ToString("X16"); using (var mmap = MemoryMappedFile.CreateFromFile(fs, mapName, 0, MemoryMappedFileAccess.Read, null, HandleInheritability.Inheritable, false)) { using (var reader = mmap.CreateViewAccessor(From, Count * 4, MemoryMappedFileAccess.Read)) { var LocatedScanTarget = UnsafeHelp.ScanBytes(reader, ScanData, Count); if (LocatedScanTarget.Count() > 0) { foreach (var ioff in LocatedScanTarget) { var target = From + ioff; //WriteColor($"Found input @ {(target):X}"); rv.Add(target); yield return(target); } } } } } yield break; }
public UnsafeHelp Clone() { var rv = new UnsafeHelp(); rv.BitmapBackingFile = BitmapBackingFile; rv.SetupHandles(); return(rv); }
public Mem(Mem parent) : this() { MD = parent.MD; ID = parent.ID; DumpedPFNBitmap = new UnsafeHelp(ID.ToString(), MaxLimit, true); StartOfMemory = parent.StartOfMemory; MemoryDump = parent.MemoryDump; FileSize = parent.FileSize; MapViewSize = parent.MapViewSize; SetupStreams(); }
public Mem(Mem parent) : this() { MD = parent.MD; ID = parent.ID; DumpedPFNBitmap = new UnsafeHelp($"{Process.GetCurrentProcess().Id}-{ID}", MaxLimit, true); StartOfMemory = parent.StartOfMemory; MemoryDump = parent.MemoryDump; FileSize = parent.FileSize; MapViewSize = parent.MapViewSize; SetupStreams(); }
public Mem(Mem parent) : this() { MD = parent.MD; ID = parent.ID; DumpedPFNBitmap = parent.DumpedPFNBitmap.Clone(); StartOfMemory = parent.StartOfMemory; MemoryDump = parent.MemoryDump; FileSize = parent.FileSize; MapViewSize = parent.MapViewSize; SetupStreams(); }
private bool disposedValue = false; // To detect redundant calls protected virtual void Dispose(bool disposing) { if (!disposedValue) { if (disposing) { if (HDBBitMap != null) { HDBBitMap.Dispose(); } HDBBitMap = null; disposedValue = true; } } }
/// <summary> /// A simple memory mapped scan over the input provided in the constructor /// </summary> /// <param name="ExitAfter">Optionally stop checking or exit early after this many candidates. 0 does not exit early.</param> /// <returns></returns> public int Analyze(int ExitAfter = 0) { CurrWindowBase = 0; mapSize = (64 * 1024 * 1024); if (File.Exists(Filename)) { using (var fs = new FileStream(Filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { var mapName = Path.GetFileNameWithoutExtension(Filename) + DateTime.Now.ToBinary().ToString("X16"); using (var mmap = MemoryMappedFile.CreateFromFile(fs, mapName, 0, MemoryMappedFileAccess.Read, null, HandleInheritability.Inheritable, false)) { if (FileSize == 0) { FileSize = new FileInfo(Filename).Length; } while (CurrWindowBase < FileSize) { using (var reader = mmap.CreateViewAccessor(CurrWindowBase, mapSize, MemoryMappedFileAccess.Read)) { CurrMapBase = 0; reader.ReadArray(CurrMapBase, buffers[filled], 0, 512); while (CurrMapBase < mapSize) { var offset = CurrWindowBase + CurrMapBase; // next page, may be faster with larger chunks but it's simple to view 1 page at a time CurrMapBase += 4096; block = buffers[filled]; filled ^= 1; #pragma warning disable HeapAnalyzerImplicitParamsRule // Array allocation for params parameter Parallel.Invoke(() => Parallel.ForEach <Func <long, bool> >(CheckMethods, (check) => { check(offset); }), () => { if (CurrMapBase < mapSize) { UnsafeHelp.ReadBytes(reader, CurrMapBase, ref buffers[filled]); } } ); if (ExitAfter > 0 && ExitAfter == DetectedProcesses.Count()) { return(DetectedProcesses.Count()); } var progress = Convert.ToInt32((Convert.ToDouble(CurrWindowBase) / Convert.ToDouble(FileSize) * 100.0) + 0.5); if (progress != ProgressBarz.Progress) { ProgressBarz.RenderConsoleProgress(progress); } } } // close current window CurrWindowBase += CurrMapBase; if (CurrWindowBase + mapSize > FileSize) { mapSize = FileSize - CurrWindowBase; } } } } // close map } // close stream return(DetectedProcesses.Count()); }
/// <summary> /// Get a pagesized block that contains the data from the byte offset specified /// </summary> /// <param name="FileOffset">byte offset of long aligned page block</param> /// <param name="block">to be filled on return optionally</param> /// <param name="DataRead">signals success</param> /// <returns>long value from fileoffset</returns> public long GetPageFromFileOffset(long FileOffset, ref long[] block, ref bool DataRead) { var rv = 0L; DataRead = false; var NewMapViewSize = MapViewSize; var CheckBase = FileOffset / MapViewSize; var NewMapViewBase = CheckBase * MapViewSize; if (FileOffset > FileSize) { return(0); } var AbsOffset = FileOffset - NewMapViewBase; var BlockOffset = AbsOffset & ~(PAGE_SIZE - 1); try { if (NewMapViewBase != MapViewBase) { cntInAccessor++; if (NewMapViewBase + MapViewSize > FileSize) { NewMapViewSize = FileSize - NewMapViewBase; } else { NewMapViewSize = MapViewSize; } mappedAccess = mappedFile.CreateViewAccessor( NewMapViewBase, NewMapViewSize, MemoryMappedFileAccess.Read); MapViewBase = NewMapViewBase; } else { cntOutAccsor++; } if (block != null) { var copy_len = block.Length; if (BlockOffset + (block.Length * 8) > NewMapViewSize) { copy_len = (int)((NewMapViewSize - BlockOffset) / 8); } UnsafeHelp.ReadBytes(mappedAccess, BlockOffset, ref block, copy_len); rv = block[((AbsOffset >> 3) & 0x1ff)]; } // FIX: ReadInt64 uses byte address so when we use it must adjust, check for other callers // assumptions since we changed this from array<long>[] maybe expecting old behavior, however // caller from getpageforphysaddr passes valid block usually so that's the main one from V2P else { rv = mappedAccess.ReadInt64(BlockOffset | (AbsOffset & 0x1ff)); } DataRead = true; } catch (Exception ex) { throw new MemoryMapWindowFailedException("Unable to map or read memory offset", ex); } return(rv); }
/// <summary> /// A simple memory mapped scan over the input provided in the constructor /// </summary> /// <param name="ExitAfter">Optionally stop checking or exit early after this many candidates. 0 does not exit early.</param> /// <returns></returns> public int Analyze(int ExitAfter = 0) { CurrWindowBase = 0; mapSize = init_map_size; long RunShift = 0; if (File.Exists(Filename)) { using (var fs = new FileStream(Filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { var mapName = Path.GetFileNameWithoutExtension(Filename) + DateTime.Now.ToBinary().ToString("X16"); using (var mmap = MemoryMappedFile.CreateFromFile(fs, mapName, 0, MemoryMappedFileAccess.Read, null, HandleInheritability.Inheritable, false)) { if (FileSize == 0) { FileSize = new FileInfo(Filename).Length; } // TODO: Clean up all the shifts while (CurrWindowBase < FileSize) { using (var reader = mmap.CreateViewAccessor(CurrWindowBase, mapSize, MemoryMappedFileAccess.Read)) { CurrMapBase = 0; //reader.ReadArray(CurrMapBase, buffers[filled], 0, block_count); UnsafeHelp.ReadBytes(reader, CurrMapBase, ref buffers[filled], block_count); while (CurrMapBase < mapSize) { // setup buffers for parallel load/read block = buffers[filled]; filled ^= 1; var CURR_BASES = CurrWindowBase + CurrMapBase; CurrMapBase += block_size; #pragma warning disable HeapAnalyzerImplicitParamsRule // Array allocation for params parameter Parallel.Invoke(() => Parallel.ForEach <Func <int, long, bool> >(CheckMethods, (check) => { for (int bo = 0; bo < block_count; bo += 512) { // Adjust for known memory run / extents mappings. // Adjust TrueOffset is actually possibly used by check fn (TODO: CLEAN UP THE GLOBALS) var offset = TrueOffset = CURR_BASES + (bo * 8); var offset_pfn = offset >> MagicNumbers.PAGE_SHIFT; // next page, may be faster with larger chunks but it's simple to view 1 page at a time long IndexedOffset_pfn = 0; do { IndexedOffset_pfn = vtero.MemAccess.OffsetToMemIndex(offset_pfn + RunShift); if (IndexedOffset_pfn == -1) { RunShift++; continue; } if (IndexedOffset_pfn == -2) { break; } } while (IndexedOffset_pfn < 0); // found shift, accumulate indexes offset_pfn += RunShift; IndexedOffset_pfn = IndexedOffset_pfn >> MagicNumbers.PAGE_SHIFT; // Calculate DIFF var diff_off_pfn = offset < IndexedOffset_pfn ? IndexedOffset_pfn - offset_pfn : offset_pfn - IndexedOffset_pfn; // Skew Offset offset += (diff_off_pfn << MagicNumbers.PAGE_SHIFT); ///// !!! DO CHECK !!! check(bo, offset); } }), () => { if (CurrMapBase < mapSize) { var total_count_remain = ((mapSize - CurrMapBase) / 8); var read_in_count = total_count_remain > block_count ? block_count : total_count_remain; UnsafeHelp.ReadBytes(reader, CurrMapBase, ref buffers[filled], (int)read_in_count); } } ); if (ExitAfter > 0 && (ExitAfter == DetectedProcesses.Count())) // || FoundValueOffsets.Count() >= ExitAfter)) { return(DetectedProcesses.Count()); } } } // close current window CurrWindowBase += CurrMapBase; if (CurrWindowBase + mapSize > FileSize) { mapSize = FileSize - CurrWindowBase; } var progress = Convert.ToInt32(Convert.ToDouble(CurrWindowBase) / Convert.ToDouble(FileSize) * 100.0); if (progress != ProgressBarz.Progress) { ProgressBarz.RenderConsoleProgress(progress); } } } } // close map } // close stream return(DetectedProcesses.Count()); }
public long GetPageFromFileOffset(long FileOffset, ref long[] block) { var rv = 0L; var NewMapViewBase = MapViewBase; var NewMapViewSize = MapViewSize; var CheckBase = FileOffset / MapViewSize; if (MapViewBase != CheckBase * MapViewSize) { NewMapViewBase = CheckBase * MapViewSize; } if (FileOffset > FileSize) { return(0); } if (FileOffset < NewMapViewBase) { throw new OverflowException("FileOffset must be >= than base"); } var AbsOffset = FileOffset - NewMapViewBase; var BlockOffset = AbsOffset & ~(PAGE_SIZE - 1); try { if (NewMapViewBase != MapViewBase) { cntInAccessor++; if (NewMapViewBase + MapViewSize > FileSize) { NewMapViewSize = FileSize - NewMapViewBase; } else { NewMapViewSize = MapViewSize; } mappedAccess = mappedFile.CreateViewAccessor( NewMapViewBase, NewMapViewSize, MemoryMappedFileAccess.Read); MapViewBase = NewMapViewBase; } else { cntOutAccsor++; } if (block != null) { UnsafeHelp.ReadBytes(mappedAccess, BlockOffset, ref block); } rv = mappedAccess.ReadInt64(AbsOffset); } catch (Exception ex) { block = null; throw new MemoryMapWindowFailedException("Unable to map or read into", ex); } return(rv); }