/// <summary>
/// HashDB manager
/// MUST BE A POWER OF 2
/// </summary>
/// <param name="minBlockSize">POWER of 2</param>
/// <param name="DB">Primary DB</param>
/// <param name="relocFolder">Relocation data</param>
/// <param name="Size">POWER OF 2!</param>
public HashDB(int minBlockSize, string DB, string relocFolder, long Size = 0)
{
HashDBFile = DB;
HashDBBitMap = DB + ".bin";
if (Size != 0)
{
DBSize = (long)FractHashTree.RoundUpPow2(Size);
}
if (!File.Exists(HashDBFile))
{
if (!FractHashTree.IsPow2(DBSize))
{
throw new InternalBufferOverflowException($"DB SIZE not a power of 2!");
}
using (var fileStream = new FileStream(HashDBFile, FileMode.Create, FileAccess.Write, FileShare.None))
fileStream.SetLength(DBSize + (DB_READ_SIZE));
}
else
{
DBSize = (long)FractHashTree.RoundDownPow2(new FileInfo(HashDBFile).Length);
}
// Divide by HASH size
DBEntries = (ulong)DBSize >> HASH_SHIFT;
DBEntriesMask = (ulong)DBEntries - 1;
MinBlockSize = minBlockSize;
ReRe = new ReReDB(relocFolder);
HDBBitMap = new UnsafeHelp(HashDBBitMap, (long)DBEntries);
}
static IEnumerable <long> MapScanFile(String File, long From, int ScanData, int Count)
{
List <long> rv = new List <long>();
// TODO: These streams should be persistent across these calls right?
// TODO: This path is only 1 time and pretty infrequent so far though
using (var fs = new FileStream(File, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
var mapName = Path.GetFileNameWithoutExtension(File) + From.ToString("X16");
using (var mmap =
MemoryMappedFile.CreateFromFile(fs, mapName, 0, MemoryMappedFileAccess.Read,
null, HandleInheritability.Inheritable, false))
{
using (var reader = mmap.CreateViewAccessor(From, Count * 4, MemoryMappedFileAccess.Read))
{
var LocatedScanTarget = UnsafeHelp.ScanBytes(reader, ScanData, Count);
if (LocatedScanTarget.Count() > 0)
{
foreach (var ioff in LocatedScanTarget)
{
var target = From + ioff;
//WriteColor($"Found input @ {(target):X}");
rv.Add(target);
yield return(target);
}
}
}
}
}
yield break;
}
public UnsafeHelp Clone()
{
var rv = new UnsafeHelp();
rv.BitmapBackingFile = BitmapBackingFile;
rv.SetupHandles();
return(rv);
}
public Mem(Mem parent) : this()
{
MD = parent.MD;
ID = parent.ID;
DumpedPFNBitmap = new UnsafeHelp(ID.ToString(), MaxLimit, true);
StartOfMemory = parent.StartOfMemory;
MemoryDump = parent.MemoryDump;
FileSize = parent.FileSize;
MapViewSize = parent.MapViewSize;
SetupStreams();
}
public Mem(Mem parent) : this()
{
MD = parent.MD;
ID = parent.ID;
DumpedPFNBitmap = new UnsafeHelp($"{Process.GetCurrentProcess().Id}-{ID}", MaxLimit, true);
StartOfMemory = parent.StartOfMemory;
MemoryDump = parent.MemoryDump;
FileSize = parent.FileSize;
MapViewSize = parent.MapViewSize;
SetupStreams();
}
public Mem(Mem parent) : this()
{
MD = parent.MD;
ID = parent.ID;
DumpedPFNBitmap = parent.DumpedPFNBitmap.Clone();
StartOfMemory = parent.StartOfMemory;
MemoryDump = parent.MemoryDump;
FileSize = parent.FileSize;
MapViewSize = parent.MapViewSize;
SetupStreams();
}
private bool disposedValue = false; // To detect redundant calls
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
if (HDBBitMap != null)
{
HDBBitMap.Dispose();
}
HDBBitMap = null;
disposedValue = true;
}
}
}
/// <summary>
/// A simple memory mapped scan over the input provided in the constructor
/// </summary>
/// <param name="ExitAfter">Optionally stop checking or exit early after this many candidates. 0 does not exit early.</param>
/// <returns></returns>
public int Analyze(int ExitAfter = 0)
{
CurrWindowBase = 0;
mapSize = (64 * 1024 * 1024);
if (File.Exists(Filename))
{
using (var fs = new FileStream(Filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
var mapName = Path.GetFileNameWithoutExtension(Filename) + DateTime.Now.ToBinary().ToString("X16");
using (var mmap =
MemoryMappedFile.CreateFromFile(fs,
mapName,
0,
MemoryMappedFileAccess.Read,
null,
HandleInheritability.Inheritable,
false))
{
if (FileSize == 0)
{
FileSize = new FileInfo(Filename).Length;
}
while (CurrWindowBase < FileSize)
{
using (var reader = mmap.CreateViewAccessor(CurrWindowBase, mapSize, MemoryMappedFileAccess.Read))
{
CurrMapBase = 0;
reader.ReadArray(CurrMapBase, buffers[filled], 0, 512);
while (CurrMapBase < mapSize)
{
var offset = CurrWindowBase + CurrMapBase;
// next page, may be faster with larger chunks but it's simple to view 1 page at a time
CurrMapBase += 4096;
block = buffers[filled];
filled ^= 1;
#pragma warning disable HeapAnalyzerImplicitParamsRule // Array allocation for params parameter
Parallel.Invoke(() =>
Parallel.ForEach <Func <long, bool> >(CheckMethods, (check) => {
check(offset);
}), () => {
if (CurrMapBase < mapSize)
{
UnsafeHelp.ReadBytes(reader, CurrMapBase, ref buffers[filled]);
}
}
);
if (ExitAfter > 0 && ExitAfter == DetectedProcesses.Count())
{
return(DetectedProcesses.Count());
}
var progress = Convert.ToInt32((Convert.ToDouble(CurrWindowBase) / Convert.ToDouble(FileSize) * 100.0) + 0.5);
if (progress != ProgressBarz.Progress)
{
ProgressBarz.RenderConsoleProgress(progress);
}
}
} // close current window
CurrWindowBase += CurrMapBase;
if (CurrWindowBase + mapSize > FileSize)
{
mapSize = FileSize - CurrWindowBase;
}
}
}
} // close map
} // close stream
return(DetectedProcesses.Count());
}
/// <summary>
/// Get a pagesized block that contains the data from the byte offset specified
/// </summary>
/// <param name="FileOffset">byte offset of long aligned page block</param>
/// <param name="block">to be filled on return optionally</param>
/// <param name="DataRead">signals success</param>
/// <returns>long value from fileoffset</returns>
public long GetPageFromFileOffset(long FileOffset, ref long[] block, ref bool DataRead)
{
var rv = 0L;
DataRead = false;
var NewMapViewSize = MapViewSize;
var CheckBase = FileOffset / MapViewSize;
var NewMapViewBase = CheckBase * MapViewSize;
if (FileOffset > FileSize)
{
return(0);
}
var AbsOffset = FileOffset - NewMapViewBase;
var BlockOffset = AbsOffset & ~(PAGE_SIZE - 1);
try
{
if (NewMapViewBase != MapViewBase)
{
cntInAccessor++;
if (NewMapViewBase + MapViewSize > FileSize)
{
NewMapViewSize = FileSize - NewMapViewBase;
}
else
{
NewMapViewSize = MapViewSize;
}
mappedAccess = mappedFile.CreateViewAccessor(
NewMapViewBase,
NewMapViewSize,
MemoryMappedFileAccess.Read);
MapViewBase = NewMapViewBase;
}
else
{
cntOutAccsor++;
}
if (block != null)
{
var copy_len = block.Length;
if (BlockOffset + (block.Length * 8) > NewMapViewSize)
{
copy_len = (int)((NewMapViewSize - BlockOffset) / 8);
}
UnsafeHelp.ReadBytes(mappedAccess, BlockOffset, ref block, copy_len);
rv = block[((AbsOffset >> 3) & 0x1ff)];
}
// FIX: ReadInt64 uses byte address so when we use it must adjust, check for other callers
// assumptions since we changed this from array<long>[] maybe expecting old behavior, however
// caller from getpageforphysaddr passes valid block usually so that's the main one from V2P
else
{
rv = mappedAccess.ReadInt64(BlockOffset | (AbsOffset & 0x1ff));
}
DataRead = true;
}
catch (Exception ex)
{
throw new MemoryMapWindowFailedException("Unable to map or read memory offset", ex);
}
return(rv);
}
/// <summary>
/// A simple memory mapped scan over the input provided in the constructor
/// </summary>
/// <param name="ExitAfter">Optionally stop checking or exit early after this many candidates. 0 does not exit early.</param>
/// <returns></returns>
public int Analyze(int ExitAfter = 0)
{
CurrWindowBase = 0;
mapSize = init_map_size;
long RunShift = 0;
if (File.Exists(Filename))
{
using (var fs = new FileStream(Filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
var mapName = Path.GetFileNameWithoutExtension(Filename) + DateTime.Now.ToBinary().ToString("X16");
using (var mmap =
MemoryMappedFile.CreateFromFile(fs,
mapName,
0,
MemoryMappedFileAccess.Read,
null,
HandleInheritability.Inheritable,
false))
{
if (FileSize == 0)
{
FileSize = new FileInfo(Filename).Length;
}
// TODO: Clean up all the shifts
while (CurrWindowBase < FileSize)
{
using (var reader = mmap.CreateViewAccessor(CurrWindowBase, mapSize, MemoryMappedFileAccess.Read))
{
CurrMapBase = 0;
//reader.ReadArray(CurrMapBase, buffers[filled], 0, block_count);
UnsafeHelp.ReadBytes(reader, CurrMapBase, ref buffers[filled], block_count);
while (CurrMapBase < mapSize)
{
// setup buffers for parallel load/read
block = buffers[filled];
filled ^= 1;
var CURR_BASES = CurrWindowBase + CurrMapBase;
CurrMapBase += block_size;
#pragma warning disable HeapAnalyzerImplicitParamsRule // Array allocation for params parameter
Parallel.Invoke(() =>
Parallel.ForEach <Func <int, long, bool> >(CheckMethods, (check) =>
{
for (int bo = 0; bo < block_count; bo += 512)
{
// Adjust for known memory run / extents mappings.
// Adjust TrueOffset is actually possibly used by check fn (TODO: CLEAN UP THE GLOBALS)
var offset = TrueOffset = CURR_BASES + (bo * 8);
var offset_pfn = offset >> MagicNumbers.PAGE_SHIFT;
// next page, may be faster with larger chunks but it's simple to view 1 page at a time
long IndexedOffset_pfn = 0;
do
{
IndexedOffset_pfn = vtero.MemAccess.OffsetToMemIndex(offset_pfn + RunShift);
if (IndexedOffset_pfn == -1)
{
RunShift++;
continue;
}
if (IndexedOffset_pfn == -2)
{
break;
}
} while (IndexedOffset_pfn < 0);
// found shift, accumulate indexes
offset_pfn += RunShift;
IndexedOffset_pfn = IndexedOffset_pfn >> MagicNumbers.PAGE_SHIFT;
// Calculate DIFF
var diff_off_pfn = offset < IndexedOffset_pfn ? IndexedOffset_pfn - offset_pfn : offset_pfn - IndexedOffset_pfn;
// Skew Offset
offset += (diff_off_pfn << MagicNumbers.PAGE_SHIFT);
///// !!! DO CHECK !!!
check(bo, offset);
}
}), () =>
{
if (CurrMapBase < mapSize)
{
var total_count_remain = ((mapSize - CurrMapBase) / 8);
var read_in_count = total_count_remain > block_count ? block_count : total_count_remain;
UnsafeHelp.ReadBytes(reader, CurrMapBase, ref buffers[filled], (int)read_in_count);
}
}
);
if (ExitAfter > 0 && (ExitAfter == DetectedProcesses.Count())) // || FoundValueOffsets.Count() >= ExitAfter))
{
return(DetectedProcesses.Count());
}
}
} // close current window
CurrWindowBase += CurrMapBase;
if (CurrWindowBase + mapSize > FileSize)
{
mapSize = FileSize - CurrWindowBase;
}
var progress = Convert.ToInt32(Convert.ToDouble(CurrWindowBase) / Convert.ToDouble(FileSize) * 100.0);
if (progress != ProgressBarz.Progress)
{
ProgressBarz.RenderConsoleProgress(progress);
}
}
}
} // close map
} // close stream
return(DetectedProcesses.Count());
}
public long GetPageFromFileOffset(long FileOffset, ref long[] block)
{
var rv = 0L;
var NewMapViewBase = MapViewBase;
var NewMapViewSize = MapViewSize;
var CheckBase = FileOffset / MapViewSize;
if (MapViewBase != CheckBase * MapViewSize)
{
NewMapViewBase = CheckBase * MapViewSize;
}
if (FileOffset > FileSize)
{
return(0);
}
if (FileOffset < NewMapViewBase)
{
throw new OverflowException("FileOffset must be >= than base");
}
var AbsOffset = FileOffset - NewMapViewBase;
var BlockOffset = AbsOffset & ~(PAGE_SIZE - 1);
try
{
if (NewMapViewBase != MapViewBase)
{
cntInAccessor++;
if (NewMapViewBase + MapViewSize > FileSize)
{
NewMapViewSize = FileSize - NewMapViewBase;
}
else
{
NewMapViewSize = MapViewSize;
}
mappedAccess = mappedFile.CreateViewAccessor(
NewMapViewBase,
NewMapViewSize,
MemoryMappedFileAccess.Read);
MapViewBase = NewMapViewBase;
}
else
{
cntOutAccsor++;
}
if (block != null)
{
UnsafeHelp.ReadBytes(mappedAccess, BlockOffset, ref block);
}
rv = mappedAccess.ReadInt64(AbsOffset);
}
catch (Exception ex)
{
block = null;
throw new MemoryMapWindowFailedException("Unable to map or read into", ex);
}
return(rv);
}
