public static PageTable AddProcess(DetectedProc dp, Mem mem)
{
long Address = 0;
int AddressIndex = 0;
// dump Page table high to low
var va = new VIRTUAL_ADDRESS(long.MaxValue - 0xfff);
var rv = new PageTable
{
Failed = new List<HARDWARE_ADDRESS_ENTRY>(),
DP = dp,
mem = mem
};
// TODO: encode VA's for self/recursive physical addr's
if (dp.PageTableType == PTType.Windows)
{
Address = MagicNumbers.Windows_SelfAsVA;
AddressIndex = MagicNumbers.Windows_SelfPtr;
}
// any output is error/warning output
var cnt = rv.FillTable(new VIRTUAL_ADDRESS(Address), AddressIndex, dp.CR3Value);
Debug.WriteLine($"extracted {cnt} PTE from process {dp.vmcs.EPTP:X16}:{dp.CR3Value:X16}");
dp.PT = rv;
return rv;
}
public static PageTable AddProcess(DetectedProc dp, Mem mem, bool RedundantKernelEntries = true, int DepthToGo = 4)
{
if(Vtero.DiagOutput)
WriteLine($"PT analysis of {dp}");
var rv = new PageTable
{
Root = new PageTableRoot() { SLAT = dp.vmcs != null ? dp.vmcs.EPTP : 0, CR3 = dp.CR3Value, Entries = new PFN() },
Failed = new List<HARDWARE_ADDRESS_ENTRY>(),
DP = dp,
mem = mem
};
if (dp.MemAccess == null || mem != null)
dp.MemAccess = mem;
/* Commenting out this block so we defer the table enumeration
// any output is error/warning output
var cnt = rv.FillTable(RedundantKernelEntries, DepthToGo);
rv.EntriesParsed += cnt;
if (cnt == 0)
{
if (dp.vmcs != null)
WriteLine($"BAD EPTP/DirectoryTable Base {dp.vmcs.EPTP:X12}, try a different candidate or this dump may lack a hypervisor. Recommend attempt PT walk W/O SLAT");
else
WriteLine($"Decoding failed for {dp.CR3Value:X12}");
//WriteLine($"Physical walk w/o SLAT yielded {cnt} entries");
}
*/
dp.PT = rv;
return rv;
}
public void DumpProc(string Folder, bool IncludeData = false, bool KernelSpace = true, bool OnlyExec = true)
{
//// TODO: BOILER PLATE check perf of using callbacks
PageTable.AddProcess(this, new Mem(MemAccess));
var cnt = PT.FillPageQueue(false, KernelSpace);
Folder = Folder + Path.DirectorySeparatorChar.ToString();
Directory.CreateDirectory(Folder);
long ContigSizeState = 0, curr = 0;
// single threaded worked best so far
//Parallel.For(0, cnt, (i, loopState) => x
foreach (var range in PT.FillPageQueue(false, KernelSpace))
{
curr++;
if (Vtero.VerboseLevel > 1)
{
//var curr = cnt - PT.PageQueue.Count;
//var done = Convert.ToDouble(curr) / Convert.ToDouble(cnt) * 100.0;
Console.CursorLeft = 0;
Console.Write($"{curr} scanned");
}
if (range.PTE.Valid)
{
// skip data as requested
if (!IncludeData && range.PTE.NoExecute)
{
continue;
}
Vtero.WriteRange(range.VA, range, Folder, ref ContigSizeState, MemAccess);
}
}
}
public int FillPageQueue(bool OnlyLarge = false)
{
PageQueue = new ConcurrentQueue <PFN>();
VIRTUAL_ADDRESS VA;
VA.Address = 0;
if (DP.PT == null)
{
PageTable.AddProcess(DP, DP.MemAccess);
}
Parallel.ForEach(DP.TopPageTablePage, (kvp) =>
//foreach (var kvp in DP.TopPageTablePage)
{
// were at the top level (4th)
VA.PML4 = kvp.Key;
var pfn = new PFN {
PTE = kvp.Value, VA = new VIRTUAL_ADDRESS(VA.PML4 << 39)
};
foreach (var DirectoryPointerOffset in DP.PT.ExtractNextLevel(pfn, true, 3))
{
if (DirectoryPointerOffset == null)
{
continue;
}
foreach (var DirectoryOffset in DP.PT.ExtractNextLevel(DirectoryPointerOffset, KernelSpace, 2))
{
if (DirectoryOffset == null)
{
continue;
}
// if we have a large page we add it now
if (DirectoryOffset.PTE.LargePage)
{
PageQueue.Enqueue(DirectoryOffset);
}
// otherwise were scanning lower level entries
else if (!OnlyLarge)
{
foreach (var TableOffset in DP.PT.ExtractNextLevel(DirectoryOffset, KernelSpace, 1))
{
if (TableOffset == null)
{
continue;
}
PageQueue.Enqueue(TableOffset);
}
}
}
}
//}
});
return(PageQueue.Count());
}
public static PageTable AddProcess(DetectedProc dp, Mem mem, bool OnlyUserSpace = false)
{
long Address = 0;
int AddressIndex = 0;
// dump Page table high to low
var va = new VIRTUAL_ADDRESS(long.MaxValue - 0xfff);
var rv = new PageTable
{
Failed = new List <HARDWARE_ADDRESS_ENTRY>(),
DP = dp,
mem = mem
};
// TODO: encode VA's for self/recursive physical addr's
if (dp.PageTableType == PTType.Windows)
{
Address = MagicNumbers.Windows_SelfAsVA;
AddressIndex = MagicNumbers.Windows_SelfPtr;
}
// any output is error/warning output
var cnt = rv.FillTable(new VIRTUAL_ADDRESS(Address), AddressIndex, dp.CR3Value, OnlyUserSpace);
if (cnt == 0)
{
if (dp.vmcs != null)
{
WriteLine($"BAD EPTP/DirectoryTable Base {dp.vmcs.EPTP:X16}, try a different candidate or this dump may lack a hypervisor. Attempting PT walk W/O SLAT");
}
else
{
WriteLine($"Decoding failed for {dp.CR3Value:X16}");
}
/*cnt = rv.FillTable(new VIRTUAL_ADDRESS(Address), AddressIndex, dp.CR3Value, OnlyUserSpace);
* WriteLine($"Physical walk w/o SLAT yielded {cnt} entries");*/
}
dp.PT = rv;
return(rv);
}
public static PageTable AddProcess(DetectedProc dp, Mem mem, bool RedundantKernelEntries = true)
{
if (Vtero.DiagOutput)
{
WriteLine($"PT analysis of {dp}");
}
var rv = new PageTable
{
Root = new PageTableRoot()
{
SLAT = dp.vmcs != null ? dp.vmcs.EPTP : 0, CR3 = dp.CR3Value, Entries = new PFN()
},
Failed = new List <HARDWARE_ADDRESS_ENTRY>(),
DP = dp,
mem = mem
};
if (dp.MemAccess == null || mem != null)
{
dp.MemAccess = mem;
}
/* Commenting out this block so we defer the table enumeration
* // any output is error/warning output
*
* var cnt = rv.FillTable(RedundantKernelEntries, DepthToGo);
* rv.EntriesParsed += cnt;
*
* if (cnt == 0)
* {
* if (dp.vmcs != null)
* WriteLine($"BAD EPTP/DirectoryTable Base {dp.vmcs.EPTP:X12}, try a different candidate or this dump may lack a hypervisor. Recommend attempt PT walk W/O SLAT");
* else
* WriteLine($"Decoding failed for {dp.CR3Value:X12}");
* //WriteLine($"Physical walk w/o SLAT yielded {cnt} entries");
* }
*/
dp.PT = rv;
return(rv);
}
public static PageTable AddProcess(DetectedProc dp, Mem mem, bool OnlyUserSpace = false)
{
long Address = 0;
int AddressIndex = 0;
// dump Page table high to low
var va = new VIRTUAL_ADDRESS(long.MaxValue - 0xfff);
var rv = new PageTable
{
Failed = new List<HARDWARE_ADDRESS_ENTRY>(),
DP = dp,
mem = mem
};
// TODO: encode VA's for self/recursive physical addr's
if (dp.PageTableType == PTType.Windows)
{
Address = MagicNumbers.Windows_SelfAsVA;
AddressIndex = MagicNumbers.Windows_SelfPtr;
}
// any output is error/warning output
var cnt = rv.FillTable(new VIRTUAL_ADDRESS(Address), AddressIndex, dp.CR3Value, OnlyUserSpace);
if (cnt == 0)
{
if (dp.vmcs != null)
WriteLine($"BAD EPTP/DirectoryTable Base {dp.vmcs.EPTP:X16}, try a different candidate or this dump may lack a hypervisor. Attempting PT walk W/O SLAT");
else
WriteLine($"Decoding failed for {dp.CR3Value:X16}");
/*cnt = rv.FillTable(new VIRTUAL_ADDRESS(Address), AddressIndex, dp.CR3Value, OnlyUserSpace);
WriteLine($"Physical walk w/o SLAT yielded {cnt} entries");*/
}
dp.PT = rv;
return rv;
}
public List <ScanResult> YaraScan(string RulesFile, bool IncludeData = false, bool KernelSpace = false)
{
var rv = new List <ScanResult>();
using (var ctx = new YaraContext())
{
Rules rules = null;
try
{
// Rules and Compiler objects must be disposed.
using (var compiler = new Compiler())
{
compiler.AddRuleFile(RulesFile);
rules = compiler.GetRules();
}
PageTable.AddProcess(this, MemAccess);
//var cnt = PT.FillPageQueue(false, KernelSpace);
var curr = 0;
YaraTotalScanned = 0;
// single threaded worked best so far
//Parallel.For(0, cnt, (i, loopState) => x
foreach (var range in PT.FillPageQueue(false, KernelSpace, true, false))
//for (int i = 0; i < cnt; i++)
{
curr++;
if (Vtero.VerboseLevel > 1)
{
//var curr = cnt - PT.PageQueue.Count;
//var done = Convert.ToDouble(curr) / Convert.ToDouble(cnt) * 100.0;
Console.CursorLeft = 0;
Console.Write($"{curr} scanned");
}
if (range.PTE.Valid)
{
// skip data as requested
if (!IncludeData && range.PTE.NoExecute)
{
continue;
}
// Scanner and ScanResults do not need to be disposed.
var scanner = new libyaraNET.Scanner();
unsafe
{
long[] block = null;
bool GotData = false;
if (range.PTE.LargePage)
{
block = new long[0x40000];
}
else
{
block = new long[0x200];
}
MemAccess.GetPageForPhysAddr(range.PTE, ref block, ref GotData);
if (GotData)
{
fixed(void *lp = block)
{
var res = scanner.ScanMemory((byte *)lp, block.Length, rules, ScanFlags.None);
rv.AddRange(res);
YaraTotalScanned += block.Length;
}
}
}
}
}
}
finally
{
// Rules and Compiler objects must be disposed.
if (rules != null)
{
rules.Dispose();
}
}
}
YaraOutput = rv;
return(YaraOutput);
}
/// <summary>
/// Currently we scan hard for only kernel regions (2MB pages + ExEC)
/// If there are kernel modules named the OnlyModule it may cause us to ignore the real one in that case
/// you can still scan for * by passing null or empty string
/// </summary>
/// <param name="OnlyModule">Stop when the first module named this is found</param>
public VirtualScanner ScanAndLoadModules(string OnlyModule = "ntkrnlmp.pdb", bool OnlyLarge = true, bool IncludeKernelSpace = true, bool OnlyValid = true, bool IncludeData = false, bool DoExtraHeaderScan = true)
{
const int LARGE_PAGE_SIZE = 1024 * 1024 * 2;
var curr = 0;
PageTable.AddProcess(this, new Mem(MemAccess));
//var cnt = PT.FillPageQueue(OnlyLarge, IncludeKernelSpace);
var KVS = new VirtualScanner(this, new Mem(MemAccess), DoExtraHeaderScan);
// single threaded worked best so far
//Parallel.For(0, cnt, (i, loopState) => x
foreach (var range in PT.FillPageQueue(OnlyLarge, IncludeKernelSpace, OnlyValid, IncludeData))
//for (int i = 0; i < cnt; i++)
{
curr++;
bool stop = false;
if (Vtero.VerboseLevel > 1)
{
//var curr = cnt - PT.PageQueue.Count;
//var done = Convert.ToDouble(curr) / Convert.ToDouble(cnt) * 100.0;
Console.CursorLeft = 0;
Console.Write($"{curr} scanned");
}
if (range.PTE.Valid && !range.PTE.NoExecute)
{
foreach (var artifact in KVS.Run(range.VA.Address, range.VA.Address + (range.PTE.LargePage ? LARGE_PAGE_SIZE : MagicNumbers.PAGE_SIZE), range))
{
var ms = new MemSection()
{
IsExec = true, Module = artifact, VA = new VIRTUAL_ADDRESS(artifact.VA), Source = range
};
var extracted = ExtractCVDebug(ms);
if (extracted == null)
{
if (Vtero.VerboseLevel > 1)
{
WriteColor(ConsoleColor.Yellow, $"failed debug info for PE @address {range.VA.Address:X}, extracted headers: {artifact}");
}
continue;
}
if (!string.IsNullOrWhiteSpace(OnlyModule) && OnlyModule != ms.Name)
{
continue;
}
if (!Sections.ContainsKey(artifact.VA))
{
Sections.TryAdd(artifact.VA, ms);
}
// we can clobber this guy all the time I guess since everything is stateless in Sym and managed
// entirely by the handle ID really which is local to our GUID so....
sym = Vtero.TryLoadSymbols(ID.GetHashCode(), ms.DebugDetails, ms.VA.Address);
if (Vtero.VerboseOutput)
{
WriteColor((sym != null) ? ConsoleColor.Green : ConsoleColor.Yellow, $" symbol loaded = [{sym != null}] PDB [{ms.DebugDetails.PDBFullPath}] @ {range.VA.Address:X}, {ms.Name}");
if (Vtero.VerboseLevel > 1)
{
WriteColor((sym != null) ? ConsoleColor.Green : ConsoleColor.Yellow, $"headers: { artifact} ");
}
}
if (!string.IsNullOrWhiteSpace(OnlyModule))
{
if (!string.IsNullOrWhiteSpace(ms.Name) && ms.Name == OnlyModule)
{
stop = true;
//loopState.Stop();
break;
}
}
//if (loopState.IsStopped)
//return;
if (stop)
{
break;
}
}
}
//if (loopState.IsStopped)
// return;e
//});
if (stop)
{
break;
}
}
return(KVS);
}
/// <summary>
/// Currently we scan hard for only kernel regions (2MB pages + ExEC)
/// If there are kernel modules named the OnlyModule it may cause us to ignore the real one in that case
/// you can still scan for * by passing null or empty string
/// </summary>
/// <param name="OnlyModule">Stop when the first module named this is found</param>
public VirtualScanner ScanAndLoadModules(string OnlyModule = "ntkrnlmp")
{
PageTable.AddProcess(this, new Mem(MemAccess));
var cnt = PT.FillPageQueue(true);
var KVS = new VirtualScanner(this, new Mem(MemAccess));
KVS.ScanMode = VAScanType.PE_FAST;
Parallel.For(0, cnt, (i, loopState) =>
{
PFN range;
var curr = cnt - PT.PageQueue.Count;
var done = (int)(Convert.ToDouble(curr) / Convert.ToDouble(cnt) * 100.0) + 0.5;
if (PT.PageQueue.TryDequeue(out range) && range.PTE.Valid)
{
var found = KVS.Run(range.VA.Address, range.VA.Address + (range.PTE.LargePage ? (1024 * 1024 * 2) : 0x1000), loopState);
// Attempt load
foreach (var artifact in found)
{
var ms = new MemSection()
{
IsExec = true, Module = artifact.Value, VA = new VIRTUAL_ADDRESS(artifact.Key)
};
var extracted = ExtractCVDebug(ms);
if (extracted == null)
{
continue;
}
if (!string.IsNullOrWhiteSpace(OnlyModule) && OnlyModule != ms.Name)
{
continue;
}
if (!Sections.ContainsKey(artifact.Key))
{
Sections.TryAdd(artifact.Key, ms);
}
// we can clobber this guy all the time I guess since everything is stateless in Sym and managed
// entirely by the handle ID really which is local to our GUID so....
sym = Vtero.TryLoadSymbols(ID.GetHashCode(), ms.DebugDetails, ms.VA.Address);
if (Vtero.VerboseOutput)
{
WriteColor(ConsoleColor.Green, $"symbol loaded [{sym != null}] from file [{ms.DebugDetails.PDBFullPath}]");
}
if (!string.IsNullOrWhiteSpace(OnlyModule))
{
if (!string.IsNullOrWhiteSpace(ms.Name) && ms.Name == OnlyModule)
{
loopState.Stop();
return;
}
}
if (loopState.IsStopped)
{
return;
}
}
}
if (loopState.IsStopped)
{
return;
}
});
return(KVS);
}
//[ProtoIgnore]
//public List<PFN> PageQueue;
//TODO: this should really take a PFN with various bit's set we can test with a .Match
//TODO: fix all callers of this to use a callback also
public IEnumerable <PFN> FillPageQueue(bool OnlyLarge = false, bool RedundantKernelSpaces = false, bool OnlyValid = true, bool OnlyExec = true)
{
KernelSpace = RedundantKernelSpaces;
//PageQueue = new List<PFN>();
VIRTUAL_ADDRESS VA;
VA.Address = 0;
if (DP.PT == null)
{
PageTable.AddProcess(DP, DP.MemAccess);
}
//Parallel.ForEach(DP.TopPageTablePage, (kvp) =>
foreach (var kvp in DP.TopPageTablePage)
{
// were at the top level (4th)
VA.PML4 = kvp.Key;
var pfn = new PFN {
PTE = kvp.Value, VA = new VIRTUAL_ADDRESS(VA.PML4 << 39)
};
// do redundant check here
if (!RedundantKernelSpaces && (kvp.Key >= MagicNumbers.KERNEL_PT_INDEX_START_USUALLY))
{
continue;
}
if (OnlyExec && pfn.PTE.NoExecute)
{
continue;
}
foreach (var DirectoryPointerOffset in DP.PT.ExtractNextLevel(pfn, 3))
{
if (DirectoryPointerOffset == null)
{
continue;
}
if (OnlyExec && DirectoryPointerOffset.PTE.NoExecute)
{
continue;
}
foreach (var DirectoryOffset in DP.PT.ExtractNextLevel(DirectoryPointerOffset, 2))
{
if (DirectoryOffset == null)
{
continue;
}
// if we have a large page we add it now
if (DirectoryOffset.PTE.LargePage || (OnlyValid && !DirectoryOffset.PTE.Valid))
{
if (OnlyExec && DirectoryOffset.PTE.NoExecute)
{
continue;
}
yield return(DirectoryOffset);
continue;
}
// otherwise were scanning lower level entries
// unless we are only large page scanning.
else if (!OnlyLarge)
{
foreach (var TableOffset in DP.PT.ExtractNextLevel(DirectoryOffset, 1))
{
if (OnlyExec && TableOffset.PTE.NoExecute)
{
continue;
}
if (TableOffset == null || (OnlyValid && !TableOffset.PTE.Valid))
{
continue;
}
yield return(TableOffset);
}
}
}
}
}
//});
yield break;
}
/// <summary>
/// This routine is fairly expensive, maybe unnecessary as well but it demo's walking the page table + EPT.
/// You can connect an address space dumper really easily
/// </summary>
/// <param name="MemSpace">The list of VMCS/EPTP configurations which will alter the page table use</param>
/// <param name="Procs">Detected procs to query</param>
/// <param name="pTypes">Type bitmas to interpreate</param>
public List <DetectedProc> ExtrtactAddressSpaces(IOrderedEnumerable <VMCS> MemSpace = null, ConcurrentBag <DetectedProc> Procs = null, PTType pTypes = PTType.UNCONFIGURED)
{
List <DetectedProc> rvList = new List <DetectedProc>();
var PT2Scan = pTypes == PTType.UNCONFIGURED ? (PTType.Windows | PTType.HyperV | PTType.GENERIC) : pTypes;
var procList = Procs == null ? Processes : Procs;
//var memSpace = MemSpace == null ? VMCSs.First() : MemSpace.First();
var memSpace = MemSpace == null?VMCSs.GroupBy(x => x.EPTP).Select(xg => xg.First()) : MemSpace;
var p = from proc in Processes
where (((proc.PageTableType & PT2Scan) == proc.PageTableType))
orderby proc.CR3Value ascending
select proc;
int pcnt = Processes.Count();
int vmcnt = memSpace.Count();
var tot = pcnt * vmcnt;
var curr = 0;
var AlikelyKernelSet = from ptes in p.First().TopPageTablePage
where ptes.Key > 255 && MagicNumbers.Each.All(ppx => ppx != ptes.Key)
select ptes.Value;
Console.ForegroundColor = ConsoleColor.Yellow;
WriteLine($"assessing {tot} address spaces");
ProgressBarz.Progress = 0;
var VMCSTriage = new Dictionary <VMCS, int>();
//Parallel.ForEach(memSpace, (space) =>
foreach (var space in memSpace)
{
// we do it this way so that parallelized tasks do not interfere with each other
// overall it may blow the cache hit ratio but will tune a single task to see the larger/better cache
// versus multicore, my suspicion is that multi-core is better
using (var memAxs = new Mem(MemFile, null, DetectedDesc))
{
var sx = 0;
foreach (var px in p)
//Parallel.ForEach(p, (proc) =>
{
try
{
var proc = px.Clone <DetectedProc>();
proc.vmcs = space;
var pt = PageTable.AddProcess(proc, memAxs, false);
if (pt != null && VerboseOutput)
{
// If we used group detection correlation a valid EPTP should work for every process
if (proc.vmcs != null && proc.PT.RootPageTable.PFNCount > proc.TopPageTablePage.Count())
{
WriteLine($"Virtualized Process PT Entries [{proc.PT.RootPageTable.PFNCount}] Type [{proc.PageTableType}] PID [{proc.vmcs.EPTP:X}:{proc.CR3Value:X}]");
rvList.Add(proc);
}
else
{
WriteLine($"canceling evaluation of bad EPTP for this group");
foreach (var pxc in Processes)
{
pxc.vmcs = null;
}
break;
}
sx++;
curr++;
var progress = Convert.ToInt32((Convert.ToDouble(curr) / Convert.ToDouble(tot) * 100.0) + 0.5);
ProgressBarz.RenderConsoleProgress(progress);
}
}
catch (ExtendedPageNotFoundException eptpX)
{
WriteLine($"Bad EPTP selection;{Environment.NewLine}\tEPTP:{eptpX.RequestedEPTP}{Environment.NewLine}\t CR3:{eptpX.RequestedCR3}{Environment.NewLine} Attempting to skip to next proc.");
memAxs.DumpPFNIndex();
}
catch (MemoryRunMismatchException mrun)
{
WriteLine($"Error in accessing memory for PFN {mrun.PageRunNumber:X16}");
memAxs.DumpPFNIndex();
}
catch (PageNotFoundException pnf)
{
WriteLine($"Error in selecting page, see {pnf}");
memAxs.DumpPFNIndex();
}
catch (Exception ex)
{
WriteLine($"Error in memspace extraction: {ex.ToString()}");
memAxs.DumpPFNIndex();
}
WriteLine($"{sx} VMCS dominated process address spaces and were decoded succsessfully.");
//});
}
}
}
//});
using (var memAxs = new Mem(MemFile, null, DetectedDesc))
{
var nonVMCSprocs = from px in Processes
where px.vmcs == null
select px;
foreach (var px in nonVMCSprocs)
{
var pmetal = px.Clone <DetectedProc>();
// this is a process on the bare metal
var pt = PageTable.AddProcess(pmetal, memAxs, false);
WriteLine($"Process {pmetal.CR3Value:X16} Physical walk w/o SLAT yielded {pmetal.PT.RootPageTable.PFNCount} entries");
rvList.Add(pmetal);
}
}
return(rvList);
}
