public void TestRegionChunking()
{
Random random = new Random();
List <SnapshotRegion> groundTruth = new List <SnapshotRegion>();
List <SnapshotRegion> chunkedRegions = new List <SnapshotRegion>();
UInt64 baseAddress = 0;
for (Int32 i = 0; i < 10000; i++)
{
baseAddress += (UInt64)random.Next(1, 1024);
UInt64 regionSize = random.Next(1, 1024).ToUInt64();
UInt64 chunkSize = random.Next(32, 128).ToUInt64();
SnapshotRegion region = new SnapshotRegion(baseAddress.ToIntPtr(), regionSize);
IEnumerable <SnapshotRegion> regions = region.ChunkNormalizedRegion(chunkSize).Select(x => new SnapshotRegion(x)).OrderBy(x => random.Next());
groundTruth.Add(region);
chunkedRegions.AddRange(regions);
baseAddress += (UInt64)regionSize;
}
UInt64 originalSize = groundTruth.Sum(x => x.RegionSize);
UInt64 newSize = chunkedRegions.Sum(x => x.RegionSize);
Assert.AreEqual(originalSize, newSize);
}
/// <summary>
/// Gets the snapshot generated by the prefilter.
/// </summary>
/// <returns>The snapshot generated by the prefilter.</returns>
public Snapshot GetPrefilteredSnapshot()
{
List <SnapshotRegion> regions = new List <SnapshotRegion>();
lock (this.ChunkLock)
{
foreach (RegionProperties virtualPage in this.ChunkList)
{
if (!virtualPage.HasChanged)
{
continue;
}
SnapshotRegion newRegion = new SnapshotRegion(virtualPage);
regions.Add(newRegion);
}
}
// Create snapshot from valid regions, do standard expand/mask operations to catch lost bytes for larger data types
Snapshot prefilteredSnapshot = new Snapshot(regions);
prefilteredSnapshot.ExpandAllRegions(PrimitiveTypes.GetLargestPrimitiveSize() - 1);
prefilteredSnapshot.MaskRegions(prefilteredSnapshot);
return(prefilteredSnapshot);
}
public void TestSnapshotMerging()
{
Random random = new Random();
List <SnapshotRegion> groundTruth = new List <SnapshotRegion>();
List <SnapshotRegion> chunkedRegions = new List <SnapshotRegion>();
SettingsViewModel.GetInstance().Alignment = 1;
UInt64 baseAddress = 0;
for (Int32 i = 0; i < 10000; i++)
{
baseAddress += (UInt64)random.Next(1, 1024);
Int32 regionSize = random.Next(1, 1024);
Int32 chunkSize = random.Next(32, 2048);
SnapshotRegion region = new SnapshotRegion(baseAddress.ToIntPtr(), regionSize);
IEnumerable <SnapshotRegion> regions = region.ChunkNormalizedRegion(chunkSize).Select(x => new SnapshotRegion(x)).OrderBy(x => random.Next());
groundTruth.Add(region);
chunkedRegions.AddRange(regions);
baseAddress += (UInt64)regionSize;
}
Snapshot snapshot = new Snapshot(chunkedRegions);
List <SnapshotRegion> reconstructedRegions = snapshot.GetSnapshotRegions().ToList();
Assert.AreEqual(groundTruth.Count, reconstructedRegions.Count);
Int32 originalSize = groundTruth.Sum(x => x.RegionSize);
Int32 newSize = reconstructedRegions.Sum(x => x.RegionSize);
Assert.AreEqual(originalSize, newSize);
}
private ExtractedPointer ExtractRandomPointer(Snapshot snapshot)
{
SnapshotRegion extractedRegion = snapshot.SnapshotRegions[Random.Next(0, snapshot.SnapshotRegions.Length)];
SnapshotElementIndexer extractedElement = extractedRegion[Random.Next(0, extractedRegion.GetElementCount(PointerSize.ToSize()))];
return(this.ExtractPointerFromElement(extractedElement));
}
/// <summary>
/// Called when the scan updates.
/// </summary>
/// <param name="cancellationToken">The cancellation token for handling canceled tasks.</param>
protected override void OnUpdate(CancellationToken cancellationToken)
{
UInt64 processedPointers = 0;
// Create a snapshot only containing the destination
SnapshotRegion destinationRegion = new SnapshotRegion(baseAddress: this.TargetAddress.ToIntPtr(), regionSize: 1);
destinationRegion.Expand(this.PointerRadius);
// Start with the previous level as the destination (as this is a back-tracing algorithm, we work backwards from the destination)
Snapshot previousLevelSnapshot = new Snapshot(destinationRegion);
// Find all pointers that point to the previous level
for (Int32 level = 0; level < this.PointerDepth; level++)
{
Boolean isModuleLevel = level == this.PointerDepth - 1;
PointerPool currentLevelPointers = new PointerPool();
// Iterate all of the heap or module pointers
Parallel.ForEach(
isModuleLevel ? this.ModulePointers : this.HeapPointers,
SettingsViewModel.GetInstance().ParallelSettingsFullCpu,
(pointer) =>
{
// Accept this pointer if it is points to the previous level snapshot
if (previousLevelSnapshot.ContainsAddress(pointer.Value))
{
currentLevelPointers[pointer.Key] = pointer.Value;
}
// Update scan progress
lock (this.ProgressLock)
{
processedPointers++;
// Limit how often we update the progress
if (processedPointers % 10000 == 0)
{
this.UpdateProgress((processedPointers / this.PointerDepth).ToInt32(), this.HeapPointers.Count + this.ModulePointers.Count, canFinalize: false);
}
}
});
// Create a snapshot from this level of pointers
previousLevelSnapshot = currentLevelPointers.ToSnapshot(this.PointerRadius);
// Add the pointer pool to the level structure
if (isModuleLevel)
{
this.LevelPointers.ModulePointerPool = currentLevelPointers;
}
else
{
this.LevelPointers.AddHeapPointerPool(currentLevelPointers);
}
}
// Add the destination pointer pool
this.LevelPointers.DestinationPointerPool = new PointerPool(new KeyValuePair <UInt64, UInt64>(this.TargetAddress, 0));
}
protected override void OnUpdate()
{
ConcurrentDictionary <dynamic, Int64> histogram = new ConcurrentDictionary <dynamic, Int64>();
Int32 processedPages = 0;
lock (this.SnapshotLock)
{
if (this.Snapshot == null)
{
this.Cancel();
return;
}
Parallel.ForEach(
this.Snapshot.Cast <Object>(),
SettingsViewModel.GetInstance().ParallelSettings,
(regionObject) =>
{
SnapshotRegion region = regionObject as SnapshotRegion;
if (((dynamic)region).GetElementLabels() == null || region.GetElementCount() <= 0)
{
return;
}
foreach (SnapshotElementRef element in region)
{
lock (this.ItemLock)
{
if (histogram.ContainsKey(((dynamic)element).GetElementLabel()))
{
histogram[((dynamic)element).GetElementLabel()]++;
}
else
{
histogram.TryAdd(((dynamic)element).GetElementLabel(), 1);
}
}
}
lock (this.ProgressLock)
{
processedPages++;
this.UpdateProgress(processedPages, this.Snapshot.GetRegionCount());
}
//// End foreach element
});
//// End foreach region
}
this.Histogram = new SortedList <dynamic, Int64>(histogram);
this.UpdateHistogram();
this.Cancel();
base.OnUpdate();
}
/// <summary>
/// Gets a random pointer from the pointer collection.
/// </summary>
/// <returns>A random discovered pointer, or null if unable to find one.</returns>
public Pointer GetRandomPointer(Int32 levelIndex)
{
if (levelIndex >= this.Levels.Count || this.Levels[levelIndex].StaticPointers == null)
{
return(null);
}
Snapshot currentSnapshot = this.Levels[levelIndex].StaticPointers;
ExtractedPointer pointer = this.ExtractRandomPointer(currentSnapshot);
UInt64 pointerBase = pointer.BaseAddress;
List <Int32> offsets = new List <Int32>();
foreach (Level level in this.Levels.Take(levelIndex + 1).Reverse())
{
IEnumerable <Int32> shuffledOffsets = Enumerable.Range(-(Int32)this.MaxOffset, (Int32)(this.MaxOffset * 2) + 1).Shuffle();
Boolean found = false;
// Brute force all possible offsets in a random order to find the next path (this guarantees uniform path probabilities)
foreach (Int32 nextRandomOffset in shuffledOffsets)
{
UInt64 newDestination = nextRandomOffset < 0 ? pointer.Destination.Subtract(-nextRandomOffset, wrapAround: false) : pointer.Destination.Add(nextRandomOffset, wrapAround: false);
SnapshotRegion snapshotRegion = level.HeapPointers.SnapshotRegions.Select(x => x).Where(y => newDestination >= y.BaseAddress && newDestination <= y.EndAddress).FirstOrDefault();
if (snapshotRegion != null)
{
// We may have sampled an offset that results in a mis-aligned index, so just randomly take an element from this snapshot rather than using the random offset
SnapshotElementIndexer randomElement = snapshotRegion[Random.Next(0, snapshotRegion.GetElementCount(PointerSize.ToSize()))];
UInt64 baseAddress = randomElement.GetBaseAddress(PointerSize.ToSize());
Int32 alignedOffset = pointer.Destination >= baseAddress ? -((Int32)(pointer.Destination - baseAddress)) : ((Int32)(baseAddress - pointer.Destination));
pointer = this.ExtractPointerFromElement(randomElement);
offsets.Add(alignedOffset);
found = true;
break;
}
}
if (!found)
{
Logger.Log(LogLevel.Error, "Unable to collect a pointer, encountered dead end path");
return(null);
}
}
return(new Pointer(pointerBase, this.PointerSize, offsets.ToArray()));
}
public Snapshot ToSnapshot(UInt64 pointerRadius)
{
Snapshot pointerPoolSnapshot = new Snapshot();
IList <SnapshotRegion> levelRegions = new List <SnapshotRegion>();
foreach (KeyValuePair <UInt64, UInt64> pointer in this)
{
SnapshotRegion levelRegion = new SnapshotRegion(pointer.Key.ToIntPtr(), 1);
levelRegion.Expand(pointerRadius);
levelRegions.Add(levelRegion);
}
pointerPoolSnapshot.AddSnapshotRegions(levelRegions);
return(pointerPoolSnapshot);
}
protected override void OnUpdate()
{
Int32 processedPages = 0;
// Read memory to get current values
this.Snapshot.ReadAllMemory();
Parallel.ForEach(
this.Snapshot.Cast <Object>(),
SettingsViewModel.GetInstance().ParallelSettings,
(Object regionObject) =>
{
SnapshotRegion region = regionObject as SnapshotRegion;
if (!region.CanCompare())
{
return;
}
foreach (SnapshotElementRef element in region)
{
if (element.Changed())
{
((dynamic)element).ElementLabel++;
}
}
lock (this.ProgressLock)
{
processedPages++;
this.UpdateProgress(processedPages, this.Snapshot.GetRegionCount());
}
});
this.UpdateScanCount?.Invoke();
base.OnUpdate();
}
/// <summary>
/// Gradually gathers pointers in the running process.
/// </summary>
private void GatherPointers()
{
Boolean isOpenedProcess32Bit = EngineCore.GetInstance().Processes.IsOpenedProcess32Bit();
dynamic invalidPointerMin = isOpenedProcess32Bit ? (UInt32)UInt16.MaxValue : (UInt64)UInt16.MaxValue;
dynamic invalidPointerMax = isOpenedProcess32Bit ? Int32.MaxValue : Int64.MaxValue;
ConcurrentHashSet <IntPtr> foundPointers = new ConcurrentHashSet <IntPtr>();
// Test for conditions where we set the final found set and take a new snapshot to parse
if (this.CurrentSnapshot == null || this.CurrentSnapshot.GetRegionCount() <= 0 || this.processedCount >= this.CurrentSnapshot.GetRegionCount())
{
this.processedCount = 0;
this.CurrentSnapshot = SnapshotManager.GetInstance().CollectSnapshot(useSettings: true, usePrefilter: false).Clone();
this.FoundPointers = this.ConstructingSet;
this.ConstructingSet = new HashSet <IntPtr>();
}
List <SnapshotRegion> sortedRegions = new List <SnapshotRegion>(this.CurrentSnapshot.GetSnapshotRegions().OrderBy(x => x.GetTimeSinceLastRead()));
// Process the allowed amount of chunks from the priority queue
Parallel.For(
0,
Math.Min(sortedRegions.Count, PointerCollector.RegionLimit),
SettingsViewModel.GetInstance().ParallelSettings,
(index) =>
{
Interlocked.Increment(ref this.processedCount);
SnapshotRegion region = sortedRegions[index];
Boolean success;
// Set to type of a pointer
region.ElementType = EngineCore.GetInstance().Processes.IsOpenedProcess32Bit() ? typeof(UInt32) : typeof(UInt64);
// Enforce 4-byte alignment of pointers
region.Alignment = sizeof(Int32);
// Read current page data for chunk
region.ReadAllRegionMemory(out success);
// Read failed; Deallocated page
if (!success)
{
return;
}
if (region.GetCurrentValues() == null || region.GetCurrentValues().Length <= 0)
{
return;
}
foreach (SnapshotElementRef element in region)
{
// Enforce user mode memory pointers
if (element.LessThanValue(invalidPointerMin) || element.GreaterThanValue(invalidPointerMax))
{
continue;
}
// Enforce 4-byte alignment of destination
if (element.GetCurrentValue() % 4 != 0)
{
continue;
}
IntPtr Value = new IntPtr(element.GetCurrentValue());
// Check if it is possible that this pointer is valid, if so keep it
if (this.CurrentSnapshot.ContainsAddress(Value))
{
foundPointers.Add(Value);
}
}
// Clear the saved values, we do not need them now
region.SetCurrentValues(null);
});
IEnumerable <IntPtr> pointers = foundPointers.ToList();
this.ConstructingSet.UnionWith(pointers);
this.FoundPointers.UnionWith(pointers);
}
protected override void OnUpdate()
{
// Read memory to update previous and current values
this.Snapshot.ReadAllMemory();
Boolean conditionValid = this.IsInputConditionValid(this.Snapshot.GetTimeSinceLastUpdate());
Int32 processedPages = 0;
// Note the duplicated code here is an optimization to minimize comparisons done per iteration
if (conditionValid)
{
Parallel.ForEach(
this.Snapshot.Cast <Object>(),
SettingsViewModel.GetInstance().ParallelSettings,
(regionObject) =>
{
SnapshotRegion region = regionObject as SnapshotRegion;
if (!region.CanCompare())
{
return;
}
foreach (SnapshotElementRef element in region)
{
if (element.Changed())
{
((dynamic)element).ElementLabel++;
}
}
lock (this.ProgressLock)
{
processedPages++;
this.UpdateProgress(processedPages, this.Snapshot.GetRegionCount());
}
});
}
else
{
Parallel.ForEach(
this.Snapshot.Cast <Object>(),
SettingsViewModel.GetInstance().ParallelSettings,
(regionObject) =>
{
SnapshotRegion region = regionObject as SnapshotRegion;
if (!region.CanCompare())
{
return;
}
foreach (SnapshotElementRef element in region)
{
if (element.Changed())
{
((dynamic)element).ElementLabel--;
}
}
lock (this.ProgressLock)
{
processedPages++;
this.UpdateProgress(processedPages, this.Snapshot.GetRegionCount());
}
});
}
this.UpdateScanCount?.Invoke();
base.OnUpdate();
}
/// <summary>
/// Called when the scan updates.
/// </summary>
protected override void OnUpdate()
{
Int32 processedPages = 0;
// Read memory to get current values
Parallel.ForEach(
this.Snapshot.Cast <Object>(),
SettingsViewModel.GetInstance().ParallelSettings,
(regionObject) =>
{
SnapshotRegion region = regionObject as SnapshotRegion;
Boolean readSuccess;
region.ReadAllRegionMemory(out readSuccess, keepValues: true);
if (!readSuccess)
{
region.SetAllValidBits(false);
return;
}
// Ignore region if it requires current & previous values, but we cannot find them
if (this.ScanConstraintManager.HasRelativeConstraint() && !region.CanCompare())
{
region.SetAllValidBits(false);
return;
}
foreach (SnapshotElementRef element in region)
{
// Enforce each value constraint on the element
foreach (ScanConstraint scanConstraint in this.ScanConstraintManager)
{
switch (scanConstraint.Constraint)
{
case ConstraintsEnum.Unchanged:
if (!element.Unchanged())
{
element.SetValid(false);
}
break;
case ConstraintsEnum.Changed:
if (!element.Changed())
{
element.SetValid(false);
}
break;
case ConstraintsEnum.Increased:
if (!element.Increased())
{
element.SetValid(false);
}
break;
case ConstraintsEnum.Decreased:
if (!element.Decreased())
{
element.SetValid(false);
}
break;
case ConstraintsEnum.IncreasedByX:
if (!element.IncreasedByValue(scanConstraint.ConstraintValue))
{
element.SetValid(false);
}
break;
case ConstraintsEnum.DecreasedByX:
if (!element.DecreasedByValue(scanConstraint.ConstraintValue))
{
element.SetValid(false);
}
break;
case ConstraintsEnum.Equal:
if (!element.EqualToValue(scanConstraint.ConstraintValue))
{
element.SetValid(false);
}
break;
case ConstraintsEnum.NotEqual:
if (!element.NotEqualToValue(scanConstraint.ConstraintValue))
{
element.SetValid(false);
}
break;
case ConstraintsEnum.GreaterThan:
if (!element.GreaterThanValue(scanConstraint.ConstraintValue))
{
element.SetValid(false);
}
break;
case ConstraintsEnum.GreaterThanOrEqual:
if (!element.GreaterThanOrEqualToValue(scanConstraint.ConstraintValue))
{
element.SetValid(false);
}
break;
case ConstraintsEnum.LessThan:
if (!element.LessThanValue(scanConstraint.ConstraintValue))
{
element.SetValid(false);
}
break;
case ConstraintsEnum.LessThanOrEqual:
if (!element.LessThanOrEqualToValue(scanConstraint.ConstraintValue))
{
element.SetValid(false);
}
break;
case ConstraintsEnum.NotScientificNotation:
if (element.IsScientificNotation())
{
element.SetValid(false);
}
break;
}
}
//// End foreach Constraint
}
//// End foreach Element
lock (this.ProgressLock)
{
processedPages++;
this.UpdateProgress(processedPages, this.Snapshot.GetRegionCount());
}
});
//// End foreach Region
base.OnUpdate();
}
