public void When_two_branches_with_two_same_children_change_one_and_change_back_next_block()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Set(_keyB, _longLeaf1);
patriciaTree.Set(_keyC, _longLeaf1);
patriciaTree.Set(_keyD, _longLeaf1);
patriciaTree.UpdateRootHash();
patriciaTree.Commit(0);
patriciaTree.Set(_keyA, _longLeaf3);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.UpdateRootHash();
patriciaTree.Commit(1);
memDb.Keys.Should().HaveCount(5);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(_keyA).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(_keyB).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(_keyC).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(_keyD).Should().BeEquivalentTo(_longLeaf1);
}
//Timer function used to get data from Performance Counters
private void OnTimerMonitor()
{
//empty the list to prevent duplicate, and thus failed deletions
_finishedInstances.Clear();
lock (_PruneInstances)
{
foreach (KeyValuePair <int, PruneProcessInstance> entry in _PruneInstances)
{
//try to get data
bool getDataSuccessful = entry.Value.GetData();
//If there was an error or if the process is marked as finished,
// we need to stop monitoring it
if (!getDataSuccessful)
{
_finishedInstances.Add(entry.Key);
}
else
{
}
}
//Loop through the finished instances and remove them from the active instance list
foreach (int i in _finishedInstances)
{
_PruneInstances.Remove(i);
_processIdToWhitelistEntry.Remove(i);
Prune.RemoveEtwCounter(i);
}
}
}
/// <summary>
/// Triggers full pruning.
/// </summary>
/// <returns>Status of triggering full pruning.</returns>
public PruningStatus Trigger()
{
PruningTriggerEventArgs args = new PruningTriggerEventArgs();
Prune?.Invoke(this, args);
return(args.Status);
}
public void Single_leaf_and_keep_for_multiple_dispatches_then_delete()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), new ConstantInterval(4), LimboLogs.Instance);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Commit(0);
patriciaTree.Commit(1);
patriciaTree.Commit(2);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Commit(3);
patriciaTree.Commit(4);
patriciaTree.Set(_keyA, Array.Empty <byte>());
patriciaTree.Commit(5);
patriciaTree.Set(_keyB, _longLeaf2);
patriciaTree.Commit(6);
patriciaTree.Commit(7);
patriciaTree.Commit(8);
patriciaTree.Commit(9);
patriciaTree.Commit(10);
patriciaTree.Commit(11);
patriciaTree.Set(_keyB, Array.Empty <byte>());
patriciaTree.Commit(12);
patriciaTree.Commit(13);
patriciaTree.UpdateRootHash();
// leaf (root)
memDb.Keys.Should().HaveCount(2);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(_keyA).Should().BeNull();
checkTree.Get(_keyB).Should().BeNull();
}
//Timer function used to get data from Performance Counters
private void OnTimerMonitor()
{
//empty the list to prevent duplicate, and thus failed deletions
_finishedInstances.Clear();
lock (_PruneInstances)
{
foreach (KeyValuePair <int, PruneProcessInstance> entry in _PruneInstances)
{
//If the instance has been flagged as finished, add it to a list
if (entry.Value.ProcessFinished)
{
_finishedInstances.Add(entry.Key);
}
else //Otherwise, we can call the get data method
{
entry.Value.GetData();
}
}
//Loop through the finished instances and remove them from the active instance list
foreach (int i in _finishedInstances)
{
_PruneInstances.Remove(i);
_processIdToWhitelistEntry.Remove(i);
Prune.RemoveEtwCounter(i);
}
}
}
public void ThreadProc()
{
int update = 0;
Prune prune = new Prune(0.7, 0.5, PruneIncrementalForm.XOR_INPUT, this.obtainIdeal(), 0.05);
prune.StartIncremental();
while (!prune.Done)
{
prune.PruneIncramental();
update++;
if (update == 10)
{
this.SetText("Cycles:" + prune.Cycles
+ ",Hidden Neurons:" + prune.HiddenNeuronCount
+ ", Current Error=" + prune.Error);
update = 0;
}
}
this.SetText("Best network found:" + prune.HiddenNeuronCount
+ ",error = " + prune.Error);
this.network = prune.CurrentNetwork;
//this.btnRun.Enabled = true;
}
public void Two_branches_exactly_same_leaf_then_one_removed()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(
memDb,
Prune.WhenCacheReaches(1.MB()),
Persist.EveryBlock,
LimboLogs.Instance);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Set(_keyB, _longLeaf1);
patriciaTree.Set(_keyC, _longLeaf1);
patriciaTree.Set(_keyD, _longLeaf1);
patriciaTree.Set(_keyA, Array.Empty <byte>());
patriciaTree.Commit(0);
// leaf (root)
memDb.Keys.Should().HaveCount(6);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(_keyA).Should().BeNull();
checkTree.Get(_keyB).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(_keyC).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(_keyD).Should().BeEquivalentTo(_longLeaf1);
}
public void Extension_branch_extension_and_leaf_then_branch_leaf_leaf()
{
/* R
* E - - - - - - - - - - - - - - -
* B B B B B B B B B B B B B B B B
* E L - - - - - - - - - - - - - -
* E - - - - - - - - - - - - - - -
* B B B B B B B B B B B B B B B B
* L L - - - - - - - - - - - - - - */
byte[] key1 = Bytes.FromHexString("000000100000000aa");
byte[] key2 = Bytes.FromHexString("000000100000000bb");
byte[] key3 = Bytes.FromHexString("000000200000000cc");
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(key1, _longLeaf1);
patriciaTree.Set(key2, _longLeaf1);
patriciaTree.Set(key3, _longLeaf1);
patriciaTree.UpdateRootHash();
patriciaTree.Commit(0);
memDb.Keys.Should().HaveCount(7);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(key1).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(key2).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(key3).Should().BeEquivalentTo(_longLeaf1);
}
private void btnPrune_Click(object sender, EventArgs e)
{
Prune prune = new Prune(this.network, PruneSelectiveForm.XOR_INPUT, this.obtainIdeal(), 0.05);
int count = prune.PruneSelective();
this.network = prune.CurrentNetwork;
this.SetText("Prune removed " + count + " neurons.");
}
private void OnTick(object?sender, EventArgs e)
{
long size = GetDbSize();
if (size >= _threshold)
{
Prune?.Invoke(this, new PruningTriggerEventArgs());
}
}
private void OnTick(object?sender, EventArgs e)
{
string driveName = _fileSystem.Path.GetPathRoot(_fileSystem.Path.GetFullPath(_path));
IDriveInfo drive = _fileSystem.DriveInfo.FromDriveName(driveName);
if (drive.AvailableFreeSpace < _threshold)
{
Prune?.Invoke(this, new PruningTriggerEventArgs());
}
}
public void Connect_extension_with_extension()
{
/* to test this case we need something like this initially */
/* R
* E - - - - - - - - - - - - - - -
* B B B B B B B B B B B B B B B B
* E L - - - - - - - - - - - - - -
* E - - - - - - - - - - - - - - -
* B B B B B B B B B B B B B B B B
* L L - - - - - - - - - - - - - - */
/* then we delete the leaf (marked as X) */
/* R
* B B B B B B B B B B B B B B B B
* E X - - - - - - - - - - - - - -
* E - - - - - - - - - - - - - - -
* B B B B B B B B B B B B B B B B
* L L - - - - - - - - - - - - - - */
/* and we end up with an extended extension replacing what was previously a top-level branch*/
/* R
* E
* E
* E - - - - - - - - - - - - - - -
* B B B B B B B B B B B B B B B B
* L L - - - - - - - - - - - - - - */
byte[] key1 = Bytes.FromHexString("000000100000000aa");
byte[] key2 = Bytes.FromHexString("000000100000000bb");
byte[] key3 = Bytes.FromHexString("000000200000000cc");
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(key1, _longLeaf1);
patriciaTree.Set(key2, _longLeaf1);
patriciaTree.Set(key3, _longLeaf1);
patriciaTree.UpdateRootHash();
patriciaTree.Commit(0);
patriciaTree.Set(key3, Array.Empty <byte>());
patriciaTree.UpdateRootHash();
patriciaTree.Commit(1);
memDb.Keys.Should().HaveCount(8);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(key1).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(key2).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(key3).Should().BeNull();
}
public override int GetHashCode()
{
unchecked
{
int hashCode = base.GetHashCode();
hashCode = (hashCode * 397) ^ DryRun.GetHashCode();
hashCode = (hashCode * 397) ^ (Manifests != null ? Manifests.GetHashCode() : 0);
hashCode = (hashCode * 397) ^ Prune.GetHashCode();
return(hashCode);
}
}
public void Single_leaf()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Commit(0);
// leaf (root)
memDb.Keys.Should().HaveCount(1);
}
//Delete logged files older than N number of Hours/Days.
private void RemoveOldLogs()
{
//Get all of the process directories
string[] directories;
try
{
directories = Directory.GetDirectories(DirectoryPath);
}
catch (Exception e)
{
Prune.HandleError(true, 1, "Error fetching process specific directories for removing pre-existing files\n" + e.Message);
return;
}
//Look through each of the process directories
foreach (string directory in directories)
{
//The logged sub-directory of the process directory
string logdir = directory + "\\logged";
//Get all json files in the process directory
string[] files = Directory.GetFiles(logdir, "*.json");
//Go to the next directory if there are no files
if (files.Length == 0)
{
continue;
}
try
{
//For each file in the process directory
foreach (string file in files)
{
//get the start and end times of the file
string[] split = file.Split('-');
DateTime tempStart = DateTime.ParseExact(split[1], "yyyyMMdd_HHmmss", null);
DateTime tempEnd = DateTime.ParseExact(split[2].Split('.')[0], "yyyyMMdd_HHmmss", null);
//Store the earliest start time and the latest finish time of all json files in this directory
if (DateTime.Compare(tempEnd, DateTime.Now.AddDays(-7)) < 0)
{
File.Delete(file);
}
}
}
catch (Exception e)
{
Prune.HandleError(true, 1, "Error deleting pre-existing files from the earliest time\n" + e.Message);
}
}
}
public void Extension_with_branch_with_two_same_children()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Set(_keyB, _longLeaf1);
patriciaTree.Commit(0);
memDb.Keys.Should().HaveCount(4);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(_keyA).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(_keyB).Should().BeEquivalentTo(_longLeaf1);
}
//Runs when the service shuts down
protected override void OnStop()
{
PruneEvents.PRUNE_EVENT_PROVIDER.EventWriteSERVICE_EXITING_EVENT();
lock (_PruneInstances)
{
//Write whatever is currently in the cache to a file for all processes being monitored
foreach (PruneProcessInstance inst in _PruneInstances.Values)
{
inst.DumpCache();
}
}
Prune.DisposeTraceSession();
}
private void OnTimerWhitelist()
{
//Parse the whitelist and return map containing all new process instances.
foreach (var instEntry in ParseWhitelist())
{
//Add the new entry to the master list and then initialize it
lock (_PruneInstances)
{
_PruneInstances.Add(instEntry.Key, instEntry.Value);
}
Prune.AddEtwCounter(instEntry.Key);
instEntry.Value.InitializeInstance();
}
}
public void Single_leaf_delete_same_block()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), No.Persistence, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Set(_keyA, Array.Empty <byte>());
patriciaTree.Commit(0);
// leaf (root)
memDb.Keys.Should().HaveCount(0);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(_keyA).Should().BeNull();
}
public void Single_leaf_update_same_block()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Set(_keyA, _longLeaf2);
patriciaTree.Commit(0);
// leaf (root)
memDb.Keys.Should().HaveCount(1);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(_keyA).Should().NotBeEquivalentTo(_longLeaf1);
checkTree.Get(_keyA).Should().BeEquivalentTo(_longLeaf2);
}
public void Branch_with_branch_and_leaf()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Set(_keyB, _longLeaf1);
patriciaTree.Set(_keyC, _longLeaf1);
patriciaTree.Commit(0);
// leaf (root)
memDb.Keys.Should().HaveCount(6);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(_keyA).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(_keyB).Should().BeEquivalentTo(_longLeaf1);
checkTree.Get(_keyC).Should().BeEquivalentTo(_longLeaf1);
}
public void When_branch_with_two_different_children_change_one_and_change_back_next_block()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Set(_keyB, _longLeaf2);
patriciaTree.UpdateRootHash();
patriciaTree.Commit(0);
patriciaTree.Set(_keyA, _longLeaf3);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.UpdateRootHash();
patriciaTree.Commit(1);
// extension
// branch
// leaf x 2
memDb.Keys.Should().HaveCount(4);
}
/// <summary>
/// Set up the context.
/// </summary>
private void Setup(Program prog, ProverContext ctx, Implementation impl = null)
{
Program = prog;
// TODO(wuestholz): Is this lock necessary?
lock (Program.TopLevelDeclarations)
{
foreach (Declaration decl in Prune.GetSuccinctDecl(prog, impl))
{
Contract.Assert(decl != null);
var typeDecl = decl as TypeCtorDecl;
var constDecl = decl as Constant;
var funDecl = decl as Function;
var axiomDecl = decl as Axiom;
var glVarDecl = decl as GlobalVariable;
if (typeDecl != null)
{
ctx.DeclareType(typeDecl, null);
}
else if (constDecl != null)
{
ctx.DeclareConstant(constDecl, constDecl.Unique, null);
}
else if (funDecl != null)
{
ctx.DeclareFunction(funDecl, null);
}
else if (axiomDecl != null)
{
ctx.AddAxiom(axiomDecl, null);
}
else if (glVarDecl != null)
{
ctx.DeclareGlobalVariable(glVarDecl, null);
}
}
}
}
public void Branch_with_branch_and_leaf_then_deleted()
{
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.EveryBlock, _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
patriciaTree.Set(_keyA, _longLeaf1);
patriciaTree.Set(_keyB, _longLeaf1);
patriciaTree.Set(_keyC, _longLeaf1);
patriciaTree.Commit(0);
patriciaTree.Set(_keyA, Array.Empty <byte>());
patriciaTree.Set(_keyB, Array.Empty <byte>());
patriciaTree.Set(_keyC, Array.Empty <byte>());
patriciaTree.Commit(1);
patriciaTree.UpdateRootHash();
// leaf (root)
memDb.Keys.Should().HaveCount(6);
PatriciaTree checkTree = CreateCheckTree(memDb, patriciaTree);
checkTree.Get(_keyA).Should().BeNull();
checkTree.Get(_keyB).Should().BeNull();
checkTree.Get(_keyC).Should().BeNull();
}
//Handle any unhandled exceptions
void UnhandedExceptionHandler(object sender, UnhandledExceptionEventArgs e)
{
Prune.HandleError(true, 1, "UNHANDLED EXCEPTION: " + (e.ExceptionObject as Exception).Message);
}
private Dictionary <int, PruneProcessInstance> ParseWhitelist()
{
Dictionary <int, PruneProcessInstance> newInstances = new Dictionary <int, PruneProcessInstance>();
if (File.Exists(WhitelistPath))
{
try
{
//if it does, read in all of it's lines
string[] lines = File.ReadAllLines(WhitelistPath);
//A dictionary so we know which of the currently monitored processes are in the whitelist
Dictionary <string, bool> programFoundInWhitelist = new Dictionary <string, bool>();
//initialize all members of the list to false
foreach (string value in _processIdToWhitelistEntry.Values)
{
if (!programFoundInWhitelist.ContainsKey(value))
{
programFoundInWhitelist.Add(value, false);
}
}
Process[] runningProcesses = Process.GetProcesses(".");
//look at each line of the whitelist
foreach (string line in lines)
{
if (!string.IsNullOrWhiteSpace(line))
{
//remove starting and ending whitespace
string processName = line.Trim();
//the idle and system "processes" should not be monitored, so if they are included they get skipped
if (processName.Equals("0") || processName.Equals("4") ||
processName.Equals("idle", StringComparison.OrdinalIgnoreCase) ||
processName.Equals("system", StringComparison.OrdinalIgnoreCase))
{
PruneEvents.PRUNE_EVENT_PROVIDER.EventWriteDISALLOWED_PROCESS_EVENT();
continue;
}
//if the line isn't a comment
if (processName.ToCharArray()[0] != '#')
{
if (processName.Contains("module=") || processName.Contains("Module="))
{
//Split 'module=' off, then trim white space
string moduleName = processName.Split('.')[0].Trim(); //module=test.dll -> module=test
string moduleFileTemp = processName.Split('=')[1].Trim(); //module=test.dll -> test.dll
string moduleFile = null;
string moduleProcess = null;
if (moduleFileTemp.Contains(","))
{
string[] moduleTempSplit = moduleFileTemp.Split(','); //module=test.dll,svchost -> test.dll,svchost
moduleFile = moduleTempSplit[0].Trim(); //module=test.dll,svchost -> test.dll
moduleProcess = moduleTempSplit[1].Trim(); //module=test.dll,svchost -> svchost
}
else
{
moduleFile = moduleFileTemp;
}
if (_processIdToWhitelistEntry.ContainsValue(moduleName))
{
programFoundInWhitelist[moduleName] = true;
//TODO: Continue here? If it is already in the list, we can skip this
}
foreach (Process proc in runningProcesses)
{
//ensure we don't already monitor this process and that this process is not the system or idle process
if (proc.Id != 4 && proc.Id != 0 && !_processIdToWhitelistEntry.ContainsKey(proc.Id))
{
if (moduleProcess == null || moduleProcess == proc.ProcessName)
{
try {
//collect the modules from for the process
List <Prune.Module> moduleList =
Prune.NativeMethods.CollectModules(proc);
foreach (Prune.Module module in moduleList)
{
//Check if this module is the one we are looking for
if (module.ModuleName.Equals(moduleFile,
StringComparison.OrdinalIgnoreCase))
{
//It is, so add a new instances and log that it was created
newInstances.Add(proc.Id, new PruneProcessInstance(true,
proc.Id,
moduleName, _writeCacheInterval, _logInterval, DirectoryPath));
_processIdToWhitelistEntry.Add(proc.Id, moduleName);
PruneEvents.PRUNE_EVENT_PROVIDER.EventWriteCREATING_INSTANCE_EVENT(moduleName + "_" + proc.Id);
//We don't need to look at the rest of the modules
break;
}
}
} catch (Exception) {
//If we fail to get any modules for some reason, we need to keep going
//We also don't need to report the error, because this will happen any time we try for a protected processes,
// which may be often
continue;
}
}
else
{
}
}
}
}
else
{
if (_processIdToWhitelistEntry.ContainsValue(processName))
{
programFoundInWhitelist[processName] = true;
}
string tempProcName;
bool nameIsId = false;
//If the name is all digits, it is treated as an ID
if (processName.All(char.IsDigit))
{
nameIsId = true;
//Get the process name from the ID to ensure there is a process tied to this ID currently active
tempProcName = Prune.GetProcessNameFromProcessId(int.Parse(processName));
if (tempProcName == null)
{
//Could not find a name for the given process ID.
// Assume the process is not active and skip it.
continue;
}
}
else //Otherwise it is treated as a process name
{
tempProcName = processName;
}
//Get all system process objects that have the specified name
Process[] processes = Prune.GetProcessesFromProcessName(tempProcName);
if (processes == null || processes.Length == 0)
{
//Could not find any processes that have the provided name.
// Assume the process is not started and skip it.
continue;
}
if (nameIsId)
{
int procId = int.Parse(processName);
processName = Prune.GetProcessNameFromProcessId(procId);
if (!_processIdToWhitelistEntry.ContainsKey(procId))
{
//Because an ID was provided, use the ID for the new instance
newInstances.Add(procId,
new PruneProcessInstance(true, procId, processName,
_writeCacheInterval, _logInterval, DirectoryPath));
_processIdToWhitelistEntry.Add(procId, processName);
PruneEvents.PRUNE_EVENT_PROVIDER.EventWriteCREATING_INSTANCE_EVENT(processName + "_" + procId);
}
}
else
{
//We were given a process name, so we create a Prune instance for each process instances
foreach (Process proc in processes)
{
int procId = proc.Id;
if (!_processIdToWhitelistEntry.ContainsKey(procId))
{
newInstances.Add(procId,
new PruneProcessInstance(true, procId, processName,
_writeCacheInterval, _logInterval, DirectoryPath));
_processIdToWhitelistEntry.Add(procId, processName);
PruneEvents.PRUNE_EVENT_PROVIDER.EventWriteCREATING_INSTANCE_EVENT(processName + "_" + procId);
}
}
}
}
}
}
}
//Loop through to see which Prune instances are tied to processes no longer in the whitelist
foreach (string key in programFoundInWhitelist.Keys)
{
if (!programFoundInWhitelist[key])
{
//Make a copy of the list to iterate over so that we can remove elements from the original list
Dictionary <int, string> tempList = new Dictionary <int, string>(_processIdToWhitelistEntry);
lock (_PruneInstances)
{
//loop through the copied list of all elements that we have Prune instances for that are no longer in the whitelist
foreach (KeyValuePair <int, string> entry in tempList)
{
if (entry.Value == key)
{
//call the finish monitoring method to wrap everything up
_PruneInstances[entry.Key].FinishMonitoring();
//Remove the process from the list of processes monitored by ETW
Prune.RemoveEtwCounter(entry.Key);
//remove it form the active Prune instances
_PruneInstances.Remove(entry.Key);
//remove it from the active whitelist entries
_processIdToWhitelistEntry.Remove(entry.Key);
}
}
}
}
}
}
catch (Exception e)
{
Prune.HandleError(true, 1, "Error while reading and parsing the whitelist file\n" + e.Message);
}
}
else
{
try
{
//If the whitelist file does not exist, create a blank text file for future use
File.CreateText(WhitelistPath);
PruneEvents.PRUNE_EVENT_PROVIDER.EventWriteNO_WHITELIST_EVENT();
}
catch (Exception e)
{
Prune.HandleError(true, 1, "Error creating whitelist file\n" + e.Message);
}
}
return(newInstances);
}
public void Fuzz_accounts_with_reorganizations(
int accountsCount,
int blocksCount,
int uniqueValuesCount,
int lookupLimit,
int?seed)
{
int usedSeed = seed ?? _random.Next(int.MaxValue);
_random = new Random(usedSeed);
_logger.Info($"RANDOM SEED {usedSeed}");
string fileName = Path.GetTempFileName();
//string fileName = "C:\\Temp\\fuzz.txt";
_logger.Info(
$"Fuzzing with accounts: {accountsCount}, " +
$"blocks {blocksCount}, " +
$"values: {uniqueValuesCount}, " +
$"lookup: {lookupLimit} into file {fileName}");
using FileStream fileStream = new FileStream(fileName, FileMode.Create);
using StreamWriter streamWriter = new StreamWriter(fileStream);
Queue <Keccak> rootQueue = new Queue <Keccak>();
Stack <Keccak> rootStack = new Stack <Keccak>();
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.IfBlockOlderThan(lookupLimit), _logManager);
PatriciaTree patriciaTree = new PatriciaTree(trieStore, _logManager);
byte[][] accounts = new byte[accountsCount][];
byte[][] randomValues = new byte[uniqueValuesCount][];
for (int i = 0; i < randomValues.Length; i++)
{
bool isEmptyValue = _random.Next(0, 2) == 0;
if (isEmptyValue)
{
randomValues[i] = Array.Empty <byte>();
}
else
{
randomValues[i] = GenerateRandomAccountRlp();
}
}
for (int accountIndex = 0; accountIndex < accounts.Length; accountIndex++)
{
byte[] key = new byte[32];
((UInt256)accountIndex).ToBigEndian(key);
accounts[accountIndex] = key;
}
int blockCount = 0;
for (int blockNumber = 0; blockNumber < blocksCount; blockNumber++)
{
int reorgDepth = _random.Next(Math.Min(5, blockCount));
_logger.Debug($"Reorganizing {reorgDepth}");
for (int i = 0; i < reorgDepth; i++)
{
try
{
// no longer need undo?
// trieStore.UndoOneBlock();
}
catch (InvalidOperationException)
{
// if memory limit hits in
blockCount = 0;
}
rootStack.Pop();
patriciaTree.RootHash = rootStack.Peek();
}
blockCount = Math.Max(0, blockCount - reorgDepth);
_logger.Debug($"Setting block count to {blockCount}");
bool isEmptyBlock = _random.Next(5) == 0;
if (!isEmptyBlock)
{
for (int i = 0; i < Math.Max(1, accountsCount / 8); i++)
{
int randomAccountIndex = _random.Next(accounts.Length);
int randomValueIndex = _random.Next(randomValues.Length);
byte[] account = accounts[randomAccountIndex];
byte[] value = randomValues[randomValueIndex];
streamWriter.WriteLine(
$"Block {blockCount} - setting {account.ToHexString()} = {value.ToHexString()}");
patriciaTree.Set(account, value);
}
}
streamWriter.WriteLine(
$"Commit block {blockCount} | empty: {isEmptyBlock}");
patriciaTree.UpdateRootHash();
patriciaTree.Commit(blockCount);
rootQueue.Enqueue(patriciaTree.RootHash);
rootStack.Push(patriciaTree.RootHash);
blockCount++;
_logger.Debug($"Setting block count to {blockCount}");
}
streamWriter.Flush();
fileStream.Seek(0, SeekOrigin.Begin);
streamWriter.WriteLine($"DB size: {memDb.Keys.Count}");
_logger.Info($"DB size: {memDb.Keys.Count}");
int verifiedBlocks = 0;
rootQueue.Clear();
Stack <Keccak> stackCopy = new Stack <Keccak>();
while (rootStack.Any())
{
stackCopy.Push(rootStack.Pop());
}
rootStack = stackCopy;
while (rootStack.TryPop(out Keccak currentRoot))
{
try
{
patriciaTree.RootHash = currentRoot;
for (int i = 0; i < accounts.Length; i++)
{
patriciaTree.Get(accounts[i]);
}
_logger.Info($"Verified positive {verifiedBlocks}");
}
catch (Exception ex)
{
if (verifiedBlocks % lookupLimit == 0)
{
throw new InvalidDataException(ex.ToString());
}
else
{
_logger.Info($"Verified negative {verifiedBlocks} (which is ok on block {verifiedBlocks})");
}
}
verifiedBlocks++;
}
}
// [TestCase(4, 16, 4, 4)]
public void Fuzz_accounts(
int accountsCount,
int blocksCount,
int uniqueValuesCount,
int lookupLimit)
{
string fileName = Path.GetTempFileName();
//string fileName = "C:\\Temp\\fuzz.txt";
_logger.Info(
$"Fuzzing with accounts: {accountsCount}, " +
$"blocks {blocksCount}, " +
$"values: {uniqueValuesCount}, " +
$"lookup: {lookupLimit} into file {fileName}");
using FileStream fileStream = new FileStream(fileName, FileMode.Create);
using StreamWriter streamWriter = new StreamWriter(fileStream);
Queue <Keccak> rootQueue = new Queue <Keccak>();
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.IfBlockOlderThan(lookupLimit), _logManager);
StateTree patriciaTree = new StateTree(trieStore, _logManager);
byte[][] accounts = new byte[accountsCount][];
byte[][] randomValues = new byte[uniqueValuesCount][];
for (int i = 0; i < randomValues.Length; i++)
{
bool isEmptyValue = _random.Next(0, 2) == 0;
if (isEmptyValue)
{
randomValues[i] = Array.Empty <byte>();
}
else
{
randomValues[i] = GenerateRandomAccountRlp();
}
}
for (int accountIndex = 0; accountIndex < accounts.Length; accountIndex++)
{
byte[] key = new byte[32];
((UInt256)accountIndex).ToBigEndian(key);
accounts[accountIndex] = key;
}
for (int blockNumber = 0; blockNumber < blocksCount; blockNumber++)
{
bool isEmptyBlock = _random.Next(5) == 0;
if (!isEmptyBlock)
{
for (int i = 0; i < Math.Max(1, accountsCount / 8); i++)
{
int randomAccountIndex = _random.Next(accounts.Length);
int randomValueIndex = _random.Next(randomValues.Length);
byte[] account = accounts[randomAccountIndex];
byte[] value = randomValues[randomValueIndex];
streamWriter.WriteLine(
$"Block {blockNumber} - setting {account.ToHexString()} = {value.ToHexString()}");
patriciaTree.Set(account, value);
}
}
streamWriter.WriteLine(
$"Commit block {blockNumber} | empty: {isEmptyBlock}");
patriciaTree.UpdateRootHash();
patriciaTree.Commit(blockNumber);
rootQueue.Enqueue(patriciaTree.RootHash);
}
streamWriter.Flush();
fileStream.Seek(0, SeekOrigin.Begin);
streamWriter.WriteLine($"DB size: {memDb.Keys.Count}");
_logger.Info($"DB size: {memDb.Keys.Count}");
int verifiedBlocks = 0;
while (rootQueue.TryDequeue(out Keccak currentRoot))
{
try
{
patriciaTree.RootHash = currentRoot;
for (int i = 0; i < accounts.Length; i++)
{
patriciaTree.Get(accounts[i]);
}
_logger.Info($"Verified positive {verifiedBlocks}");
}
catch (Exception ex)
{
if (verifiedBlocks % lookupLimit == 0)
{
throw new InvalidDataException(ex.ToString());
}
else
{
_logger.Info($"Verified negative {verifiedBlocks}");
}
}
verifiedBlocks++;
}
}
protected override void OnStart(string[] args)
{
//register our unhandled exception handler
AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(UnhandedExceptionHandler);
//Log that the service is starting
bool returnVal = PruneEvents.PRUNE_EVENT_PROVIDER.EventWriteSERVICE_STARTING_EVENT();
//create the ProgramData directory if it does not already exist
Directory.CreateDirectory(DirectoryPath);
//Read the config file and get it's values
ReadConfigFile();
lock (_PruneInstances)
{
//Parse the whitelist and add all of the Prune process instances to the master list, and to the ETW list
Dictionary <int, PruneProcessInstance> list = ParseWhitelist();
foreach (var instEntry in list)
{
_PruneInstances.Add(instEntry.Key, instEntry.Value);
Prune.AddEtwCounter(instEntry.Key);
}
}
//initialize each of the Prune instances
try
{
lock (_PruneInstances)
{
foreach (PruneProcessInstance inst in _PruneInstances.Values)
{
inst.InitializeInstance();
}
}
}
catch (Exception e)
{
Prune.HandleError(true, 1, "Error initializing Prune instances\n" + e.Message + "\n" + e.Source);
}
//Start the etw sesstion
Prune.StartEtwSession(true);
try
{
//Check for old, unlogged files as the service starts
ReadOldFiles();
}
catch (Exception e)
{
Prune.HandleError(true, 1, "Error reading old files\n" + e.Message);
}
//create the timer for gathering data from the performance counters
try
{
_monitorTimer.Elapsed += (sender, e) => { OnTimerMonitor(); };
_monitorTimer.Start();
}
catch (Exception e)
{
Prune.HandleError(true, 1, "Error starting the process monitoring timer\n" + e.Message);
return;
}
//Create and start the timer to monitor the whitelist file
try
{
_whitelistTimer.Elapsed += (sender, e) => { OnTimerWhitelist(); };
_whitelistTimer.Start();
}
catch (Exception e)
{
Prune.HandleError(true, 1, "Error starting the whitelist check timer\n" + e.Message);
//cleanup already running timers before exiting
_monitorTimer.Stop();
return;
}
//Create and start the timer to monitor the config file
try
{
_configTimer.Elapsed += (sender, e) => { ReadConfigFile(); };
_configTimer.Start();
}
catch (Exception e)
{
Prune.HandleError(true, 1, "Error starting the configuration file check timer\n" + e.Message);
//cleanup already running timers before exiting
_monitorTimer.Stop();
_whitelistTimer.Stop();
}
}
// [TestCase(8, 32, 8, 1543322391)]
public void Fuzz_accounts_with_storage(
int accountsCount,
int blocksCount,
int lookupLimit,
int?seed)
{
int usedSeed = seed ?? _random.Next(int.MaxValue);
_random = new Random(usedSeed);
_logger.Info($"RANDOM SEED {usedSeed}");
string fileName = Path.GetTempFileName();
//string fileName = "C:\\Temp\\fuzz.txt";
_logger.Info(
$"Fuzzing with accounts: {accountsCount}, " +
$"blocks {blocksCount}, " +
$"lookup: {lookupLimit} into file {fileName}");
using FileStream fileStream = new FileStream(fileName, FileMode.Create);
using StreamWriter streamWriter = new StreamWriter(fileStream);
Queue <Keccak> rootQueue = new Queue <Keccak>();
MemDb memDb = new MemDb();
TrieStore trieStore = new TrieStore(memDb, Prune.WhenCacheReaches(1.MB()), Persist.IfBlockOlderThan(lookupLimit), _logManager);
StateProvider stateProvider = new StateProvider(trieStore, new MemDb(), _logManager);
StorageProvider storageProvider = new StorageProvider(trieStore, stateProvider, _logManager);
Account[] accounts = new Account[accountsCount];
Address[] addresses = new Address[accountsCount];
for (int i = 0; i < accounts.Length; i++)
{
bool isEmptyValue = _random.Next(0, 2) == 0;
if (isEmptyValue)
{
accounts[i] = Account.TotallyEmpty;
}
else
{
accounts[i] = GenerateRandomAccount();
}
addresses[i] = TestItem.GetRandomAddress(_random);
}
for (int blockNumber = 0; blockNumber < blocksCount; blockNumber++)
{
bool isEmptyBlock = _random.Next(5) == 0;
if (!isEmptyBlock)
{
for (int i = 0; i < Math.Max(1, accountsCount / 8); i++)
{
int randomAddressIndex = _random.Next(addresses.Length);
int randomAccountIndex = _random.Next(accounts.Length);
Address address = addresses[randomAddressIndex];
Account account = accounts[randomAccountIndex];
if (stateProvider.AccountExists(address))
{
Account existing = stateProvider.GetAccount(address);
if (existing.Balance != account.Balance)
{
if (account.Balance > existing.Balance)
{
stateProvider.AddToBalance(
address, account.Balance - existing.Balance, MuirGlacier.Instance);
}
else
{
stateProvider.SubtractFromBalance(
address, existing.Balance - account.Balance, MuirGlacier.Instance);
}
stateProvider.IncrementNonce(address);
}
byte[] storage = new byte[1];
_random.NextBytes(storage);
storageProvider.Set(new StorageCell(address, 1), storage);
}
else if (!account.IsTotallyEmpty)
{
stateProvider.CreateAccount(address, account.Balance);
byte[] storage = new byte[1];
_random.NextBytes(storage);
storageProvider.Set(new StorageCell(address, 1), storage);
}
}
}
streamWriter.WriteLine(
$"Commit block {blockNumber} | empty: {isEmptyBlock}");
storageProvider.Commit();
stateProvider.Commit(MuirGlacier.Instance);
storageProvider.CommitTrees(blockNumber);
stateProvider.CommitTree(blockNumber);
rootQueue.Enqueue(stateProvider.StateRoot);
}
streamWriter.Flush();
fileStream.Seek(0, SeekOrigin.Begin);
streamWriter.WriteLine($"DB size: {memDb.Keys.Count}");
_logger.Info($"DB size: {memDb.Keys.Count}");
int verifiedBlocks = 0;
while (rootQueue.TryDequeue(out Keccak currentRoot))
{
try
{
stateProvider.StateRoot = currentRoot;
for (int i = 0; i < addresses.Length; i++)
{
Account account = stateProvider.GetAccount(addresses[i]);
if (account != null)
{
for (int j = 0; j < 256; j++)
{
storageProvider.Get(new StorageCell(addresses[i], (UInt256)j));
}
}
}
_logger.Info($"Verified positive {verifiedBlocks}");
}
catch (Exception ex)
{
if (verifiedBlocks % lookupLimit == 0)
{
throw new InvalidDataException(ex.ToString());
}
else
{
_logger.Info($"Verified negative {verifiedBlocks} which is ok here");
}
}
verifiedBlocks++;
}
}
