static void Main(string[] args)
{
Console.CancelKeyPress += Console_CancelKeyPress;
if (args.Length < 1)
{
Console.WriteLine("Usage: RunParServer.exe file.xml");
return;
}
var force = true;
var boogie = false;
var debug = false;
var test = false;
if (args.Any(s => s == "/break"))
{
System.Diagnostics.Debugger.Launch();
}
if (args.Any(s => s == "/debug"))
{
debug = true;
}
if (args.Any(s => s == "/noforce"))
{
force = false;
}
if (args.Any(s => s == "/resume"))
{
force = false;
resume = true;
}
if (args.Any(s => s == "/test"))
{
test = true;
}
if (args.Any(s => s == "/boogie"))
{
boogie = true;
}
// Read in the config
config = RunParConfig.DeSerialize(args[0]);
config.Dump();
log = new StreamWriter("server.log");
// Check install
Console.WriteLine("Checking self installation");
try
{
Installer.CheckSelfInstall(config);
}
catch (Exception e)
{
Console.WriteLine("{0}", e.Message);
return;
}
Console.WriteLine("Done");
// Find the files and their arguments
var bpls = new List <Tuple <string, string> >();
var filemaps = new List <string>();
Util.ReadFiles(config, out bpls, out filemaps);
Console.WriteLine("Found {0} files", bpls.Count);
var machineToFiles = new Dictionary <int, List <string> >();
// Split across machines
var M = config.RemoteRoots.Length + 1;
for (int i = 0; i < M; i++)
{
machineToFiles.Add(i, new List <string>());
}
for (int i = 0; i < bpls.Count; i++)
{
machineToFiles[i % M].Add(bpls[i].Item1);
}
// Do remote installation
Console.WriteLine("Doing remote installation");
var rm = 1;
foreach (var r in config.RemoteRoots)
{
Console.WriteLine("Installing {0} files on {1}", machineToFiles[rm].Count, r.value);
if (machineToFiles[rm].Count != 0)
{
Installer.RemoteInstall(config.root, r.value, config.Utils.Select(u => u.dir).Distinct(), machineToFiles[rm], filemaps, boogie, force);
}
rm++;
}
Console.WriteLine("Done");
if (debug)
{
Util.debugOutput = true;
}
var threads = new List <Thread>();
var workers = new List <Worker>();
var starttime = DateTime.Now;
Console.WriteLine("Spawning clients");
// spawn client on own machine
config.DumpClientConfig(config.root, machineToFiles[0].Select(f => Util.GetFileName(f, config.root)), System.IO.Path.Combine(config.root, "config-client.xml"));
var w0 = new Worker(false, config.root, resume ? "/resume" : "", test ? "/test" : "");
workers.Add(w0);
threads.Add(new Thread(new ThreadStart(w0.Run)));
// spawn clients on remote machines
rm = 0;
foreach (var r in config.RemoteRoots)
{
rm++;
if (machineToFiles[rm].Count == 0)
{
continue;
}
config.DumpClientConfig(Util.GetRemoteFolder(r.value), machineToFiles[rm].Select(f => Util.GetFileName(f, config.root)), System.IO.Path.Combine(r.value, "config-client.xml"));
var w1 = new Worker(true, r.value, resume ? "/resume" : "", test ? "/test" : "");
threads.Add(new Thread(new ThreadStart(w1.Run)));
workers.Add(w1);
}
// start threads
threads.ForEach(t => t.Start());
threads.ForEach(t => t.Join());
Console.WriteLine("Time taken = {0} seconds", (DateTime.Now - starttime).TotalSeconds.ToString("F2"));
Console.WriteLine("Collating results");
// put the results together
foreach (var u in config.Utils)
{
MergeResults(u, workers, u.name + ".db");
}
// text result
var output = new System.IO.StreamWriter(System.IO.Path.Combine(config.root, "results.txt"));
// file -> util -> output
var resdict = new Dictionary <string, Dictionary <string, string> >();
var defaultDelim = 0;
foreach (var w in workers)
{
foreach (var line in w.textresult)
{
var tokens = line.Split(new char[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries);
if (tokens.Length < 2)
{
continue;
}
if (!resdict.ContainsKey(tokens[1]))
{
resdict.Add(tokens[1], new Dictionary <string, string>());
}
var result = "";
for (int i = 2; i < tokens.Length; i++)
{
result += tokens[i] + "\t";
}
defaultDelim = (tokens.Length - 2); // Ideally, this should be the most frequest distribution of this number
resdict[tokens[1]][tokens[0]] = result;
}
}
var defaultStr = "";
for (int i = 0; i < defaultDelim; i++)
{
defaultStr += "XXX\t";
}
foreach (var file in bpls)
{
var key = Util.GetFileName(file.Item1, config.root);
if (!resdict.ContainsKey(key))
{
continue;
}
output.Write("{0}\t", key);
foreach (var u in config.Utils)
{
if (!resdict[key].ContainsKey(u.name))
{
output.Write("{0}", defaultStr);
}
else
{
output.Write("{0}", resdict[key][u.name]);
}
}
output.WriteLine();
}
output.Close();
log.Close();
Console.WriteLine("Done");
}
static void Main(string[] args)
{
Console.CancelKeyPress += Console_CancelKeyPress;
if (args.Length < 1)
{
Console.WriteLine("Usage: RunParClient.exe rootfolder");
return;
}
var test = false;
var root = args[0];
if (args.Any(a => a == "/break"))
{
System.Diagnostics.Debugger.Launch();
}
if (args.Any(a => a == "/resume"))
{
resume = true;
}
if (args.Any(a => a == "/test"))
{
test = true;
}
// Get the config
var configfile = Path.Combine(root, "config-client.xml");
if (!File.Exists(configfile))
{
Console.WriteLine("Config file doesn't exist");
return;
}
config = RunParConfig.DeSerialize(configfile);
log = new StreamWriter("client.log", resume);
log.AutoFlush = true;
// Are we already running?
var procs =
System.Diagnostics.Process.GetProcessesByName(System.Diagnostics.Process.GetCurrentProcess().ProcessName);
if (procs.Count() > 1)
{
log.WriteLine("Detected another instance of RunParClient running on the machine, aborting");
log.Close();
return;
}
var workcompleted = new Dictionary <string, string>();
if (resume)
{
log.WriteLine("Resuming work from where we left off!");
// Find the work that we've completed already
// Format:
// workitem1
// result1
// workitem2
// result2
// ...
if (File.Exists("client.work.log"))
{
var lines = new List <string>(File.ReadAllLines("client.work.log"));
lines.RemoveAll(s => s == "" || s == " ");
for (int i = 0; i < lines.Count - 1; i += 2)
{
workcompleted[lines[i]] = lines[i + 1];
}
}
}
worklog = new StreamWriter("client.work.log", resume);
// Get the files
var bpls = new List <Tuple <string, string> >();
var filemaps = new List <string>();
Util.ReadFiles(config, out bpls, out filemaps);
// load up the work
foreach (var u in config.Utils)
{
for (int i = 0; i < bpls.Count; i++)
{
var key = string.Format("{0}\t{1}", u.name, Util.GetFileName(bpls[i].Item1, config.root));
var wi = new WorkItem(key, u.name, Path.Combine(config.root, u.value), bpls[i].Item1 + " " + u.arguments + " " + bpls[i].Item2);
if (workcompleted.ContainsKey(wi.ToString()))
{
fileToResult.Add(key, workcompleted[wi.ToString()]);
continue;
}
work.Enqueue(wi);
fileToResult.Add(key, null);
}
}
log.WriteLine("Got {0} workitems", work.Count);
#region Debugging
if (test)
{
// Drop half the workitems -- for debugging
var q = new System.Collections.Concurrent.ConcurrentQueue <WorkItem>();
var n = (work.Count / 2) + 1;
if (n > work.Count)
{
n = work.Count;
}
for (int i = 0; i < n; i++)
{
WorkItem wi;
work.TryDequeue(out wi);
q.Enqueue(wi);
}
work = q;
}
#endregion
resultsdir = Path.Combine(Environment.CurrentDirectory, "RunParResults");
// delete the results directory, if one exists
if (!resume && Directory.Exists(resultsdir))
{
Directory.Delete(resultsdir, true);
}
// create a results directory for each util
foreach (var util in config.Utils)
{
var ud = Path.Combine(resultsdir, util.name);
if (!resume && Directory.Exists(ud))
{
Util.CleanDirectory(ud);
}
else
{
Directory.CreateDirectory(ud);
}
}
log.WriteLine("Directories are in place");
var threads = new List <Thread>();
var numthreads = config.MaxThreads == 0 ? Environment.ProcessorCount - 1 : config.MaxThreads;
for (int i = 0; i < numthreads; i++)
{
var worker = new Worker(i);
threads.Add(new Thread(new ThreadStart(worker.Run)));
}
var outputThread = new Thread(new ThreadStart(DumpResults));
// start threads
threads.ForEach(t => t.Start());
outputThread.Start();
threads.ForEach(t => t.Join());
outputThread.Interrupt();
outputThread.Join();
// put results together
foreach (var util in config.Utils)
{
var ud = Path.Combine(resultsdir, util.name);
if (!Directory.Exists(ud))
{
continue;
}
log.WriteLine("Putting together results for {0}", util.name);
Util.run(ud, Path.Combine(config.root, util.value),
util.arguments + " " + GlobalConfig.merge_flag + " " + ud);
if (File.Exists(Path.Combine(ud, GlobalConfig.util_result_file)))
{
File.Move(Path.Combine(ud, GlobalConfig.util_result_file),
Path.Combine(Environment.CurrentDirectory, util.name + ".db"));
}
else
{
log.WriteLine("Merge failed!");
}
}
worklog.Close();
log.WriteLine("Parallel computation finished");
log.Close();
}
