static void Main(string[] args) { Console.CancelKeyPress += Console_CancelKeyPress; if (args.Length < 1) { Console.WriteLine("Usage: RunParServer.exe file.xml"); return; } var force = true; var boogie = false; var debug = false; var test = false; if (args.Any(s => s == "/break")) { System.Diagnostics.Debugger.Launch(); } if (args.Any(s => s == "/debug")) { debug = true; } if (args.Any(s => s == "/noforce")) { force = false; } if (args.Any(s => s == "/resume")) { force = false; resume = true; } if (args.Any(s => s == "/test")) { test = true; } if (args.Any(s => s == "/boogie")) { boogie = true; } // Read in the config config = RunParConfig.DeSerialize(args[0]); config.Dump(); log = new StreamWriter("server.log"); // Check install Console.WriteLine("Checking self installation"); try { Installer.CheckSelfInstall(config); } catch (Exception e) { Console.WriteLine("{0}", e.Message); return; } Console.WriteLine("Done"); // Find the files and their arguments var bpls = new List <Tuple <string, string> >(); var filemaps = new List <string>(); Util.ReadFiles(config, out bpls, out filemaps); Console.WriteLine("Found {0} files", bpls.Count); var machineToFiles = new Dictionary <int, List <string> >(); // Split across machines var M = config.RemoteRoots.Length + 1; for (int i = 0; i < M; i++) { machineToFiles.Add(i, new List <string>()); } for (int i = 0; i < bpls.Count; i++) { machineToFiles[i % M].Add(bpls[i].Item1); } // Do remote installation Console.WriteLine("Doing remote installation"); var rm = 1; foreach (var r in config.RemoteRoots) { Console.WriteLine("Installing {0} files on {1}", machineToFiles[rm].Count, r.value); if (machineToFiles[rm].Count != 0) { Installer.RemoteInstall(config.root, r.value, config.Utils.Select(u => u.dir).Distinct(), machineToFiles[rm], filemaps, boogie, force); } rm++; } Console.WriteLine("Done"); if (debug) { Util.debugOutput = true; } var threads = new List <Thread>(); var workers = new List <Worker>(); var starttime = DateTime.Now; Console.WriteLine("Spawning clients"); // spawn client on own machine config.DumpClientConfig(config.root, machineToFiles[0].Select(f => Util.GetFileName(f, config.root)), System.IO.Path.Combine(config.root, "config-client.xml")); var w0 = new Worker(false, config.root, resume ? "/resume" : "", test ? "/test" : ""); workers.Add(w0); threads.Add(new Thread(new ThreadStart(w0.Run))); // spawn clients on remote machines rm = 0; foreach (var r in config.RemoteRoots) { rm++; if (machineToFiles[rm].Count == 0) { continue; } config.DumpClientConfig(Util.GetRemoteFolder(r.value), machineToFiles[rm].Select(f => Util.GetFileName(f, config.root)), System.IO.Path.Combine(r.value, "config-client.xml")); var w1 = new Worker(true, r.value, resume ? "/resume" : "", test ? "/test" : ""); threads.Add(new Thread(new ThreadStart(w1.Run))); workers.Add(w1); } // start threads threads.ForEach(t => t.Start()); threads.ForEach(t => t.Join()); Console.WriteLine("Time taken = {0} seconds", (DateTime.Now - starttime).TotalSeconds.ToString("F2")); Console.WriteLine("Collating results"); // put the results together foreach (var u in config.Utils) { MergeResults(u, workers, u.name + ".db"); } // text result var output = new System.IO.StreamWriter(System.IO.Path.Combine(config.root, "results.txt")); // file -> util -> output var resdict = new Dictionary <string, Dictionary <string, string> >(); var defaultDelim = 0; foreach (var w in workers) { foreach (var line in w.textresult) { var tokens = line.Split(new char[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries); if (tokens.Length < 2) { continue; } if (!resdict.ContainsKey(tokens[1])) { resdict.Add(tokens[1], new Dictionary <string, string>()); } var result = ""; for (int i = 2; i < tokens.Length; i++) { result += tokens[i] + "\t"; } defaultDelim = (tokens.Length - 2); // Ideally, this should be the most frequest distribution of this number resdict[tokens[1]][tokens[0]] = result; } } var defaultStr = ""; for (int i = 0; i < defaultDelim; i++) { defaultStr += "XXX\t"; } foreach (var file in bpls) { var key = Util.GetFileName(file.Item1, config.root); if (!resdict.ContainsKey(key)) { continue; } output.Write("{0}\t", key); foreach (var u in config.Utils) { if (!resdict[key].ContainsKey(u.name)) { output.Write("{0}", defaultStr); } else { output.Write("{0}", resdict[key][u.name]); } } output.WriteLine(); } output.Close(); log.Close(); Console.WriteLine("Done"); }
static void Main(string[] args) { Console.CancelKeyPress += Console_CancelKeyPress; if (args.Length < 1) { Console.WriteLine("Usage: RunParClient.exe rootfolder"); return; } var test = false; var root = args[0]; if (args.Any(a => a == "/break")) { System.Diagnostics.Debugger.Launch(); } if (args.Any(a => a == "/resume")) { resume = true; } if (args.Any(a => a == "/test")) { test = true; } // Get the config var configfile = Path.Combine(root, "config-client.xml"); if (!File.Exists(configfile)) { Console.WriteLine("Config file doesn't exist"); return; } config = RunParConfig.DeSerialize(configfile); log = new StreamWriter("client.log", resume); log.AutoFlush = true; // Are we already running? var procs = System.Diagnostics.Process.GetProcessesByName(System.Diagnostics.Process.GetCurrentProcess().ProcessName); if (procs.Count() > 1) { log.WriteLine("Detected another instance of RunParClient running on the machine, aborting"); log.Close(); return; } var workcompleted = new Dictionary <string, string>(); if (resume) { log.WriteLine("Resuming work from where we left off!"); // Find the work that we've completed already // Format: // workitem1 // result1 // workitem2 // result2 // ... if (File.Exists("client.work.log")) { var lines = new List <string>(File.ReadAllLines("client.work.log")); lines.RemoveAll(s => s == "" || s == " "); for (int i = 0; i < lines.Count - 1; i += 2) { workcompleted[lines[i]] = lines[i + 1]; } } } worklog = new StreamWriter("client.work.log", resume); // Get the files var bpls = new List <Tuple <string, string> >(); var filemaps = new List <string>(); Util.ReadFiles(config, out bpls, out filemaps); // load up the work foreach (var u in config.Utils) { for (int i = 0; i < bpls.Count; i++) { var key = string.Format("{0}\t{1}", u.name, Util.GetFileName(bpls[i].Item1, config.root)); var wi = new WorkItem(key, u.name, Path.Combine(config.root, u.value), bpls[i].Item1 + " " + u.arguments + " " + bpls[i].Item2); if (workcompleted.ContainsKey(wi.ToString())) { fileToResult.Add(key, workcompleted[wi.ToString()]); continue; } work.Enqueue(wi); fileToResult.Add(key, null); } } log.WriteLine("Got {0} workitems", work.Count); #region Debugging if (test) { // Drop half the workitems -- for debugging var q = new System.Collections.Concurrent.ConcurrentQueue <WorkItem>(); var n = (work.Count / 2) + 1; if (n > work.Count) { n = work.Count; } for (int i = 0; i < n; i++) { WorkItem wi; work.TryDequeue(out wi); q.Enqueue(wi); } work = q; } #endregion resultsdir = Path.Combine(Environment.CurrentDirectory, "RunParResults"); // delete the results directory, if one exists if (!resume && Directory.Exists(resultsdir)) { Directory.Delete(resultsdir, true); } // create a results directory for each util foreach (var util in config.Utils) { var ud = Path.Combine(resultsdir, util.name); if (!resume && Directory.Exists(ud)) { Util.CleanDirectory(ud); } else { Directory.CreateDirectory(ud); } } log.WriteLine("Directories are in place"); var threads = new List <Thread>(); var numthreads = config.MaxThreads == 0 ? Environment.ProcessorCount - 1 : config.MaxThreads; for (int i = 0; i < numthreads; i++) { var worker = new Worker(i); threads.Add(new Thread(new ThreadStart(worker.Run))); } var outputThread = new Thread(new ThreadStart(DumpResults)); // start threads threads.ForEach(t => t.Start()); outputThread.Start(); threads.ForEach(t => t.Join()); outputThread.Interrupt(); outputThread.Join(); // put results together foreach (var util in config.Utils) { var ud = Path.Combine(resultsdir, util.name); if (!Directory.Exists(ud)) { continue; } log.WriteLine("Putting together results for {0}", util.name); Util.run(ud, Path.Combine(config.root, util.value), util.arguments + " " + GlobalConfig.merge_flag + " " + ud); if (File.Exists(Path.Combine(ud, GlobalConfig.util_result_file))) { File.Move(Path.Combine(ud, GlobalConfig.util_result_file), Path.Combine(Environment.CurrentDirectory, util.name + ".db")); } else { log.WriteLine("Merge failed!"); } } worklog.Close(); log.WriteLine("Parallel computation finished"); log.Close(); }