internal object syncExec(Job job, object obj)
{
if (Thread.CurrentThread.IsThreadPoolThread)
return job.run(obj);
else
return asyncExec(job, obj).waitFor();
}
internal Future asyncExec(Job job, object obj)
{
Future future = new Future();
Runner runner = new Runner(job, obj, future);
lock (runnables)
{
runnables.AddLast(runner);
Monitor.Pulse(runnables);
}
return future;
}
// Try the following?
//private string ConvertToString(char[] buffer, int count)
//{
// StringBuilder data = new StringBuilder(buffer.Length);
// for (int i = 0; i < count; i++)
// {
// data.Append(buffer[i]);
// }
// return data.ToString();
//}
//void replace_checksat(Experiment e, Job j)
//{
// string fn = j.localFilename;
// string tr = "(check-sat)";
// string rp = e.custom_check_sat;
// int cl = tr.Length;
// int ll = cl + rp.Length;
// char[] buffer = new char[ll];
// int index = 0;
// int rl = tr.Length;
// StreamReader streamReader = new StreamReader(fn);
// StreamWriter streamWriter = new StreamWriter(fn + ".tmp");
// while (true)
// {
// streamReader.DiscardBufferedData();
// streamReader.BaseStream.Seek(index, SeekOrigin.Begin);
// int count = streamReader.ReadBlock(buffer, 0, ll);
// if (count == 0) break;
// string data = ConvertToString(buffer, count);
// bool isEndReplaced = false;
// if (count >= cl)
// isEndReplaced = (data.LastIndexOf(tr, cl) > 0);
// data = data.Replace(tr, rp);
// if (isEndReplaced)
// {
// streamWriter.Write(data);
// index += count;
// }
// else
// {
// if (count >= cl)
// {
// streamWriter.Write(data.Substring(0, data.Length - rl));
// index += cl;
// }
// else
// {
// streamWriter.Write(data);
// index += cl;
// }
// }
// }
// streamReader.Close();
// streamWriter.Close();
// File.Delete(fn);
// File.Move(fn + ".tmp", fn);
//}
void runJob(Experiment e, Job j)
{
retry_from_scratch:
try
{
getBinary(e);
Result r = new Result();
r.j = j;
try
{
// Console.WriteLine("Running job #" + j.ID);
File.Copy(j.filename, j.localFilename, true);
if (e.custom_check_sat != null)
replace_checksat(e, j);
}
catch (System.OutOfMemoryException)
{
r.exitCode = "MEMORY";
r.runtime = 0;
results.Add(r);
return;
}
int output_limit = 134217728; // 128 MB
int error_limit = 262144; // 256 KB
StreamWriter out_writer = new StreamWriter(r.stdout);
StreamWriter err_writer = new StreamWriter(r.stderr);
Process p = new Process();
p.StartInfo.FileName = e.localExecutable;
p.StartInfo.WorkingDirectory = e.localDir;
p.StartInfo.Arguments = j.localFilename + " " + e.Parameters;
//p.StartInfo.Arguments = e.Parameters + " " + j.localFilename;
//p.StartInfo.Arguments = e.Parameters;
//p.StartInfo.Arguments = " " + j.filename;
p.StartInfo.CreateNoWindow = true;
p.StartInfo.RedirectStandardOutput = true;
p.StartInfo.RedirectStandardError = true;
p.StartInfo.UseShellExecute = false;
p.OutputDataReceived += (sender, args) => WriteToStream(sender, args, out_writer, ref output_limit);
p.ErrorDataReceived += (sender, args) => WriteToStream(sender, args, err_writer, ref error_limit);
bool exhausted_time = false, exhausted_memory = false;
if (e.localExecutable.EndsWith(".cmd") || e.localExecutable.EndsWith(".bat"))
{
p.StartInfo.FileName = @"C:\Windows\System32\cmd.exe";
p.StartInfo.Arguments = "/c " + e.localExecutable + " " + p.StartInfo.Arguments;
}
// For stdin-only programs like mathsat:
//p.StartInfo.RedirectStandardInput = true;
//p.StartInfo.Arguments = e.Parameters;
//StreamReader fin = new StreamReader(j.localFilename);
retry:
try
{
p.Start();
p.BeginOutputReadLine();
p.BeginErrorReadLine();
// For stdin-only programs like mathsat:
//while (!fin.EndOfStream)
// p.StandardInput.WriteLine(fin.ReadLine());
//fin.Close();
//p.StandardInput.Close();
}
catch (System.ComponentModel.Win32Exception ex)
{
if (ex.Message == "The process cannot access the file because it is being used by another process")
{
Console.WriteLine("Retrying to execute binary...");
Thread.Sleep(500);
goto retry;
}
else
throw ex;
}
try
{
do
{
p.Refresh();
if (!p.HasExited)
{
if (processTime(p).TotalSeconds >= e.timeout.TotalSeconds)
{
Console.WriteLine("Job timed out; killing.");
exhausted_time = true;
processKill(p);
}
else if (e.memout != 0 && processMemory(p) > e.memout)
{
Console.WriteLine("Job uses too much memory; killing.");
exhausted_memory = true;
processKill(p);
}
else if (output_limit <= 0 || error_limit <= 0)
{
Console.WriteLine("Job produced too much output; killing.");
processKill(p);
throw new Exception("Job produced too much output.");
}
}
}
while (!p.WaitForExit(500));
}
catch (InvalidOperationException ex)
{
Console.WriteLine("Invalid Operation: " + ex.Message);
Console.WriteLine("Assuming process has ended.");
}
p.WaitForExit();
int excode = p.ExitCode;
if (excode == -1073741515)
logInfrastructureError(j, "Binary could not be executed.");
double runtime = (exhausted_time ? e.timeout.TotalSeconds : processTime(p).TotalSeconds);
p.Close();
Thread.Sleep(500); // Give the asynch stdout/stderr events a chance to finish.
out_writer.Flush();
err_writer.Flush();
r.exitCode = ((exhausted_time) ? "TIME" : (exhausted_memory) ? "MEMORY" : excode.ToString());
r.runtime = runtime;
results.Add(r);
}
catch (Exception ex)
{
if (logInfrastructureError(j, ex.Message + "\n" + ex.StackTrace))
goto retry_from_scratch;
}
}
void replace_checksat(Experiment e, Job j)
{
string tmpf = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
FileStream f = new FileStream(j.localFilename, FileMode.Open, FileAccess.Read);
FileStream ft = new FileStream(tmpf, FileMode.Create, FileAccess.Write);
StreamReader fr = new StreamReader(f);
StreamWriter ftw = new StreamWriter(ft);
while (!fr.EndOfStream)
{
string s = fr.ReadLine();
ftw.WriteLine(s.Replace("(check-sat)", e.custom_check_sat));
}
ftw.Close();
ft.Close();
fr.Close();
f.Close();
File.Copy(tmpf, j.localFilename, true);
try
{
File.Delete(tmpf);
}
catch { }
}
bool logInfrastructureError(Job j, string message)
{
if (--infrastructure_errors_max == 0)
{
string x = "INFRASTRUCTURE ERROR: " + message;
ensureConnected();
SqlCommand cmd = new SqlCommand("INSERT INTO Data " +
"(ExperimentID,FilenameP,ResultCode,stderr,Worker,SAT,UNSAT,UNKNOWN,TargetSAT,TargetUNSAT,TargetUNKNOWN) VALUES (" +
j.experimentID + ", " +
j.filenameP + ", " +
"4," +
"@ERRORMESSAGE, " +
"'" + myName + "'," +
"0,0,0,0,0,0); " +
"DELETE FROM JobQueue WHERE ID=" + j.ID + ";", sql);
cmd.CommandTimeout = 0;
SqlParameter p = cmd.Parameters.Add("@ERRORMESSAGE", System.Data.SqlDbType.VarChar);
p.Size = x.Length;
p.Value = x;
cmd.ExecuteNonQuery();
return false;
}
else
{
Thread.Sleep(1000);
return true;
}
}
Job getJob(Experiment e, int jID = 0)
{
Job j = new Job();
if (jID == -1)
{
retry:
Dictionary<string, Object> r =
SQLRead("SELECT TOP 1 ID FROM JobQueue WHERE ExperimentID=" + e.ID, sql);
if (r.Count() == 0) return null;
jID = (int)r["ID"];
ensureConnected();
SqlCommand cmd = new SqlCommand("UPDATE JobQueue SET Worker='" + myName + "/Recovery',AcquireTime=GETDATE() WHERE ID=" + jID, sql);
if (cmd.ExecuteNonQuery() == 0)
jID = 0;
if (jID == 0) goto retry;
Dictionary<string, Object> rd =
SQLRead("SELECT a.ID as ID,a.FilenameP as FileP,b.s as Filename FROM JobQueue as a,Strings as b WHERE Worker='" + myName + "/Recovery' AND a.FilenameP=b.ID AND ExperimentID=" + e.ID + ";", sql);
if (rd.Count == 0)
return null;
else
{
jID = (int)rd["ID"];
j.ID = jID.ToString();
j.experimentID = e.ID;
j.filename = e.sharedDir + "\\" + (string)rd["Filename"];
j.filenameP = (int)rd["FileP"];
j.localFilename = e.localDir + "\\" + j.ID + Path.GetExtension(j.filename);
}
if (e.ID != j.experimentID)
throw new Exception("Got job from wrong experiment.");
}
else
{
while (jID == 0)
{
TimeSpan x = e.timeout;
int limit = Convert.ToInt32(x.TotalSeconds);
limit += 300; // +5 minutes
string cond = "((Worker IS NULL) OR (DATEDIFF(second, AcquireTime, GETDATE()) >= " + limit.ToString() + ")";
//string cond = "(Worker IS NULL)";
List<Result>.Enumerator en = results.GetEnumerator();
while (en.MoveNext())
{
cond += " AND (ID <> " + en.Current.j.ID + ")";
}
cond += ")";
Dictionary<string, Object> rd = new Dictionary<string, object>();
rd = SQLRead("DECLARE @jid INT;" +
"BEGIN TRANSACTION; " +
"SELECT TOP 1 @jid = ID FROM JobQueue WITH (ROWLOCK,UPDLOCK,READPAST) WHERE " +
"ExperimentID=" + e.ID + " AND " + cond + ";" +
"UPDATE JobQueue SET Worker='" + myName + "',AcquireTime=GETDATE() WHERE ID=@jid; COMMIT; " +
"SELECT a.ID as ID,a.FilenameP as FileP,b.s as Filename FROM JobQueue as a, Strings as b " +
"WHERE a.ID=@jid AND a.FilenameP=b.ID;", sql);
if (rd.Count == 0)
return null;
else
{
jID = (int)rd["ID"];
j.ID = jID.ToString();
j.experimentID = e.ID;
j.filename = e.sharedDir + "\\" + (string)rd["Filename"];
j.filenameP = (int)rd["FileP"];
j.localFilename = e.localDir + "\\" + j.ID + Path.GetExtension(j.filename);
}
if (e.ID != j.experimentID)
throw new Exception("Got job from wrong experiment.");
}
}
return j;
}
public Runner(Job job, object obj, Future future1)
{
future = future1;
args = obj;
runnable = job;
}