public static void ExecOneLocal(SourceCode.Job cfgj, string[] ExecArgs, bool verbose)
{
if (verbose)
{
Console.WriteLine("[{0}] [Local: {2}]", System.DateTime.Now.ToString(), System.DateTime.Now.Millisecond, cfgj.NarrativeName);
}
int BlockID = 0;
string SlaveIP = null;
string logname = Surrogate.SafeTextPath(cfgj.NarrativeName) + "_" + Guid.NewGuid().ToString() + ".j" + sjid + "_log.txt";
bool aborting = false;
try
{
dfs dc = LoadDfsConfig();
string firstslave = dc.Slaves.GetFirstSlave();
string SlaveHost = cfgj.IOSettings.LocalHost;
if (SlaveHost == null || SlaveHost.Length == 0)
{
SlaveHost = firstslave;
}
SlaveIP = IPAddressUtil.GetIPv4Address(SlaveHost);
MySpace.DataMining.DistributedObjects5.Remote rem = new MySpace.DataMining.DistributedObjects5.Remote(cfgj.NarrativeName + "_local");
rem.OutputStartingPoint = BlockID;
rem.LocalCompile = true;
rem.CompilerOptions = cfgj.IOSettings.CompilerOptions;
rem.CompilerVersion = cfgj.IOSettings.CompilerVersion;
if (cfgj.OpenCVExtension != null)
{
rem.AddOpenCVExtension();
}
if (cfgj.MemCache != null)
{
rem.AddMemCacheExtension();
}
if (cfgj.Unsafe != null)
{
rem.AddUnsafe();
}
if (cfgj.AssemblyReferencesCount > 0)
{
cfgj.AddAssemblyReferences(rem.CompilerAssemblyReferences, Surrogate.NetworkPathForHost(firstslave));
}
rem.SetJID(jid, CurrentJobFileName + " Local: " + cfgj.NarrativeName);
rem.AddBlock(SlaveHost + @"|"
+ (cfgj.ForceStandardError != null ? "&" : "") + logname + @"|slaveid=0");
rem.Open();
string codectx = (@"
public const int DSpace_BlockID = " + BlockID.ToString() + @";
public const int DSpace_ProcessID = DSpace_BlockID;
public const int Qizmt_ProcessID = DSpace_ProcessID;
public const int DSpace_BlocksTotalCount = 1;
public const int DSpace_ProcessCount = DSpace_BlocksTotalCount;
public const int Qizmt_ProcessCount = DSpace_ProcessCount;
public const string DSpace_SlaveHost = `" + SlaveHost + @"`;
public const string DSpace_MachineHost = DSpace_SlaveHost;
public const string Qizmt_MachineHost = DSpace_MachineHost;
public const string DSpace_SlaveIP = `" + SlaveIP + @"`;
public const string DSpace_MachineIP = DSpace_SlaveIP;
public const string Qizmt_MachineIP = DSpace_MachineIP;
public static readonly string[] DSpace_ExecArgs = new string[] { " + ExecArgsCode(ExecArgs) + @" };
public static readonly string[] Qizmt_ExecArgs = DSpace_ExecArgs;
public const string DSpace_ExecDir = @`" + System.Environment.CurrentDirectory + @"`;
public const string Qizmt_ExecDir = DSpace_ExecDir;
static string Shell(string line, bool suppresserrors)
{
return MySpace.DataMining.DistributedObjects.Exec.Shell(line, suppresserrors);
}
static string Shell(string line)
{
return MySpace.DataMining.DistributedObjects.Exec.Shell(line, false);
}
const string _userlogname = `" + logname + @"`;
static System.Threading.Mutex _logmutex = new System.Threading.Mutex(false, `distobjlog`);
private static int userlogsremain = " + AELight.maxuserlogs.ToString() + @";
public static void Qizmt_Log(string line) { DSpace_Log(line); }
public static void DSpace_Log(string line)
{
if(--userlogsremain < 0)
{
return;
}
try
{
_logmutex.WaitOne();
}
catch (System.Threading.AbandonedMutexException)
{
}
try
{
using (System.IO.StreamWriter fstm = System.IO.File.AppendText(_userlogname))
{
fstm.WriteLine(`{0}`, line);
}
}
finally
{
_logmutex.ReleaseMutex();
}
}
public static void Qizmt_LogResult(string line, bool passed) { DSpace_LogResult(line, passed); }
public static void DSpace_LogResult(string name, bool passed)
{
if(passed)
{
DSpace_Log(`[\u00012PASSED\u00010] - ` + name);
}
else
{
DSpace_Log(`[\u00014FAILED\u00010] - ` + name);
}
}
").Replace('`', '"') + CommonDynamicCsCode;
rem.LocalExec(codectx + cfgj.Local, cfgj.Usings);
rem.Close();
if (verbose)
{
Console.Write('*');
ConsoleFlush();
}
}
catch (System.Threading.ThreadAbortException)
{
aborting = true;
}
finally
{
if (!aborting)
{
CheckUserLogs(new string[] { SlaveIP }, logname);
}
}
if (verbose)
{
Console.WriteLine();
Console.WriteLine("[{0}] Done", System.DateTime.Now.ToString(), System.DateTime.Now.Millisecond);
}
}
internal void threadproc()
{
try
{
string reason;
if (!Surrogate.IsHealthySlaveMachine(SlaveHost, out reason))
{
if (explicithost)
{
throw new Exception("Remote cannot connect to explicit host '" + SlaveHost + "': " + reason);
}
/*if (!failoverenabled)
* {
* throw new Exception("Remote cannot connect to host '" + SlaveHost + "': " + reason);
* }
* else*/
{
SlaveHost = null;
int startsi = (new Random(DateTime.Now.Millisecond / (BlockID + 2))).Next() % slaves.Count;
for (int i = startsi; i < slaves.Count; i++)
{
if (Surrogate.IsHealthySlaveMachine(slaves[i], out reason))
{
SlaveHost = slaves[i];
rem.OutputStartingPoint = i;
break;
}
}
if (null == SlaveHost)
{
for (int i = 0; i < startsi; i++)
{
if (Surrogate.IsHealthySlaveMachine(slaves[i], out reason))
{
SlaveHost = slaves[i];
rem.OutputStartingPoint = i;
break;
}
}
if (null == SlaveHost)
{
throw new Exception("Remote cannot connect to any hosts; last reason: " + reason);
}
}
}
}
SlaveIP = IPAddressUtil.GetIPv4Address(SlaveHost); // !
rem.SetJID(jid, CurrentJobFileName + " Remote: " + cfgj.NarrativeName);
rem.AddBlock(SlaveIP + @"|"
+ (cfgj.ForceStandardError != null ? "&" : "") + logname + @"|slaveid=0");
string codectx = (@"
public const int DSpace_BlockID = " + BlockID.ToString() + @";
public const int DSpace_ProcessID = DSpace_BlockID;
public const int Qizmt_ProcessID = DSpace_ProcessID;
public const int DSpace_BlocksTotalCount = " + blockcount.ToString() + @";
public const int DSpace_ProcessCount = DSpace_BlocksTotalCount;
public const int Qizmt_ProcessCount = DSpace_ProcessCount;
public const string DSpace_SlaveHost = `" + SlaveHost + @"`;
public const string DSpace_MachineHost = DSpace_SlaveHost;
public const string Qizmt_MachineHost = DSpace_MachineHost;
public const string DSpace_SlaveIP = `" + SlaveIP + @"`;
public const string DSpace_MachineIP = DSpace_SlaveIP;
public const string Qizmt_MachineIP = DSpace_MachineIP;
public static readonly string[] DSpace_ExecArgs = new string[] { " + ExecArgsCode(ExecArgs) + @" };
public static readonly string[] Qizmt_ExecArgs = DSpace_ExecArgs;
public const string DSpace_OutputFilePath = `" + DFSWriter + @"`; // Includes `dfs://` if in DFS.
public const string Qizmt_OutputFilePath = DSpace_OutputFilePath;
public static int DSpace_InputRecordLength { get { return MySpace.DataMining.DistributedObjects.StaticGlobals.DSpace_InputRecordLength; } }
public static int Qizmt_InputRecordLength { get { return MySpace.DataMining.DistributedObjects.StaticGlobals.DSpace_InputRecordLength; } }
public static int DSpace_OutputRecordLength { get { return MySpace.DataMining.DistributedObjects.StaticGlobals.DSpace_OutputRecordLength; } }
public static int Qizmt_OutputRecordLength { get { return MySpace.DataMining.DistributedObjects.StaticGlobals.DSpace_OutputRecordLength; } }
public const string Qizmt_Meta = @`" + Meta + @"`;
const string _userlogname = `" + logname + @"`;
static System.Threading.Mutex _logmutex = new System.Threading.Mutex(false, `distobjlog`);
static string Shell(string line, bool suppresserrors)
{
return MySpace.DataMining.DistributedObjects.Exec.Shell(line, suppresserrors);
}
static string Shell(string line)
{
return MySpace.DataMining.DistributedObjects.Exec.Shell(line, false);
}
private static int userlogsremain = " + AELight.maxuserlogs.ToString() + @";
public static void Qizmt_Log(string line) { DSpace_Log(line); }
public static void DSpace_Log(string line)
{
if(--userlogsremain < 0)
{
return;
}
try
{
_logmutex.WaitOne();
}
catch (System.Threading.AbandonedMutexException)
{
}
try
{
using (System.IO.StreamWriter fstm = System.IO.File.AppendText(_userlogname))
{
fstm.WriteLine(`{0}`, line);
}
}
finally
{
_logmutex.ReleaseMutex();
}
}
public static void Qizmt_LogResult(string line, bool passed) { DSpace_LogResult(line, passed); }
public static void DSpace_LogResult(string name, bool passed)
{
if(passed)
{
DSpace_Log(`[\u00012PASSED\u00010] - ` + name);
}
else
{
DSpace_Log(`[\u00014FAILED\u00010] - ` + name);
}
}
").Replace('`', '"') + CommonDynamicCsCode;
List <string> outputbasenames = new List <string>(DFSWriters.Count);
foreach (string writer in DFSWriters)
{
string dfsname = writer;
if (dfsname.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
dfsname = dfsname.Substring(6);
}
outputbasenames.Add(GenerateZdFileDataNodeBaseName(dfsname));
}
rem.Open();
outputsizes = new List <long>();
outputsizeses = new List <List <long> >();
outputdfsnodeses = new List <List <string> >();
rem.RemoteExec(dfsinputpaths, outputdfsdirs, outputbasenames, baseoutputfilesize, codectx + cfgj.Remote, cfgj.Usings, outputsizeses, outputdfsnodeses, dfsinputfilenames, dfsinputnodesoffsets);
blockfail = false; // !
if (verbose)
{
Console.Write('*');
ConsoleFlush();
}
}
catch (Exception e)
{
LogOutput("RemoteBlockInfo.threadproc exception: " + e.ToString());
}
}
static void DoSurrogate(string hostsinfo, string mypassword, bool participating)
{
string[] hinfos = hostsinfo.Split((char)1);
string myipaddr = IPAddressUtil.GetIPv4Address(System.Net.Dns.GetHostName());
string myhost = GetHostnameInternal(myipaddr);
try
{
if (myipaddr != IPAddressUtil.GetIPv4Address(myhost))
{
throw new Exception("IP addresses do not match");
}
}
catch (Exception e)
{
throw new Exception("Error with internal IP address and host name"
+ " (" + myipaddr + " != " + myhost + ")", e);
}
// File in the format:
// <WindowsUser>
// <machine>=<password>
string logondatContent = "administrator" + Environment.NewLine;
logondatContent += myhost + "=" + mypassword + Environment.NewLine;
string formatmachines = "";
if (participating)
{
formatmachines = myhost;
}
string appendhostsfiles = Environment.NewLine
+ myipaddr + " " + myhost + Environment.NewLine;
foreach (string hinfo in hinfos)
{
int ieq = hinfo.IndexOf('=');
if (-1 != ieq)
{
// hinfo already in form <machine>=<password>
//logondatContent += hinfo + Environment.NewLine;
// I want the hostname, not the IPaddr.
string rhost = hinfo.Substring(0, ieq); // Probably an internal IP address.
string rpasswd = hinfo.Substring(ieq + 1);
{
if (formatmachines.Length > 0)
{
formatmachines += ";";
}
string fmhost = rhost;
try
{
fmhost = GetHostnameInternal(rhost);
appendhostsfiles += rhost + " " + fmhost + Environment.NewLine;
}
catch
{
}
formatmachines += fmhost;
logondatContent += fmhost + "=" + rpasswd + Environment.NewLine;
}
}
}
// Append my appendhostsfiles to hosts file.
string hostsfilepath = Environment.ExpandEnvironmentVariables(
@"%SystemRoot%\system32\drivers\etc\hosts");
System.IO.File.Copy(hostsfilepath, "hosts.old", true); // Backup old one.
System.IO.File.AppendAllText(ToNetworkPath(hostsfilepath, myipaddr), appendhostsfiles);
System.Threading.Thread.Sleep(1000);
// Write my logon.dat; and logon.
string thisnetpath = NetworkPathForHost(myipaddr);
if (!System.IO.Directory.Exists(thisnetpath))
{
System.IO.Directory.CreateDirectory(thisnetpath);
}
string thislogonfile = thisnetpath + @"\logon.dat";
for (int itries = 0; ;)
{
try
{
System.IO.File.WriteAllText(thislogonfile, logondatContent);
}
catch
{
if (++itries > 3)
{
throw;
}
System.Threading.Thread.Sleep(1000 * itries);
try
{
System.IO.File.Delete(thislogonfile);
}
catch
{
}
continue;
}
break;
}
XLog.statuslog("Logging on all machines in cluster");
if (!LogonMachines(thislogonfile))
{
throw new Exception("Was not able to LogonMachines");
}
XLog.statuslog("Logged on all machines in cluster");
XLog.statuslog("Configuring machines in cluster");
// Write logon.dat to other machines.
// Append appendhostsfiles to other machines' hosts file.
foreach (string hinfo in hinfos)
{
int ieq = hinfo.IndexOf('=');
if (-1 != ieq)
{
string rhost = hinfo.Substring(0, ieq); // Probably an internal IP address.
string fmhost = rhost;
try
{
fmhost = GetHostnameInternal(rhost);
}
catch
{
}
string rnetpath = NetworkPathForHost(fmhost);
if (!System.IO.Directory.Exists(rnetpath))
{
System.IO.Directory.CreateDirectory(rnetpath);
}
string rlogonfile = rnetpath + @"\logon.dat";
for (int itries = 0; ;)
{
try
{
System.IO.File.WriteAllText(rlogonfile, logondatContent);
}
catch
{
if (++itries > 3)
{
throw;
}
System.Threading.Thread.Sleep(1000 * itries);
try
{
System.IO.File.Delete(rlogonfile);
}
catch
{
}
continue;
}
break;
}
System.IO.File.AppendAllText(ToNetworkPath(hostsfilepath, fmhost), appendhostsfiles);
}
}
XLog.statuslog("Configured machines in cluster");
{
string fmtcmd = "Qizmt format Machines=" + formatmachines;
XLog.statuslog("Formatting Qizmt cluster: " + fmtcmd);
Exec.Shell(fmtcmd);
}
}
