/// <summary>
/// Fetch next non-ping able IPAddress
/// If we want to assign an IPAddress to the printer manually
/// this method fetches the non-ping-able IPAddress from the subnet
/// </summary>
/// <param name="subnetMask">subnet mask of the printer</param>
/// <param name="printerIP">IPAddress of the printer</param>
/// <returns>Returns a new non-ping-able IPAddress</returns>
public static IPAddress FetchNextIPAddress(IPAddress subnetMask, IPAddress printerIP)
{
IPAddress newIPAddress = IPAddress.None;
newIPAddress = IPAddressUtil.NextAssignable(printerIP, subnetMask);
while (PingUntilTimeout(newIPAddress, 0))
{
newIPAddress = IPAddressUtil.NextAssignable(newIPAddress, subnetMask);
}
return(newIPAddress);
}
static bool _GetStatusFromLine(string line, out string host, out string statusmsg)
{
line = line.Trim();
int ix = line.IndexOf(':');
if (-1 != ix)
{
string xhost = line.Substring(0, ix);
host = IPAddressUtil.GetName(xhost);
statusmsg = line.Substring(ix + 1).TrimStart();
return(true);
}
host = null;
statusmsg = null;
return(false);
}
static void BlankTest(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = dc.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
{
// Test logic:
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
static void Main(string[] args)
{
string hostname = Dns.GetHostName();
IPAddress[] iplist = Dns.GetHostAddresses(hostname);
foreach (IPAddress ipa in iplist)
{
if (ipa.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork)
{
Console.WriteLine(ipa.ToString());
}
}
string hostName;
if (IPAddressUtil.GetLocalIP() != null)
{
hostName = IPAddressUtil.GetLocalIP();
Console.WriteLine("Get Local IP :" + hostName);
}
else
{
hostName = "127.0.0.1";
}
Server server = new Server
{
Services = { Greeter.BindService(new DemoServerImpl()) },
Ports = { new ServerPort(hostName, PORT, ServerCredentials.Insecure) }
};
server.Start();
Console.WriteLine("Greeter server listening on port " + PORT);
Console.WriteLine("Press any key to stop the server ...");
string flag = Console.ReadKey().Key.ToString();
server.ShutdownAsync().Wait();
}
static void RemoveSurrogate(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
bool incluster;
if (args[2] == "incluster")
{
incluster = true;
}
else if (args[2] == "isolated")
{
incluster = false;
}
else
{
throw new Exception("Expected: incluster or isolated");
}
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
string masterslavedat = masterdir + @"\slave.dat";
dfs olddfs = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
List <string> otherhosts = new List <string>(); // Non-surrogate machines, reguardless if participating surrogate or not.
foreach (string slave in olddfs.Slaves.SlaveList.Split(';'))
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
otherhosts.Add(slave);
}
}
string newmaster = otherhosts[0];
Console.WriteLine("Backing up DFS.xml to: {0} ...", dfsxmlpathbackup);
try
{
System.IO.File.Delete(dfsxmlpathbackup);
}
catch
{
}
System.IO.File.Move(dfsxmlpath, dfsxmlpathbackup);
try
{
StringBuilder sbmachines = new StringBuilder(1000);
if (incluster)
{
sbmachines.Append(masterhost);
}
foreach (string host in otherhosts)
{
if (0 != sbmachines.Length)
{
sbmachines.Append(',');
}
sbmachines.Append(host);
}
Console.WriteLine("Formatting DFS for test...");
Exec.Shell("Qizmt @format Machines=" + sbmachines.ToString());
Console.WriteLine("Adding some files to DFS...");
Console.Write(" ");
Exec.Shell("Qizmt bingen 1MB 1MB 50");
Console.Write("10%");
Exec.Shell("Qizmt examples");
Console.Write("..15%");
Exec.Shell("Qizmt wordgen 10MB 10MB 100");
Console.Write("..50%");
Exec.Shell("Qizmt asciigen 50MB 50MB 500");
Console.Write("..100%");
Console.WriteLine();
int ls_output_linecount = Exec.Shell("Qizmt ls").Split('\n').Length;
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Run test job, save output...");
string md5_10MB_output = Exec.Shell("Qizmt md5 10MB");
Console.WriteLine("Removing Surrogate (removemachine {0}) ...", masterhost);
Console.WriteLine(Exec.Shell("Qizmt removemachine " + masterhost));
Console.WriteLine("Interface with new surrogate...");
System.IO.File.WriteAllText(masterslavedat, "master=" + newmaster + Environment.NewLine);
{
// Not comparing contents because of the free disk space line.
int new_ls_output_linecount = Exec.Shell("Qizmt ls").Split('\n').Length;
if (ls_output_linecount != new_ls_output_linecount)
{
throw new Exception("Cluster does not contain the same files as before removemachine " + masterdir + ", or problem issuing commands on new surrogate");
}
}
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Run test job, confirm output...");
if (md5_10MB_output != Exec.Shell("Qizmt md5 10MB"))
{
throw new Exception("Test job output does not match previous run");
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
finally
{
Console.WriteLine("Restoring DFS.xml backup...");
// Note: these are safe; the try/finally only wraps the new dfs.
try
{
Exec.Shell("Qizmt del *");
}
catch
{
}
try
{
System.IO.File.Delete(masterslavedat);
}
catch
{
}
for (int si = 0; si < otherhosts.Count; si++)
{
try
{
System.IO.File.Delete(Surrogate.NetworkPathForHost(otherhosts[si]) + @"\slave.dat");
// Deleting dfs.xml should go last because it'll usually fail.
System.IO.File.Delete(Surrogate.NetworkPathForHost(otherhosts[si]) + @"\dfs.xml");
}
catch
{
}
}
try
{
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
try
{
// Reformat the cluster so stuff like slave.dat is correct...
Exec.Shell("Qizmt @format Machines=" + string.Join(",", otherhosts.ToArray()));
}
catch (Exception exf)
{
Console.Error.WriteLine("Problem during reformat, there may be an issue with the cluster", exf);
}
try
{
// Delete the dfs.xml just written, it's being replaced with the good one.
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
System.IO.File.Move(dfsxmlpathbackup, dfsxmlpath);
}
}
static void ReplicationFailover(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs olddfs = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = olddfs.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
Console.WriteLine("Backing up DFS.xml to: {0} ...", dfsxmlpathbackup);
try
{
System.IO.File.Delete(dfsxmlpathbackup);
}
catch
{
}
System.IO.File.Move(dfsxmlpath, dfsxmlpathbackup);
try
{
{
// Note: added replication and much lower DataNodeBaseSize!
Console.WriteLine("Formatting DFS with Replication=3 for test...");
Exec.Shell("Qizmt @format Machines=" + string.Join(",", allmachines) + " Replication=3 DataNodeBaseSize=1048576");
}
{
// Test logic:
if (allmachines.Length < 3)
{
throw new Exception("This test needs a cluster of at least 3 machines!");
}
long XBYTES = (long)4194304 * (long)allmachines.Length;
Console.WriteLine("Generating data...");
Console.Write(" ");
Exec.Shell("Qizmt gen data{476D6FE8-D645-41cc-83A1-3AB5E2DE23E7} " + (XBYTES / 4).ToString());
Console.Write("25%");
Exec.Shell("Qizmt gen data{61136275-16EC-4ff9-84CE-ACC967550181} " + (XBYTES / 4).ToString());
Console.Write("..50%");
Exec.Shell("Qizmt gen data{C76F6C06-EFC8-4808-B214-DB4D167171EB} " + (XBYTES / 2).ToString());
Console.Write("..100%");
Console.WriteLine();
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Running job on healthy cluster...");
string exec_md5;
{
string exectempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_ReplicationFailover-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(exectempdir))
{
System.IO.Directory.CreateDirectory(exectempdir);
}
string execfp = exectempdir + @"\exec{FA19CAB0-5225-4cc8-8728-9BFC3A1B834C}";
System.IO.File.WriteAllText(execfp, (@"<?xml version=`1.0` encoding=`utf-8`?>
<SourceCode>
<Jobs>
<Job Name=`exec{FA19CAB0-5225-4cc8-8728-9BFC3A1B834C}`>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>100</KeyLength>
<DFSInput>dfs://data{*}</DFSInput>
<DFSOutput>dfs://output{04454992-E2CD-4342-AEEB-1D0607B32D84}</DFSOutput>
<KeyMajor>8</KeyMajor>
<OutputMethod>sorted</OutputMethod>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
output.Add(line, ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output)
{
for(int i = 0; i < values.Length; i++)
{
output.Add(key);
}
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"'));
Exec.Shell("Qizmt importdir " + exectempdir);
try
{
System.IO.File.Delete(execfp);
System.IO.Directory.Delete(exectempdir);
}
catch
{
}
Exec.Shell("Qizmt exec exec{FA19CAB0-5225-4cc8-8728-9BFC3A1B834C}");
exec_md5 = DfsSum("md5", "output{04454992-E2CD-4342-AEEB-1D0607B32D84}");
Exec.Shell("Qizmt del output{04454992-E2CD-4342-AEEB-1D0607B32D84}");
}
try
{
Console.WriteLine("Disrupting 2 machines...");
{
string badmachine = allmachines[allmachines.Length - 1];
Console.WriteLine(" Bad disk on {0}", badmachine);
string netpath = Surrogate.NetworkPathForHost(badmachine);
foreach (System.IO.FileInfo fi in (new System.IO.DirectoryInfo(netpath)).GetFiles("zd.*.zd"))
{
if (!fi.Name.StartsWith("zd.!."))
{
System.IO.File.Move(fi.FullName, fi.DirectoryName + @"\zd.!." + fi.Name.Substring(3));
}
}
}
{
string badmachine = allmachines[allmachines.Length - 2];
Console.WriteLine(" Bad network connection on {0}", badmachine);
Exec.Shell(@"sc \\" + badmachine + @" stop DistributedObjects");
}
Console.WriteLine("Ensure the cluster is NOT perfectly healthy...");
{
bool healthy;
try
{
EnsurePerfectQizmtHealtha();
healthy = true;
}
catch
{
healthy = false;
}
if (healthy)
{
throw new Exception("Cluster is still healthy");
}
}
Console.WriteLine("Running job on unhealthy cluster...");
{
try
{
Exec.Shell("Qizmt exec exec{FA19CAB0-5225-4cc8-8728-9BFC3A1B834C}");
}
catch
{
// Replication will output a warning and throw an exception,
// so we need to ignore that exception.
// The MD5 check will ensure it ran fine.
}
string new_exec_md5 = DfsSum("md5", "output{04454992-E2CD-4342-AEEB-1D0607B32D84}");
Exec.Shell("Qizmt del output{04454992-E2CD-4342-AEEB-1D0607B32D84}");
if (new_exec_md5 != exec_md5)
{
throw new Exception("Output files from before and after disrupting cluster do not match");
}
}
}
finally
{
{
Console.WriteLine("Repairing disrupted disk");
string badmachine = allmachines[allmachines.Length - 1];
string netpath = Surrogate.NetworkPathForHost(badmachine);
foreach (System.IO.FileInfo fi in (new System.IO.DirectoryInfo(netpath)).GetFiles("zd.!.*.zd"))
{
System.IO.File.Move(fi.FullName, fi.DirectoryName + @"\zd." + fi.Name.Substring(5));
}
}
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
finally
{
Console.WriteLine("Restoring DFS.xml backup...");
// Note: these are safe; the try/finally only wraps the new dfs.
try
{
Exec.Shell("Qizmt del *");
}
catch
{
}
try
{
// Delete temp dfs.xml, it's being replaced with the good one.
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
System.IO.File.Move(dfsxmlpathbackup, dfsxmlpath);
{
// Note: killall issued to fix disrupted machines
Console.WriteLine("Running killall to repair");
Exec.Shell("Qizmt killall -f");
}
}
}
static void DoSurrogate(string hostsinfo, string mypassword, bool participating)
{
string[] hinfos = hostsinfo.Split((char)1);
string myipaddr = IPAddressUtil.GetIPv4Address(System.Net.Dns.GetHostName());
string myhost = GetHostnameInternal(myipaddr);
try
{
if (myipaddr != IPAddressUtil.GetIPv4Address(myhost))
{
throw new Exception("IP addresses do not match");
}
}
catch (Exception e)
{
throw new Exception("Error with internal IP address and host name"
+ " (" + myipaddr + " != " + myhost + ")", e);
}
// File in the format:
// <WindowsUser>
// <machine>=<password>
string logondatContent = "administrator" + Environment.NewLine;
logondatContent += myhost + "=" + mypassword + Environment.NewLine;
string formatmachines = "";
if (participating)
{
formatmachines = myhost;
}
string appendhostsfiles = Environment.NewLine
+ myipaddr + " " + myhost + Environment.NewLine;
foreach (string hinfo in hinfos)
{
int ieq = hinfo.IndexOf('=');
if (-1 != ieq)
{
// hinfo already in form <machine>=<password>
//logondatContent += hinfo + Environment.NewLine;
// I want the hostname, not the IPaddr.
string rhost = hinfo.Substring(0, ieq); // Probably an internal IP address.
string rpasswd = hinfo.Substring(ieq + 1);
{
if (formatmachines.Length > 0)
{
formatmachines += ";";
}
string fmhost = rhost;
try
{
fmhost = GetHostnameInternal(rhost);
appendhostsfiles += rhost + " " + fmhost + Environment.NewLine;
}
catch
{
}
formatmachines += fmhost;
logondatContent += fmhost + "=" + rpasswd + Environment.NewLine;
}
}
}
// Append my appendhostsfiles to hosts file.
string hostsfilepath = Environment.ExpandEnvironmentVariables(
@"%SystemRoot%\system32\drivers\etc\hosts");
System.IO.File.Copy(hostsfilepath, "hosts.old", true); // Backup old one.
System.IO.File.AppendAllText(ToNetworkPath(hostsfilepath, myipaddr), appendhostsfiles);
System.Threading.Thread.Sleep(1000);
// Write my logon.dat; and logon.
string thisnetpath = NetworkPathForHost(myipaddr);
if (!System.IO.Directory.Exists(thisnetpath))
{
System.IO.Directory.CreateDirectory(thisnetpath);
}
string thislogonfile = thisnetpath + @"\logon.dat";
for (int itries = 0; ;)
{
try
{
System.IO.File.WriteAllText(thislogonfile, logondatContent);
}
catch
{
if (++itries > 3)
{
throw;
}
System.Threading.Thread.Sleep(1000 * itries);
try
{
System.IO.File.Delete(thislogonfile);
}
catch
{
}
continue;
}
break;
}
XLog.statuslog("Logging on all machines in cluster");
if (!LogonMachines(thislogonfile))
{
throw new Exception("Was not able to LogonMachines");
}
XLog.statuslog("Logged on all machines in cluster");
XLog.statuslog("Configuring machines in cluster");
// Write logon.dat to other machines.
// Append appendhostsfiles to other machines' hosts file.
foreach (string hinfo in hinfos)
{
int ieq = hinfo.IndexOf('=');
if (-1 != ieq)
{
string rhost = hinfo.Substring(0, ieq); // Probably an internal IP address.
string fmhost = rhost;
try
{
fmhost = GetHostnameInternal(rhost);
}
catch
{
}
string rnetpath = NetworkPathForHost(fmhost);
if (!System.IO.Directory.Exists(rnetpath))
{
System.IO.Directory.CreateDirectory(rnetpath);
}
string rlogonfile = rnetpath + @"\logon.dat";
for (int itries = 0; ;)
{
try
{
System.IO.File.WriteAllText(rlogonfile, logondatContent);
}
catch
{
if (++itries > 3)
{
throw;
}
System.Threading.Thread.Sleep(1000 * itries);
try
{
System.IO.File.Delete(rlogonfile);
}
catch
{
}
continue;
}
break;
}
System.IO.File.AppendAllText(ToNetworkPath(hostsfilepath, fmhost), appendhostsfiles);
}
}
XLog.statuslog("Configured machines in cluster");
{
string fmtcmd = "Qizmt format Machines=" + formatmachines;
XLog.statuslog("Formatting Qizmt cluster: " + fmtcmd);
Exec.Shell(fmtcmd);
}
}
static void ReplicationChecks(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs olddfs = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = olddfs.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
Console.WriteLine("Backing up DFS.xml to: {0} ...", dfsxmlpathbackup);
try
{
System.IO.File.Delete(dfsxmlpathbackup);
}
catch
{
}
System.IO.File.Move(dfsxmlpath, dfsxmlpathbackup);
try
{
{
Console.WriteLine("Formatting DFS for test...");
Exec.Shell("Qizmt @format Machines=" + string.Join(",", allmachines));
}
{
// Test logic:
{
long XBYTES = (long)4194304 * (long)allmachines.Length;
Console.WriteLine("Generating data...");
Console.Write(" ");
Exec.Shell("Qizmt gen data{476D6FE8-D645-41cc-83A1-3AB5E2DE23E7} " + (XBYTES / 4).ToString());
Console.Write("25%");
Exec.Shell("Qizmt gen data{61136275-16EC-4ff9-84CE-ACC967550181} " + (XBYTES / 4).ToString());
Console.Write("..50%");
Exec.Shell("Qizmt gen data{C76F6C06-EFC8-4808-B214-DB4D167171EB} " + (XBYTES / 2).ToString());
Console.Write("..100%");
Console.WriteLine();
}
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Raising replication factor to 2...");
Exec.Shell("Qizmt replicationupdate 2");
{
Console.WriteLine("Raising replication factor too high (ensure fail)...");
bool ok = false;
System.Threading.Thread thd = new System.Threading.Thread(
new System.Threading.ThreadStart(
delegate
{
try
{
Exec.Shell("Qizmt replicationupdate 999999999");
}
catch (Exception e)
{
ok = true;
Console.WriteLine("Got exception as expected: {0}", e.Message);
}
}));
thd.Start();
if (!thd.Join(1000 * 10))
{
thd.Abort();
}
if (!ok)
{
throw new Exception("Test failed: expected exception");
}
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
finally
{
Console.WriteLine("Restoring DFS.xml backup...");
// Note: these are safe; the try/finally only wraps the new dfs.
try
{
Exec.Shell("Qizmt del *");
}
catch
{
}
try
{
// Delete temp dfs.xml, it's being replaced with the good one.
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
System.IO.File.Move(dfsxmlpathbackup, dfsxmlpath);
}
}
/// <summary>
/// Parses the string representation of a {@code URL} into a
/// {@code URL} object.
/// <para>
/// If there is any inherited context, then it has already been
/// copied into the {@code URL} argument.
/// </para>
/// <para>
/// The {@code parseURL} method of {@code URLStreamHandler}
/// parses the string representation as if it were an
/// {@code http} specification. Most URL protocol families have a
/// similar parsing. A stream protocol handler for a protocol that has
/// a different syntax must override this routine.
///
/// </para>
/// </summary>
/// <param name="u"> the {@code URL} to receive the result of parsing
/// the spec. </param>
/// <param name="spec"> the {@code String} representing the URL that
/// must be parsed. </param>
/// <param name="start"> the character index at which to begin parsing. This is
/// just past the '{@code :}' (if there is one) that
/// specifies the determination of the protocol name. </param>
/// <param name="limit"> the character position to stop parsing at. This is the
/// end of the string or the position of the
/// "{@code #}" character, if present. All information
/// after the sharp sign indicates an anchor. </param>
protected internal virtual void ParseURL(URL u, String spec, int start, int limit)
{
// These fields may receive context content if this was relative URL
String protocol = u.Protocol;
String authority = u.Authority;
String userInfo = u.UserInfo;
String host = u.Host;
int port = u.Port;
String path = u.Path;
String query = u.Query;
// This field has already been parsed
String @ref = u.Ref;
bool isRelPath = false;
bool queryOnly = false;
// FIX: should not assume query if opaque
// Strip off the query part
if (start < limit)
{
int queryStart = spec.IndexOf('?');
queryOnly = queryStart == start;
if ((queryStart != -1) && (queryStart < limit))
{
query = StringHelperClass.SubstringSpecial(spec, queryStart + 1, limit);
if (limit > queryStart)
{
limit = queryStart;
}
spec = spec.Substring(0, queryStart);
}
}
int i = 0;
// Parse the authority part if any
bool isUNCName = (start <= limit - 4) && (spec.CharAt(start) == '/') && (spec.CharAt(start + 1) == '/') && (spec.CharAt(start + 2) == '/') && (spec.CharAt(start + 3) == '/');
if (!isUNCName && (start <= limit - 2) && (spec.CharAt(start) == '/') && (spec.CharAt(start + 1) == '/'))
{
start += 2;
i = spec.IndexOf('/', start);
if (i < 0)
{
i = spec.IndexOf('?', start);
if (i < 0)
{
i = limit;
}
}
host = authority = spec.Substring(start, i - start);
int ind = authority.IndexOf('@');
if (ind != -1)
{
userInfo = authority.Substring(0, ind);
host = authority.Substring(ind + 1);
}
else
{
userInfo = null;
}
if (host != null)
{
// If the host is surrounded by [ and ] then its an IPv6
// literal address as specified in RFC2732
if (host.Length() > 0 && (host.CharAt(0) == '['))
{
if ((ind = host.IndexOf(']')) > 2)
{
String nhost = host;
host = nhost.Substring(0, ind + 1);
if (!IPAddressUtil.isIPv6LiteralAddress(host.Substring(1, ind - 1)))
{
throw new IllegalArgumentException("Invalid host: " + host);
}
port = -1;
if (nhost.Length() > ind + 1)
{
if (nhost.CharAt(ind + 1) == ':')
{
++ind;
// port can be null according to RFC2396
if (nhost.Length() > (ind + 1))
{
port = Convert.ToInt32(nhost.Substring(ind + 1));
}
}
else
{
throw new IllegalArgumentException("Invalid authority field: " + authority);
}
}
}
else
{
throw new IllegalArgumentException("Invalid authority field: " + authority);
}
}
else
{
ind = host.IndexOf(':');
port = -1;
if (ind >= 0)
{
// port can be null according to RFC2396
if (host.Length() > (ind + 1))
{
port = Convert.ToInt32(host.Substring(ind + 1));
}
host = host.Substring(0, ind);
}
}
}
else
{
host = "";
}
if (port < -1)
{
throw new IllegalArgumentException("Invalid port number :" + port);
}
start = i;
// If the authority is defined then the path is defined by the
// spec only; See RFC 2396 Section 5.2.4.
if (authority != null && authority.Length() > 0)
{
path = "";
}
}
if (host == null)
{
host = "";
}
// Parse the file path if any
if (start < limit)
{
if (spec.CharAt(start) == '/')
{
path = spec.Substring(start, limit - start);
}
else if (path != null && path.Length() > 0)
{
isRelPath = true;
int ind = path.LastIndexOf('/');
String seperator = "";
if (ind == -1 && authority != null)
{
seperator = "/";
}
path = path.Substring(0, ind + 1) + seperator + spec.Substring(start, limit - start);
}
else
{
String seperator = (authority != null) ? "/" : "";
path = seperator + spec.Substring(start, limit - start);
}
}
else if (queryOnly && path != null)
{
int ind = path.LastIndexOf('/');
if (ind < 0)
{
ind = 0;
}
path = path.Substring(0, ind) + "/";
}
if (path == null)
{
path = "";
}
if (isRelPath)
{
// Remove embedded /./
while ((i = path.IndexOf("/./")) >= 0)
{
path = path.Substring(0, i) + path.Substring(i + 2);
}
// Remove embedded /../ if possible
i = 0;
while ((i = path.IndexOf("/../", i)) >= 0)
{
/*
* A "/../" will cancel the previous segment and itself,
* unless that segment is a "/../" itself
* i.e. "/a/b/../c" becomes "/a/c"
* but "/../../a" should stay unchanged
*/
if (i > 0 && (limit = path.LastIndexOf('/', i - 1)) >= 0 && (path.IndexOf("/../", limit) != 0))
{
path = path.Substring(0, limit) + path.Substring(i + 3);
i = 0;
}
else
{
i = i + 3;
}
}
// Remove trailing .. if possible
while (path.EndsWith("/.."))
{
i = path.IndexOf("/..");
if ((limit = path.LastIndexOf('/', i - 1)) >= 0)
{
path = path.Substring(0, limit + 1);
}
else
{
break;
}
}
// Remove starting .
if (path.StartsWith("./") && path.Length() > 2)
{
path = path.Substring(2);
}
// Remove trailing .
if (path.EndsWith("/."))
{
path = path.Substring(0, path.Length() - 1);
}
}
SetURL(u, protocol, host, port, authority, userInfo, path, query, @ref);
}
internal HostPortrange(String scheme, String str)
{
// Parse the host name. A name has up to three components, the
// hostname, a port number, or two numbers representing a port
// range. "www.sun.com:8080-9090" is a valid host name.
// With IPv6 an address can be 2010:836B:4179::836B:4179
// An IPv6 address needs to be enclose in []
// For ex: [2010:836B:4179::836B:4179]:8080-9090
// Refer to RFC 2732 for more information.
// first separate string into two fields: hoststr, portstr
String hoststr, portstr = null;
this.Scheme = scheme;
// check for IPv6 address
if (str.CharAt(0) == '[')
{
Ipv6 = Literal_Renamed = true;
int rb = str.IndexOf(']');
if (rb != -1)
{
hoststr = str.Substring(1, rb - 1);
}
else
{
throw new IllegalArgumentException("invalid IPv6 address: " + str);
}
int sep = str.IndexOf(':', rb + 1);
if (sep != -1 && str.Length() > sep)
{
portstr = str.Substring(sep + 1);
}
// need to normalize hoststr now
sbyte[] ip = IPAddressUtil.textToNumericFormatV6(hoststr);
if (ip == null)
{
throw new IllegalArgumentException("illegal IPv6 address");
}
StringBuilder sb = new StringBuilder();
Formatter formatter = new Formatter(sb, Locale.US);
formatter.format("%02x%02x:%02x%02x:%02x%02x:%02x" + "%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x", ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], ip[8], ip[9], ip[10], ip[11], ip[12], ip[13], ip[14], ip[15]);
Hostname_Renamed = sb.ToString();
}
else
{
// not IPv6 therefore ':' is the port separator
int sep = str.IndexOf(':');
if (sep != -1 && str.Length() > sep)
{
hoststr = str.Substring(0, sep);
portstr = str.Substring(sep + 1);
}
else
{
hoststr = sep == -1 ? str : str.Substring(0, sep);
}
// is this a domain wildcard specification?
if (hoststr.LastIndexOf('*') > 0)
{
throw new IllegalArgumentException("invalid host wildcard specification");
}
else if (hoststr.StartsWith("*"))
{
Wildcard_Renamed = true;
if (hoststr.Equals("*"))
{
hoststr = "";
}
else if (hoststr.StartsWith("*."))
{
hoststr = ToLowerCase(hoststr.Substring(1));
}
else
{
throw new IllegalArgumentException("invalid host wildcard specification");
}
}
else
{
// check if ipv4 (if rightmost label a number)
// The normal way to specify ipv4 is 4 decimal labels
// but actually three, two or single label formats valid also
// So, we recognise ipv4 by just testing the rightmost label
// being a number.
int lastdot = hoststr.LastIndexOf('.');
if (lastdot != -1 && (hoststr.Length() > 1))
{
bool ipv4 = true;
for (int i = lastdot + 1, len = hoststr.Length(); i < len; i++)
{
char c = hoststr.CharAt(i);
if (c < '0' || c > '9')
{
ipv4 = false;
break;
}
}
this.Ipv4 = this.Literal_Renamed = ipv4;
if (ipv4)
{
sbyte[] ip = IPAddressUtil.textToNumericFormatV4(hoststr);
if (ip == null)
{
throw new IllegalArgumentException("illegal IPv4 address");
}
StringBuilder sb = new StringBuilder();
Formatter formatter = new Formatter(sb, Locale.US);
formatter.format("%d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3]);
hoststr = sb.ToString();
}
else
{
// regular domain name
hoststr = ToLowerCase(hoststr);
}
}
}
Hostname_Renamed = hoststr;
}
try
{
Portrange_Renamed = ParsePort(portstr);
}
catch (Exception)
{
throw new IllegalArgumentException("invalid port range: " + portstr);
}
}
static void ReduceInitializeThrow(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = dc.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
{
// Test logic:
{
string exectempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_ReduceInitializeThrow-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(exectempdir))
{
System.IO.Directory.CreateDirectory(exectempdir);
}
string execfp = exectempdir + @"\baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}";
System.IO.File.WriteAllText(execfp, (@"<?xml version=`1.0` encoding=`utf-8`?>
<SourceCode>
<Jobs>
<Job Name=`PrepJob` Custodian=`` email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt -dfs del baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}_*.txt`,true); // Clean previous run.
}
]]>
</Local>
</Job>
<Job description=`Load sample data` Name=`baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}_LoadData` Custodian=`` email=``>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO>
<DFSReader></DFSReader>
<DFSWriter>dfs://baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}_Input.txt</DFSWriter>
</DFS_IO>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
dfsoutput.WriteLine(@`
MySpace is for everyone:
Friends who want to talk Online
Single people who want to meet other Singles
Matchmakers who want to connect their friends with other friends
Families who want to keep in touch--map your Family Tree
Business people and co-workers interested in networking
Classmates and study partners
Anyone looking for long lost friends!
`);
}
]]>
</Remote>
</Job>
<Job Name=`baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}` Custodian=`` email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>16</KeyLength>
<DFSInput>dfs://baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}_Input.txt</DFSInput>
<DFSOutput>dfs://baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}_Output.txt</DFSOutput>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
mstring sLine= mstring.Prepare(line);
mstringarray parts = sLine.SplitM(' ');
for(int i=0; i < parts.Length; i++)
{
mstring word = parts[i];
if(word.Length > 0 && word.Length <= 16) // Word cannot be longer than the KeyLength!
{
output.Add(word.ToLowerM(), mstring.Prepare(1));
}
}
}
]]>
</Map>
<ReduceInitialize>
<![CDATA[
public void ReduceInitialize()
{
Qizmt_Log(`ReduceInitialize {3AF77789-4D38-4fde-B5DD-DC5115A909F5}`);
throw new Exception(`Exception from ReduceInitialize {D5C5B240-537E-4b80-AD70-6463F632EE7B}`);
}
]]>
</ReduceInitialize>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
mstring sLine = mstring.Prepare(UnpadKey(key));
sLine = sLine.AppendM(',').AppendM(values.Length);
output.Add(sLine);
}
]]>
</Reduce>
<ReduceFinalize>
<![CDATA[
public void ReduceFinalize()
{
Qizmt_Log(`ReduceFinalize {3C677456-22C5-46cd-A1E7-47383274C0C5}`);
}
]]>
</ReduceFinalize>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"'));
Exec.Shell("Qizmt importdir " + exectempdir);
try
{
System.IO.File.Delete(execfp);
System.IO.Directory.Delete(exectempdir);
}
catch
{
}
// Run it twice: first suppressing errors to get stdout, then again to get stderr exception.
string output = Exec.Shell("Qizmt exec baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}", true);
try
{
Exec.Shell("Qizmt exec baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}");
throw new Exception("<<< Job completed without errors; this is wrong! >>>");
}
catch (Exception e)
{
string err = e.ToString();
string badstr = "System.FormatException: Expected 16 hex digits, got \"-000000000000000\"";
if (-1 != err.IndexOf(badstr) || -1 != output.IndexOf(badstr))
{
throw new Exception("Test failed: broken protocol!", e);
}
if (-1 == err.IndexOf("{D5C5B240-537E-4b80-AD70-6463F632EE7B}"))
{
throw new Exception("Test failed: did not get expected exception from exec", e);
}
}
finally
{
Exec.Shell("Qizmt del baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}*");
}
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
public static void ExecOneLocal(SourceCode.Job cfgj, string[] ExecArgs, bool verbose)
{
if (verbose)
{
Console.WriteLine("[{0}] [Local: {2}]", System.DateTime.Now.ToString(), System.DateTime.Now.Millisecond, cfgj.NarrativeName);
}
int BlockID = 0;
string SlaveIP = null;
string logname = Surrogate.SafeTextPath(cfgj.NarrativeName) + "_" + Guid.NewGuid().ToString() + ".j" + sjid + "_log.txt";
bool aborting = false;
try
{
dfs dc = LoadDfsConfig();
string firstslave = dc.Slaves.GetFirstSlave();
string SlaveHost = cfgj.IOSettings.LocalHost;
if (SlaveHost == null || SlaveHost.Length == 0)
{
SlaveHost = firstslave;
}
SlaveIP = IPAddressUtil.GetIPv4Address(SlaveHost);
MySpace.DataMining.DistributedObjects5.Remote rem = new MySpace.DataMining.DistributedObjects5.Remote(cfgj.NarrativeName + "_local");
rem.OutputStartingPoint = BlockID;
rem.LocalCompile = true;
rem.CompilerOptions = cfgj.IOSettings.CompilerOptions;
rem.CompilerVersion = cfgj.IOSettings.CompilerVersion;
if (cfgj.OpenCVExtension != null)
{
rem.AddOpenCVExtension();
}
if (cfgj.MemCache != null)
{
rem.AddMemCacheExtension();
}
if (cfgj.Unsafe != null)
{
rem.AddUnsafe();
}
if (cfgj.AssemblyReferencesCount > 0)
{
cfgj.AddAssemblyReferences(rem.CompilerAssemblyReferences, Surrogate.NetworkPathForHost(firstslave));
}
rem.SetJID(jid, CurrentJobFileName + " Local: " + cfgj.NarrativeName);
rem.AddBlock(SlaveHost + @"|"
+ (cfgj.ForceStandardError != null ? "&" : "") + logname + @"|slaveid=0");
rem.Open();
string codectx = (@"
public const int DSpace_BlockID = " + BlockID.ToString() + @";
public const int DSpace_ProcessID = DSpace_BlockID;
public const int Qizmt_ProcessID = DSpace_ProcessID;
public const int DSpace_BlocksTotalCount = 1;
public const int DSpace_ProcessCount = DSpace_BlocksTotalCount;
public const int Qizmt_ProcessCount = DSpace_ProcessCount;
public const string DSpace_SlaveHost = `" + SlaveHost + @"`;
public const string DSpace_MachineHost = DSpace_SlaveHost;
public const string Qizmt_MachineHost = DSpace_MachineHost;
public const string DSpace_SlaveIP = `" + SlaveIP + @"`;
public const string DSpace_MachineIP = DSpace_SlaveIP;
public const string Qizmt_MachineIP = DSpace_MachineIP;
public static readonly string[] DSpace_ExecArgs = new string[] { " + ExecArgsCode(ExecArgs) + @" };
public static readonly string[] Qizmt_ExecArgs = DSpace_ExecArgs;
public const string DSpace_ExecDir = @`" + System.Environment.CurrentDirectory + @"`;
public const string Qizmt_ExecDir = DSpace_ExecDir;
static string Shell(string line, bool suppresserrors)
{
return MySpace.DataMining.DistributedObjects.Exec.Shell(line, suppresserrors);
}
static string Shell(string line)
{
return MySpace.DataMining.DistributedObjects.Exec.Shell(line, false);
}
const string _userlogname = `" + logname + @"`;
static System.Threading.Mutex _logmutex = new System.Threading.Mutex(false, `distobjlog`);
private static int userlogsremain = " + AELight.maxuserlogs.ToString() + @";
public static void Qizmt_Log(string line) { DSpace_Log(line); }
public static void DSpace_Log(string line)
{
if(--userlogsremain < 0)
{
return;
}
try
{
_logmutex.WaitOne();
}
catch (System.Threading.AbandonedMutexException)
{
}
try
{
using (System.IO.StreamWriter fstm = System.IO.File.AppendText(_userlogname))
{
fstm.WriteLine(`{0}`, line);
}
}
finally
{
_logmutex.ReleaseMutex();
}
}
public static void Qizmt_LogResult(string line, bool passed) { DSpace_LogResult(line, passed); }
public static void DSpace_LogResult(string name, bool passed)
{
if(passed)
{
DSpace_Log(`[\u00012PASSED\u00010] - ` + name);
}
else
{
DSpace_Log(`[\u00014FAILED\u00010] - ` + name);
}
}
").Replace('`', '"') + CommonDynamicCsCode;
rem.LocalExec(codectx + cfgj.Local, cfgj.Usings);
rem.Close();
if (verbose)
{
Console.Write('*');
ConsoleFlush();
}
}
catch (System.Threading.ThreadAbortException)
{
aborting = true;
}
finally
{
if (!aborting)
{
CheckUserLogs(new string[] { SlaveIP }, logname);
}
}
if (verbose)
{
Console.WriteLine();
Console.WriteLine("[{0}] Done", System.DateTime.Now.ToString(), System.DateTime.Now.Millisecond);
}
}
static void EnableReplication(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
bool withcache = -1 != args[0].IndexOf("withcache", StringComparison.OrdinalIgnoreCase);
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
long bytes_to_add = 0;
if (args.Length > 2)
{
// To-do: ParseCapacity.
bytes_to_add = long.Parse(args[2]);
if (bytes_to_add < 0)
{
throw new Exception("Invalid bytes-to-add (" + bytes_to_add.ToString() + " bytes)");
}
if (bytes_to_add < 1048576)
{
throw new Exception("bytes-to-add must be at least 1 MB");
}
}
int num_files = 0;
if (args.Length > 3)
{
num_files = int.Parse(args[3]);
if (num_files < 0 || num_files > bytes_to_add / 20)
{
throw new Exception("Invalid #files");
}
}
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs olddfs = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = olddfs.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
Console.WriteLine("Backing up DFS.xml to: {0} ...", dfsxmlpathbackup);
try
{
System.IO.File.Delete(dfsxmlpathbackup);
}
catch
{
}
System.IO.File.Move(dfsxmlpath, dfsxmlpathbackup);
try
{
{
Console.WriteLine("Formatting DFS for test...");
Exec.Shell("Qizmt @format Machines=" + string.Join(",", allmachines));
}
{
// Test logic:
Console.WriteLine("Adding some files to DFS...");
Console.Write(" ");
Exec.Shell("Qizmt bingen 1MB 1MB 50");
Console.Write("10%");
Exec.Shell("Qizmt examples");
Console.Write("..15%");
Exec.Shell("Qizmt wordgen 10MB 10MB 100"); // Note: also used by Cacher.
Console.Write("..50%");
Exec.Shell("Qizmt asciigen 50MB 50MB 500");
Console.Write("..100%");
Console.WriteLine();
if (bytes_to_add > 0)
{
Console.WriteLine("Adding {0} bytes as requested (bytes-to-add)...", bytes_to_add);
long bta10 = bytes_to_add / 10;
Console.Write(" ");
Exec.Shell("Qizmt gen bta10-" + Guid.NewGuid().ToString() + " " + bta10.ToString());
Console.Write("10%");
Exec.Shell("Qizmt gen bta20-" + Guid.NewGuid().ToString() + " " + (bta10 * 2).ToString());
Console.Write("..30%");
{
long totsz = (bta10 * 3);
if (num_files > 1)
{
long onesz = totsz / num_files;
//for (int inf = 0; inf < num_files; inf++)
MySpace.DataMining.Threading.ThreadTools.Parallel(
new Action <int>(
delegate(int inf)
{
Exec.Shell("Qizmt gen bta30." + inf.ToString() + "-" + Guid.NewGuid().ToString() + " " + onesz.ToString());
}), num_files, 15);
}
else
{
Exec.Shell("Qizmt gen bta30-" + Guid.NewGuid().ToString() + " " + totsz.ToString());
Console.Write("..60%");
}
}
Exec.Shell("Qizmt gen bta40-" + Guid.NewGuid().ToString() + " " + (bta10 * 4).ToString());
Console.Write("..100%");
Console.WriteLine();
}
if (withcache)
{
Console.WriteLine("Generating cache files...");
string cachertempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_Cacher-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(cachertempdir))
{
System.IO.Directory.CreateDirectory(cachertempdir);
}
string cacherfp = cachertempdir + @"\Cacher";
System.IO.File.WriteAllText(cacherfp, (@"<?xml version=`1.0` encoding=`utf-8`?>
<SourceCode>
<Jobs>
<Job Name=`Cacher`>
<Delta>
<Name>Cacher_cache</Name>
<DFSInput>dfs://10MB</DFSInput>
</Delta>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>100</KeyLength>
<DFSInput></DFSInput>
<DFSOutput>dfs://Cacher_output</DFSOutput>
<KeyMajor>8</KeyMajor>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
output.Add(line, ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output)
{
for(int i = 0; i < values.Length; i++)
{
output.Add(key);
}
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"'));
Exec.Shell("Qizmt importdir " + cachertempdir);
try
{
System.IO.File.Delete(cacherfp);
System.IO.Directory.Delete(cachertempdir);
}
catch
{
}
Exec.Shell("Qizmt exec Cacher"); // Creates cache file Cacher_cache
Exec.Shell("Qizmt del Cacher_output");
}
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
string ls_output = Exec.Shell("Qizmt ls");
int ls_output_linecount = ls_output.Split('\n').Length;
Console.WriteLine("*** ls output before replication:");
Console.WriteLine(ls_output);
Console.WriteLine("Updating Replication Factor...");
const int replicationfactor = 2;
Console.WriteLine(Exec.Shell("Qizmt replicationupdate " + replicationfactor.ToString()));
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
if (withcache)
{
Console.WriteLine("Validate cache files...");
Exec.Shell("Qizmt exec Cacher"); // Uses existing cache file Cacher_cache
Exec.Shell("Qizmt del Cacher_output");
}
Console.WriteLine("Ensure data is replicated...");
EnsureReplication(dfsxmlpath, replicationfactor);
{
// Not comparing contents because of the free disk space line.
string new_ls_output = Exec.Shell("Qizmt ls");
Console.WriteLine("*** ls output after replication:");
Console.WriteLine(new_ls_output);
int new_ls_output_linecount = new_ls_output.Split('\n').Length;
if (ls_output_linecount != new_ls_output_linecount)
{
throw new Exception("Cluster does not contain the same files as before replication");
}
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
finally
{
Console.WriteLine("Restoring DFS.xml backup...");
// Note: these are safe; the try/finally only wraps the new dfs.
try
{
Exec.Shell("Qizmt del *");
}
catch
{
}
try
{
// Delete temp dfs.xml, it's being replaced with the good one.
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
System.IO.File.Move(dfsxmlpathbackup, dfsxmlpath);
}
}
static void NewDFS(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs olddfs = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = olddfs.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
Console.WriteLine("Backing up DFS.xml to: {0} ...", dfsxmlpathbackup);
try
{
System.IO.File.Delete(dfsxmlpathbackup);
}
catch
{
}
System.IO.File.Move(dfsxmlpath, dfsxmlpathbackup);
try
{
{
Console.WriteLine("Formatting DFS for test...");
Exec.Shell("Qizmt @format Machines=" + string.Join(",", allmachines));
}
{
// Test logic:
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
finally
{
Console.WriteLine("Restoring DFS.xml backup...");
// Note: these are safe; the try/finally only wraps the new dfs.
try
{
Exec.Shell("Qizmt del *");
}
catch
{
}
try
{
// Delete temp dfs.xml, it's being replaced with the good one.
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
System.IO.File.Move(dfsxmlpathbackup, dfsxmlpath);
}
}
static void SortedCache(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = dc.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
{
// Test logic:
string fguid = "{" + Guid.NewGuid().ToString() + "}";
string jobfn = "regression_test_SortedCache-" + Guid.NewGuid().ToString();
SortedCacheCleanup(jobfn); // Cleanup previous run.
try
{
{
Console.WriteLine("Generating data and jobs...");
string exectempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_SortedCache" + fguid;
if (!System.IO.Directory.Exists(exectempdir))
{
System.IO.Directory.CreateDirectory(exectempdir);
}
string execfp = exectempdir + @"\" + jobfn;
string scguid = Guid.NewGuid().ToString();
string ECODE = (@"<SourceCode>
<Jobs>
<Job Name=`CS` Custodian=`Chris Miller` Email=``>
<Delta>
<Name>{D7D3A6FE-8472-4320-9144-486E436D4542}CS_cache</Name>
<DFSInput>{D7D3A6FE-8472-4320-9144-486E436D4542}a.txt;dfs://{D7D3A6FE-8472-4320-9144-486E436D4542}b?.txt</DFSInput>
</Delta>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>1</KeyLength>
<DFSInput></DFSInput>
<DFSOutput>dfs://{D7D3A6FE-8472-4320-9144-486E436D4542}CS_Output.txt</DFSOutput>
<OutputMethod>{9235036E-4A47-4ee5-985F-F19D2F2DE85C}</OutputMethod>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
List<byte> foo = new List<byte>();
List<byte> bar = new List<byte>();
public virtual void Map(ByteSlice line, MapOutput output)
{
foo.Clear();
bar.Clear();
foo.Add((byte)('A' + line[0] % 16)); // A-F only.
bar.Add(line[1]);
output.Add(ByteSlice.Prepare(foo), ByteSlice.Prepare(bar));
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
long result = 0;
while(values.MoveNext())
{
ByteSlice v = values.Current;
result += v[0];
}
mstring ms = mstring.Prepare();
ms.AppendM((char)key[0]);
ms.AppendM(result);
output.Add(ms);
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"');
System.IO.File.WriteAllText(execfp + ".grouped", ECODE.Replace("{9235036E-4A47-4ee5-985F-F19D2F2DE85C}", "grouped"));
System.IO.File.WriteAllText(execfp + ".sorted", ECODE.Replace("{9235036E-4A47-4ee5-985F-F19D2F2DE85C}", "sorted"));
Exec.Shell("Qizmt importdir " + exectempdir);
try
{
System.IO.File.Delete(execfp + ".grouped");
System.IO.File.Delete(execfp + ".sorted");
System.IO.Directory.Delete(exectempdir);
}
catch
{
}
Exec.Shell("Qizmt asciigen {D7D3A6FE-8472-4320-9144-486E436D4542}a.txt 16KB 2B");
Exec.Shell("Qizmt asciigen {D7D3A6FE-8472-4320-9144-486E436D4542}b1.txt 8KB 2B");
Exec.Shell("Qizmt asciigen {D7D3A6FE-8472-4320-9144-486E436D4542}b2.txt_ 8KB 2B");
}
string checksum_grouped = "";
{
Console.WriteLine("Running grouped job...");
Exec.Shell("Qizmt del {D7D3A6FE-8472-4320-9144-486E436D4542}CS_cache");
Exec.Shell("Qizmt del {D7D3A6FE-8472-4320-9144-486E436D4542}CS_Output.txt");
Exec.Shell("Qizmt exec " + jobfn + ".grouped");
Exec.Shell("Qizmt rename {D7D3A6FE-8472-4320-9144-486E436D4542}b2.txt_ {D7D3A6FE-8472-4320-9144-486E436D4542}b2.txt");
Exec.Shell("Qizmt del {D7D3A6FE-8472-4320-9144-486E436D4542}CS_Output.txt");
Exec.Shell("Qizmt exec " + jobfn + ".grouped");
Exec.Shell("Qizmt rename {D7D3A6FE-8472-4320-9144-486E436D4542}b2.txt {D7D3A6FE-8472-4320-9144-486E436D4542}b2.txt_");
checksum_grouped = DfsSum("Sum2", "{D7D3A6FE-8472-4320-9144-486E436D4542}CS_Output.txt");
Console.WriteLine(" checksum2 = {0}", checksum_grouped);
}
string checksum_sorted = "";
{
Console.WriteLine("Running sorted job...");
Exec.Shell("Qizmt del {D7D3A6FE-8472-4320-9144-486E436D4542}CS_cache");
Exec.Shell("Qizmt del {D7D3A6FE-8472-4320-9144-486E436D4542}CS_Output.txt");
Exec.Shell("Qizmt exec " + jobfn + ".sorted");
Exec.Shell("Qizmt rename {D7D3A6FE-8472-4320-9144-486E436D4542}b2.txt_ {D7D3A6FE-8472-4320-9144-486E436D4542}b2.txt");
Exec.Shell("Qizmt del {D7D3A6FE-8472-4320-9144-486E436D4542}CS_Output.txt");
Exec.Shell("Qizmt exec " + jobfn + ".sorted");
Exec.Shell("Qizmt rename {D7D3A6FE-8472-4320-9144-486E436D4542}b2.txt {D7D3A6FE-8472-4320-9144-486E436D4542}b2.txt_");
checksum_sorted = DfsSum("Sum2", "{D7D3A6FE-8472-4320-9144-486E436D4542}CS_Output.txt");
Console.WriteLine(" checksum2 = {0}", checksum_sorted);
}
if (checksum_grouped != checksum_sorted)
{
throw new Exception("Checksums do not match; sort with cache test failed!");
}
}
finally
{
SortedCacheCleanup(jobfn);
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
static void RangeSort(string[] args)
{
string sortmethod = args[0];
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
// Checking for arg[2]==pause later...
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] slaves = dc.Slaves.SlaveList.Split(';');
string[] allmachines;
{
List <string> aml = new List <string>(slaves.Length + 1);
aml.Add(masterhost);
foreach (string slave in slaves)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
string pausefile = "";
if (args.Length > 2 && "pause" == args[2])
{
string pausedir = @"\\" + masterhost + @"\c$\temp\qizmt";
try
{
System.IO.Directory.CreateDirectory(pausedir);
}
catch
{
}
pausefile = pausedir + @"\" + sortmethod + @"-pause.txt";
System.IO.File.WriteAllText(pausefile, "Delete this file to un-pause..." + Environment.NewLine);
Console.WriteLine();
Console.WriteLine("Delete the file '{0}' to un-pause...", pausefile);
Console.WriteLine();
}
{
Console.WriteLine("Ensure cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
{
string fguid = "{" + Guid.NewGuid().ToString() + "}";
// Generate some data to operate on.
Console.WriteLine("Generating data...");
// Note: this test depends on wordgen, and wordgen lines always starting with uppercase!
long gensize = 1048576 * dc.Blocks.SortedTotalCount; // 1MB * #processes
string gencmd = "wordgen";
int keymajor = 8;
if (-1 != sortmethod.IndexOf("hash", StringComparison.OrdinalIgnoreCase))
{
//gencmd = "bingen"; // Will write crazy files to c:\temp
keymajor = 2;
}
Exec.Shell("Qizmt " + gencmd + " data" + fguid + " " + gensize.ToString());
try
{
string exectempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_" + sortmethod + @"-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(exectempdir))
{
System.IO.Directory.CreateDirectory(exectempdir);
}
string execfp = exectempdir + @"\exec" + fguid;
// Note: using c:\temp instead of IOUtils.GetTempDirectory() in the following test
// because I can't get the IOUtils.GetTempDirectory() for other machines.
string scguid = Guid.NewGuid().ToString();
System.IO.File.WriteAllText(execfp, (@"<?xml version=`1.0` encoding=`utf-8`?>
<SourceCode>
<Jobs>
<Job Name=`exec" + fguid + @"`>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>100</KeyLength>
<DFSInput>dfs://data" + fguid + @"</DFSInput>
<DFSOutput>dfs://output" + fguid + @"</DFSOutput>
<KeyMajor>" + keymajor.ToString() + @"</KeyMajor>
<OutputMethod>" + sortmethod + @"</OutputMethod>
<Setting name=`Subprocess_TotalPrime` value=`0` /> <!-- Don't use grouped. -->
<Setting name=`Subprocess_SortedTotalCount` value=`" + slaves.Length.ToString() + @"` />
<!-- ^ One process per participating machine. -->
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
output.Add(line, ByteSlice.Prepare());
}
]]>
</Map>
<ReduceInitialize><![CDATA[
public virtual void ReduceInitialize() { }
]]></ReduceInitialize>
<Reduce>
<![CDATA[
string dir = null;
Dictionary<char, System.IO.StreamWriter> files = new Dictionary<char, System.IO.StreamWriter>();
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
if(null == dir)
{
dir = @`\\` + Qizmt_MachineHost + @`\c$\temp\qizmt\" + sortmethod + @"-" + scguid + @"`;
if(!System.IO.Directory.Exists(dir))
{
System.IO.Directory.CreateDirectory(dir);
}
}
System.IO.StreamWriter stmw;
if(!files.ContainsKey((char)key[0]))
{
stmw = new System.IO.StreamWriter(dir + @`\` + (char)key[0] + `.txt`, true); // append=true
files[(char)key[0]] = stmw;
}
stmw = files[(char)key[0]];
stmw.WriteLine(key.ToString());
}
]]>
</Reduce>
<ReduceFinalize><![CDATA[
public virtual void ReduceFinalize()
{
foreach(KeyValuePair<char, System.IO.StreamWriter> kvp in files)
{
kvp.Value.Close();
}
}
]]></ReduceFinalize>
</MapReduce>
</Job>
<Job Name=`Verify Sort Range` >
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
readonly string[] slaves = `" + dc.Slaves.SlaveList + @"`.Split(';');
readonly string pausefile = @`" + pausefile + @"`;
public virtual void Local()
{
if(!string.IsNullOrEmpty(pausefile))
{
bool bb = false;
while(System.IO.File.Exists(pausefile))
{
if(!bb)
{
bb = true;
try
{
System.IO.File.AppendAllText(pausefile, `Ready!` + Environment.NewLine);
}
catch
{
}
}
System.Threading.Thread.Sleep(1000);
}
}
bool failed = false;
char bound = '\0';
foreach(string slave in slaves)
{
string dir = @`\\` + slave + @`\c$\temp\qizmt\" + sortmethod + @"-" + scguid + @"`;
try
{
char thishighest = '\0';
foreach(System.IO.FileInfo fi in (new System.IO.DirectoryInfo(dir).GetFiles()))
{
char c = fi.Name[0];
if(c < bound)
{
failed = true;
throw new Exception(`Data is not range sorted (" + sortmethod + @") starting on machine ` + slave + ` (Error 1FC8AB58-4DBD-4d56-9587-96312F9A5886)`);
}
if(c > thishighest)
{
thishighest = c;
}
fi.Delete();
}
if(thishighest > bound)
{
bound = thishighest;
}
System.IO.Directory.Delete(dir);
}
catch(Exception e)
{
Qizmt_Log(`Exception: ` + e.ToString());
}
}
if(!failed && bound > '\0')
{
Qizmt_Log(`Success! (OK 55D106EA-AD09-4503-96BA-387795EDEECB)`);
}
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"'));
Exec.Shell("Qizmt importdir " + exectempdir);
try
{
System.IO.File.Delete(execfp);
System.IO.Directory.Delete(exectempdir);
}
catch
{
}
try
{
Console.WriteLine("Running " + sortmethod + " job...");
string output = Exec.Shell("Qizmt exec exec" + fguid);
Console.WriteLine(output.Trim());
if (-1 == output.IndexOf("55D106EA-AD09-4503-96BA-387795EDEECB"))
{
throw new Exception("Sort range order verification (" + sortmethod + ") did not succeed");
}
}
finally
{
Exec.Shell("Qizmt del output" + fguid);
Exec.Shell("Qizmt del exec" + fguid);
}
}
finally
{
Exec.Shell("Qizmt del data" + fguid);
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
}
static void Deploy(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = dc.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
{
Console.WriteLine("Ensure cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
// Run a job...
string exec_md5;
{
// Generate some data to operate on.
Exec.Shell("Qizmt gen data{AE7E8F7E-AE48-40e7-B5B2-7E07E39B46F9} " + 1048576.ToString());
string exectempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_Deploy-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(exectempdir))
{
System.IO.Directory.CreateDirectory(exectempdir);
}
string execfp = exectempdir + @"\exec{07E2B469-80F9-4776-908F-E504A906E3B6}";
System.IO.File.WriteAllText(execfp, (@"<?xml version=`1.0` encoding=`utf-8`?>
<SourceCode>
<Jobs>
<Job Name=`exec{07E2B469-80F9-4776-908F-E504A906E3B6}`>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>100</KeyLength>
<DFSInput>dfs://data{*}</DFSInput>
<DFSOutput>dfs://output{A785E7D1-9017-45fe-9E07-57695192A5DC}</DFSOutput>
<KeyMajor>8</KeyMajor>
<OutputMethod>sorted</OutputMethod>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
output.Add(line, ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
while(values.MoveNext())
{
output.Add(key);
}
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"'));
Exec.Shell("Qizmt importdir " + exectempdir);
try
{
System.IO.File.Delete(execfp);
System.IO.Directory.Delete(exectempdir);
}
catch
{
}
Exec.Shell("Qizmt exec exec{07E2B469-80F9-4776-908F-E504A906E3B6}");
exec_md5 = DfsSum("md5", "output{A785E7D1-9017-45fe-9E07-57695192A5DC}");
Exec.Shell("Qizmt del output{A785E7D1-9017-45fe-9E07-57695192A5DC}");
}
try
{
const string TEMP_DLLS_PATTERN = "temp_????????-????-????-????-????????????.dll";
// Prepare to detect leaked DLLs:
string lmachine = allmachines[allmachines.Length - 1];
string[] dummyleaknames = new string[] {
TEMP_DLLS_PATTERN.Replace('?', 'x'),
//"dummy1D48A66FD2EF41e3B6266C06D320A17D.dll",
//"dummy1D48A66FD2EF41e3B6266C06D320A17D.exe"
};
try
{
// Delete leaked DLLs on lmachine...
foreach (string fn in System.IO.Directory.GetFiles(Surrogate.NetworkPathForHost(lmachine), TEMP_DLLS_PATTERN))
{
System.IO.File.Delete(fn);
}
// Delete planted files from lmachine...
//foreach (string host in allmachines)
{
string host = lmachine;
string netdir = Surrogate.NetworkPathForHost(host);
foreach (string dummyleakname in dummyleaknames)
{
try
{
System.IO.File.Delete(netdir + @"\" + dummyleakname);
}
catch
{
}
}
}
// Plant some new leaked files on surrogate...
foreach (string dummyleakname in dummyleaknames)
{
System.IO.File.WriteAllText(masterdir + @"\" + dummyleakname, "Dummy file for deploy leak detector" + Environment.NewLine);
}
}
catch (Exception e)
{
lmachine = null;
throw new Exception("Failed to prepare for deploy leak detector", e);
}
{
Console.WriteLine("Deleting critical files across cluster to ensure deploy will succeed...");
int nfailed = 0;
string failreason = "";
//foreach (string host in allmachines)
if (allmachines.Length > 1) // Important; can't delete slave.exe on surrogate or it can't deploy it.
{
string host = allmachines[allmachines.Length - 1];
try
{
string netdir = Surrogate.NetworkPathForHost(host);
System.IO.File.Delete(netdir + @"\MySpace.DataMining.DistributedObjects.DistributedObjectsSlave.exe");
}
catch (Exception fe)
{
nfailed++;
failreason = fe.ToString();
}
}
if (nfailed > 0)
{
Console.WriteLine("Warning: {0} files failed to be deleted; {0}", failreason);
}
}
try
{
Console.WriteLine("Deploying...");
Exec.Shell("aelight deploy");
System.Threading.Thread.Sleep(1000 * 5); // Wait a bit for the services to come back up.
}
catch (Exception e)
{
Console.Error.WriteLine(e.ToString());
Console.Error.WriteLine(" WARNING: cluster may be in a bad state; may need to reinstall");
throw;
}
Console.WriteLine("Ensuring deploy succeeded...");
Console.WriteLine("(Note: if this hangs indefinitely, deploy failed and need to reinstall)");
if (lmachine != null)
{
//foreach (string host in allmachines)
{
string host = lmachine;
string netdir = Surrogate.NetworkPathForHost(host);
foreach (string dummyleakname in dummyleaknames)
{
{
string fp = netdir + @"\" + dummyleakname;
if (System.IO.File.Exists(fp))
{
throw new Exception("Deployed dummy/leaked file: " + fp);
}
}
}
}
{
string[] leaks = System.IO.Directory.GetFiles(Surrogate.NetworkPathForHost(lmachine), TEMP_DLLS_PATTERN);
if (leaks.Length > 0)
{
throw new Exception("Deployed leaked dll: " + leaks[0] + " (" + leaks.Length.ToString() + " in total)");
}
}
// Delete the planted dummy files from surrogate!
foreach (string dummyleakname in dummyleaknames)
{
System.IO.File.Delete(masterdir + @"\" + dummyleakname);
}
}
Console.WriteLine("Ensure cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
// Re-run job, confirm good...
{
Exec.Shell("Qizmt exec exec{07E2B469-80F9-4776-908F-E504A906E3B6}");
string new_exec_md5 = DfsSum("md5", "output{A785E7D1-9017-45fe-9E07-57695192A5DC}");
Exec.Shell("Qizmt del output{A785E7D1-9017-45fe-9E07-57695192A5DC}");
if (new_exec_md5 != exec_md5)
{
throw new Exception("Output files from before and after deploy do not match");
}
}
}
finally
{
try
{
Console.WriteLine("Cleaning temporary test data...");
Exec.Shell("Qizmt del exec{07E2B469-80F9-4776-908F-E504A906E3B6}");
Exec.Shell("Qizmt del data{AE7E8F7E-AE48-40e7-B5B2-7E07E39B46F9}");
Exec.Shell("Qizmt del output{A785E7D1-9017-45fe-9E07-57695192A5DC}");
}
catch
{
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
}
static void CacheWithRedundancy(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs olddfs = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = olddfs.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
Console.WriteLine("Backing up DFS.xml to: {0} ...", dfsxmlpathbackup);
try
{
System.IO.File.Delete(dfsxmlpathbackup);
}
catch
{
}
System.IO.File.Move(dfsxmlpath, dfsxmlpathbackup);
try
{
{
// Note: added replication.
Console.WriteLine("Formatting DFS with Replication for test...");
Exec.Shell("Qizmt @format Machines=" + string.Join(",", allmachines) + " Replication=2");
}
{
// Test logic:
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Adding files to DFS...");
Exec.Shell("Qizmt wordgen Cacher_input 10MB 100");
Console.WriteLine("Generating cache files...");
string cachertempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_Cacher-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(cachertempdir))
{
System.IO.Directory.CreateDirectory(cachertempdir);
}
string cacherfp = cachertempdir + @"\Cacher";
System.IO.File.WriteAllText(cacherfp, (@"<?xml version=`1.0` encoding=`utf-8`?>
<SourceCode>
<Jobs>
<Job Name=`Cacher`>
<Delta>
<Name>Cacher_cache</Name>
<DFSInput>dfs://Cacher_input</DFSInput>
</Delta>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>100</KeyLength>
<DFSInput></DFSInput>
<DFSOutput>dfs://Cacher_output</DFSOutput>
<KeyMajor>8</KeyMajor>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
output.Add(line, ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output)
{
for(int i = 0; i < values.Length; i++)
{
output.Add(key);
}
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"'));
Exec.Shell("Qizmt importdir " + cachertempdir);
try
{
System.IO.File.Delete(cacherfp);
System.IO.Directory.Delete(cachertempdir);
}
catch
{
}
Exec.Shell("Qizmt exec Cacher"); // Creates cache file Cacher_cache
string cacher_output_sum = DfsSum("Cacher_output");
string cacher_output_md5 = DfsSum("md5", "Cacher_output");
if (cacher_output_sum != DfsSum("Cacher_input"))
{
throw new Exception("Output file does not have same checksum as input file");
}
Exec.Shell("Qizmt del Cacher_output");
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Validate cache files...");
Exec.Shell("Qizmt exec Cacher"); // Uses existing cache file Cacher_cache
if (cacher_output_sum != DfsSum("Cacher_output"))
{
throw new Exception("Output file not the same when using cache (sum)");
}
if (cacher_output_md5 != DfsSum("md5", "Cacher_output"))
{
throw new Exception("Output file not the same when using cache (md5)");
}
Exec.Shell("Qizmt del Cacher_output");
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
finally
{
Console.WriteLine("Restoring DFS.xml backup...");
// Note: these are safe; the try/finally only wraps the new dfs.
try
{
Exec.Shell("Qizmt del *");
}
catch
{
}
try
{
// Delete temp dfs.xml, it's being replaced with the good one.
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
System.IO.File.Move(dfsxmlpathbackup, dfsxmlpath);
}
}
static void DfsUpdateStressTest(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = dc.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
{
Console.WriteLine("Ensure cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Stressing DFS updates...");
//System.Threading.Thread.Sleep(1000 * 8);
string exectempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_DfsUpdateStressTest-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(exectempdir))
{
System.IO.Directory.CreateDirectory(exectempdir);
}
string execfp = exectempdir + @"\exec{8B8F731B-3BEC-4e99-B08F-BDEB81525172}";
const int NUMBER_OF_JOBS = 50; // <--STRESS-NUMBER--
for (int njob = 0; njob < NUMBER_OF_JOBS; njob++)
{
System.IO.File.WriteAllText(execfp + njob.ToString(), (@"<SourceCode><Jobs></Jobs></SourceCode>").Replace('`', '"'));
}
try
{
Exec.Shell("Qizmt importdirmt " + exectempdir);
Console.WriteLine("Confirming updates...");
{
string lsoutput = Exec.Shell("Qizmt ls");
int njobs = 0;
for (int i = 0; ;)
{
if (njobs == NUMBER_OF_JOBS)
{
break; // Good!
}
i = lsoutput.IndexOf("{8B8F731B-3BEC-4e99-B08F-BDEB81525172}", i);
if (-1 == i)
{
throw new Exception("Not all updates to DFS were written (only found " + njobs.ToString() + " jobs imported, expected " + NUMBER_OF_JOBS.ToString() + ")");
}
i += "{8B8F731B-3BEC-4e99-B08F-BDEB81525172}".Length;
njobs++;
}
}
}
finally
{
try
{
for (int njob = 0; njob < NUMBER_OF_JOBS; njob++)
{
System.IO.File.Delete(execfp + njob.ToString());
}
System.IO.Directory.Delete(exectempdir);
}
catch
{
}
try
{
Exec.Shell("Qizmt del exec{8B8F731B-3BEC-4e99-B08F-BDEB81525172}*");
}
catch
{
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
}
internal void threadproc()
{
try
{
string reason;
if (!Surrogate.IsHealthySlaveMachine(SlaveHost, out reason))
{
if (explicithost)
{
throw new Exception("Remote cannot connect to explicit host '" + SlaveHost + "': " + reason);
}
/*if (!failoverenabled)
* {
* throw new Exception("Remote cannot connect to host '" + SlaveHost + "': " + reason);
* }
* else*/
{
SlaveHost = null;
int startsi = (new Random(DateTime.Now.Millisecond / (BlockID + 2))).Next() % slaves.Count;
for (int i = startsi; i < slaves.Count; i++)
{
if (Surrogate.IsHealthySlaveMachine(slaves[i], out reason))
{
SlaveHost = slaves[i];
rem.OutputStartingPoint = i;
break;
}
}
if (null == SlaveHost)
{
for (int i = 0; i < startsi; i++)
{
if (Surrogate.IsHealthySlaveMachine(slaves[i], out reason))
{
SlaveHost = slaves[i];
rem.OutputStartingPoint = i;
break;
}
}
if (null == SlaveHost)
{
throw new Exception("Remote cannot connect to any hosts; last reason: " + reason);
}
}
}
}
SlaveIP = IPAddressUtil.GetIPv4Address(SlaveHost); // !
rem.SetJID(jid, CurrentJobFileName + " Remote: " + cfgj.NarrativeName);
rem.AddBlock(SlaveIP + @"|"
+ (cfgj.ForceStandardError != null ? "&" : "") + logname + @"|slaveid=0");
string codectx = (@"
public const int DSpace_BlockID = " + BlockID.ToString() + @";
public const int DSpace_ProcessID = DSpace_BlockID;
public const int Qizmt_ProcessID = DSpace_ProcessID;
public const int DSpace_BlocksTotalCount = " + blockcount.ToString() + @";
public const int DSpace_ProcessCount = DSpace_BlocksTotalCount;
public const int Qizmt_ProcessCount = DSpace_ProcessCount;
public const string DSpace_SlaveHost = `" + SlaveHost + @"`;
public const string DSpace_MachineHost = DSpace_SlaveHost;
public const string Qizmt_MachineHost = DSpace_MachineHost;
public const string DSpace_SlaveIP = `" + SlaveIP + @"`;
public const string DSpace_MachineIP = DSpace_SlaveIP;
public const string Qizmt_MachineIP = DSpace_MachineIP;
public static readonly string[] DSpace_ExecArgs = new string[] { " + ExecArgsCode(ExecArgs) + @" };
public static readonly string[] Qizmt_ExecArgs = DSpace_ExecArgs;
public const string DSpace_OutputFilePath = `" + DFSWriter + @"`; // Includes `dfs://` if in DFS.
public const string Qizmt_OutputFilePath = DSpace_OutputFilePath;
public static int DSpace_InputRecordLength { get { return MySpace.DataMining.DistributedObjects.StaticGlobals.DSpace_InputRecordLength; } }
public static int Qizmt_InputRecordLength { get { return MySpace.DataMining.DistributedObjects.StaticGlobals.DSpace_InputRecordLength; } }
public static int DSpace_OutputRecordLength { get { return MySpace.DataMining.DistributedObjects.StaticGlobals.DSpace_OutputRecordLength; } }
public static int Qizmt_OutputRecordLength { get { return MySpace.DataMining.DistributedObjects.StaticGlobals.DSpace_OutputRecordLength; } }
public const string Qizmt_Meta = @`" + Meta + @"`;
const string _userlogname = `" + logname + @"`;
static System.Threading.Mutex _logmutex = new System.Threading.Mutex(false, `distobjlog`);
static string Shell(string line, bool suppresserrors)
{
return MySpace.DataMining.DistributedObjects.Exec.Shell(line, suppresserrors);
}
static string Shell(string line)
{
return MySpace.DataMining.DistributedObjects.Exec.Shell(line, false);
}
private static int userlogsremain = " + AELight.maxuserlogs.ToString() + @";
public static void Qizmt_Log(string line) { DSpace_Log(line); }
public static void DSpace_Log(string line)
{
if(--userlogsremain < 0)
{
return;
}
try
{
_logmutex.WaitOne();
}
catch (System.Threading.AbandonedMutexException)
{
}
try
{
using (System.IO.StreamWriter fstm = System.IO.File.AppendText(_userlogname))
{
fstm.WriteLine(`{0}`, line);
}
}
finally
{
_logmutex.ReleaseMutex();
}
}
public static void Qizmt_LogResult(string line, bool passed) { DSpace_LogResult(line, passed); }
public static void DSpace_LogResult(string name, bool passed)
{
if(passed)
{
DSpace_Log(`[\u00012PASSED\u00010] - ` + name);
}
else
{
DSpace_Log(`[\u00014FAILED\u00010] - ` + name);
}
}
").Replace('`', '"') + CommonDynamicCsCode;
List <string> outputbasenames = new List <string>(DFSWriters.Count);
foreach (string writer in DFSWriters)
{
string dfsname = writer;
if (dfsname.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
dfsname = dfsname.Substring(6);
}
outputbasenames.Add(GenerateZdFileDataNodeBaseName(dfsname));
}
rem.Open();
outputsizes = new List <long>();
outputsizeses = new List <List <long> >();
outputdfsnodeses = new List <List <string> >();
rem.RemoteExec(dfsinputpaths, outputdfsdirs, outputbasenames, baseoutputfilesize, codectx + cfgj.Remote, cfgj.Usings, outputsizeses, outputdfsnodeses, dfsinputfilenames, dfsinputnodesoffsets);
blockfail = false; // !
if (verbose)
{
Console.Write('*');
ConsoleFlush();
}
}
catch (Exception e)
{
LogOutput("RemoteBlockInfo.threadproc exception: " + e.ToString());
}
}
static void RemoveMachine(string[] args)
{
bool two2one = -1 != args[0].IndexOf("2to1", StringComparison.OrdinalIgnoreCase);
bool three2two = -1 != args[0].IndexOf("3to2", StringComparison.OrdinalIgnoreCase);
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
string masterslavedat = masterdir + @"\slave.dat";
dfs olddfs = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
int iarg = 2;
string sreplication = "1";
if (args.Length > iarg)
{
if (args[iarg].StartsWith("#"))
{
sreplication = args[iarg++].Substring(1);
}
}
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = olddfs.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
if (two2one)
{
if (allmachines.Length < 2)
{
throw new Exception("Need >= 2 machines for 2to1");
}
allmachines = new string[] { allmachines[0], allmachines[1] };
}
if (three2two)
{
if (allmachines.Length < 3)
{
throw new Exception("Need >= 3 machines for 3to2");
}
allmachines = new string[] { allmachines[0], allmachines[1], allmachines[2] };
}
if (allmachines.Length < 2)
{
throw new Exception("Cluster needs at least 2 machines for this test");
}
Console.WriteLine("Backing up DFS.xml to: {0} ...", dfsxmlpathbackup);
try
{
System.IO.File.Delete(dfsxmlpathbackup);
}
catch
{
}
System.IO.File.Move(dfsxmlpath, dfsxmlpathbackup);
try
{
Console.WriteLine("Formatting DFS for test...");
{
string fmtcmd = "Qizmt @format Machines=" + string.Join(",", allmachines) + " Replication=" + sreplication;
Console.WriteLine(" {0}", fmtcmd);
Exec.Shell(fmtcmd);
}
Console.WriteLine("Adding some files to DFS...");
Console.Write(" ");
Exec.Shell("Qizmt bingen 1MB 1MB 50");
Console.Write("10%");
Exec.Shell("Qizmt examples");
Console.Write("..15%");
Exec.Shell("Qizmt wordgen 10MB 10MB 100");
Console.Write("..50%");
Exec.Shell("Qizmt asciigen 50MB 50MB 500");
Console.Write("..100%");
Console.WriteLine();
int ls_output_linecount = Exec.Shell("Qizmt ls").Split('\n').Length;
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Run test job, save output...");
string md5_10MB_output = Exec.Shell("Qizmt md5 10MB");
string rmachine = allmachines[allmachines.Length - 1];
Console.WriteLine("Removing machine (removemachine {0}) ...", rmachine);
Console.WriteLine(Exec.Shell("Qizmt removemachine " + rmachine));
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Run test job, confirm output...");
if (md5_10MB_output != Exec.Shell("Qizmt md5 10MB"))
{
throw new Exception("Test job output does not match previous run");
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
finally
{
Console.WriteLine("Restoring DFS.xml backup...");
// Note: these are safe; the try/finally only wraps the new dfs.
try
{
Exec.Shell("Qizmt del *");
}
catch
{
}
try
{
// Delete temp dfs.xml, it's being replaced with the good one.
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
System.IO.File.Move(dfsxmlpathbackup, dfsxmlpath);
}
}
public static MemoryStream DownloadDataFromUri(Uri uri, long expectedMaxResponseSize, Func <long, bool, bool> responseValidationCallback, bool isUrlUserInput = true, bool isBposUser = true)
{
string text = Guid.NewGuid().ToString();
if (isBposUser && isUrlUserInput && IPAddressUtil.IsIntranetAddress(uri))
{
throw new DownloadPermanentException();
}
string text2 = uri.OriginalString;
if (text2.Contains("?"))
{
text2 = text2 + "&corr=" + text;
}
else
{
text2 = text2 + "?corr=" + text;
}
uri = new Uri(text2);
MemoryStream result = null;
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(uri);
httpWebRequest.AllowAutoRedirect = true;
httpWebRequest.Timeout = 30000;
httpWebRequest.CachePolicy = SynchronousDownloadData.NoCachePolicy;
try
{
using (HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse())
{
using (Stream responseStream = httpWebResponse.GetResponseStream())
{
long num = expectedMaxResponseSize + 1L;
if (responseStream.CanSeek)
{
num = Math.Min(num, responseStream.Length);
}
byte[] array = new byte[num];
int num2 = 0;
int num3;
do
{
num3 = responseStream.Read(array, num2, array.Length - num2);
num2 += num3;
responseValidationCallback((long)num2, true);
}while (num3 > 0);
result = new MemoryStream(array, 0, num2);
}
}
}
catch (Exception ex)
{
ExtensionDiagnostics.Logger.LogEvent(ApplicationLogicEventLogConstants.Tuple_DownloadDataFromOfficeMarketPlaceFailed, null, new object[]
{
"DownloadNewApp",
text,
uri,
ExtensionDiagnostics.GetLoggedExceptionString(ex)
});
throw ex;
}
ExtensionDiagnostics.LogToDatacenterOnly(ApplicationLogicEventLogConstants.Tuple_DownloadDataFromOfficeMarketPlaceSucceeded, null, new object[]
{
"DownloadNewApp",
text,
uri
});
return(result);
}
static void ServiceCommands(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = IPAddressUtil.GetName(Surrogate.MasterHost);
string[] allmachines;
{
string[] sl = dc.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(IPAddressUtil.GetName(slave));
}
}
allmachines = aml.ToArray();
}
string tempdir = @"\\" + System.Net.Dns.GetHostName() + @"\c$\temp";
if (!System.IO.Directory.Exists(tempdir))
{
System.IO.Directory.CreateDirectory(tempdir);
}
string hoststxtpath = tempdir + @"\dspace-hosts-" + Guid.NewGuid().ToString() + ".txt";
System.IO.File.WriteAllText(hoststxtpath, "# Test" + Environment.NewLine + string.Join(Environment.NewLine, allmachines));
try
{
Console.WriteLine("Confirm initial cluster status...");
{
Dictionary <string, string> status = _GetStatusAll("");
if (status.Count != allmachines.Length)
{
throw new Exception("Qizmt servicestatusall: (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("Qizmt servicestatusall: " + e.Message, e);
}
}
{
string arg = string.Join(",", allmachines);
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
{
string arg = "\"@" + hoststxtpath + "\"";
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
Console.WriteLine("Stopping surrogate...");
Console.WriteLine(Exec.Shell("Qizmt stopsurrogate"));
Console.WriteLine("Confirm stopped status of surrogate...");
{
string host, statusmsg;
if (!_GetStatusFromLine(Exec.Shell("Qizmt servicestatussurrogate"), out host, out statusmsg) ||
statusmsg.StartsWith("RUNNING"))
{
throw new Exception("Qizmt servicestatussurrogate");
}
}
{
Dictionary <string, string> status = _GetStatusAll("");
if (status.Count != allmachines.Length)
{
throw new Exception("(surrogate down check) Qizmt servicestatusall: (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
throw new Exception("(surrogate down check) Qizmt servicestatusall: cluster reported healthy, but surrogate should be down");
}
catch
{
// Expected!
}
}
Console.WriteLine("Starting surrogate...");
Console.WriteLine(Exec.Shell("Qizmt startsurrogate"));
Console.WriteLine("Confirm started status of surrogate...");
{
string host, statusmsg;
if (!_GetStatusFromLine(Exec.Shell("Qizmt servicestatussurrogate"), out host, out statusmsg) ||
!statusmsg.StartsWith("RUNNING"))
{
throw new Exception("(restarted surrogate check) Qizmt servicestatussurrogate");
}
}
{
Dictionary <string, string> status = _GetStatusAll("");
if (status.Count != allmachines.Length)
{
throw new Exception("(restarted surrogate check) Qizmt servicestatusall: (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("(restarted surrogate check) Qizmt servicestatusall: " + e.Message, e);
}
}
{
string arg = string.Join(",", allmachines);
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("(restarted surrogate check) Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("(restarted surrogate check) Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
{
string arg = "\"@" + hoststxtpath + "\"";
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("(restarted surrogate check) Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("(restarted surrogate check) Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
foreach (string xallargs in new string[] { "", string.Join(",", allmachines), "\"@" + hoststxtpath + "\"" })
{
Console.WriteLine("Stopping all services in cluster, including surrogate...");
Console.WriteLine(Exec.Shell("Qizmt stopall " + xallargs));
Console.WriteLine("Confirming stopped status of all services...");
{
Dictionary <string, string> status = _GetStatusAll("");
if (status.Count != allmachines.Length)
{
throw new Exception("stopall " + xallargs + ": (stopped all services check) Qizmt servicestatusall: (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "STOPPED");
}
catch (Exception e)
{
throw new Exception("stopall " + xallargs + ": (stopped all services check) Qizmt servicestatusall: " + e.Message, e);
}
}
{
string arg = string.Join(",", allmachines);
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("stopall " + xallargs + ": (stopped all services check) Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "STOPPED");
}
catch (Exception e)
{
throw new Exception("stopall " + xallargs + ": (stopped all services check) Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
{
string arg = "\"@" + hoststxtpath + "\"";
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("stopall " + xallargs + ": (stopped all services check) Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "STOPPED");
}
catch (Exception e)
{
throw new Exception("stopall " + xallargs + ": (stopped all services check) Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
Console.WriteLine("Starting all services in cluster, including surrogate...");
Console.WriteLine(Exec.Shell("Qizmt startall " + xallargs));
Console.WriteLine("Confirm started status of all services...");
{
Dictionary <string, string> status = _GetStatusAll("");
if (status.Count != allmachines.Length)
{
throw new Exception("startall " + xallargs + ": (restarted all services check) Qizmt servicestatusall: (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("startall " + xallargs + ": (restarted all services check) Qizmt servicestatusall: " + e.Message, e);
}
}
{
string arg = string.Join(",", allmachines);
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("startall " + xallargs + ": (restarted all services check) Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("startall " + xallargs + ": (restarted all services check) Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
{
string arg = "\"@" + hoststxtpath + "\"";
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("startall " + xallargs + ": (restarted all services check) Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("startall " + xallargs + ": (restarted all services check) Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
}
}
finally
{
try
{
System.IO.File.Delete(hoststxtpath);
}
catch
{
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
static void Kill(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = dc.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
{
Console.WriteLine("Ensure cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Run job in one thread, kill from another...");
//System.Threading.Thread.Sleep(1000 * 8);
string exectempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_Kill-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(exectempdir))
{
System.IO.Directory.CreateDirectory(exectempdir);
}
string execfp = exectempdir + @"\exec{3EAE6884-28BB-4340-8F94-6E9421B68C92}";
System.IO.File.WriteAllText(execfp, (@"<SourceCode>
<Jobs>
<Job Name=`exec{3EAE6884-28BB-4340-8F94-6E9421B68C92}_Preprocessing 1`>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
// Sleep forever so that kill will take it down.
System.Threading.Thread.Sleep(System.Threading.Timeout.Infinite);
}
]]>
</Local>
</Job>
<Job Name=`exec{3EAE6884-28BB-4340-8F94-6E9421B68C92}_Preprocessing 2`>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Qizmt_Log(`{DC46FA81-A69F-4d46-9A30-54869168916B}`);
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"'));
Exec.Shell("Qizmt importdir " + exectempdir);
try
{
try
{
System.IO.File.Delete(execfp);
System.IO.Directory.Delete(exectempdir);
}
catch
{
}
bool execdone = false;
bool execx = false;
System.Threading.Thread execthread = new System.Threading.Thread(
new System.Threading.ThreadStart(
delegate()
{
try
{
Console.WriteLine(" Running exec...");
string output = Exec.Shell("Qizmt exec exec{3EAE6884-28BB-4340-8F94-6E9421B68C92}");
Console.WriteLine("exec output: {0}", output.Trim());
if (-1 != output.IndexOf("{DC46FA81-A69F-4d46-9A30-54869168916B}"))
{
lock (typeof(Program))
{
execx = true;
}
}
}
catch (Exception e)
{
Console.Error.WriteLine("Warning: exec exception: {0}", e.ToString());
}
lock (typeof(Program))
{
execdone = true;
}
}));
execthread.Start();
// Wait a few seconds to give the exec a chance to get started.
System.Threading.Thread.Sleep(1000 * 5);
lock (typeof(Program))
{
if (execx)
{
throw new Exception("exec completed; problem with test");
}
if (execdone)
{
throw new Exception("exec finished too early, did not get a chance to call kill");
}
}
int execjid = 0;
string execsjid = "0";
string psexecline = "N/A";
{
foreach (string psln in Exec.Shell("qizmt ps").Split('\n'))
{
if (-1 != psln.IndexOf("exec{3EAE6884-28BB-4340-8F94-6E9421B68C92}"))
{
psexecline = psln.Trim();
{
int isp = psexecline.IndexOf(' ');
if (-1 != isp)
{
execsjid = psexecline.Substring(0, isp);
}
}
break;
}
}
}
if (!int.TryParse(execsjid, out execjid) || execjid < 1)
{
throw new Exception("JID for job not valid: " + execsjid);
}
execsjid = execjid.ToString(); // Normalize.
Console.WriteLine(" Running kill... ({0})", psexecline);
Exec.Shell("Qizmt kill " + execsjid);
lock (typeof(Program))
{
if (execx)
{
throw new Exception("exec completed; problem with test");
}
if (!execdone)
{
throw new Exception("kill completed but exec has not yet returned");
}
}
// Wait a couple seconds to give the services a chance to come back up.
System.Threading.Thread.Sleep(1000 * 2);
}
finally
{
try
{
Exec.Shell("Qizmt del exec{3EAE6884-28BB-4340-8F94-6E9421B68C92}");
}
catch
{
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
}
/// <summary>
/// Create an InetAddress with a fully qualified hostname of the given
/// hostname.
/// </summary>
/// <remarks>
/// Create an InetAddress with a fully qualified hostname of the given
/// hostname. InetAddress does not qualify an incomplete hostname that
/// is resolved via the domain search list.
/// <see cref="System.Net.IPAddress.ToString()"/>
/// will fully qualify the
/// hostname, but it always return the A record whereas the given hostname
/// may be a CNAME.
/// </remarks>
/// <param name="host">a hostname or ip address</param>
/// <returns>InetAddress with the fully qualified hostname or ip</returns>
/// <exception cref="UnknownHostException">if host does not exist</exception>
public virtual IPAddress GetByName(string host)
{
IPAddress addr = null;
if (IPAddressUtil.IsIPv4LiteralAddress(host))
{
// use ipv4 address as-is
byte[] ip = IPAddressUtil.TextToNumericFormatV4(host);
addr = IPAddress.GetByAddress(host, ip);
}
else
{
if (IPAddressUtil.IsIPv6LiteralAddress(host))
{
// use ipv6 address as-is
byte[] ip = IPAddressUtil.TextToNumericFormatV6(host);
addr = IPAddress.GetByAddress(host, ip);
}
else
{
if (host.EndsWith("."))
{
// a rooted host ends with a dot, ex. "host."
// rooted hosts never use the search path, so only try an exact lookup
addr = GetByExactName(host);
}
else
{
if (host.Contains("."))
{
// the host contains a dot (domain), ex. "host.domain"
// try an exact host lookup, then fallback to search list
addr = GetByExactName(host);
if (addr == null)
{
addr = GetByNameWithSearch(host);
}
}
else
{
// it's a simple host with no dots, ex. "host"
// try the search list, then fallback to exact host
IPAddress loopback = Extensions.GetAddressByName(null);
if (Runtime.EqualsIgnoreCase(host, loopback.GetHostName()))
{
addr = IPAddress.GetByAddress(host, loopback.GetAddressBytes());
}
else
{
addr = GetByNameWithSearch(host);
if (addr == null)
{
addr = GetByExactName(host);
}
}
}
}
}
}
// unresolvable!
if (addr == null)
{
throw new UnknownHostException(host);
}
return(addr);
}
