public static void TestCriticalSection(string[] args)
{
const int threadcount = 2;
string dir = Environment.CurrentDirectory + @"\TestCriticalSection\";
if (System.IO.Directory.Exists(dir))
{
System.IO.Directory.Delete(dir, true);
}
System.IO.Directory.CreateDirectory(dir);
dir = dir.Replace(":", "$");
dir = @"\\" + System.Net.Dns.GetHostName() + @"\" + dir;
dumpfn = dir + "dump.txt";
Console.WriteLine("-");
Console.WriteLine("Testing critical section...");
for (int i = 0; i < threadcount; i++)
{
string mr = (@"<SourceCode>
<Jobs>
<Job Name=`regression_test_CriticalSection_Preprocessing` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt del regression_test_CriticalSection_Input" + i.ToString() + @".txt`);
Shell(@`Qizmt del regression_test_CriticalSection_Output" + i.ToString() + @".txt`);
using(GlobalCriticalSection.GetLock())
{
Increment();
}
}
private void Increment()
{
System.IO.StreamReader r = new System.IO.StreamReader(Qizmt_ExecArgs[0]);
int count = Int32.Parse(r.ReadToEnd());
r.Close();
count++;
System.IO.FileStream fs = new System.IO.FileStream(Qizmt_ExecArgs[0], System.IO.FileMode.Open, System.IO.FileAccess.Write, System.IO.FileShare.None);
byte[] buf = System.Text.Encoding.UTF8.GetBytes(count.ToString());
fs.Write(buf, 0, buf.Length);
fs.Close();
}
]]>
</Local>
</Job>
<Job Name=`regression_test_CriticalSection_CreateSampleData` Custodian=`` Email=``>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO>
<DFSReader></DFSReader>
<DFSWriter>dfs://regression_test_CriticalSection_Input" + i.ToString() + @".txt</DFSWriter>
</DFS_IO>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
//Create sample data.
dfsoutput.WriteLine(`1498`);
dfsoutput.WriteLine(`1503`);
dfsoutput.WriteLine(`1501`);
dfsoutput.WriteLine(`1501`);
using(GlobalCriticalSection.GetLock())
{
Increment();
}
}
private void Increment()
{
System.IO.StreamReader r = new System.IO.StreamReader(Qizmt_ExecArgs[0]);
int count = Int32.Parse(r.ReadToEnd());
r.Close();
count++;
System.IO.FileStream fs = new System.IO.FileStream(Qizmt_ExecArgs[0], System.IO.FileMode.Open, System.IO.FileAccess.Write, System.IO.FileShare.None);
byte[] buf = System.Text.Encoding.UTF8.GetBytes(count.ToString());
fs.Write(buf, 0, buf.Length);
fs.Close();
}
]]>
</Remote>
</Job>
<Job Name=`regression_test_CriticalSection` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>4</KeyLength>
<DFSInput>dfs://regression_test_CriticalSection_Input" + i.ToString() + @".txt</DFSInput>
<DFSOutput>dfs://regression_test_CriticalSection_Output" + i.ToString() + @".txt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
output.Add(line, line);
using(GlobalCriticalSection.GetLock())
{
Increment();
}
}
private void Increment()
{
System.IO.StreamReader r = new System.IO.StreamReader(Qizmt_ExecArgs[0]);
int count = Int32.Parse(r.ReadToEnd());
r.Close();
count++;
System.IO.FileStream fs = new System.IO.FileStream(Qizmt_ExecArgs[0], System.IO.FileMode.Open, System.IO.FileAccess.Write, System.IO.FileShare.None);
byte[] buf = System.Text.Encoding.UTF8.GetBytes(count.ToString());
fs.Write(buf, 0, buf.Length);
fs.Close();
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
for(int i = 0; i < values.Length; i++)
{
output.Add(key);
using(GlobalCriticalSection.GetLock())
{
Increment();
}
}
}
private void Increment()
{
System.IO.StreamReader r = new System.IO.StreamReader(Qizmt_ExecArgs[0]);
int count = Int32.Parse(r.ReadToEnd());
r.Close();
count++;
System.IO.FileStream fs = new System.IO.FileStream(Qizmt_ExecArgs[0], System.IO.FileMode.Open, System.IO.FileAccess.Write, System.IO.FileShare.None);
byte[] buf = System.Text.Encoding.UTF8.GetBytes(count.ToString());
fs.Write(buf, 0, buf.Length);
fs.Close();
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`regression_test_CriticalSection_Preprocessing` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt del regression_test_CriticalSection_Input" + i.ToString() + @".txt`);
Shell(@`Qizmt del regression_test_CriticalSection_Output" + i.ToString() + @".txt`);
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>").Replace('`', '"');
string fn = "regressionTest_criticalSection" + i.ToString() + ".xml";
System.IO.File.WriteAllText(dir + fn, mr);
}
Exec.Shell(exe + @" del regressionTest_criticalSection*.xml");
Exec.Shell(exe + @" importdirmt " + dir);
System.IO.File.WriteAllText(dumpfn, "0");
System.Threading.Thread[] ths = new System.Threading.Thread[threadcount];
for (int i = 0; i < threadcount; i++)
{
System.Threading.Thread th = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(ThreadProc));
ths[i] = th;
th.Start(i);
}
evt.Set();
for (int i = 0; i < threadcount; i++)
{
ths[i].Join();
}
int result = Int32.Parse(System.IO.File.ReadAllText(dumpfn).Trim());
if (result == threadcount * 10)
{
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
else
{
Console.WriteLine("[FAILED] - " + string.Join(" ", args));
}
Exec.Shell(exe + @" del regressionTest_criticalSection*.xml");
System.IO.Directory.Delete(dir, true);
}
public static void TestHost(string[] args)
{
if (args.Length < 2)
{
Console.Error.WriteLine("Error: LocalJobHost command needs argument: <dfsXmlPath>");
return;
}
string dfspath = args[1];
if (!dfspath.StartsWith(@"\\"))
{
Console.Error.WriteLine("Argument: <dfsXmlPath> must be a network path");
return;
}
string surrogate = dfspath.Substring(2, dfspath.IndexOf(@"\", 2) - 2).ToUpper();
System.Xml.XmlDocument dfs = new System.Xml.XmlDocument();
string slavelist = null;
try
{
dfs.Load(dfspath);
System.Xml.XmlNode node = dfs.SelectSingleNode("//SlaveList");
if (node == null)
{
Console.Error.WriteLine("SlaveList node is not found in dfs.xml");
return;
}
slavelist = node.InnerText.ToUpper();
string[] parts = slavelist.Split(new char[] { ',', ';' });
if (parts.Length < 2)
{
Console.Error.WriteLine("Must have at least 2 hosts in SlaveList tag in dfs.xml");
return;
}
}
catch (Exception e)
{
Console.Error.WriteLine("Error loading dfs.xml: {0}", e.Message);
return;
}
string exe = Exec.GetQizmtExe();
//Test Host tag, empty and non-empty.
{
string[] hosts = slavelist.Split(new char[] { ',', ';' });
string nonsurrogate = null;
string firsthost = hosts[0];
foreach (string host in hosts)
{
if (string.Compare(surrogate, host, true) != 0)
{
nonsurrogate = host;
break;
}
}
if (nonsurrogate == null)
{
Console.Error.WriteLine("Non-surrogate is not found from dfs.xml");
return;
}
string mr = @"<SourceCode>
<Jobs>
<Job Name=`mash_Preprocessing` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
<Host>xxx</Host>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Qizmt_Log(`DNS=` + System.Net.Dns.GetHostName());
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>".Replace('`', '"').Replace("xxx", nonsurrogate);
string dir = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\RegressionTest\LocalJobHostTest\";
if (System.IO.Directory.Exists(dir))
{
System.IO.Directory.Delete(dir, true);
}
System.IO.Directory.CreateDirectory(dir);
System.IO.File.WriteAllText(dir + "regressionTest_localJobHost_nonEmptyHost.xml", mr);
mr = @"<SourceCode>
<Jobs>
<Job Name=`mash_Preprocessing` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Qizmt_Log(`DNS=` + System.Net.Dns.GetHostName());
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>".Replace('`', '"');
System.IO.File.WriteAllText(dir + "regressionTest_localJobHost_emptyHost.xml", mr);
Exec.Shell(exe + " del regressionTest_localJobHost_nonEmptyHost.xml");
Exec.Shell(exe + " del regressionTest_localJobHost_emptyHost.xml");
Exec.Shell(exe + " importdir " + dir);
Console.WriteLine("-");
Console.WriteLine("Testing non-empty host in local job...");
string output = Exec.Shell(exe + " exec regressionTest_localJobHost_nonEmptyHost.xml");
string expected = "DNS=" + nonsurrogate;
if (output.IndexOf(expected, StringComparison.OrdinalIgnoreCase) > -1)
{
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
else
{
Console.WriteLine("[FAILED] - " + string.Join(" ", args));
}
Console.WriteLine("-");
Console.WriteLine("Testing empty host in local job...");
output = Exec.Shell(exe + " exec regressionTest_localJobHost_emptyHost.xml");
expected = "DNS=" + firsthost;
if (output.IndexOf(expected, StringComparison.OrdinalIgnoreCase) > -1)
{
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
else
{
Console.WriteLine("[FAILED] - " + string.Join(" ", args));
}
//Clean up
System.IO.Directory.Delete(dir, true);
Exec.Shell(exe + " del regressionTest_localJobHost_nonEmptyHost.xml");
Exec.Shell(exe + " del regressionTest_localJobHost_emptyHost.xml");
}
}
public static void TestAddRefNonParticipatingCluster(string[] args)
{
if (args.Length < 2)
{
Console.Error.WriteLine("Error: LocalJobAddRefNonParticipatingCluster command needs argument: <dfsXmlPath>");
return;
}
string dfspath = args[1];
if (!dfspath.StartsWith(@"\\"))
{
Console.Error.WriteLine("Argument: <dfsXmlPath> must be a network path");
return;
}
string surrogate = dfspath.Substring(2, dfspath.IndexOf(@"\", 2) - 2).ToUpper();
System.Xml.XmlDocument dfs = new System.Xml.XmlDocument();
string slavelist = null;
try
{
dfs.Load(dfspath);
System.Xml.XmlNode node = dfs.SelectSingleNode("//SlaveList");
if (node == null)
{
Console.Error.WriteLine("SlaveList node is not found in dfs.xml");
return;
}
slavelist = node.InnerText.ToUpper();
string[] parts = slavelist.Split(new char[] { ',', ';' });
if (parts.Length < 2)
{
Console.Error.WriteLine("Must have at least 2 hosts in SlaveList tag in dfs.xml");
return;
}
}
catch (Exception e)
{
Console.Error.WriteLine("Error loading dfs.xml: {0}", e.Message);
return;
}
string dfsback = null;
bool ok = false;
try
{
if (slavelist.IndexOf(surrogate, StringComparison.OrdinalIgnoreCase) > -1)
{
if (!DFSUtils.MakeFileBackup(dfspath, ref dfsback))
{
Console.Error.WriteLine("Error while backing up dfs.xml.");
return;
}
string newslavelist = "";
string[] hosts = slavelist.Split(new char[] { ',', ';' });
foreach (string host in hosts)
{
if (string.Compare(surrogate, host, true) != 0)
{
newslavelist = ";" + host;
}
}
newslavelist = newslavelist.Trim(';');
DFSUtils.ChangeDFSXMLSlaveList(dfs, dfspath, newslavelist);
}
string mr = @"<SourceCode>
<Jobs>
<Job>
<Narrative>
<Name>regression_test_put_dll_Preprocessing</Name>
<Custodian></Custodian>
<email></email>
</Narrative>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt del regressionTestLocalJobAddRef4A30DE94_testdll.dll`);
}
]]>
</Local>
</Job>
<Job>
<Narrative>
<Name>regression_test_put_dll PUT DLL</Name>
<Custodian></Custodian>
<email></email>
</Narrative>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
string localdir = IOUtils.GetTempDirectory();
string fn = `regressionTestLocalJobAddRef4A30DE94_testdll.dll`;
string localfn = localdir + @`\` + Guid.NewGuid().ToString() + fn;
string testdlldatab64 = `TVqQAAMAAAAEAAAA//8AALgAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAA4fug4AtAnNIbgBTM0hVGhpcyBwcm9ncmFtIGNhbm5vdCBiZSBydW4gaW4gRE9TIG1vZGUuDQ0KJAAAAAAAAABQRQAATAEDAOMDJ0oAAAAAAAAAAOAAAiELAQgAAAgAAAAIAAAAAAAALicAAAAgAAAAQAAAAABAAAAgAAAAAgAABAAAAAAAAAAEAAAAAAAAAACAAAAAAgAAAAAAAAMAQIUAABAAABAAAAAAEAAAEAAAAAAAABAAAAAAAAAAAAAAAOAmAABLAAAAAEAAACgEAAAAAAAAAAAAAAAAAAAAAAAAAGAAAAwAAAAsJgAAHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAACAAAAAAAAAAAAAAACCAAAEgAAAAAAAAAAAAAAC50ZXh0AAAANAcAAAAgAAAACAAAAAIAAAAAAAAAAAAAAAAAACAAAGAucnNyYwAAACgEAAAAQAAAAAYAAAAKAAAAAAAAAAAAAAAAAABAAABALnJlbG9jAAAMAAAAAGAAAAACAAAAEAAAAAAAAAAAAAAAAAAAQAAAQgAAAAAAAAAAAAAAAAAAAAAQJwAAAAAAAEgAAAACAAUAcCAAALwFAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABMwAQALAAAAAQAAEQByAQAAcAorAAYqHgIoEAAACioAQlNKQgEAAQAAAAAADAAAAHYyLjAuNTA3MjcAAAAABQBsAAAAwAEAACN+AAAsAgAAiAIAACNTdHJpbmdzAAAAALQEAAAUAAAAI1VTAMgEAAAQAAAAI0dVSUQAAADYBAAA5AAAACNCbG9iAAAAAAAAAAIAAAFHFAIACQAAAAD6ATMAFgAAAQAAABEAAAACAAAAAgAAABAAAAANAAAAAQAAAAEAAAABAAAAAAAKAAEAAAAAAAYAggB7AAYApwCVAAYAvgCVAAYA2wCVAAYA+gCVAAYAEwGVAAYALAGVAAYARwGVAAYAYgGVAAYAmgF7AQYArgF7AQYAvAGVAAYA1QGVAAYABQLyATsAGQIAAAYASAIoAgYAaAIoAgAAAAABAAAAAAABAAEAAQAQADsARQAFAAEAAQBQIAAAAACWAIkACgABAGcgAAAAAIYYjwAOAAEAEQCPABIAGQCPABIAIQCPABIAKQCPABIAMQCPABIAOQCPABIAQQCPABIASQCPABIAUQCPABcAWQCPABIAYQCPABIAaQCPABIAcQCPABwAgQCPACIAiQCPAA4ACQCPAA4ALgALACsALgATAF0ALgAbAF0ALgAjAF0ALgArACsALgAzAGMALgA7AF0ALgBLAF0ALgBTAHsALgBjAKUALgBrALIALgBzALsALgB7AMQAJwAEgAAAAQAAAAAAAAAAAAAAAABFAAAAAgAAAAAAAAAAAAAAAQByAAAAAAAAAAA8TW9kdWxlPgByZWdyZXNzaW9uVGVzdExvY2FsSm9iQWRkUmVmNEEzMERFOTRfdGVzdGRsbC5kbGwAVGVzdENsYXNzAHJlZ3Jlc3Npb25UZXN0TG9jYWxKb2JBZGRSZWY0QTMwREU5NF90ZXN0ZGxsAG1zY29ybGliAFN5c3RlbQBPYmplY3QAU2F5SGkALmN0b3IAU3lzdGVtLlJlZmxlY3Rpb24AQXNzZW1ibHlUaXRsZUF0dHJpYnV0ZQBBc3NlbWJseURlc2NyaXB0aW9uQXR0cmlidXRlAEFzc2VtYmx5Q29uZmlndXJhdGlvbkF0dHJpYnV0ZQBBc3NlbWJseUNvbXBhbnlBdHRyaWJ1dGUAQXNzZW1ibHlQcm9kdWN0QXR0cmlidXRlAEFzc2VtYmx5Q29weXJpZ2h0QXR0cmlidXRlAEFzc2VtYmx5VHJhZGVtYXJrQXR0cmlidXRlAEFzc2VtYmx5Q3VsdHVyZUF0dHJpYnV0ZQBTeXN0ZW0uUnVudGltZS5JbnRlcm9wU2VydmljZXMAQ29tVmlzaWJsZUF0dHJpYnV0ZQBHdWlkQXR0cmlidXRlAEFzc2VtYmx5VmVyc2lvbkF0dHJpYnV0ZQBBc3NlbWJseUZpbGVWZXJzaW9uQXR0cmlidXRlAFN5c3RlbS5EaWFnbm9zdGljcwBEZWJ1Z2dhYmxlQXR0cmlidXRlAERlYnVnZ2luZ01vZGVzAFN5c3RlbS5SdW50aW1lLkNvbXBpbGVyU2VydmljZXMAQ29tcGlsYXRpb25SZWxheGF0aW9uc0F0dHJpYnV0ZQBSdW50aW1lQ29tcGF0aWJpbGl0eUF0dHJpYnV0ZQAAAAARaABpACAAdABoAGUAcgBlAAAA+LohWe+Eo0GsrsHvHDwd5wAIt3pcVhk04IkDAAAOAyAAAQQgAQEOBCABAQIFIAEBET0EIAEBCAMHAQ4xAQAscmVncmVzc2lvblRlc3RMb2NhbEpvYkFkZFJlZjRBMzBERTk0X3Rlc3RkbGwAAAUBAAAAABcBABJDb3B5cmlnaHQgwqkgIDIwMDkAACkBACQwN2Y1Njc4Zi04YzJlLTQ3OWUtOTg1Ny1iMzBhNjFkZDczN2UAAAwBAAcxLjAuMC4wAAAIAQAHAQAAAAAIAQAIAAAAAAAeAQABAFQCFldyYXBOb25FeGNlcHRpb25UaHJvd3MBAAAAAADjAydKAAAAAAIAAACWAAAASCYAAEgIAABSU0RTJqyw+VFRU0yJX+9RNtIOogEAAABDOlxzb3VyY2VcY29uc29sZUFwcHNccmVncmVzc2lvblRlc3RMb2NhbEpvYkFkZFJlZjRBMzBERTk0X3Rlc3RkbGxcb2JqXERlYnVnXHJlZ3Jlc3Npb25UZXN0TG9jYWxKb2JBZGRSZWY0QTMwREU5NF90ZXN0ZGxsLnBkYgAAAAgnAAAAAAAAAAAAAB4nAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQJwAAAAAAAAAAX0NvckRsbE1haW4AbXNjb3JlZS5kbGwAAAAAAP8lACBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAQAAAAGAAAgAAAAAAAAAAAAAAAAAAAAQABAAAAMAAAgAAAAAAAAAAAAAAAAAAAAQAAAAAASAAAAFhAAADQAwAAAAAAAAAAAADQAzQAAABWAFMAXwBWAEUAUgBTAEkATwBOAF8ASQBOAEYATwAAAAAAvQTv/gAAAQAAAAEAAAAAAAAAAQAAAAAAPwAAAAAAAAAEAAAAAgAAAAAAAAAAAAAAAAAAAEQAAAABAFYAYQByAEYAaQBsAGUASQBuAGYAbwAAAAAAJAAEAAAAVAByAGEAbgBzAGwAYQB0AGkAbwBuAAAAAAAAALAEMAMAAAEAUwB0AHIAaQBuAGcARgBpAGwAZQBJAG4AZgBvAAAADAMAAAEAMAAwADAAMAAwADQAYgAwAAAAhAAtAAEARgBpAGwAZQBEAGUAcwBjAHIAaQBwAHQAaQBvAG4AAAAAAHIAZQBnAHIAZQBzAHMAaQBvAG4AVABlAHMAdABMAG8AYwBhAGwASgBvAGIAQQBkAGQAUgBlAGYANABBADMAMABEAEUAOQA0AF8AdABlAHMAdABkAGwAbAAAAAAAMAAIAAEARgBpAGwAZQBWAGUAcgBzAGkAbwBuAAAAAAAxAC4AMAAuADAALgAwAAAAhAAxAAEASQBuAHQAZQByAG4AYQBsAE4AYQBtAGUAAAByAGUAZwByAGUAcwBzAGkAbwBuAFQAZQBzAHQATABvAGMAYQBsAEoAbwBiAEEAZABkAFIAZQBmADQAQQAzADAARABFADkANABfAHQAZQBzAHQAZABsAGwALgBkAGwAbAAAAAAASAASAAEATABlAGcAYQBsAEMAbwBwAHkAcgBpAGcAaAB0AAAAQwBvAHAAeQByAGkAZwBoAHQAIACpACAAIAAyADAAMAA5AAAAjAAxAAEATwByAGkAZwBpAG4AYQBsAEYAaQBsAGUAbgBhAG0AZQAAAHIAZQBnAHIAZQBzAHMAaQBvAG4AVABlAHMAdABMAG8AYwBhAGwASgBvAGIAQQBkAGQAUgBlAGYANABBADMAMABEAEUAOQA0AF8AdABlAHMAdABkAGwAbAAuAGQAbABsAAAAAAB8AC0AAQBQAHIAbwBkAHUAYwB0AE4AYQBtAGUAAAAAAHIAZQBnAHIAZQBzAHMAaQBvAG4AVABlAHMAdABMAG8AYwBhAGwASgBvAGIAQQBkAGQAUgBlAGYANABBADMAMABEAEUAOQA0AF8AdABlAHMAdABkAGwAbAAAAAAANAAIAAEAUAByAG8AZAB1AGMAdABWAGUAcgBzAGkAbwBuAAAAMQAuADAALgAwAC4AMAAAADgACAABAEEAcwBzAGUAbQBiAGwAeQAgAFYAZQByAHMAaQBvAG4AAAAxAC4AMAAuADAALgAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAMAAAAMDcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`;
byte[] testdlldata = System.Convert.FromBase64String(testdlldatab64);
System.IO.File.WriteAllBytes(localfn, testdlldata);
try
{
Shell(@`Qizmt dfs put ` + localfn + ` ` + fn);
}
finally
{
System.IO.File.Delete(localfn);
}
}
]]>
</Local>
</Job>
<Job Name=`testdll` Custodian=`` email=``>
<Add Reference=`regressionTestLocalJobAddRef4A30DE94_testdll.dll` Type=`dfs` />
<Using>regressionTestLocalJobAddRef4A30DE94_testdll</Using>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Qizmt_Log(TestClass.SayHi());
}
]]>
</Local>
</Job>
<Job Name=`testdll` Custodian=`` email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt del regressionTestLocalJobAddRef4A30DE94_testdll.dll`);
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>
".Replace('`', '"');
string dir = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\RegressionTest\LocalJobAddRefTest\";
if (System.IO.Directory.Exists(dir))
{
System.IO.Directory.Delete(dir, true);
}
System.IO.Directory.CreateDirectory(dir);
System.IO.File.WriteAllText(dir + "regressionTest_localJobAddRef.xml", mr);
string exe = Exec.GetQizmtExe();
Exec.Shell(exe + " del regressionTest_localJobAddRef.xml");
Exec.Shell(exe + " importdir " + dir);
Console.WriteLine("-");
Console.WriteLine("Testing add reference in a local job on a non-participating cluster...");
string output = Exec.Shell(exe + " exec regressionTest_localJobAddRef.xml");
string expected = "hi there";
if (output.IndexOf(expected, StringComparison.OrdinalIgnoreCase) > -1)
{
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
else
{
Console.WriteLine("[FAILED] - " + string.Join(" ", args));
}
//Clean up
Exec.Shell(exe + " del regressionTest_localJobAddRef.xml");
System.IO.Directory.Delete(dir, true);
}
catch (Exception e)
{
Console.Error.WriteLine("Error during local job add reference testing: {0}", e.Message);
}
finally
{
if (dfsback != null)
{
ok = DFSUtils.UndoFileChanges(dfspath, dfsback);
}
}
if (dfsback != null && ok)
{
System.IO.File.Delete(dfsback);
}
}
static void FaultTolerantExecutionTest(string[] args)
{
bool verbose = false;
bool skipsplitsort = false;
for (int ai = 1; ai < args.Length; ai++)
{
switch (args[ai].ToLower())
{
case "-verbose":
verbose = true;
Console.WriteLine("verbose=true");
break;
case "-skipsplitsort":
skipsplitsort = true;
Console.WriteLine("skipsplitsort=true");
break;
default:
throw new Exception("Unknown argument: " + args[ai]);
}
}
Console.WriteLine("====FaultTolerantExecutionTest====");
if (!MySpace.DataMining.AELight.FTTest.enabled)
{
throw new Exception("TESTFAULTTOLERANT is not #defined. Need Qizmt build with all #define TESTFAULTTOLERANT uncommented.");
}
if (GetReplicationFactor() < 2)
{
throw new Exception("Replication factor must be 2 or greater.");
}
if (!IsClusterHealthy())
{
throw new Exception("Cluster must be 100% healthy to begin with.");
}
{
Console.WriteLine("Importing jobs...");
string tempdir = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\" + Guid.NewGuid().ToString();
System.IO.Directory.CreateDirectory(tempdir);
#region job1
string txt = @"<SourceCode>
<Jobs>
<Job Name=`Preprocessing` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt del oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input*`);
Shell(@`Qizmt del oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Output*`);
}
]]>
</Local>
</Job>
<Job Name=`CreateSampleData` Custodian=`` Email=`` Description=``>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO_Multi>
<DFSReader></DFSReader>
<DFSWriter>oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input1_####.txt</DFSWriter>
<Mode>ALL CORES</Mode>
</DFS_IO_Multi>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
int max = 50000;
Random rnd = new Random(System.DateTime.Now.Millisecond / 2 + System.Diagnostics.Process.GetCurrentProcess().Id / 2);
for(int i = 0; i < max; i++)
{
if(i % Qizmt_ProcessCount == Qizmt_ProcessID)
{
int num = rnd.Next(Int32.MinValue, Int32.MaxValue);
int valuescount = rnd.Next(10,50);
for(int vi = 0; vi < valuescount; vi++)
{
int num2 = rnd.Next(Int32.MinValue, Int32.MaxValue);
dfsoutput.WriteLine(num.ToString() + `,apple,` + num2.ToString() + `,lemon`);
}
}
}
}
]]>
</Remote>
</Job>
<Job Name=`CreateSampleData` Custodian=`` Email=`` Description=``>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO_Multi>
<DFSReader></DFSReader>
<DFSWriter>oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input2_####.txt</DFSWriter>
<Mode>ALL CORES</Mode>
</DFS_IO_Multi>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
int max = 50000;
Random rnd = new Random(System.DateTime.Now.Millisecond / 2 + System.Diagnostics.Process.GetCurrentProcess().Id / 2);
for(int i = 0; i < max; i++)
{
if(i % Qizmt_ProcessCount == Qizmt_ProcessID)
{
int num = rnd.Next(Int32.MinValue, Int32.MaxValue);
int valuescount = rnd.Next(10,50);
for(int vi = 0; vi < valuescount; vi++)
{
int num2 = rnd.Next(Int32.MinValue, Int32.MaxValue);
dfsoutput.WriteLine(num.ToString() + `,apple,` + num2.ToString() + `,lemon`);
}
}
}
}
]]>
</Remote>
</Job>
<Job Name=`Preprocessing` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt combine oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input1_*.txt +oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input1.txt`);
Shell(@`Qizmt combine oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input2_*.txt +oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input2.txt`);
}
]]>
</Local>
</Job>
<Job Name=`mr` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>int</KeyLength>
<DFSInput>dfs://oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input1.txt;oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input2.txt</DFSInput>
<DFSOutput>dfs://oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Output_not_sp.txt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
mstring sLine = mstring.Prepare(line);
int num = sLine.NextItemToInt(',');
mstring title = sLine.NextItemToString(',');
int num2 = sLine.NextItemToInt(',');
mstring title2 = sLine.NextItemToString(',');
recordset rKey = recordset.Prepare();
rKey.PutInt(num);
int num3 = -1;
if(StaticGlobals.DSpace_InputFileName == `oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input1.txt`)
{
num3 = 1;
}
else if(StaticGlobals.DSpace_InputFileName == `oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input2.txt`)
{
num3 = 2;
}
else if(StaticGlobals.DSpace_InputFileName == `oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input3.txt`)
{
num3 = 3;
}
recordset rValue = recordset.Prepare();
rValue.PutInt(num2);
rValue.PutString(title);
rValue.PutString(title2);
rValue.PutInt(num3);
output.Add(rKey, rValue);
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
recordset rKey = recordset.Prepare(key);
int num = rKey.GetInt();
for(int i = 0; i < values.Length; i++)
{
recordset rValue = recordset.Prepare(values.Items[i]);
int num2 = rValue.GetInt();
mstring title = rValue.GetString();
mstring title2 = rValue.GetString();
int num3 = rValue.GetInt();
mstring sLine = mstring.Prepare(num);
sLine = sLine.AppendM(',')
.AppendM(num2)
.AppendM(',')
.AppendM(title)
.AppendM(',')
.AppendM(title2)
.AppendM(',')
.AppendM(num3);
output.Add(sLine);
}
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>".Replace('`', '"');
System.IO.File.WriteAllText(tempdir + @"\reg_job1_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml", txt);
#endregion
#region job2
txt = @"<SourceCode>
<Jobs>
<Job Name=`Preprocessing` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt del oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Output_sp.txt`);
}
]]>
</Local>
</Job>
<Job Name=`mr` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>int</KeyLength>
<DFSInput>dfs://oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input1.txt;oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input2.txt</DFSInput>
<DFSOutput>dfs://oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Output_sp.txt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<FaultTolerantExecution>
<Mode>enabled</Mode>
<MapInputOrder>shuffle</MapInputOrder>
</FaultTolerantExecution>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
mstring sLine = mstring.Prepare(line);
int num = sLine.NextItemToInt(',');
mstring title = sLine.NextItemToString(',');
int num2 = sLine.NextItemToInt(',');
mstring title2 = sLine.NextItemToString(',');
recordset rKey = recordset.Prepare();
rKey.PutInt(num);
int num3 = -1;
if(StaticGlobals.DSpace_InputFileName == `oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input1.txt`)
{
num3 = 1;
}
else if(StaticGlobals.DSpace_InputFileName == `oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input2.txt`)
{
num3 = 2;
}
else if(StaticGlobals.DSpace_InputFileName == `oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Input3.txt`)
{
num3 = 3;
}
recordset rValue = recordset.Prepare();
rValue.PutInt(num2);
rValue.PutString(title);
rValue.PutString(title2);
rValue.PutInt(num3);
output.Add(rKey, rValue);
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
recordset rKey = recordset.Prepare(key);
int num = rKey.GetInt();
List<KeyValuePair<int, string>> list = new List<KeyValuePair<int, string>>(values.Length);
for(int i = 0; i < values.Length; i++)
{
recordset rValue = recordset.Prepare(values.Items[i]);
int num2 = rValue.GetInt();
mstring title = rValue.GetString();
mstring title2 = rValue.GetString();
int num3 = rValue.GetInt();
mstring sLine = mstring.Prepare(num);
sLine = sLine.AppendM(',')
.AppendM(num2)
.AppendM(',')
.AppendM(title)
.AppendM(',')
.AppendM(title2)
.AppendM(',')
.AppendM(num3);
output.Add(sLine);
}
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
".Replace('`', '"');
System.IO.File.WriteAllText(tempdir + @"\reg_job2_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml", txt);
#endregion
#region job3
txt = @"<SourceCode>
<Jobs>
<Job Name=`checkresults` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
string f1 = IOUtils.GetTempDirectory() + @`\` + Guid.NewGuid().ToString();
string f2 = IOUtils.GetTempDirectory() + @`\` + Guid.NewGuid().ToString();
Shell(@`qizmt get oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Output_not_sp.txt ` + f1);
Shell(@`qizmt get oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_Output_sp.txt ` + f2);
string[] lines1 = System.IO.File.ReadAllLines(f1);
string[] lines2 = System.IO.File.ReadAllLines(f2);
if(lines1.Length != lines2.Length)
{
throw new Exception(`lines counts are different. lines1.len=` + lines1.Length.ToString() + `; lines2.len=` + lines2.Length.ToString());
}
if(!CheckGroupedKeys(lines1))
{
throw new Exception(`lines1 not grouped properly`);
}
if(!CheckGroupedKeys(lines2))
{
throw new Exception(`lines2 not grouped properly`);
}
{
List<string> list1 = new List<string>(lines1);
List<string> list2 = new List<string>(lines2);
list1.Sort();
list2.Sort();
for(int i=0; i < list1.Count; i++)
{
if(list1[i] != list2[i])
{
throw new Exception(`line different at: ` + i.ToString());
}
}
}
System.IO.File.Delete(f1);
System.IO.File.Delete(f2);
Qizmt_Log(`error count=0`);
}
public bool CheckGroupedKeys(string[] lines)
{
//Make sure the keys are grouped.
Dictionary<int, List<string>> dic = new Dictionary<int, List<string>>(lines.Length);
int prevkey = -1;
for(int i = 0; i < lines.Length; i++)
{
string line = lines[i];
string[] parts = line.Split(',');
int key = Int32.Parse(parts[0]);
bool keychanged = false;
if(i == 0)
{
keychanged = true;
}
else if(prevkey != key)
{
keychanged = true;
}
if(keychanged)
{
prevkey = key;
dic.Add(key, new List<string>()); //will error out if this key has been seen before. thus not grouped properly.
}
dic[key].Add(line);
}
return true;
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>
".Replace('`', '"');
System.IO.File.WriteAllText(tempdir + @"\reg_job3_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml", txt);
#endregion
#region job1splitsort
txt = @"<SourceCode>
<Jobs>
<Job Name=`Preprocessing` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt del oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Input*`);
Shell(@`Qizmt del oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Output*`);
}
]]>
</Local>
</Job>
<Job Name=`CreateSampleData` Custodian=`` Email=`` Description=``>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO_Multi>
<DFSReader></DFSReader>
<DFSWriter>oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Input1_####.txt</DFSWriter>
<Mode>ALL CORES</Mode>
</DFS_IO_Multi>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
int max = 2097152;
Random rnd = new Random(System.DateTime.Now.Millisecond / 2 + System.Diagnostics.Process.GetCurrentProcess().Id / 2);
for(int i = 0; i < max; i++)
{
if(i % Qizmt_ProcessCount == Qizmt_ProcessID)
{
int num = 1;
int valuescount = 1;
for(int vi = 0; vi < valuescount; vi++)
{
int num2 = rnd.Next(0,9);
dfsoutput.WriteLine(num.ToString() + `,` + num2.ToString());
}
}
}
}
]]>
</Remote>
</Job>
<Job Name=`CreateSampleData` Custodian=`` Email=`` Description=``>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO_Multi>
<DFSReader></DFSReader>
<DFSWriter>oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Input2_####.txt</DFSWriter>
<Mode>ALL CORES</Mode>
</DFS_IO_Multi>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
int max = 2097152;
Random rnd = new Random(System.DateTime.Now.Millisecond / 2 + System.Diagnostics.Process.GetCurrentProcess().Id / 2);
for(int i = 0; i < max; i++)
{
if(i % Qizmt_ProcessCount == Qizmt_ProcessID)
{
int num = 1;
int valuescount = 1;
for(int vi = 0; vi < valuescount; vi++)
{
int num2 = rnd.Next(0,9);
dfsoutput.WriteLine(num.ToString() + `,` + num2.ToString());
}
}
}
}
]]>
</Remote>
</Job>
<Job Name=`Preprocessing` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt combine oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Input1_*.txt +oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Input1.txt`);
Shell(@`Qizmt combine oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Input2_*.txt +oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Input2.txt`);
}
]]>
</Local>
</Job>
<Job Name=`mr` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>int</KeyLength>
<DFSInput>dfs://oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Input1.txt;oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Input2.txt</DFSInput>
<DFSOutput>dfs://oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Output_not_sp.txt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
mstring sLine = mstring.Prepare(line);
int num = sLine.NextItemToInt(',');
int num2 = sLine.NextItemToInt(',');
recordset rKey = recordset.Prepare();
num = 1; // constant key to cause splitsort
rKey.PutInt(num);
recordset rValue = recordset.Prepare();
rValue.PutInt(num2);
for(int i = 0; i < 32; i++)
{
output.Add(rKey, rValue);
}
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
recordset rKey = recordset.Prepare(key);
int num = rKey.GetInt();
for(int i = 0; i < values.Length; i++)
{
recordset rValue = recordset.Prepare(values.Items[i]);
int num2 = rValue.GetInt();
mstring sLine = mstring.Prepare(num);
sLine = sLine.AppendM(',')
.AppendM(num2);
output.Add(sLine);
}
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
".Replace('`', '"');
System.IO.File.WriteAllText(tempdir + @"\reg_splitsort_job1_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml", txt);
#endregion
#region job2splitsort
txt = @"<SourceCode>
<Jobs>
<Job Name=`Preprocessing` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt del oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Output_sp.txt`);
}
]]>
</Local>
</Job>
<Job Name=`mr` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>int</KeyLength>
<DFSInput>dfs://oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Input1.txt;oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Input2.txt</DFSInput>
<DFSOutput>dfs://oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Output_sp.txt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<FaultTolerantExecution>
<Mode>enabled</Mode>
<MapInputOrder>shuffle</MapInputOrder>
</FaultTolerantExecution>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
mstring sLine = mstring.Prepare(line);
int num = sLine.NextItemToInt(',');
int num2 = sLine.NextItemToInt(',');
recordset rKey = recordset.Prepare();
num = 1; // constant key to cause splitsort
rKey.PutInt(num);
recordset rValue = recordset.Prepare();
rValue.PutInt(num2);
for(int i = 0; i < 32; i++)
{
output.Add(rKey, rValue);
}
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
recordset rKey = recordset.Prepare(key);
int num = rKey.GetInt();
for(int i = 0; i < values.Length; i++)
{
recordset rValue = recordset.Prepare(values.Items[i]);
int num2 = rValue.GetInt();
mstring sLine = mstring.Prepare(num);
sLine = sLine.AppendM(',')
.AppendM(num2);
output.Add(sLine);
}
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
".Replace('`', '"');
System.IO.File.WriteAllText(tempdir + @"\reg_splitsort_job2_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml", txt);
#endregion
#region job3splitsort
txt = @"<SourceCode>
<Jobs>
<Job Name=`checkresults` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
string f1 = IOUtils.GetTempDirectory() + @`\` + Guid.NewGuid().ToString();
string f2 = IOUtils.GetTempDirectory() + @`\` + Guid.NewGuid().ToString();
Shell(@`qizmt get oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Output_not_sp.txt ` + f1);
Shell(@`qizmt get oo_D3FA879A-5AC7-48c5-80BC-0168550B9A11_splitsort_Output_sp.txt ` + f2);
string[] lines1 = System.IO.File.ReadAllLines(f1);
string[] lines2 = System.IO.File.ReadAllLines(f2);
if(lines1.Length != lines2.Length)
{
throw new Exception(`lines counts are different. lines1.len=` + lines1.Length.ToString() + `; lines2.len=` + lines2.Length.ToString());
}
{
List<string> list1 = new List<string>(lines1);
List<string> list2 = new List<string>(lines2);
list1.Sort();
list2.Sort();
for(int i=0; i < list1.Count; i++)
{
if(list1[i] != list2[i])
{
throw new Exception(`line different at: ` + i.ToString());
}
}
}
System.IO.File.Delete(f1);
System.IO.File.Delete(f2);
Qizmt_Log(`error count=0`);
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>
".Replace('`', '"');
System.IO.File.WriteAllText(tempdir + @"\reg_splitsort_job3_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml", txt);
#endregion
Exec.Shell("qizmt del reg_*job*_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
Exec.Shell("qizmt importdir \"" + tempdir + "\"");
System.IO.Directory.Delete(tempdir, true);
Console.WriteLine("Done");
}
string controlfile = @"\\" + MySpace.DataMining.AELight.Surrogate.MasterHost + @"\c$\temp\"
+ MySpace.DataMining.AELight.FTTest.controlfilename;
try
{
Console.WriteLine("Running job in normal mode...");
string output = Exec.Shell("qizmt exec reg_job1_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
//Test failure at map
{
string phase = "map";
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
Console.WriteLine("Running job in FTE mode with failure at {0}...", phase);
output = Exec.Shell("qizmt exec reg_job2_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
Console.WriteLine("Checking results...");
output = Exec.Shell("qizmt exec reg_job3_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (output.IndexOf("error count=0", StringComparison.OrdinalIgnoreCase) == -1)
{
throw new Exception("Test failed");
}
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
}
//Test failure at exchangeremote
{
string phase = "exchangeremote";
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
Console.WriteLine("Running job in FTE mode with failure at {0}...", phase);
output = Exec.Shell("qizmt exec reg_job2_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
Console.WriteLine("Checking results...");
output = Exec.Shell("qizmt exec reg_job3_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (output.IndexOf("error count=0", StringComparison.OrdinalIgnoreCase) == -1)
{
throw new Exception("Test failed");
}
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
}
//Test failure at exchangeowned
{
string phase = "exchangeowned";
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
Console.WriteLine("Running job in FTE mode with failure at {0}...", phase);
output = Exec.Shell("qizmt exec reg_job2_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
Console.WriteLine("Checking results...");
output = Exec.Shell("qizmt exec reg_job3_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (output.IndexOf("error count=0", StringComparison.OrdinalIgnoreCase) == -1)
{
throw new Exception("Test failed");
}
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
}
//Test failure at sort
{
string phase = "sort";
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
Console.WriteLine("Running job in FTE mode with failure at {0}...", phase);
output = Exec.Shell("qizmt exec reg_job2_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
Console.WriteLine("Checking results...");
output = Exec.Shell("qizmt exec reg_job3_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (output.IndexOf("error count=0", StringComparison.OrdinalIgnoreCase) == -1)
{
throw new Exception("Test failed");
}
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
}
//Test failure at reduce
{
string phase = "reduce";
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
Console.WriteLine("Running job in FTE mode with failure at {0}...", phase);
output = Exec.Shell("qizmt exec reg_job2_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
Console.WriteLine("Checking results...");
output = Exec.Shell("qizmt exec reg_job3_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (output.IndexOf("error count=0", StringComparison.OrdinalIgnoreCase) == -1)
{
throw new Exception("Test failed");
}
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
}
//Test failure at replication
{
string phase = "replication";
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
Console.WriteLine("Running job in FTE mode with failure at {0}...", phase);
output = Exec.Shell("qizmt exec reg_job2_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
Console.WriteLine("Checking results...");
output = Exec.Shell("qizmt exec reg_job3_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (output.IndexOf("error count=0", StringComparison.OrdinalIgnoreCase) == -1)
{
throw new Exception("Test failed");
}
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
}
if (!skipsplitsort)
{
System.IO.File.Delete(controlfile); //!
Console.WriteLine("Running job in normal mode with splitsort...");
Exec.Shell("qizmt exec reg_splitsort_job1_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
Console.WriteLine("Done");
//Test failure at splitsort
{
string phase = "splitsort";
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
Console.WriteLine("Running job in FTE mode with failure at {0}...", phase);
output = Exec.Shell("qizmt exec reg_splitsort_job2_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
Console.WriteLine("Checking results...");
output = Exec.Shell("qizmt exec reg_splitsort_job3_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (output.IndexOf("error count=0", StringComparison.OrdinalIgnoreCase) == -1)
{
throw new Exception("Test failed");
}
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
}
//Test failure at reducelargezblock, reduce when splitsort occurrs.
{
string phase = "reducelargezblock";
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
Console.WriteLine("Running job in FTE mode with failure at {0}...", phase);
output = Exec.Shell("qizmt exec reg_splitsort_job2_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
Console.WriteLine("Checking results...");
output = Exec.Shell("qizmt exec reg_splitsort_job3_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
if (output.IndexOf("error count=0", StringComparison.OrdinalIgnoreCase) == -1)
{
throw new Exception("Test failed");
}
if (verbose)
{
Console.WriteLine(output);
}
Console.WriteLine("Done");
}
}
}
finally
{
Exec.Shell("qizmt del reg_*job*_D3FA879A-5AC7-48c5-80BC-0168550B9A11.xml");
Exec.Shell("qizmt del *_D3FA879A-5AC7-48c5-80BC-0168550B9A11_*.txt");
System.IO.File.Delete(controlfile);
}
}
static bool IsClusterHealthy()
{
string output = Exec.Shell("qizmt health");
return(output.IndexOf("100%") != -1);
}
public static void TestKillallProxy(string[] args)
{
string job = @"<SourceCode>
<Jobs>
<Job Name=`m_Preprocessing`>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
System.Threading.Thread.Sleep(System.Threading.Timeout.Infinite);
}
]]>
</Local>
</Job>
<Job Name=`m_Preprocessing`>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Qizmt_Log(`B40735E5-8B8D-4638-8908-CD2AB024C3A7`);
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>".Replace('`', '"');
string dir = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\RegressionTest\B40735E5-8B8D-4638-8908-CD2AB024C3A7\";
if (System.IO.Directory.Exists(dir))
{
System.IO.Directory.Delete(dir, true);
}
System.IO.Directory.CreateDirectory(dir);
System.IO.File.WriteAllText(dir + @"B40735E5-8B8D-4638-8908-CD2AB024C3A7.xml", job);
string exe = Exec.GetQizmtExe();
Exec.Shell(exe + @" del B40735E5-8B8D-4638-8908-CD2AB024C3A7.xml");
Exec.Shell(exe + @" importdir " + dir);
System.IO.Directory.Delete(dir, true);
bool guidfound = false;
bool jobdone = false;
System.Threading.Thread th = new Thread(new ThreadStart(delegate()
{
try
{
Console.WriteLine("Running job...");
string results = Exec.Shell(exe + @" exec B40735E5-8B8D-4638-8908-CD2AB024C3A7.xml");
Console.WriteLine("Job output: {0}", results);
if (results.IndexOf("B40735E5-8B8D-4638-8908-CD2AB024C3A7") > -1)
{
guidfound = true;
}
}
catch
{
}
jobdone = true;
}));
th.Start();
System.Threading.Thread.Sleep(5000);
if (guidfound)
{
throw new Exception("Job exited normally. Job error.");
}
if (jobdone)
{
throw new Exception("Job finished too early, cannot run killall to kill it.");
}
Console.WriteLine("Running killall proxy...");
Exec.Shell(exe + " killall -f proxy");
System.Threading.Thread.Sleep(2000);
if (guidfound)
{
throw new Exception("Job exited normally. Job error.");
}
if (!jobdone)
{
throw new Exception("Job didn't return. Killall didn't kill the job.");
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
Exec.Shell(exe + @" del B40735E5-8B8D-4638-8908-CD2AB024C3A7.xml");
}
