public virtual void TestPipesPartitioner()
{
PipesPartitioner <IntWritable, Text> partitioner = new PipesPartitioner <IntWritable
, Text>();
JobConf configuration = new JobConf();
Submitter.GetJavaPartitioner(configuration);
partitioner.Configure(new JobConf());
IntWritable iw = new IntWritable(4);
// the cache empty
NUnit.Framework.Assert.AreEqual(0, partitioner.GetPartition(iw, new Text("test"),
2));
// set data into cache
PipesPartitioner.SetNextPartition(3);
// get data from cache
NUnit.Framework.Assert.AreEqual(3, partitioner.GetPartition(iw, new Text("test"),
2));
}
public virtual void TestSubmitter()
{
JobConf conf = new JobConf();
FilePath[] psw = CleanTokenPasswordFile();
Runtime.SetProperty("test.build.data", "target/tmp/build/TEST_SUBMITTER_MAPPER/data"
);
conf.Set("hadoop.log.dir", "target/tmp");
// prepare configuration
Submitter.SetIsJavaMapper(conf, false);
Submitter.SetIsJavaReducer(conf, false);
Submitter.SetKeepCommandFile(conf, false);
Submitter.SetIsJavaRecordReader(conf, false);
Submitter.SetIsJavaRecordWriter(conf, false);
PipesPartitioner <IntWritable, Text> partitioner = new PipesPartitioner <IntWritable
, Text>();
partitioner.Configure(conf);
Submitter.SetJavaPartitioner(conf, partitioner.GetType());
NUnit.Framework.Assert.AreEqual(typeof(PipesPartitioner), (Submitter.GetJavaPartitioner
(conf)));
// test going to call main method with System.exit(). Change Security
SecurityManager securityManager = Runtime.GetSecurityManager();
// store System.out
TextWriter oldps = System.Console.Out;
ByteArrayOutputStream @out = new ByteArrayOutputStream();
ExitUtil.DisableSystemExit();
// test without parameters
try
{
Runtime.SetOut(new TextWriter(@out));
Submitter.Main(new string[0]);
NUnit.Framework.Assert.Fail();
}
catch (ExitUtil.ExitException)
{
// System.exit prohibited! output message test
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains(string.Empty));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("bin/hadoop pipes"));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-input <path>] // Input directory"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-output <path>] // Output directory"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-jar <jar file> // jar filename"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-inputformat <class>] // InputFormat class"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-map <class>] // Java Map class"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-partitioner <class>] // Java Partitioner"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-reduce <class>] // Java Reduce class"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-writer <class>] // Java RecordWriter"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-program <executable>] // executable URI"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-reduces <num>] // number of reduces"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("[-lazyOutput <true/false>] // createOutputLazily"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-conf <configuration file> specify an application configuration file"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-D <property=value> use value for given property"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-fs <local|namenode:port> specify a namenode"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-jt <local|resourcemanager:port> specify a ResourceManager"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-files <comma separated list of files> specify comma separated files to be copied to the map reduce cluster"
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-libjars <comma separated list of jars> specify comma separated jar files to include in the classpath."
));
NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("-archives <comma separated list of archives> specify comma separated archives to be unarchived on the compute machines."
));
}
finally
{
Runtime.SetOut(oldps);
// restore
Runtime.SetSecurityManager(securityManager);
if (psw != null)
{
// remove password files
foreach (FilePath file in psw)
{
file.DeleteOnExit();
}
}
}
// test call Submitter form command line
try
{
FilePath fCommand = GetFileCommand(null);
string[] args = new string[22];
FilePath input = new FilePath(workSpace + FilePath.separator + "input");
if (!input.Exists())
{
NUnit.Framework.Assert.IsTrue(input.CreateNewFile());
}
FilePath outPut = new FilePath(workSpace + FilePath.separator + "output");
FileUtil.FullyDelete(outPut);
args[0] = "-input";
args[1] = input.GetAbsolutePath();
// "input";
args[2] = "-output";
args[3] = outPut.GetAbsolutePath();
// "output";
args[4] = "-inputformat";
args[5] = "org.apache.hadoop.mapred.TextInputFormat";
args[6] = "-map";
args[7] = "org.apache.hadoop.mapred.lib.IdentityMapper";
args[8] = "-partitioner";
args[9] = "org.apache.hadoop.mapred.pipes.PipesPartitioner";
args[10] = "-reduce";
args[11] = "org.apache.hadoop.mapred.lib.IdentityReducer";
args[12] = "-writer";
args[13] = "org.apache.hadoop.mapred.TextOutputFormat";
args[14] = "-program";
args[15] = fCommand.GetAbsolutePath();
// "program";
args[16] = "-reduces";
args[17] = "2";
args[18] = "-lazyOutput";
args[19] = "lazyOutput";
args[20] = "-jobconf";
args[21] = "mapreduce.pipes.isjavarecordwriter=false,mapreduce.pipes.isjavarecordreader=false";
Submitter.Main(args);
NUnit.Framework.Assert.Fail();
}
catch (ExitUtil.ExitException e)
{
// status should be 0
NUnit.Framework.Assert.AreEqual(e.status, 0);
}
finally
{
Runtime.SetOut(oldps);
Runtime.SetSecurityManager(securityManager);
}
}
/// <summary>The task output a record with a partition number attached.</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void PartitionedOutput(int reduce, K key, V value)
{
PipesPartitioner.SetNextPartition(reduce);
collector.Collect(key, value);
}
