public virtual void TestPipesReduser()
{
FilePath[] psw = CleanTokenPasswordFile();
JobConf conf = new JobConf();
try
{
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(Sharpen.Runtime.GetBytesForString("user"), Sharpen.Runtime.GetBytesForString
("password"), new Text("kind"), new Text("service"));
TokenCache.SetJobToken(token, conf.GetCredentials());
FilePath fCommand = GetFileCommand("org.apache.hadoop.mapred.pipes.PipeReducerStub"
);
conf.Set(MRJobConfig.CacheLocalfiles, fCommand.GetAbsolutePath());
PipesReducer <BooleanWritable, Text, IntWritable, Text> reducer = new PipesReducer
<BooleanWritable, Text, IntWritable, Text>();
reducer.Configure(conf);
BooleanWritable bw = new BooleanWritable(true);
conf.Set(MRJobConfig.TaskAttemptId, taskName);
InitStdOut(conf);
conf.SetBoolean(MRJobConfig.SkipRecords, true);
TestPipeApplication.CombineOutputCollector <IntWritable, Text> output = new TestPipeApplication.CombineOutputCollector
<IntWritable, Text>(this, new Counters.Counter(), new TestPipeApplication.Progress
(this));
Reporter reporter = new TestPipeApplication.TestTaskReporter(this);
IList <Text> texts = new AList <Text>();
texts.AddItem(new Text("first"));
texts.AddItem(new Text("second"));
texts.AddItem(new Text("third"));
reducer.Reduce(bw, texts.GetEnumerator(), output, reporter);
reducer.Close();
string stdOut = ReadStdOut(conf);
// test data: key
NUnit.Framework.Assert.IsTrue(stdOut.Contains("reducer key :true"));
// and values
NUnit.Framework.Assert.IsTrue(stdOut.Contains("reduce value :first"));
NUnit.Framework.Assert.IsTrue(stdOut.Contains("reduce value :second"));
NUnit.Framework.Assert.IsTrue(stdOut.Contains("reduce value :third"));
}
finally
{
if (psw != null)
{
// remove password files
foreach (FilePath file in psw)
{
file.DeleteOnExit();
}
}
}
}
public virtual void TestRunner()
{
// clean old password files
FilePath[] psw = CleanTokenPasswordFile();
try
{
RecordReader <FloatWritable, NullWritable> rReader = new TestPipeApplication.ReaderPipesMapRunner
(this);
JobConf conf = new JobConf();
conf.Set(Submitter.IsJavaRr, "true");
// for stdour and stderror
conf.Set(MRJobConfig.TaskAttemptId, taskName);
TestPipeApplication.CombineOutputCollector <IntWritable, Text> output = new TestPipeApplication.CombineOutputCollector
<IntWritable, Text>(this, new Counters.Counter(), new TestPipeApplication.Progress
(this));
FileSystem fs = new RawLocalFileSystem();
fs.SetConf(conf);
IFile.Writer <IntWritable, Text> wr = new IFile.Writer <IntWritable, Text>(conf, fs
.Create(new Path(workSpace + FilePath.separator + "outfile")), typeof(IntWritable
), typeof(Text), null, null, true);
output.SetWriter(wr);
// stub for client
FilePath fCommand = GetFileCommand("org.apache.hadoop.mapred.pipes.PipeApplicationRunnableStub"
);
conf.Set(MRJobConfig.CacheLocalfiles, fCommand.GetAbsolutePath());
// token for authorization
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(Sharpen.Runtime.GetBytesForString("user"), Sharpen.Runtime.GetBytesForString
("password"), new Text("kind"), new Text("service"));
TokenCache.SetJobToken(token, conf.GetCredentials());
conf.SetBoolean(MRJobConfig.SkipRecords, true);
TestPipeApplication.TestTaskReporter reporter = new TestPipeApplication.TestTaskReporter
(this);
PipesMapRunner <FloatWritable, NullWritable, IntWritable, Text> runner = new PipesMapRunner
<FloatWritable, NullWritable, IntWritable, Text>();
InitStdOut(conf);
runner.Configure(conf);
runner.Run(rReader, output, reporter);
string stdOut = ReadStdOut(conf);
// test part of translated data. As common file for client and test -
// clients stdOut
// check version
NUnit.Framework.Assert.IsTrue(stdOut.Contains("CURRENT_PROTOCOL_VERSION:0"));
// check key and value classes
NUnit.Framework.Assert.IsTrue(stdOut.Contains("Key class:org.apache.hadoop.io.FloatWritable"
));
NUnit.Framework.Assert.IsTrue(stdOut.Contains("Value class:org.apache.hadoop.io.NullWritable"
));
// test have sent all data from reader
NUnit.Framework.Assert.IsTrue(stdOut.Contains("value:0.0"));
NUnit.Framework.Assert.IsTrue(stdOut.Contains("value:9.0"));
}
finally
{
if (psw != null)
{
// remove password files
foreach (FilePath file in psw)
{
file.DeleteOnExit();
}
}
}
}