public virtual void TestPipesReduser()
{
FilePath[] psw = CleanTokenPasswordFile();
JobConf conf = new JobConf();
try
{
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(Sharpen.Runtime.GetBytesForString("user"), Sharpen.Runtime.GetBytesForString
("password"), new Text("kind"), new Text("service"));
TokenCache.SetJobToken(token, conf.GetCredentials());
FilePath fCommand = GetFileCommand("org.apache.hadoop.mapred.pipes.PipeReducerStub"
);
conf.Set(MRJobConfig.CacheLocalfiles, fCommand.GetAbsolutePath());
PipesReducer <BooleanWritable, Text, IntWritable, Text> reducer = new PipesReducer
<BooleanWritable, Text, IntWritable, Text>();
reducer.Configure(conf);
BooleanWritable bw = new BooleanWritable(true);
conf.Set(MRJobConfig.TaskAttemptId, taskName);
InitStdOut(conf);
conf.SetBoolean(MRJobConfig.SkipRecords, true);
TestPipeApplication.CombineOutputCollector <IntWritable, Text> output = new TestPipeApplication.CombineOutputCollector
<IntWritable, Text>(this, new Counters.Counter(), new TestPipeApplication.Progress
(this));
Reporter reporter = new TestPipeApplication.TestTaskReporter(this);
IList <Text> texts = new AList <Text>();
texts.AddItem(new Text("first"));
texts.AddItem(new Text("second"));
texts.AddItem(new Text("third"));
reducer.Reduce(bw, texts.GetEnumerator(), output, reporter);
reducer.Close();
string stdOut = ReadStdOut(conf);
// test data: key
NUnit.Framework.Assert.IsTrue(stdOut.Contains("reducer key :true"));
// and values
NUnit.Framework.Assert.IsTrue(stdOut.Contains("reduce value :first"));
NUnit.Framework.Assert.IsTrue(stdOut.Contains("reduce value :second"));
NUnit.Framework.Assert.IsTrue(stdOut.Contains("reduce value :third"));
}
finally
{
if (psw != null)
{
// remove password files
foreach (FilePath file in psw)
{
file.DeleteOnExit();
}
}
}
}
public virtual void TestApplication()
{
JobConf conf = new JobConf();
RecordReader <FloatWritable, NullWritable> rReader = new TestPipeApplication.Reader
(this);
// client for test
FilePath fCommand = GetFileCommand("org.apache.hadoop.mapred.pipes.PipeApplicationStub"
);
TestPipeApplication.TestTaskReporter reporter = new TestPipeApplication.TestTaskReporter
(this);
FilePath[] psw = CleanTokenPasswordFile();
try
{
conf.Set(MRJobConfig.TaskAttemptId, taskName);
conf.Set(MRJobConfig.CacheLocalfiles, fCommand.GetAbsolutePath());
// token for authorization
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(Sharpen.Runtime.GetBytesForString("user"), Sharpen.Runtime.GetBytesForString
("password"), new Text("kind"), new Text("service"));
TokenCache.SetJobToken(token, conf.GetCredentials());
TestPipeApplication.FakeCollector output = new TestPipeApplication.FakeCollector(
this, new Counters.Counter(), new TestPipeApplication.Progress(this));
FileSystem fs = new RawLocalFileSystem();
fs.SetConf(conf);
IFile.Writer <IntWritable, Text> wr = new IFile.Writer <IntWritable, Text>(conf, fs
.Create(new Path(workSpace.GetAbsolutePath() + FilePath.separator + "outfile")),
typeof(IntWritable), typeof(Text), null, null, true);
output.SetWriter(wr);
conf.Set(Submitter.PreserveCommandfile, "true");
InitStdOut(conf);
Application <WritableComparable <IntWritable>, Writable, IntWritable, Text> application
= new Application <WritableComparable <IntWritable>, Writable, IntWritable, Text>
(conf, rReader, output, reporter, typeof(IntWritable), typeof(Text));
application.GetDownlink().Flush();
application.GetDownlink().MapItem(new IntWritable(3), new Text("txt"));
application.GetDownlink().Flush();
application.WaitForFinish();
wr.Close();
// test getDownlink().mapItem();
string stdOut = ReadStdOut(conf);
NUnit.Framework.Assert.IsTrue(stdOut.Contains("key:3"));
NUnit.Framework.Assert.IsTrue(stdOut.Contains("value:txt"));
// reporter test counter, and status should be sended
// test MessageType.REGISTER_COUNTER and INCREMENT_COUNTER
NUnit.Framework.Assert.AreEqual(1.0, reporter.GetProgress(), 0.01);
NUnit.Framework.Assert.IsNotNull(reporter.GetCounter("group", "name"));
// test status MessageType.STATUS
NUnit.Framework.Assert.AreEqual(reporter.GetStatus(), "PROGRESS");
stdOut = ReadFile(new FilePath(workSpace.GetAbsolutePath() + FilePath.separator +
"outfile"));
// check MessageType.PROGRESS
NUnit.Framework.Assert.AreEqual(0.55f, rReader.GetProgress(), 0.001);
application.GetDownlink().Close();
// test MessageType.OUTPUT
KeyValuePair <IntWritable, Text> entry = output.GetCollect().GetEnumerator().Next(
);
NUnit.Framework.Assert.AreEqual(123, entry.Key.Get());
NUnit.Framework.Assert.AreEqual("value", entry.Value.ToString());
try
{
// try to abort
application.Abort(new Exception());
NUnit.Framework.Assert.Fail();
}
catch (IOException e)
{
// abort works ?
NUnit.Framework.Assert.AreEqual("pipe child exception", e.Message);
}
}
finally
{
if (psw != null)
{
// remove password files
foreach (FilePath file in psw)
{
file.DeleteOnExit();
}
}
}
}
public virtual void TestRunner()
{
// clean old password files
FilePath[] psw = CleanTokenPasswordFile();
try
{
RecordReader <FloatWritable, NullWritable> rReader = new TestPipeApplication.ReaderPipesMapRunner
(this);
JobConf conf = new JobConf();
conf.Set(Submitter.IsJavaRr, "true");
// for stdour and stderror
conf.Set(MRJobConfig.TaskAttemptId, taskName);
TestPipeApplication.CombineOutputCollector <IntWritable, Text> output = new TestPipeApplication.CombineOutputCollector
<IntWritable, Text>(this, new Counters.Counter(), new TestPipeApplication.Progress
(this));
FileSystem fs = new RawLocalFileSystem();
fs.SetConf(conf);
IFile.Writer <IntWritable, Text> wr = new IFile.Writer <IntWritable, Text>(conf, fs
.Create(new Path(workSpace + FilePath.separator + "outfile")), typeof(IntWritable
), typeof(Text), null, null, true);
output.SetWriter(wr);
// stub for client
FilePath fCommand = GetFileCommand("org.apache.hadoop.mapred.pipes.PipeApplicationRunnableStub"
);
conf.Set(MRJobConfig.CacheLocalfiles, fCommand.GetAbsolutePath());
// token for authorization
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<AMRMTokenIdentifier>(Sharpen.Runtime.GetBytesForString("user"), Sharpen.Runtime.GetBytesForString
("password"), new Text("kind"), new Text("service"));
TokenCache.SetJobToken(token, conf.GetCredentials());
conf.SetBoolean(MRJobConfig.SkipRecords, true);
TestPipeApplication.TestTaskReporter reporter = new TestPipeApplication.TestTaskReporter
(this);
PipesMapRunner <FloatWritable, NullWritable, IntWritable, Text> runner = new PipesMapRunner
<FloatWritable, NullWritable, IntWritable, Text>();
InitStdOut(conf);
runner.Configure(conf);
runner.Run(rReader, output, reporter);
string stdOut = ReadStdOut(conf);
// test part of translated data. As common file for client and test -
// clients stdOut
// check version
NUnit.Framework.Assert.IsTrue(stdOut.Contains("CURRENT_PROTOCOL_VERSION:0"));
// check key and value classes
NUnit.Framework.Assert.IsTrue(stdOut.Contains("Key class:org.apache.hadoop.io.FloatWritable"
));
NUnit.Framework.Assert.IsTrue(stdOut.Contains("Value class:org.apache.hadoop.io.NullWritable"
));
// test have sent all data from reader
NUnit.Framework.Assert.IsTrue(stdOut.Contains("value:0.0"));
NUnit.Framework.Assert.IsTrue(stdOut.Contains("value:9.0"));
}
finally
{
if (psw != null)
{
// remove password files
foreach (FilePath file in psw)
{
file.DeleteOnExit();
}
}
}
}