public virtual void TestFormatWithCustomSeparator()
{
JobConf job = new JobConf();
string separator = "\u0001";
job.Set("mapreduce.output.textoutputformat.separator", separator);
job.Set(JobContext.TaskAttemptId, attempt);
FileOutputFormat.SetOutputPath(job, workDir.GetParent().GetParent());
FileOutputFormat.SetWorkOutputPath(job, workDir);
FileSystem fs = workDir.GetFileSystem(job);
if (!fs.Mkdirs(workDir))
{
NUnit.Framework.Assert.Fail("Failed to create output directory");
}
string file = "test_custom.txt";
// A reporter that does nothing
Reporter reporter = Reporter.Null;
TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat <object, object
>();
RecordWriter <object, object> theRecordWriter = theOutputFormat.GetRecordWriter(localFs
, job, file, reporter);
Org.Apache.Hadoop.IO.Text key1 = new Org.Apache.Hadoop.IO.Text("key1");
Org.Apache.Hadoop.IO.Text key2 = new Org.Apache.Hadoop.IO.Text("key2");
Org.Apache.Hadoop.IO.Text val1 = new Org.Apache.Hadoop.IO.Text("val1");
Org.Apache.Hadoop.IO.Text val2 = new Org.Apache.Hadoop.IO.Text("val2");
NullWritable nullWritable = NullWritable.Get();
try
{
theRecordWriter.Write(key1, val1);
theRecordWriter.Write(null, nullWritable);
theRecordWriter.Write(null, val1);
theRecordWriter.Write(nullWritable, val2);
theRecordWriter.Write(key2, nullWritable);
theRecordWriter.Write(key1, null);
theRecordWriter.Write(null, null);
theRecordWriter.Write(key2, val2);
}
finally
{
theRecordWriter.Close(reporter);
}
FilePath expectedFile = new FilePath(new Path(workDir, file).ToString());
StringBuilder expectedOutput = new StringBuilder();
expectedOutput.Append(key1).Append(separator).Append(val1).Append("\n");
expectedOutput.Append(val1).Append("\n");
expectedOutput.Append(val2).Append("\n");
expectedOutput.Append(key2).Append("\n");
expectedOutput.Append(key1).Append("\n");
expectedOutput.Append(key2).Append(separator).Append(val2).Append("\n");
string output = UtilsForTests.Slurp(expectedFile);
NUnit.Framework.Assert.AreEqual(expectedOutput.ToString(), output);
}
/// <exception cref="System.Exception"/>
public virtual void TestCommitter()
{
JobConf job = new JobConf();
SetConfForFileOutputCommitter(job);
JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
committer.SetupJob(jContext);
committer.SetupTask(tContext);
string file = "test.txt";
// A reporter that does nothing
Reporter reporter = Reporter.Null;
// write output
FileSystem localFs = FileSystem.GetLocal(job);
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file
, reporter);
WriteOutput(theRecordWriter, reporter);
// do commit
committer.CommitTask(tContext);
committer.CommitJob(jContext);
// validate output
FilePath expectedFile = new FilePath(new Path(outDir, file).ToString());
StringBuilder expectedOutput = new StringBuilder();
expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n");
expectedOutput.Append(val1).Append("\n");
expectedOutput.Append(val2).Append("\n");
expectedOutput.Append(key2).Append("\n");
expectedOutput.Append(key1).Append("\n");
expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n");
string output = UtilsForTests.Slurp(expectedFile);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
/// <exception cref="System.Exception"/>
public virtual void TestFormat()
{
JobConf job = new JobConf();
job.Set(JobContext.TaskAttemptId, attempt);
FileOutputFormat.SetOutputPath(job, workDir.GetParent().GetParent());
FileOutputFormat.SetWorkOutputPath(job, workDir);
FileSystem fs = workDir.GetFileSystem(job);
if (!fs.Mkdirs(workDir))
{
Fail("Failed to create output directory");
}
//System.out.printf("workdir: %s\n", workDir.toString());
TestMultipleTextOutputFormat.Test1(job);
TestMultipleTextOutputFormat.Test2(job);
string file_11 = "1-part-00000";
FilePath expectedFile_11 = new FilePath(new Path(workDir, file_11).ToString());
//System.out.printf("expectedFile_11: %s\n", new Path(workDir, file_11).toString());
StringBuilder expectedOutput = new StringBuilder();
for (int i = 10; i < 20; i++)
{
expectedOutput.Append(string.Empty + i).Append('\t').Append(string.Empty + i).Append
("\n");
}
string output = UtilsForTests.Slurp(expectedFile_11);
//System.out.printf("File_2 output: %s\n", output);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
string file_12 = "2-part-00000";
FilePath expectedFile_12 = new FilePath(new Path(workDir, file_12).ToString());
//System.out.printf("expectedFile_12: %s\n", new Path(workDir, file_12).toString());
expectedOutput = new StringBuilder();
for (int i_1 = 20; i_1 < 30; i_1++)
{
expectedOutput.Append(string.Empty + i_1).Append('\t').Append(string.Empty + i_1)
.Append("\n");
}
output = UtilsForTests.Slurp(expectedFile_12);
//System.out.printf("File_2 output: %s\n", output);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
string file_13 = "3-part-00000";
FilePath expectedFile_13 = new FilePath(new Path(workDir, file_13).ToString());
//System.out.printf("expectedFile_13: %s\n", new Path(workDir, file_13).toString());
expectedOutput = new StringBuilder();
for (int i_2 = 30; i_2 < 40; i_2++)
{
expectedOutput.Append(string.Empty + i_2).Append('\t').Append(string.Empty + i_2)
.Append("\n");
}
output = UtilsForTests.Slurp(expectedFile_13);
//System.out.printf("File_2 output: %s\n", output);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
string file_2 = "2/3";
FilePath expectedFile_2 = new FilePath(new Path(workDir, file_2).ToString());
//System.out.printf("expectedFile_2: %s\n", new Path(workDir, file_2).toString());
expectedOutput = new StringBuilder();
for (int i_3 = 10; i_3 < 40; i_3++)
{
expectedOutput.Append(string.Empty + i_3).Append('\t').Append(string.Empty + i_3)
.Append("\n");
}
output = UtilsForTests.Slurp(expectedFile_2);
//System.out.printf("File_2 output: %s\n", output);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
}