public virtual void TestFormatWithCustomSeparator() { JobConf job = new JobConf(); string separator = "\u0001"; job.Set("mapreduce.output.textoutputformat.separator", separator); job.Set(JobContext.TaskAttemptId, attempt); FileOutputFormat.SetOutputPath(job, workDir.GetParent().GetParent()); FileOutputFormat.SetWorkOutputPath(job, workDir); FileSystem fs = workDir.GetFileSystem(job); if (!fs.Mkdirs(workDir)) { NUnit.Framework.Assert.Fail("Failed to create output directory"); } string file = "test_custom.txt"; // A reporter that does nothing Reporter reporter = Reporter.Null; TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat <object, object >(); RecordWriter <object, object> theRecordWriter = theOutputFormat.GetRecordWriter(localFs , job, file, reporter); Org.Apache.Hadoop.IO.Text key1 = new Org.Apache.Hadoop.IO.Text("key1"); Org.Apache.Hadoop.IO.Text key2 = new Org.Apache.Hadoop.IO.Text("key2"); Org.Apache.Hadoop.IO.Text val1 = new Org.Apache.Hadoop.IO.Text("val1"); Org.Apache.Hadoop.IO.Text val2 = new Org.Apache.Hadoop.IO.Text("val2"); NullWritable nullWritable = NullWritable.Get(); try { theRecordWriter.Write(key1, val1); theRecordWriter.Write(null, nullWritable); theRecordWriter.Write(null, val1); theRecordWriter.Write(nullWritable, val2); theRecordWriter.Write(key2, nullWritable); theRecordWriter.Write(key1, null); theRecordWriter.Write(null, null); theRecordWriter.Write(key2, val2); } finally { theRecordWriter.Close(reporter); } FilePath expectedFile = new FilePath(new Path(workDir, file).ToString()); StringBuilder expectedOutput = new StringBuilder(); expectedOutput.Append(key1).Append(separator).Append(val1).Append("\n"); expectedOutput.Append(val1).Append("\n"); expectedOutput.Append(val2).Append("\n"); expectedOutput.Append(key2).Append("\n"); expectedOutput.Append(key1).Append("\n"); expectedOutput.Append(key2).Append(separator).Append(val2).Append("\n"); string output = UtilsForTests.Slurp(expectedFile); NUnit.Framework.Assert.AreEqual(expectedOutput.ToString(), output); }
/// <exception cref="System.Exception"/> public virtual void TestCommitter() { JobConf job = new JobConf(); SetConfForFileOutputCommitter(job); JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID())); TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID); FileOutputCommitter committer = new FileOutputCommitter(); FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext)); committer.SetupJob(jContext); committer.SetupTask(tContext); string file = "test.txt"; // A reporter that does nothing Reporter reporter = Reporter.Null; // write output FileSystem localFs = FileSystem.GetLocal(job); TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, file , reporter); WriteOutput(theRecordWriter, reporter); // do commit committer.CommitTask(tContext); committer.CommitJob(jContext); // validate output FilePath expectedFile = new FilePath(new Path(outDir, file).ToString()); StringBuilder expectedOutput = new StringBuilder(); expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n"); expectedOutput.Append(val1).Append("\n"); expectedOutput.Append(val2).Append("\n"); expectedOutput.Append(key2).Append("\n"); expectedOutput.Append(key1).Append("\n"); expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n"); string output = UtilsForTests.Slurp(expectedFile); NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString()); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
/// <exception cref="System.Exception"/> public virtual void TestFormat() { JobConf job = new JobConf(); job.Set(JobContext.TaskAttemptId, attempt); FileOutputFormat.SetOutputPath(job, workDir.GetParent().GetParent()); FileOutputFormat.SetWorkOutputPath(job, workDir); FileSystem fs = workDir.GetFileSystem(job); if (!fs.Mkdirs(workDir)) { Fail("Failed to create output directory"); } //System.out.printf("workdir: %s\n", workDir.toString()); TestMultipleTextOutputFormat.Test1(job); TestMultipleTextOutputFormat.Test2(job); string file_11 = "1-part-00000"; FilePath expectedFile_11 = new FilePath(new Path(workDir, file_11).ToString()); //System.out.printf("expectedFile_11: %s\n", new Path(workDir, file_11).toString()); StringBuilder expectedOutput = new StringBuilder(); for (int i = 10; i < 20; i++) { expectedOutput.Append(string.Empty + i).Append('\t').Append(string.Empty + i).Append ("\n"); } string output = UtilsForTests.Slurp(expectedFile_11); //System.out.printf("File_2 output: %s\n", output); NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString()); string file_12 = "2-part-00000"; FilePath expectedFile_12 = new FilePath(new Path(workDir, file_12).ToString()); //System.out.printf("expectedFile_12: %s\n", new Path(workDir, file_12).toString()); expectedOutput = new StringBuilder(); for (int i_1 = 20; i_1 < 30; i_1++) { expectedOutput.Append(string.Empty + i_1).Append('\t').Append(string.Empty + i_1) .Append("\n"); } output = UtilsForTests.Slurp(expectedFile_12); //System.out.printf("File_2 output: %s\n", output); NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString()); string file_13 = "3-part-00000"; FilePath expectedFile_13 = new FilePath(new Path(workDir, file_13).ToString()); //System.out.printf("expectedFile_13: %s\n", new Path(workDir, file_13).toString()); expectedOutput = new StringBuilder(); for (int i_2 = 30; i_2 < 40; i_2++) { expectedOutput.Append(string.Empty + i_2).Append('\t').Append(string.Empty + i_2) .Append("\n"); } output = UtilsForTests.Slurp(expectedFile_13); //System.out.printf("File_2 output: %s\n", output); NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString()); string file_2 = "2/3"; FilePath expectedFile_2 = new FilePath(new Path(workDir, file_2).ToString()); //System.out.printf("expectedFile_2: %s\n", new Path(workDir, file_2).toString()); expectedOutput = new StringBuilder(); for (int i_3 = 10; i_3 < 40; i_3++) { expectedOutput.Append(string.Empty + i_3).Append('\t').Append(string.Empty + i_3) .Append("\n"); } output = UtilsForTests.Slurp(expectedFile_2); //System.out.printf("File_2 output: %s\n", output); NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString()); }