public void OutputFormatCollectionTest()
{
var exls = new[] {
new ExampleChild {
BaseId = 1,
BaseTitle = "Base element",
Amount = 23.5439,
ChildId = 11,
Complex = new ExampleComplex { Title = "Antoni", Value = 7 }
},
new ExampleChild {
BaseId = 2,
BaseTitle = "Base element 2",
Amount = 6543,
ChildId = 21,
Complex = new ExampleComplex { Title = "Jarek", Value = 2 }
}};
IOutputFormat formatter = new TextOutputFormat();
var result = formatter.FormatArray(exls);
var expected = "---------- # 1 # ----------" + Environment.NewLine +
"Base ID: 1" + Environment.NewLine +
"Base title: Base element" + Environment.NewLine +
"Inner ID: 11" + Environment.NewLine +
"Amount: 23,54 zł" + Environment.NewLine +
"Complex:" + Environment.NewLine + "Antoni: 7,00" + Environment.NewLine +
"---------- # 2 # ----------" + Environment.NewLine +
"Base ID: 2" + Environment.NewLine +
"Base title: Base element 2" + Environment.NewLine +
"Inner ID: 21" + Environment.NewLine +
"Amount: 6 543,00 zł" + Environment.NewLine +
"Complex:" + Environment.NewLine + "Jarek: 2,00";
Assert.Equal(expected, result);
}
/// <exception cref="System.IO.IOException"/>
protected internal override RecordWriter <K, V> GetBaseRecordWriter(FileSystem fs,
JobConf job, string name, Progressable arg3)
{
if (theTextOutputFormat == null)
{
theTextOutputFormat = new TextOutputFormat <K, V>();
}
return(theTextOutputFormat.GetRecordWriter(fs, job, name, arg3));
}
/// <summary>Sets up a job conf for the given job using the given config object.</summary>
/// <remarks>
/// Sets up a job conf for the given job using the given config object. Ensures
/// that the correct input format is set, the mapper and and reducer class and
/// the input and output keys and value classes along with any other job
/// configuration.
/// </remarks>
/// <param name="config"/>
/// <returns>JobConf representing the job to be ran</returns>
/// <exception cref="System.IO.IOException"/>
private JobConf GetJob(ConfigExtractor config)
{
JobConf job = new JobConf(config.GetConfig(), typeof(Org.Apache.Hadoop.FS.Slive.SliveTest
));
job.SetInputFormat(typeof(DummyInputFormat));
FileOutputFormat.SetOutputPath(job, config.GetOutputPath());
job.SetMapperClass(typeof(SliveMapper));
job.SetPartitionerClass(typeof(SlivePartitioner));
job.SetReducerClass(typeof(SliveReducer));
job.SetOutputKeyClass(typeof(Text));
job.SetOutputValueClass(typeof(Text));
job.SetOutputFormat(typeof(TextOutputFormat));
TextOutputFormat.SetCompressOutput(job, false);
job.SetNumReduceTasks(config.GetReducerAmount());
job.SetNumMapTasks(config.GetMapAmount());
return(job);
}
public void OutputFormatSingleWithTemplateFormatInfoTest()
{
var exl = new CleanExampleChild
{
BaseId = 1,
BaseTitle = "Base element",
Amount = 23.5439,
ChildId = 11,
Complex = new CleanExampleComplex { Title = "Antoni", Value = 7 }
};
IOutputFormat formatter = new TextOutputFormat(new ExampleFormatProvider());
var result = formatter.FormatSingle(exl);
var expected = "ROOT ID: 1" + Environment.NewLine +
"BaseTitle: Base element" + Environment.NewLine +
"Inner ID: 11" + Environment.NewLine +
"Amount: 23,54 zł" + Environment.NewLine +
"Employ receive: Antoni: 7,00";
Assert.Equal(expected, result);
}
public void OutputFormatSingleTest()
{
var exl = new ExampleChild
{
BaseId = 1,
BaseTitle = "Base element",
Amount = 23.5439,
ChildId = 11,
Complex = new ExampleComplex { Title = "Antoni", Value = 7 }
};
IOutputFormat formatter = new TextOutputFormat();
var result = formatter.FormatSingle(exl);
var expected = "Base ID: 1" + Environment.NewLine +
"Base title: Base element" + Environment.NewLine +
"Inner ID: 11" + Environment.NewLine +
"Amount: 23,54 zł" + Environment.NewLine +
"Complex:" + Environment.NewLine + "Antoni: 7,00";
Assert.Equal(expected, result);
}
public virtual void TestCombiner()
{
if (!new FilePath(TestRootDir).Mkdirs())
{
throw new RuntimeException("Could not create test dir: " + TestRootDir);
}
FilePath @in = new FilePath(TestRootDir, "input");
if ([email protected]())
{
throw new RuntimeException("Could not create test dir: " + @in);
}
FilePath @out = new FilePath(TestRootDir, "output");
PrintWriter pw = new PrintWriter(new FileWriter(new FilePath(@in, "data.txt")));
pw.WriteLine("A|a,1");
pw.WriteLine("A|b,2");
pw.WriteLine("B|a,3");
pw.WriteLine("B|b,4");
pw.WriteLine("B|c,5");
pw.Close();
JobConf conf = new JobConf();
conf.Set("mapreduce.framework.name", "local");
Job job = new Job(conf);
TextInputFormat.SetInputPaths(job, new Path(@in.GetPath()));
TextOutputFormat.SetOutputPath(job, new Path(@out.GetPath()));
job.SetMapperClass(typeof(TestNewCombinerGrouping.Map));
job.SetReducerClass(typeof(TestNewCombinerGrouping.Reduce));
job.SetInputFormatClass(typeof(TextInputFormat));
job.SetMapOutputKeyClass(typeof(Text));
job.SetMapOutputValueClass(typeof(LongWritable));
job.SetOutputFormatClass(typeof(TextOutputFormat));
job.SetGroupingComparatorClass(typeof(TestNewCombinerGrouping.GroupComparator));
job.SetCombinerKeyGroupingComparatorClass(typeof(TestNewCombinerGrouping.GroupComparator
));
job.SetCombinerClass(typeof(TestNewCombinerGrouping.Combiner));
job.GetConfiguration().SetInt("min.num.spills.for.combine", 0);
job.Submit();
job.WaitForCompletion(false);
if (job.IsSuccessful())
{
Counters counters = job.GetCounters();
long combinerInputRecords = counters.FindCounter("org.apache.hadoop.mapreduce.TaskCounter"
, "COMBINE_INPUT_RECORDS").GetValue();
long combinerOutputRecords = counters.FindCounter("org.apache.hadoop.mapreduce.TaskCounter"
, "COMBINE_OUTPUT_RECORDS").GetValue();
NUnit.Framework.Assert.IsTrue(combinerInputRecords > 0);
NUnit.Framework.Assert.IsTrue(combinerInputRecords > combinerOutputRecords);
BufferedReader br = new BufferedReader(new FileReader(new FilePath(@out, "part-r-00000"
)));
ICollection <string> output = new HashSet <string>();
string line = br.ReadLine();
NUnit.Framework.Assert.IsNotNull(line);
output.AddItem(Sharpen.Runtime.Substring(line, 0, 1) + Sharpen.Runtime.Substring(
line, 4, 5));
line = br.ReadLine();
NUnit.Framework.Assert.IsNotNull(line);
output.AddItem(Sharpen.Runtime.Substring(line, 0, 1) + Sharpen.Runtime.Substring(
line, 4, 5));
line = br.ReadLine();
NUnit.Framework.Assert.IsNull(line);
br.Close();
ICollection <string> expected = new HashSet <string>();
expected.AddItem("A2");
expected.AddItem("B5");
NUnit.Framework.Assert.AreEqual(expected, output);
}
else
{
NUnit.Framework.Assert.Fail("Job failed");
}
}