/// <exception cref="System.Exception"/>
private static void RunTestLazyOutput(JobConf job, Path output, int numReducers,
bool createLazily)
{
job.SetJobName("test-lazy-output");
FileInputFormat.SetInputPaths(job, Input);
FileOutputFormat.SetOutputPath(job, output);
job.SetInputFormat(typeof(TextInputFormat));
job.SetMapOutputKeyClass(typeof(LongWritable));
job.SetMapOutputValueClass(typeof(Text));
job.SetOutputKeyClass(typeof(LongWritable));
job.SetOutputValueClass(typeof(Text));
job.SetMapperClass(typeof(TestLazyOutput.TestMapper));
job.SetReducerClass(typeof(TestLazyOutput.TestReducer));
JobClient client = new JobClient(job);
job.SetNumReduceTasks(numReducers);
if (createLazily)
{
LazyOutputFormat.SetOutputFormatClass(job, typeof(TextOutputFormat));
}
else
{
job.SetOutputFormat(typeof(TextOutputFormat));
}
JobClient.RunJob(job);
}
/// <exception cref="System.Exception"/>
private static void RunTestLazyOutput(Configuration conf, Path output, int numReducers
, bool createLazily)
{
Job job = Job.GetInstance(conf, "Test-Lazy-Output");
FileInputFormat.SetInputPaths(job, Input);
FileOutputFormat.SetOutputPath(job, output);
job.SetJarByClass(typeof(TestMapReduceLazyOutput));
job.SetInputFormatClass(typeof(TextInputFormat));
job.SetOutputKeyClass(typeof(LongWritable));
job.SetOutputValueClass(typeof(Text));
job.SetNumReduceTasks(numReducers);
job.SetMapperClass(typeof(TestMapReduceLazyOutput.TestMapper));
job.SetReducerClass(typeof(TestMapReduceLazyOutput.TestReducer));
if (createLazily)
{
LazyOutputFormat.SetOutputFormatClass(job, typeof(TextOutputFormat));
}
else
{
job.SetOutputFormatClass(typeof(TextOutputFormat));
}
NUnit.Framework.Assert.IsTrue(job.WaitForCompletion(true));
}
/// <exception cref="System.Exception"/>
public virtual int Run(string[] args)
{
Submitter.CommandLineParser cli = new Submitter.CommandLineParser();
if (args.Length == 0)
{
cli.PrintUsage();
return(1);
}
cli.AddOption("input", false, "input path to the maps", "path");
cli.AddOption("output", false, "output path from the reduces", "path");
cli.AddOption("jar", false, "job jar file", "path");
cli.AddOption("inputformat", false, "java classname of InputFormat", "class");
//cli.addArgument("javareader", false, "is the RecordReader in Java");
cli.AddOption("map", false, "java classname of Mapper", "class");
cli.AddOption("partitioner", false, "java classname of Partitioner", "class");
cli.AddOption("reduce", false, "java classname of Reducer", "class");
cli.AddOption("writer", false, "java classname of OutputFormat", "class");
cli.AddOption("program", false, "URI to application executable", "class");
cli.AddOption("reduces", false, "number of reduces", "num");
cli.AddOption("jobconf", false, "\"n1=v1,n2=v2,..\" (Deprecated) Optional. Add or override a JobConf property."
, "key=val");
cli.AddOption("lazyOutput", false, "Optional. Create output lazily", "boolean");
Parser parser = cli.CreateParser();
try
{
GenericOptionsParser genericParser = new GenericOptionsParser(GetConf(), args);
CommandLine results = parser.Parse(cli.options, genericParser.GetRemainingArgs());
JobConf job = new JobConf(GetConf());
if (results.HasOption("input"))
{
FileInputFormat.SetInputPaths(job, results.GetOptionValue("input"));
}
if (results.HasOption("output"))
{
FileOutputFormat.SetOutputPath(job, new Path(results.GetOptionValue("output")));
}
if (results.HasOption("jar"))
{
job.SetJar(results.GetOptionValue("jar"));
}
if (results.HasOption("inputformat"))
{
SetIsJavaRecordReader(job, true);
job.SetInputFormat(GetClass <InputFormat>(results, "inputformat", job));
}
if (results.HasOption("javareader"))
{
SetIsJavaRecordReader(job, true);
}
if (results.HasOption("map"))
{
SetIsJavaMapper(job, true);
job.SetMapperClass(GetClass <Mapper>(results, "map", job));
}
if (results.HasOption("partitioner"))
{
job.SetPartitionerClass(GetClass <Partitioner>(results, "partitioner", job));
}
if (results.HasOption("reduce"))
{
SetIsJavaReducer(job, true);
job.SetReducerClass(GetClass <Reducer>(results, "reduce", job));
}
if (results.HasOption("reduces"))
{
job.SetNumReduceTasks(System.Convert.ToInt32(results.GetOptionValue("reduces")));
}
if (results.HasOption("writer"))
{
SetIsJavaRecordWriter(job, true);
job.SetOutputFormat(GetClass <OutputFormat>(results, "writer", job));
}
if (results.HasOption("lazyOutput"))
{
if (System.Boolean.Parse(results.GetOptionValue("lazyOutput")))
{
LazyOutputFormat.SetOutputFormatClass(job, job.GetOutputFormat().GetType());
}
}
if (results.HasOption("program"))
{
SetExecutable(job, results.GetOptionValue("program"));
}
if (results.HasOption("jobconf"))
{
Log.Warn("-jobconf option is deprecated, please use -D instead.");
string options = results.GetOptionValue("jobconf");
StringTokenizer tokenizer = new StringTokenizer(options, ",");
while (tokenizer.HasMoreTokens())
{
string keyVal = tokenizer.NextToken().Trim();
string[] keyValSplit = keyVal.Split("=");
job.Set(keyValSplit[0], keyValSplit[1]);
}
}
// if they gave us a jar file, include it into the class path
string jarFile = job.GetJar();
if (jarFile != null)
{
Uri[] urls = new Uri[] { FileSystem.GetLocal(job).PathToFile(new Path(jarFile)).ToURL
() };
//FindBugs complains that creating a URLClassLoader should be
//in a doPrivileged() block.
ClassLoader loader = AccessController.DoPrivileged(new _PrivilegedAction_494(urls
));
job.SetClassLoader(loader);
}
RunJob(job);
return(0);
}
catch (ParseException pe)
{
Log.Info("Error : " + pe);
cli.PrintUsage();
return(1);
}
}
