/// <summary>Runs a MapReduce task, given number of times.</summary>
/// <remarks>
/// Runs a MapReduce task, given number of times. The input to each run
/// is the same file.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private AList<long> RunJobInSequence(JobConf masterJobConf, int numRuns)
{
Random rand = new Random();
AList<long> execTimes = new AList<long>();
for (int i = 0; i < numRuns; i++)
{
// create a new job conf every time, reusing same object does not work
JobConf jobConf = new JobConf(masterJobConf);
// reset the job jar because the copy constructor doesn't
jobConf.SetJar(masterJobConf.GetJar());
// give a new random name to output of the mapred tasks
FileOutputFormat.SetOutputPath(jobConf, new Path(OutputDir, "output_" + rand.Next
()));
Log.Info("Running job " + i + ":" + " input=" + FileInputFormat.GetInputPaths(jobConf
)[0] + " output=" + FileOutputFormat.GetOutputPath(jobConf));
// run the mapred task now
long curTime = Runtime.CurrentTimeMillis();
JobClient.RunJob(jobConf);
execTimes.AddItem(Runtime.CurrentTimeMillis() - curTime);
}
return execTimes;
}
/// <summary>Get the pathname of the job's jar.</summary>
/// <returns>the pathname</returns>
public virtual string GetJar()
{
return(conf.GetJar());
}
/// <exception cref="System.Exception"/>
public virtual int Run(string[] args)
{
Submitter.CommandLineParser cli = new Submitter.CommandLineParser();
if (args.Length == 0)
{
cli.PrintUsage();
return(1);
}
cli.AddOption("input", false, "input path to the maps", "path");
cli.AddOption("output", false, "output path from the reduces", "path");
cli.AddOption("jar", false, "job jar file", "path");
cli.AddOption("inputformat", false, "java classname of InputFormat", "class");
//cli.addArgument("javareader", false, "is the RecordReader in Java");
cli.AddOption("map", false, "java classname of Mapper", "class");
cli.AddOption("partitioner", false, "java classname of Partitioner", "class");
cli.AddOption("reduce", false, "java classname of Reducer", "class");
cli.AddOption("writer", false, "java classname of OutputFormat", "class");
cli.AddOption("program", false, "URI to application executable", "class");
cli.AddOption("reduces", false, "number of reduces", "num");
cli.AddOption("jobconf", false, "\"n1=v1,n2=v2,..\" (Deprecated) Optional. Add or override a JobConf property."
, "key=val");
cli.AddOption("lazyOutput", false, "Optional. Create output lazily", "boolean");
Parser parser = cli.CreateParser();
try
{
GenericOptionsParser genericParser = new GenericOptionsParser(GetConf(), args);
CommandLine results = parser.Parse(cli.options, genericParser.GetRemainingArgs());
JobConf job = new JobConf(GetConf());
if (results.HasOption("input"))
{
FileInputFormat.SetInputPaths(job, results.GetOptionValue("input"));
}
if (results.HasOption("output"))
{
FileOutputFormat.SetOutputPath(job, new Path(results.GetOptionValue("output")));
}
if (results.HasOption("jar"))
{
job.SetJar(results.GetOptionValue("jar"));
}
if (results.HasOption("inputformat"))
{
SetIsJavaRecordReader(job, true);
job.SetInputFormat(GetClass <InputFormat>(results, "inputformat", job));
}
if (results.HasOption("javareader"))
{
SetIsJavaRecordReader(job, true);
}
if (results.HasOption("map"))
{
SetIsJavaMapper(job, true);
job.SetMapperClass(GetClass <Mapper>(results, "map", job));
}
if (results.HasOption("partitioner"))
{
job.SetPartitionerClass(GetClass <Partitioner>(results, "partitioner", job));
}
if (results.HasOption("reduce"))
{
SetIsJavaReducer(job, true);
job.SetReducerClass(GetClass <Reducer>(results, "reduce", job));
}
if (results.HasOption("reduces"))
{
job.SetNumReduceTasks(System.Convert.ToInt32(results.GetOptionValue("reduces")));
}
if (results.HasOption("writer"))
{
SetIsJavaRecordWriter(job, true);
job.SetOutputFormat(GetClass <OutputFormat>(results, "writer", job));
}
if (results.HasOption("lazyOutput"))
{
if (System.Boolean.Parse(results.GetOptionValue("lazyOutput")))
{
LazyOutputFormat.SetOutputFormatClass(job, job.GetOutputFormat().GetType());
}
}
if (results.HasOption("program"))
{
SetExecutable(job, results.GetOptionValue("program"));
}
if (results.HasOption("jobconf"))
{
Log.Warn("-jobconf option is deprecated, please use -D instead.");
string options = results.GetOptionValue("jobconf");
StringTokenizer tokenizer = new StringTokenizer(options, ",");
while (tokenizer.HasMoreTokens())
{
string keyVal = tokenizer.NextToken().Trim();
string[] keyValSplit = keyVal.Split("=");
job.Set(keyValSplit[0], keyValSplit[1]);
}
}
// if they gave us a jar file, include it into the class path
string jarFile = job.GetJar();
if (jarFile != null)
{
Uri[] urls = new Uri[] { FileSystem.GetLocal(job).PathToFile(new Path(jarFile)).ToURL
() };
//FindBugs complains that creating a URLClassLoader should be
//in a doPrivileged() block.
ClassLoader loader = AccessController.DoPrivileged(new _PrivilegedAction_494(urls
));
job.SetClassLoader(loader);
}
RunJob(job);
return(0);
}
catch (ParseException pe)
{
Log.Info("Error : " + pe);
cli.PrintUsage();
return(1);
}
}