/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
Configuration conf = new Configuration();
string[] otherArgs = new GenericOptionsParser(conf, args).GetRemainingArgs();
if (otherArgs.Length < 2)
{
System.Console.Error.WriteLine("Usage: wordcount <in> [<in>...] <out>");
System.Environment.Exit(2);
}
Job job = Job.GetInstance(conf, "word count");
job.SetJarByClass(typeof(WordCount));
job.SetMapperClass(typeof(WordCount.TokenizerMapper));
job.SetCombinerClass(typeof(WordCount.IntSumReducer));
job.SetReducerClass(typeof(WordCount.IntSumReducer));
job.SetOutputKeyClass(typeof(Text));
job.SetOutputValueClass(typeof(IntWritable));
for (int i = 0; i < otherArgs.Length - 1; ++i)
{
FileInputFormat.AddInputPath(job, new Path(otherArgs[i]));
}
FileOutputFormat.SetOutputPath(job, new Path(otherArgs[otherArgs.Length - 1]));
System.Environment.Exit(job.WaitForCompletion(true) ? 0 : 1);
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
Configuration conf = new Configuration();
string[] otherArgs = new GenericOptionsParser(conf, args).GetRemainingArgs();
if (otherArgs.Length != 2)
{
System.Console.Error.WriteLine("Usage: secondarysort <in> <out>");
System.Environment.Exit(2);
}
Job job = Job.GetInstance(conf, "secondary sort");
job.SetJarByClass(typeof(SecondarySort));
job.SetMapperClass(typeof(SecondarySort.MapClass));
job.SetReducerClass(typeof(SecondarySort.Reduce));
// group and partition by the first int in the pair
job.SetPartitionerClass(typeof(SecondarySort.FirstPartitioner));
job.SetGroupingComparatorClass(typeof(SecondarySort.FirstGroupingComparator));
// the map output is IntPair, IntWritable
job.SetMapOutputKeyClass(typeof(SecondarySort.IntPair));
job.SetMapOutputValueClass(typeof(IntWritable));
// the reduce output is Text, IntWritable
job.SetOutputKeyClass(typeof(Text));
job.SetOutputValueClass(typeof(IntWritable));
FileInputFormat.AddInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.SetOutputPath(job, new Path(otherArgs[1]));
System.Environment.Exit(job.WaitForCompletion(true) ? 0 : 1);
}
protected internal static int PrintUsage()
{
System.Console.Error.WriteLine("Usage: [-m <maps>] [-r <reduces>]\n" + " [-keepmap <percent>] [-keepred <percent>]\n"
+ " [-indir <path>] [-outdir <path]\n" + " [-inFormat[Indirect] <InputFormat>] [-outFormat <OutputFormat>]\n"
+ " [-outKey <WritableComparable>] [-outValue <Writable>]\n");
GenericOptionsParser.PrintGenericCommandUsage(System.Console.Error);
return(-1);
}
/// <summary>Command-line interface</summary>
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
Configuration conf = new HdfsConfiguration();
Options fetcherOptions = new Options();
fetcherOptions.AddOption(Webservice, true, "HTTP url to reach the NameNode at");
fetcherOptions.AddOption(Renewer, true, "Name of the delegation token renewer");
fetcherOptions.AddOption(Cancel, false, "cancel the token");
fetcherOptions.AddOption(Renew, false, "renew the token");
fetcherOptions.AddOption(Print, false, "print the token");
fetcherOptions.AddOption(HelpShort, Help, false, "print out help information");
GenericOptionsParser parser = new GenericOptionsParser(conf, fetcherOptions, args
);
CommandLine cmd = parser.GetCommandLine();
// get options
string webUrl = cmd.HasOption(Webservice) ? cmd.GetOptionValue(Webservice) : null;
string renewer = cmd.HasOption(Renewer) ? cmd.GetOptionValue(Renewer) : null;
bool cancel = cmd.HasOption(Cancel);
bool renew = cmd.HasOption(Renew);
bool print = cmd.HasOption(Print);
bool help = cmd.HasOption(Help);
string[] remaining = parser.GetRemainingArgs();
// check option validity
if (help)
{
PrintUsage(System.Console.Out);
System.Environment.Exit(0);
}
if (cancel && renew || cancel && print || renew && print || cancel && renew && print)
{
System.Console.Error.WriteLine("ERROR: Only specify cancel, renew or print.");
PrintUsage(System.Console.Error);
}
if (remaining.Length != 1 || remaining[0][0] == '-')
{
System.Console.Error.WriteLine("ERROR: Must specify exacltly one token file");
PrintUsage(System.Console.Error);
}
// default to using the local file system
FileSystem local = FileSystem.GetLocal(conf);
Path tokenFile = new Path(local.GetWorkingDirectory(), remaining[0]);
URLConnectionFactory connectionFactory = URLConnectionFactory.DefaultSystemConnectionFactory;
// Login the current user
UserGroupInformation.GetCurrentUser().DoAs(new _PrivilegedExceptionAction_152(print
, tokenFile, conf, renew, cancel, webUrl, connectionFactory, renewer));
}
private static void PrintUsage(TextWriter err)
{
err.WriteLine("fetchdt retrieves delegation tokens from the NameNode");
err.WriteLine();
err.WriteLine("fetchdt <opts> <token file>");
err.WriteLine("Options:");
err.WriteLine(" --webservice <url> Url to contact NN on");
err.WriteLine(" --renewer <name> Name of the delegation token renewer");
err.WriteLine(" --cancel Cancel the delegation token");
err.WriteLine(" --renew Renew the delegation token. Delegation " +
"token must have been fetched using the --renewer <name> option.");
err.WriteLine(" --print Print the delegation token");
err.WriteLine();
GenericOptionsParser.PrintGenericCommandUsage(err);
ExitUtil.Terminate(1);
}
internal virtual void PrintUsage()
{
// The CLI package should do this for us, but I can't figure out how
// to make it print something reasonable.
System.Console.Out.WriteLine("bin/hadoop pipes");
System.Console.Out.WriteLine(" [-input <path>] // Input directory");
System.Console.Out.WriteLine(" [-output <path>] // Output directory");
System.Console.Out.WriteLine(" [-jar <jar file> // jar filename");
System.Console.Out.WriteLine(" [-inputformat <class>] // InputFormat class");
System.Console.Out.WriteLine(" [-map <class>] // Java Map class");
System.Console.Out.WriteLine(" [-partitioner <class>] // Java Partitioner");
System.Console.Out.WriteLine(" [-reduce <class>] // Java Reduce class");
System.Console.Out.WriteLine(" [-writer <class>] // Java RecordWriter");
System.Console.Out.WriteLine(" [-program <executable>] // executable URI");
System.Console.Out.WriteLine(" [-reduces <num>] // number of reduces");
System.Console.Out.WriteLine(" [-lazyOutput <true/false>] // createOutputLazily"
);
System.Console.Out.WriteLine();
GenericOptionsParser.PrintGenericCommandUsage(System.Console.Out);
}
/// <exception cref="System.Exception"/>
public virtual int Run(string[] args)
{
Configuration conf = GetConf();
if ("local".Equals(conf.Get(JTConfig.JtIpcAddress, "local")))
{
DisplayUsage();
}
string[] otherArgs = new GenericOptionsParser(conf, args).GetRemainingArgs();
if (otherArgs.Length == 2)
{
if (otherArgs[0].Equals("-scratchdir"))
{
dir = otherArgs[1];
}
else
{
DisplayUsage();
}
}
else
{
if (otherArgs.Length == 0)
{
dir = Runtime.GetProperty("user.dir");
}
else
{
DisplayUsage();
}
}
//to protect against the case of jobs failing even when multiple attempts
//fail, set some high values for the max attempts
conf.SetInt(JobContext.MapMaxAttempts, 10);
conf.SetInt(JobContext.ReduceMaxAttempts, 10);
RunSleepJobTest(new JobClient(new JobConf(conf)), conf);
RunSortJobTests(new JobClient(new JobConf(conf)), conf);
return(0);
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
if (DFSUtil.ParseHelpArgument(args, ZKFailoverController.Usage, System.Console.Out
, true))
{
System.Environment.Exit(0);
}
GenericOptionsParser parser = new GenericOptionsParser(new HdfsConfiguration(), args
);
Org.Apache.Hadoop.Hdfs.Tools.DFSZKFailoverController zkfc = Org.Apache.Hadoop.Hdfs.Tools.DFSZKFailoverController
.Create(parser.GetConfiguration());
int retCode = 0;
try
{
retCode = zkfc.Run(parser.GetRemainingArgs());
}
catch (Exception t)
{
Log.Fatal("Got a fatal error, exiting now", t);
}
System.Environment.Exit(retCode);
}
/// <summary>Create an Aggregate based map/reduce job.</summary>
/// <param name="args">
/// the arguments used for job creation. Generic hadoop
/// arguments are accepted.
/// </param>
/// <param name="caller">the the caller class.</param>
/// <returns>a JobConf object ready for submission.</returns>
/// <exception cref="System.IO.IOException"/>
/// <seealso cref="Org.Apache.Hadoop.Util.GenericOptionsParser"/>
public static JobConf CreateValueAggregatorJob(string[] args, Type caller)
{
Configuration conf = new Configuration();
GenericOptionsParser genericParser = new GenericOptionsParser(conf, args);
args = genericParser.GetRemainingArgs();
if (args.Length < 2)
{
System.Console.Out.WriteLine("usage: inputDirs outDir " + "[numOfReducer [textinputformat|seq [specfile [jobName]]]]"
);
GenericOptionsParser.PrintGenericCommandUsage(System.Console.Out);
System.Environment.Exit(1);
}
string inputDir = args[0];
string outputDir = args[1];
int numOfReducers = 1;
if (args.Length > 2)
{
numOfReducers = System.Convert.ToInt32(args[2]);
}
Type theInputFormat = typeof(TextInputFormat);
if (args.Length > 3 && args[3].CompareToIgnoreCase("textinputformat") == 0)
{
theInputFormat = typeof(TextInputFormat);
}
else
{
theInputFormat = typeof(SequenceFileInputFormat);
}
Path specFile = null;
if (args.Length > 4)
{
specFile = new Path(args[4]);
}
string jobName = string.Empty;
if (args.Length > 5)
{
jobName = args[5];
}
JobConf theJob = new JobConf(conf);
if (specFile != null)
{
theJob.AddResource(specFile);
}
string userJarFile = theJob.Get("user.jar.file");
if (userJarFile == null)
{
theJob.SetJarByClass(caller != null ? caller : typeof(ValueAggregatorJob));
}
else
{
theJob.SetJar(userJarFile);
}
theJob.SetJobName("ValueAggregatorJob: " + jobName);
FileInputFormat.AddInputPaths(theJob, inputDir);
theJob.SetInputFormat(theInputFormat);
theJob.SetMapperClass(typeof(ValueAggregatorMapper));
FileOutputFormat.SetOutputPath(theJob, new Path(outputDir));
theJob.SetOutputFormat(typeof(TextOutputFormat));
theJob.SetMapOutputKeyClass(typeof(Text));
theJob.SetMapOutputValueClass(typeof(Text));
theJob.SetOutputKeyClass(typeof(Text));
theJob.SetOutputValueClass(typeof(Text));
theJob.SetReducerClass(typeof(ValueAggregatorReducer));
theJob.SetCombinerClass(typeof(ValueAggregatorCombiner));
theJob.SetNumMapTasks(1);
theJob.SetNumReduceTasks(numOfReducers);
return(theJob);
}
/// <exception cref="System.Exception"/>
public virtual int Run(string[] args)
{
Submitter.CommandLineParser cli = new Submitter.CommandLineParser();
if (args.Length == 0)
{
cli.PrintUsage();
return(1);
}
cli.AddOption("input", false, "input path to the maps", "path");
cli.AddOption("output", false, "output path from the reduces", "path");
cli.AddOption("jar", false, "job jar file", "path");
cli.AddOption("inputformat", false, "java classname of InputFormat", "class");
//cli.addArgument("javareader", false, "is the RecordReader in Java");
cli.AddOption("map", false, "java classname of Mapper", "class");
cli.AddOption("partitioner", false, "java classname of Partitioner", "class");
cli.AddOption("reduce", false, "java classname of Reducer", "class");
cli.AddOption("writer", false, "java classname of OutputFormat", "class");
cli.AddOption("program", false, "URI to application executable", "class");
cli.AddOption("reduces", false, "number of reduces", "num");
cli.AddOption("jobconf", false, "\"n1=v1,n2=v2,..\" (Deprecated) Optional. Add or override a JobConf property."
, "key=val");
cli.AddOption("lazyOutput", false, "Optional. Create output lazily", "boolean");
Parser parser = cli.CreateParser();
try
{
GenericOptionsParser genericParser = new GenericOptionsParser(GetConf(), args);
CommandLine results = parser.Parse(cli.options, genericParser.GetRemainingArgs());
JobConf job = new JobConf(GetConf());
if (results.HasOption("input"))
{
FileInputFormat.SetInputPaths(job, results.GetOptionValue("input"));
}
if (results.HasOption("output"))
{
FileOutputFormat.SetOutputPath(job, new Path(results.GetOptionValue("output")));
}
if (results.HasOption("jar"))
{
job.SetJar(results.GetOptionValue("jar"));
}
if (results.HasOption("inputformat"))
{
SetIsJavaRecordReader(job, true);
job.SetInputFormat(GetClass <InputFormat>(results, "inputformat", job));
}
if (results.HasOption("javareader"))
{
SetIsJavaRecordReader(job, true);
}
if (results.HasOption("map"))
{
SetIsJavaMapper(job, true);
job.SetMapperClass(GetClass <Mapper>(results, "map", job));
}
if (results.HasOption("partitioner"))
{
job.SetPartitionerClass(GetClass <Partitioner>(results, "partitioner", job));
}
if (results.HasOption("reduce"))
{
SetIsJavaReducer(job, true);
job.SetReducerClass(GetClass <Reducer>(results, "reduce", job));
}
if (results.HasOption("reduces"))
{
job.SetNumReduceTasks(System.Convert.ToInt32(results.GetOptionValue("reduces")));
}
if (results.HasOption("writer"))
{
SetIsJavaRecordWriter(job, true);
job.SetOutputFormat(GetClass <OutputFormat>(results, "writer", job));
}
if (results.HasOption("lazyOutput"))
{
if (System.Boolean.Parse(results.GetOptionValue("lazyOutput")))
{
LazyOutputFormat.SetOutputFormatClass(job, job.GetOutputFormat().GetType());
}
}
if (results.HasOption("program"))
{
SetExecutable(job, results.GetOptionValue("program"));
}
if (results.HasOption("jobconf"))
{
Log.Warn("-jobconf option is deprecated, please use -D instead.");
string options = results.GetOptionValue("jobconf");
StringTokenizer tokenizer = new StringTokenizer(options, ",");
while (tokenizer.HasMoreTokens())
{
string keyVal = tokenizer.NextToken().Trim();
string[] keyValSplit = keyVal.Split("=");
job.Set(keyValSplit[0], keyValSplit[1]);
}
}
// if they gave us a jar file, include it into the class path
string jarFile = job.GetJar();
if (jarFile != null)
{
Uri[] urls = new Uri[] { FileSystem.GetLocal(job).PathToFile(new Path(jarFile)).ToURL
() };
//FindBugs complains that creating a URLClassLoader should be
//in a doPrivileged() block.
ClassLoader loader = AccessController.DoPrivileged(new _PrivilegedAction_494(urls
));
job.SetClassLoader(loader);
}
RunJob(job);
return(0);
}
catch (ParseException pe)
{
Log.Info("Error : " + pe);
cli.PrintUsage();
return(1);
}
}
