public TaskAttemptContextImpl(Configuration conf, TaskAttemptID taskId, StatusReporter
reporter)
: base(conf, taskId.GetJobID())
{
this.taskId = taskId;
this.reporter = reporter;
}
internal ShuffleClientMetrics(TaskAttemptID reduceId, JobConf jobConf)
{
this.numCopiers = jobConf.GetInt(MRJobConfig.ShuffleParallelCopies, 5);
MetricsContext metricsContext = MetricsUtil.GetContext("mapred");
this.shuffleMetrics = MetricsUtil.CreateRecord(metricsContext, "shuffleInput");
this.shuffleMetrics.SetTag("user", jobConf.GetUser());
this.shuffleMetrics.SetTag("jobName", jobConf.GetJobName());
this.shuffleMetrics.SetTag("jobId", reduceId.GetJobID().ToString());
this.shuffleMetrics.SetTag("taskId", reduceId.ToString());
this.shuffleMetrics.SetTag("sessionId", jobConf.GetSessionId());
metricsContext.RegisterUpdater(this);
}
/// <exception cref="System.Exception"/>
public virtual void TestCommitter()
{
Job job = Job.GetInstance();
FileOutputFormat.SetOutputPath(job, outDir);
Configuration conf = job.GetConfiguration();
conf.Set(MRJobConfig.TaskAttemptId, attempt);
JobContext jContext = new JobContextImpl(conf, taskID.GetJobID());
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);
// setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(tContext);
WriteOutput(theRecordWriter, tContext);
// do commit
committer.CommitTask(tContext);
committer.CommitJob(jContext);
// validate output
FilePath expectedFile = new FilePath(new Path(outDir, partFile).ToString());
StringBuilder expectedOutput = new StringBuilder();
expectedOutput.Append(key1).Append('\t').Append(val1).Append("\n");
expectedOutput.Append(val1).Append("\n");
expectedOutput.Append(val2).Append("\n");
expectedOutput.Append(key2).Append("\n");
expectedOutput.Append(key1).Append("\n");
expectedOutput.Append(key2).Append('\t').Append(val2).Append("\n");
string output = UtilsForTests.Slurp(expectedFile);
NUnit.Framework.Assert.AreEqual(output, expectedOutput.ToString());
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
internal static URI GetBaseURI(TaskAttemptID reduceId, string url)
{
StringBuilder baseUrl = new StringBuilder(url);
if (!url.EndsWith("/"))
{
baseUrl.Append("/");
}
baseUrl.Append("mapOutput?job=");
baseUrl.Append(reduceId.GetJobID());
baseUrl.Append("&reduce=");
baseUrl.Append(reduceId.GetTaskID().GetId());
baseUrl.Append("&map=");
URI u = URI.Create(baseUrl.ToString());
return(u);
}
/// <summary>
/// Queries the
/// <see cref="TaskTracker"/>
/// for a set of map-completion events
/// from a given event ID.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal virtual int GetMapCompletionEvents()
{
int numNewMaps = 0;
TaskCompletionEvent[] events = null;
do
{
MapTaskCompletionEventsUpdate update = umbilical.GetMapCompletionEvents((JobID)reduce
.GetJobID(), fromEventIdx, maxEventsToFetch, (TaskAttemptID)reduce);
events = update.GetMapTaskCompletionEvents();
Log.Debug("Got " + events.Length + " map completion events from " + fromEventIdx);
System.Diagnostics.Debug.Assert(!update.ShouldReset(), "Unexpected legacy state");
// Update the last seen event ID
fromEventIdx += events.Length;
// Process the TaskCompletionEvents:
// 1. Save the SUCCEEDED maps in knownOutputs to fetch the outputs.
// 2. Save the OBSOLETE/FAILED/KILLED maps in obsoleteOutputs to stop
// fetching from those maps.
// 3. Remove TIPFAILED maps from neededOutputs since we don't need their
// outputs at all.
foreach (TaskCompletionEvent @event in events)
{
scheduler.Resolve(@event);
if (TaskCompletionEvent.Status.Succeeded == @event.GetTaskStatus())
{
++numNewMaps;
}
}
}while (events.Length == maxEventsToFetch);
return(numNewMaps);
}
/// <exception cref="System.Exception"/>
public virtual int Run(string[] argv)
{
int exitCode = -1;
if (argv.Length < 1)
{
DisplayUsage(string.Empty);
return(exitCode);
}
// process arguments
string cmd = argv[0];
string submitJobFile = null;
string jobid = null;
string taskid = null;
string historyFile = null;
string counterGroupName = null;
string counterName = null;
JobPriority jp = null;
string taskType = null;
string taskState = null;
int fromEvent = 0;
int nEvents = 0;
bool getStatus = false;
bool getCounter = false;
bool killJob = false;
bool listEvents = false;
bool viewHistory = false;
bool viewAllHistory = false;
bool listJobs = false;
bool listAllJobs = false;
bool listActiveTrackers = false;
bool listBlacklistedTrackers = false;
bool displayTasks = false;
bool killTask = false;
bool failTask = false;
bool setJobPriority = false;
bool logs = false;
if ("-submit".Equals(cmd))
{
if (argv.Length != 2)
{
DisplayUsage(cmd);
return(exitCode);
}
submitJobFile = argv[1];
}
else
{
if ("-status".Equals(cmd))
{
if (argv.Length != 2)
{
DisplayUsage(cmd);
return(exitCode);
}
jobid = argv[1];
getStatus = true;
}
else
{
if ("-counter".Equals(cmd))
{
if (argv.Length != 4)
{
DisplayUsage(cmd);
return(exitCode);
}
getCounter = true;
jobid = argv[1];
counterGroupName = argv[2];
counterName = argv[3];
}
else
{
if ("-kill".Equals(cmd))
{
if (argv.Length != 2)
{
DisplayUsage(cmd);
return(exitCode);
}
jobid = argv[1];
killJob = true;
}
else
{
if ("-set-priority".Equals(cmd))
{
if (argv.Length != 3)
{
DisplayUsage(cmd);
return(exitCode);
}
jobid = argv[1];
try
{
jp = JobPriority.ValueOf(argv[2]);
}
catch (ArgumentException iae)
{
Log.Info(iae);
DisplayUsage(cmd);
return(exitCode);
}
setJobPriority = true;
}
else
{
if ("-events".Equals(cmd))
{
if (argv.Length != 4)
{
DisplayUsage(cmd);
return(exitCode);
}
jobid = argv[1];
fromEvent = System.Convert.ToInt32(argv[2]);
nEvents = System.Convert.ToInt32(argv[3]);
listEvents = true;
}
else
{
if ("-history".Equals(cmd))
{
if (argv.Length != 2 && !(argv.Length == 3 && "all".Equals(argv[1])))
{
DisplayUsage(cmd);
return(exitCode);
}
viewHistory = true;
if (argv.Length == 3 && "all".Equals(argv[1]))
{
viewAllHistory = true;
historyFile = argv[2];
}
else
{
historyFile = argv[1];
}
}
else
{
if ("-list".Equals(cmd))
{
if (argv.Length != 1 && !(argv.Length == 2 && "all".Equals(argv[1])))
{
DisplayUsage(cmd);
return(exitCode);
}
if (argv.Length == 2 && "all".Equals(argv[1]))
{
listAllJobs = true;
}
else
{
listJobs = true;
}
}
else
{
if ("-kill-task".Equals(cmd))
{
if (argv.Length != 2)
{
DisplayUsage(cmd);
return(exitCode);
}
killTask = true;
taskid = argv[1];
}
else
{
if ("-fail-task".Equals(cmd))
{
if (argv.Length != 2)
{
DisplayUsage(cmd);
return(exitCode);
}
failTask = true;
taskid = argv[1];
}
else
{
if ("-list-active-trackers".Equals(cmd))
{
if (argv.Length != 1)
{
DisplayUsage(cmd);
return(exitCode);
}
listActiveTrackers = true;
}
else
{
if ("-list-blacklisted-trackers".Equals(cmd))
{
if (argv.Length != 1)
{
DisplayUsage(cmd);
return(exitCode);
}
listBlacklistedTrackers = true;
}
else
{
if ("-list-attempt-ids".Equals(cmd))
{
if (argv.Length != 4)
{
DisplayUsage(cmd);
return(exitCode);
}
jobid = argv[1];
taskType = argv[2];
taskState = argv[3];
displayTasks = true;
if (!taskTypes.Contains(StringUtils.ToUpperCase(taskType)))
{
System.Console.Out.WriteLine("Error: Invalid task-type: " + taskType);
DisplayUsage(cmd);
return(exitCode);
}
if (!taskStates.Contains(StringUtils.ToLowerCase(taskState)))
{
System.Console.Out.WriteLine("Error: Invalid task-state: " + taskState);
DisplayUsage(cmd);
return(exitCode);
}
}
else
{
if ("-logs".Equals(cmd))
{
if (argv.Length == 2 || argv.Length == 3)
{
logs = true;
jobid = argv[1];
if (argv.Length == 3)
{
taskid = argv[2];
}
else
{
taskid = null;
}
}
else
{
DisplayUsage(cmd);
return(exitCode);
}
}
else
{
DisplayUsage(cmd);
return(exitCode);
}
}
}
}
}
}
}
}
}
}
}
}
}
}
// initialize cluster
cluster = CreateCluster();
// Submit the request
try
{
if (submitJobFile != null)
{
Job job = Job.GetInstance(new JobConf(submitJobFile));
job.Submit();
System.Console.Out.WriteLine("Created job " + job.GetJobID());
exitCode = 0;
}
else
{
if (getStatus)
{
Job job = cluster.GetJob(JobID.ForName(jobid));
if (job == null)
{
System.Console.Out.WriteLine("Could not find job " + jobid);
}
else
{
Counters counters = job.GetCounters();
System.Console.Out.WriteLine();
System.Console.Out.WriteLine(job);
if (counters != null)
{
System.Console.Out.WriteLine(counters);
}
else
{
System.Console.Out.WriteLine("Counters not available. Job is retired.");
}
exitCode = 0;
}
}
else
{
if (getCounter)
{
Job job = cluster.GetJob(JobID.ForName(jobid));
if (job == null)
{
System.Console.Out.WriteLine("Could not find job " + jobid);
}
else
{
Counters counters = job.GetCounters();
if (counters == null)
{
System.Console.Out.WriteLine("Counters not available for retired job " + jobid);
exitCode = -1;
}
else
{
System.Console.Out.WriteLine(GetCounter(counters, counterGroupName, counterName));
exitCode = 0;
}
}
}
else
{
if (killJob)
{
Job job = cluster.GetJob(JobID.ForName(jobid));
if (job == null)
{
System.Console.Out.WriteLine("Could not find job " + jobid);
}
else
{
JobStatus jobStatus = job.GetStatus();
if (jobStatus.GetState() == JobStatus.State.Failed)
{
System.Console.Out.WriteLine("Could not mark the job " + jobid + " as killed, as it has already failed."
);
exitCode = -1;
}
else
{
if (jobStatus.GetState() == JobStatus.State.Killed)
{
System.Console.Out.WriteLine("The job " + jobid + " has already been killed.");
exitCode = -1;
}
else
{
if (jobStatus.GetState() == JobStatus.State.Succeeded)
{
System.Console.Out.WriteLine("Could not kill the job " + jobid + ", as it has already succeeded."
);
exitCode = -1;
}
else
{
job.KillJob();
System.Console.Out.WriteLine("Killed job " + jobid);
exitCode = 0;
}
}
}
}
}
else
{
if (setJobPriority)
{
Job job = cluster.GetJob(JobID.ForName(jobid));
if (job == null)
{
System.Console.Out.WriteLine("Could not find job " + jobid);
}
else
{
job.SetPriority(jp);
System.Console.Out.WriteLine("Changed job priority.");
exitCode = 0;
}
}
else
{
if (viewHistory)
{
ViewHistory(historyFile, viewAllHistory);
exitCode = 0;
}
else
{
if (listEvents)
{
ListEvents(cluster.GetJob(JobID.ForName(jobid)), fromEvent, nEvents);
exitCode = 0;
}
else
{
if (listJobs)
{
ListJobs(cluster);
exitCode = 0;
}
else
{
if (listAllJobs)
{
ListAllJobs(cluster);
exitCode = 0;
}
else
{
if (listActiveTrackers)
{
ListActiveTrackers(cluster);
exitCode = 0;
}
else
{
if (listBlacklistedTrackers)
{
ListBlacklistedTrackers(cluster);
exitCode = 0;
}
else
{
if (displayTasks)
{
DisplayTasks(cluster.GetJob(JobID.ForName(jobid)), taskType, taskState);
exitCode = 0;
}
else
{
if (killTask)
{
TaskAttemptID taskID = TaskAttemptID.ForName(taskid);
Job job = cluster.GetJob(taskID.GetJobID());
if (job == null)
{
System.Console.Out.WriteLine("Could not find job " + jobid);
}
else
{
if (job.KillTask(taskID, false))
{
System.Console.Out.WriteLine("Killed task " + taskid);
exitCode = 0;
}
else
{
System.Console.Out.WriteLine("Could not kill task " + taskid);
exitCode = -1;
}
}
}
else
{
if (failTask)
{
TaskAttemptID taskID = TaskAttemptID.ForName(taskid);
Job job = cluster.GetJob(taskID.GetJobID());
if (job == null)
{
System.Console.Out.WriteLine("Could not find job " + jobid);
}
else
{
if (job.KillTask(taskID, true))
{
System.Console.Out.WriteLine("Killed task " + taskID + " by failing it");
exitCode = 0;
}
else
{
System.Console.Out.WriteLine("Could not fail task " + taskid);
exitCode = -1;
}
}
}
else
{
if (logs)
{
try
{
JobID jobID = JobID.ForName(jobid);
TaskAttemptID taskAttemptID = TaskAttemptID.ForName(taskid);
LogParams logParams = cluster.GetLogParams(jobID, taskAttemptID);
LogCLIHelpers logDumper = new LogCLIHelpers();
logDumper.SetConf(GetConf());
exitCode = logDumper.DumpAContainersLogs(logParams.GetApplicationId(), logParams.
GetContainerId(), logParams.GetNodeId(), logParams.GetOwner());
}
catch (IOException e)
{
if (e is RemoteException)
{
throw;
}
System.Console.Out.WriteLine(e.Message);
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
catch (RemoteException re)
{
IOException unwrappedException = re.UnwrapRemoteException();
if (unwrappedException is AccessControlException)
{
System.Console.Out.WriteLine(unwrappedException.Message);
}
else
{
throw;
}
}
finally
{
cluster.Close();
}
return(exitCode);
}
/// <exception cref="System.Exception"/>
private void TestRecoveryInternal(int commitVersion, int recoveryVersion)
{
Job job = Job.GetInstance();
FileOutputFormat.SetOutputPath(job, outDir);
Configuration conf = job.GetConfiguration();
conf.Set(MRJobConfig.TaskAttemptId, attempt);
conf.SetInt(MRJobConfig.ApplicationAttemptId, 1);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, commitVersion
);
JobContext jContext = new JobContextImpl(conf, taskID.GetJobID());
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);
// setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
// write output
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(tContext);
WriteOutput(theRecordWriter, tContext);
// do commit
committer.CommitTask(tContext);
Path jobTempDir1 = committer.GetCommittedTaskPath(tContext);
FilePath jtd = new FilePath(jobTempDir1.ToUri().GetPath());
if (commitVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 1 commits to temporary dir " + jtd, jtd.Exists
());
ValidateContent(jtd);
}
else
{
NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd, jtd.Exists
());
}
//now while running the second app attempt,
//recover the task output from first attempt
Configuration conf2 = job.GetConfiguration();
conf2.Set(MRJobConfig.TaskAttemptId, attempt);
conf2.SetInt(MRJobConfig.ApplicationAttemptId, 2);
conf2.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, recoveryVersion
);
JobContext jContext2 = new JobContextImpl(conf2, taskID.GetJobID());
TaskAttemptContext tContext2 = new TaskAttemptContextImpl(conf2, taskID);
FileOutputCommitter committer2 = new FileOutputCommitter(outDir, tContext2);
committer2.SetupJob(tContext2);
Path jobTempDir2 = committer2.GetCommittedTaskPath(tContext2);
FilePath jtd2 = new FilePath(jobTempDir2.ToUri().GetPath());
committer2.RecoverTask(tContext2);
if (recoveryVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 1 recovers to " + jtd2, jtd2.Exists());
ValidateContent(jtd2);
}
else
{
NUnit.Framework.Assert.IsFalse("Version 2 commits to output dir " + jtd2, jtd2.Exists
());
if (commitVersion == 1)
{
NUnit.Framework.Assert.IsTrue("Version 2 recovery moves to output dir from " + jtd
, jtd.List().Length == 0);
}
}
committer2.CommitJob(jContext2);
ValidateContent(outDir);
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}