/// <exception cref="System.IO.IOException"/>
public virtual JvmTask GetTask(JvmContext context)
{
// A rough imitation of code from TaskTracker.
JVMId jvmId = context.jvmId;
Log.Info("JVM with ID : " + jvmId + " asked for a task");
JvmTask jvmTask = null;
// TODO: Is it an authorized container to get a task? Otherwise return null.
// TODO: Child.java's firstTaskID isn't really firstTaskID. Ask for update
// to jobId and task-type.
WrappedJvmID wJvmID = new WrappedJvmID(jvmId.GetJobId(), jvmId.isMap, jvmId.GetId
());
// Try to look up the task. We remove it directly as we don't give
// multiple tasks to a JVM
if (!jvmIDToActiveAttemptMap.Contains(wJvmID))
{
Log.Info("JVM with ID: " + jvmId + " is invalid and will be killed.");
jvmTask = TaskForInvalidJvm;
}
else
{
if (!launchedJVMs.Contains(wJvmID))
{
jvmTask = null;
Log.Info("JVM with ID: " + jvmId + " asking for task before AM launch registered. Given null task"
);
}
else
{
// remove the task as it is no more needed and free up the memory.
// Also we have already told the JVM to process a task, so it is no
// longer pending, and further request should ask it to exit.
Task task = Sharpen.Collections.Remove(jvmIDToActiveAttemptMap, wJvmID);
launchedJVMs.Remove(wJvmID);
Log.Info("JVM with ID: " + jvmId + " given task: " + task.GetTaskID());
task.SetEncryptedSpillKey(encryptedSpillKey);
jvmTask = new JvmTask(task, false);
}
}
return(jvmTask);
}
/// <exception cref="System.IO.IOException"/>
public override JvmTask GetTask(JvmContext context)
{
return(null);
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
Log.Debug("Child starting");
JobConf job = new JobConf(MRJobConfig.JobConfFile);
// Initing with our JobConf allows us to avoid loading confs twice
Limits.Init(job);
UserGroupInformation.SetConfiguration(job);
string host = args[0];
int port = System.Convert.ToInt32(args[1]);
IPEndPoint address = NetUtils.CreateSocketAddrForHost(host, port);
TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2]));
long jvmIdLong = long.Parse(args[3]);
JVMId jvmId = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType(
) == TaskType.Map, jvmIdLong);
// initialize metrics
DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString
()) + "Task");
// Security framework already loaded the tokens into current ugi
Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials();
Log.Info("Executing with tokens:");
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens
())
{
Log.Info(token);
}
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid
.GetJobID()).ToString());
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken
(credentials);
SecurityUtil.SetTokenService(jt, address);
taskOwner.AddToken(jt);
TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108
(address, job));
// report non-pid to application master
JvmContext context = new JvmContext(jvmId, "-1000");
Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]);
Task task = null;
UserGroupInformation childUGI = null;
ScheduledExecutorService logSyncer = null;
try
{
int idleLoopCount = 0;
JvmTask myTask = null;
// poll for new task
for (int idle = 0; null == myTask; ++idle)
{
long sleepTimeMilliSecs = Math.Min(idle * 500, 1500);
Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now."
);
TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs);
myTask = umbilical.GetTask(context);
}
if (myTask.ShouldDie())
{
return;
}
task = myTask.GetTask();
YarnChild.taskid = task.GetTaskID();
// Create the job-conf and set credentials
ConfigureTask(job, task, credentials, jt);
// Initiate Java VM metrics
JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId());
childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment
.User.ToString()));
// Add tokens to new user so that it may execute its task correctly.
childUGI.AddCredentials(credentials);
// set job classloader if configured before invoking the task
MRApps.SetJobClassLoader(job);
logSyncer = TaskLog.CreateLogSyncer();
// Create a final reference to the task for the doAs block
Task taskFinal = task;
childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical));
}
catch (FSError e)
{
// use job-specified working directory
// run the task
Log.Fatal("FSError from child", e);
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
umbilical.FsError(taskid, e.Message);
}
}
catch (Exception exception)
{
Log.Warn("Exception running child : " + StringUtils.StringifyException(exception)
);
try
{
if (task != null)
{
// do cleanup for the task
if (childUGI == null)
{
// no need to job into doAs block
task.TaskCleanup(umbilical);
}
else
{
Task taskFinal = task;
childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical));
}
}
}
catch (Exception e)
{
Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e));
}
// Report back any failures, for diagnostic purposes
if (taskid != null)
{
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
umbilical.FatalError(taskid, StringUtils.StringifyException(exception));
}
}
}
catch (Exception throwable)
{
Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable));
if (taskid != null)
{
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
Exception tCause = throwable.InnerException;
string cause = tCause == null ? throwable.Message : StringUtils.StringifyException
(tCause);
umbilical.FatalError(taskid, cause);
}
}
}
finally
{
RPC.StopProxy(umbilical);
DefaultMetricsSystem.Shutdown();
TaskLog.SyncLogsShutdown(logSyncer);
}
}
/// <summary>Called when a child task process starts, to get its task.</summary> /// <param name="context"> /// the JvmContext of the JVM w.r.t the TaskTracker that /// launched it /// </param> /// <returns>Task object</returns> /// <exception cref="System.IO.IOException"></exception> public abstract JvmTask GetTask(JvmContext context);
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetTask()
{
AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>();
JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>();
RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler
>();
TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>();
TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl listener = new TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl
(appCtx, secret, rmHeartbeatHandler, hbHandler);
Configuration conf = new Configuration();
listener.Init(conf);
listener.Start();
JVMId id = new JVMId("foo", 1, true, 1);
WrappedJvmID wid = new WrappedJvmID(id.GetJobId(), id.isMap, id.GetId());
// Verify ask before registration.
//The JVM ID has not been registered yet so we should kill it.
JvmContext context = new JvmContext();
context.jvmId = id;
JvmTask result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.shouldDie);
// Verify ask after registration but before launch.
// Don't kill, should be null.
TaskAttemptId attemptID = Org.Mockito.Mockito.Mock <TaskAttemptId>();
Task task = Org.Mockito.Mockito.Mock <Task>();
//Now put a task with the ID
listener.RegisterPendingTask(task, wid);
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNull(result);
// Unregister for more testing.
listener.Unregister(attemptID, wid);
// Verify ask after registration and launch
//Now put a task with the ID
listener.RegisterPendingTask(task, wid);
listener.RegisterLaunchedTask(attemptID, wid);
Org.Mockito.Mockito.Verify(hbHandler).Register(attemptID);
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsFalse(result.shouldDie);
// Don't unregister yet for more testing.
//Verify that if we call it again a second time we are told to die.
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.shouldDie);
listener.Unregister(attemptID, wid);
// Verify after unregistration.
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.shouldDie);
listener.Stop();
// test JVMID
JVMId jvmid = JVMId.ForName("jvm_001_002_m_004");
NUnit.Framework.Assert.IsNotNull(jvmid);
try
{
JVMId.ForName("jvm_001_002_m_004_006");
Assert.Fail();
}
catch (ArgumentException e)
{
NUnit.Framework.Assert.AreEqual(e.Message, "TaskId string : jvm_001_002_m_004_006 is not properly formed"
);
}
}