/// <exception cref="System.IO.IOException"/>
public virtual JvmTask GetTask(JvmContext context)
{
// A rough imitation of code from TaskTracker.
JVMId jvmId = context.jvmId;
Log.Info("JVM with ID : " + jvmId + " asked for a task");
JvmTask jvmTask = null;
// TODO: Is it an authorized container to get a task? Otherwise return null.
// TODO: Child.java's firstTaskID isn't really firstTaskID. Ask for update
// to jobId and task-type.
WrappedJvmID wJvmID = new WrappedJvmID(jvmId.GetJobId(), jvmId.isMap, jvmId.GetId
());
// Try to look up the task. We remove it directly as we don't give
// multiple tasks to a JVM
if (!jvmIDToActiveAttemptMap.Contains(wJvmID))
{
Log.Info("JVM with ID: " + jvmId + " is invalid and will be killed.");
jvmTask = TaskForInvalidJvm;
}
else
{
if (!launchedJVMs.Contains(wJvmID))
{
jvmTask = null;
Log.Info("JVM with ID: " + jvmId + " asking for task before AM launch registered. Given null task"
);
}
else
{
// remove the task as it is no more needed and free up the memory.
// Also we have already told the JVM to process a task, so it is no
// longer pending, and further request should ask it to exit.
Task task = Sharpen.Collections.Remove(jvmIDToActiveAttemptMap, wJvmID);
launchedJVMs.Remove(wJvmID);
Log.Info("JVM with ID: " + jvmId + " given task: " + task.GetTaskID());
task.SetEncryptedSpillKey(encryptedSpillKey);
jvmTask = new JvmTask(task, false);
}
}
return(jvmTask);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetTask()
{
AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>();
JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>();
RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler
>();
TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>();
TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl listener = new TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl
(appCtx, secret, rmHeartbeatHandler, hbHandler);
Configuration conf = new Configuration();
listener.Init(conf);
listener.Start();
JVMId id = new JVMId("foo", 1, true, 1);
WrappedJvmID wid = new WrappedJvmID(id.GetJobId(), id.isMap, id.GetId());
// Verify ask before registration.
//The JVM ID has not been registered yet so we should kill it.
JvmContext context = new JvmContext();
context.jvmId = id;
JvmTask result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.shouldDie);
// Verify ask after registration but before launch.
// Don't kill, should be null.
TaskAttemptId attemptID = Org.Mockito.Mockito.Mock <TaskAttemptId>();
Task task = Org.Mockito.Mockito.Mock <Task>();
//Now put a task with the ID
listener.RegisterPendingTask(task, wid);
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNull(result);
// Unregister for more testing.
listener.Unregister(attemptID, wid);
// Verify ask after registration and launch
//Now put a task with the ID
listener.RegisterPendingTask(task, wid);
listener.RegisterLaunchedTask(attemptID, wid);
Org.Mockito.Mockito.Verify(hbHandler).Register(attemptID);
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsFalse(result.shouldDie);
// Don't unregister yet for more testing.
//Verify that if we call it again a second time we are told to die.
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.shouldDie);
listener.Unregister(attemptID, wid);
// Verify after unregistration.
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.shouldDie);
listener.Stop();
// test JVMID
JVMId jvmid = JVMId.ForName("jvm_001_002_m_004");
NUnit.Framework.Assert.IsNotNull(jvmid);
try
{
JVMId.ForName("jvm_001_002_m_004_006");
Assert.Fail();
}
catch (ArgumentException e)
{
NUnit.Framework.Assert.AreEqual(e.Message, "TaskId string : jvm_001_002_m_004_006 is not properly formed"
);
}
}
public static IList <string> GetVMCommand(IPEndPoint taskAttemptListenerAddr, Task
task, JVMId jvmID)
{
TaskAttemptID attemptID = task.GetTaskID();
JobConf conf = task.conf;
Vector <string> vargs = new Vector <string>(8);
vargs.AddItem(MRApps.CrossPlatformifyMREnv(task.conf, ApplicationConstants.Environment
.JavaHome) + "/bin/java");
// Add child (task) java-vm options.
//
// The following symbols if present in mapred.{map|reduce}.child.java.opts
// value are replaced:
// + @taskid@ is interpolated with value of TaskID.
// Other occurrences of @ will not be altered.
//
// Example with multiple arguments and substitutions, showing
// jvm GC logging, and start of a passwordless JVM JMX agent so can
// connect with jconsole and the likes to watch child memory, threads
// and get thread dumps.
//
// <property>
// <name>mapred.map.child.java.opts</name>
// <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \
// -Dcom.sun.management.jmxremote.authenticate=false \
// -Dcom.sun.management.jmxremote.ssl=false \
// </value>
// </property>
//
// <property>
// <name>mapred.reduce.child.java.opts</name>
// <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \
// -Dcom.sun.management.jmxremote.authenticate=false \
// -Dcom.sun.management.jmxremote.ssl=false \
// </value>
// </property>
//
string javaOpts = GetChildJavaOpts(conf, task.IsMapTask());
javaOpts = javaOpts.Replace("@taskid@", attemptID.ToString());
string[] javaOptsSplit = javaOpts.Split(" ");
for (int i = 0; i < javaOptsSplit.Length; i++)
{
vargs.AddItem(javaOptsSplit[i]);
}
Path childTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
.Pwd), YarnConfiguration.DefaultContainerTempDir);
vargs.AddItem("-Djava.io.tmpdir=" + childTmpDir);
MRApps.AddLog4jSystemProperties(task, vargs, conf);
if (conf.GetProfileEnabled())
{
if (conf.GetProfileTaskRange(task.IsMapTask()).IsIncluded(task.GetPartition()))
{
string profileParams = conf.Get(task.IsMapTask() ? MRJobConfig.TaskMapProfileParams
: MRJobConfig.TaskReduceProfileParams, conf.GetProfileParams());
vargs.AddItem(string.Format(profileParams, GetTaskLogFile(TaskLog.LogName.Profile
)));
}
}
// Add main class and its arguments
vargs.AddItem(typeof(YarnChild).FullName);
// main of Child
// pass TaskAttemptListener's address
vargs.AddItem(taskAttemptListenerAddr.Address.GetHostAddress());
vargs.AddItem(Sharpen.Extensions.ToString(taskAttemptListenerAddr.Port));
vargs.AddItem(attemptID.ToString());
// pass task identifier
// Finally add the jvmID
vargs.AddItem(jvmID.GetId().ToString());
vargs.AddItem("1>" + GetTaskLogFile(TaskLog.LogName.Stdout));
vargs.AddItem("2>" + GetTaskLogFile(TaskLog.LogName.Stderr));
// Final commmand
StringBuilder mergedCommand = new StringBuilder();
foreach (CharSequence str in vargs)
{
mergedCommand.Append(str).Append(" ");
}
Vector <string> vargsFinal = new Vector <string>(1);
vargsFinal.AddItem(mergedCommand.ToString());
return(vargsFinal);
}