public virtual void TestJVMId() { JVMId jvmid = new JVMId("test", 1, true, 2); JVMId jvmid1 = JVMId.ForName("jvm_test_0001_m_000002"); // test compare methot should be the same NUnit.Framework.Assert.AreEqual(0, jvmid.CompareTo(jvmid1)); }
/// <exception cref="System.IO.IOException"/> public virtual JvmTask GetTask(JvmContext context) { // A rough imitation of code from TaskTracker. JVMId jvmId = context.jvmId; Log.Info("JVM with ID : " + jvmId + " asked for a task"); JvmTask jvmTask = null; // TODO: Is it an authorized container to get a task? Otherwise return null. // TODO: Child.java's firstTaskID isn't really firstTaskID. Ask for update // to jobId and task-type. WrappedJvmID wJvmID = new WrappedJvmID(jvmId.GetJobId(), jvmId.isMap, jvmId.GetId ()); // Try to look up the task. We remove it directly as we don't give // multiple tasks to a JVM if (!jvmIDToActiveAttemptMap.Contains(wJvmID)) { Log.Info("JVM with ID: " + jvmId + " is invalid and will be killed."); jvmTask = TaskForInvalidJvm; } else { if (!launchedJVMs.Contains(wJvmID)) { jvmTask = null; Log.Info("JVM with ID: " + jvmId + " asking for task before AM launch registered. Given null task" ); } else { // remove the task as it is no more needed and free up the memory. // Also we have already told the JVM to process a task, so it is no // longer pending, and further request should ask it to exit. Task task = Sharpen.Collections.Remove(jvmIDToActiveAttemptMap, wJvmID); launchedJVMs.Remove(wJvmID); Log.Info("JVM with ID: " + jvmId + " given task: " + task.GetTaskID()); task.SetEncryptedSpillKey(encryptedSpillKey); jvmTask = new JvmTask(task, false); } } return(jvmTask); }
/// <exception cref="System.Exception"/> public static void Main(string[] args) { Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler ()); Log.Debug("Child starting"); JobConf job = new JobConf(MRJobConfig.JobConfFile); // Initing with our JobConf allows us to avoid loading confs twice Limits.Init(job); UserGroupInformation.SetConfiguration(job); string host = args[0]; int port = System.Convert.ToInt32(args[1]); IPEndPoint address = NetUtils.CreateSocketAddrForHost(host, port); TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2])); long jvmIdLong = long.Parse(args[3]); JVMId jvmId = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType( ) == TaskType.Map, jvmIdLong); // initialize metrics DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString ()) + "Task"); // Security framework already loaded the tokens into current ugi Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials(); Log.Info("Executing with tokens:"); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens ()) { Log.Info(token); } // Create TaskUmbilicalProtocol as actual task owner. UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid .GetJobID()).ToString()); Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken (credentials); SecurityUtil.SetTokenService(jt, address); taskOwner.AddToken(jt); TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108 (address, job)); // report non-pid to application master JvmContext context = new JvmContext(jvmId, "-1000"); Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]); Task task = null; UserGroupInformation childUGI = null; ScheduledExecutorService logSyncer = null; try { int idleLoopCount = 0; JvmTask myTask = null; // poll for new task for (int idle = 0; null == myTask; ++idle) { long sleepTimeMilliSecs = Math.Min(idle * 500, 1500); Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now." ); TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs); myTask = umbilical.GetTask(context); } if (myTask.ShouldDie()) { return; } task = myTask.GetTask(); YarnChild.taskid = task.GetTaskID(); // Create the job-conf and set credentials ConfigureTask(job, task, credentials, jt); // Initiate Java VM metrics JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId()); childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment .User.ToString())); // Add tokens to new user so that it may execute its task correctly. childUGI.AddCredentials(credentials); // set job classloader if configured before invoking the task MRApps.SetJobClassLoader(job); logSyncer = TaskLog.CreateLogSyncer(); // Create a final reference to the task for the doAs block Task taskFinal = task; childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical)); } catch (FSError e) { // use job-specified working directory // run the task Log.Fatal("FSError from child", e); if (!ShutdownHookManager.Get().IsShutdownInProgress()) { umbilical.FsError(taskid, e.Message); } } catch (Exception exception) { Log.Warn("Exception running child : " + StringUtils.StringifyException(exception) ); try { if (task != null) { // do cleanup for the task if (childUGI == null) { // no need to job into doAs block task.TaskCleanup(umbilical); } else { Task taskFinal = task; childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical)); } } } catch (Exception e) { Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e)); } // Report back any failures, for diagnostic purposes if (taskid != null) { if (!ShutdownHookManager.Get().IsShutdownInProgress()) { umbilical.FatalError(taskid, StringUtils.StringifyException(exception)); } } } catch (Exception throwable) { Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable)); if (taskid != null) { if (!ShutdownHookManager.Get().IsShutdownInProgress()) { Exception tCause = throwable.InnerException; string cause = tCause == null ? throwable.Message : StringUtils.StringifyException (tCause); umbilical.FatalError(taskid, cause); } } } finally { RPC.StopProxy(umbilical); DefaultMetricsSystem.Shutdown(); TaskLog.SyncLogsShutdown(logSyncer); } }
internal JvmContext(JVMId id, string pid) { jvmId = id; this.pid = pid; }
internal JvmContext() { jvmId = new JVMId(); pid = string.Empty; }
/// <exception cref="System.IO.IOException"/> public virtual void TestGetTask() { AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>(); JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>(); RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler >(); TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>(); TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl listener = new TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl (appCtx, secret, rmHeartbeatHandler, hbHandler); Configuration conf = new Configuration(); listener.Init(conf); listener.Start(); JVMId id = new JVMId("foo", 1, true, 1); WrappedJvmID wid = new WrappedJvmID(id.GetJobId(), id.isMap, id.GetId()); // Verify ask before registration. //The JVM ID has not been registered yet so we should kill it. JvmContext context = new JvmContext(); context.jvmId = id; JvmTask result = listener.GetTask(context); NUnit.Framework.Assert.IsNotNull(result); NUnit.Framework.Assert.IsTrue(result.shouldDie); // Verify ask after registration but before launch. // Don't kill, should be null. TaskAttemptId attemptID = Org.Mockito.Mockito.Mock <TaskAttemptId>(); Task task = Org.Mockito.Mockito.Mock <Task>(); //Now put a task with the ID listener.RegisterPendingTask(task, wid); result = listener.GetTask(context); NUnit.Framework.Assert.IsNull(result); // Unregister for more testing. listener.Unregister(attemptID, wid); // Verify ask after registration and launch //Now put a task with the ID listener.RegisterPendingTask(task, wid); listener.RegisterLaunchedTask(attemptID, wid); Org.Mockito.Mockito.Verify(hbHandler).Register(attemptID); result = listener.GetTask(context); NUnit.Framework.Assert.IsNotNull(result); NUnit.Framework.Assert.IsFalse(result.shouldDie); // Don't unregister yet for more testing. //Verify that if we call it again a second time we are told to die. result = listener.GetTask(context); NUnit.Framework.Assert.IsNotNull(result); NUnit.Framework.Assert.IsTrue(result.shouldDie); listener.Unregister(attemptID, wid); // Verify after unregistration. result = listener.GetTask(context); NUnit.Framework.Assert.IsNotNull(result); NUnit.Framework.Assert.IsTrue(result.shouldDie); listener.Stop(); // test JVMID JVMId jvmid = JVMId.ForName("jvm_001_002_m_004"); NUnit.Framework.Assert.IsNotNull(jvmid); try { JVMId.ForName("jvm_001_002_m_004_006"); Assert.Fail(); } catch (ArgumentException e) { NUnit.Framework.Assert.AreEqual(e.Message, "TaskId string : jvm_001_002_m_004_006 is not properly formed" ); } }
public static IList <string> GetVMCommand(IPEndPoint taskAttemptListenerAddr, Task task, JVMId jvmID) { TaskAttemptID attemptID = task.GetTaskID(); JobConf conf = task.conf; Vector <string> vargs = new Vector <string>(8); vargs.AddItem(MRApps.CrossPlatformifyMREnv(task.conf, ApplicationConstants.Environment .JavaHome) + "/bin/java"); // Add child (task) java-vm options. // // The following symbols if present in mapred.{map|reduce}.child.java.opts // value are replaced: // + @taskid@ is interpolated with value of TaskID. // Other occurrences of @ will not be altered. // // Example with multiple arguments and substitutions, showing // jvm GC logging, and start of a passwordless JVM JMX agent so can // connect with jconsole and the likes to watch child memory, threads // and get thread dumps. // // <property> // <name>mapred.map.child.java.opts</name> // <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // // <property> // <name>mapred.reduce.child.java.opts</name> // <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // string javaOpts = GetChildJavaOpts(conf, task.IsMapTask()); javaOpts = javaOpts.Replace("@taskid@", attemptID.ToString()); string[] javaOptsSplit = javaOpts.Split(" "); for (int i = 0; i < javaOptsSplit.Length; i++) { vargs.AddItem(javaOptsSplit[i]); } Path childTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment .Pwd), YarnConfiguration.DefaultContainerTempDir); vargs.AddItem("-Djava.io.tmpdir=" + childTmpDir); MRApps.AddLog4jSystemProperties(task, vargs, conf); if (conf.GetProfileEnabled()) { if (conf.GetProfileTaskRange(task.IsMapTask()).IsIncluded(task.GetPartition())) { string profileParams = conf.Get(task.IsMapTask() ? MRJobConfig.TaskMapProfileParams : MRJobConfig.TaskReduceProfileParams, conf.GetProfileParams()); vargs.AddItem(string.Format(profileParams, GetTaskLogFile(TaskLog.LogName.Profile ))); } } // Add main class and its arguments vargs.AddItem(typeof(YarnChild).FullName); // main of Child // pass TaskAttemptListener's address vargs.AddItem(taskAttemptListenerAddr.Address.GetHostAddress()); vargs.AddItem(Sharpen.Extensions.ToString(taskAttemptListenerAddr.Port)); vargs.AddItem(attemptID.ToString()); // pass task identifier // Finally add the jvmID vargs.AddItem(jvmID.GetId().ToString()); vargs.AddItem("1>" + GetTaskLogFile(TaskLog.LogName.Stdout)); vargs.AddItem("2>" + GetTaskLogFile(TaskLog.LogName.Stderr)); // Final commmand StringBuilder mergedCommand = new StringBuilder(); foreach (CharSequence str in vargs) { mergedCommand.Append(str).Append(" "); } Vector <string> vargsFinal = new Vector <string>(1); vargsFinal.AddItem(mergedCommand.ToString()); return(vargsFinal); }