예제 #1
0
        /// <exception cref="System.IO.IOException"/>
        public virtual JvmTask GetTask(JvmContext context)
        {
            // A rough imitation of code from TaskTracker.
            JVMId jvmId = context.jvmId;

            Log.Info("JVM with ID : " + jvmId + " asked for a task");
            JvmTask jvmTask = null;
            // TODO: Is it an authorized container to get a task? Otherwise return null.
            // TODO: Child.java's firstTaskID isn't really firstTaskID. Ask for update
            // to jobId and task-type.
            WrappedJvmID wJvmID = new WrappedJvmID(jvmId.GetJobId(), jvmId.isMap, jvmId.GetId
                                                       ());

            // Try to look up the task. We remove it directly as we don't give
            // multiple tasks to a JVM
            if (!jvmIDToActiveAttemptMap.Contains(wJvmID))
            {
                Log.Info("JVM with ID: " + jvmId + " is invalid and will be killed.");
                jvmTask = TaskForInvalidJvm;
            }
            else
            {
                if (!launchedJVMs.Contains(wJvmID))
                {
                    jvmTask = null;
                    Log.Info("JVM with ID: " + jvmId + " asking for task before AM launch registered. Given null task"
                             );
                }
                else
                {
                    // remove the task as it is no more needed and free up the memory.
                    // Also we have already told the JVM to process a task, so it is no
                    // longer pending, and further request should ask it to exit.
                    Task task = Sharpen.Collections.Remove(jvmIDToActiveAttemptMap, wJvmID);
                    launchedJVMs.Remove(wJvmID);
                    Log.Info("JVM with ID: " + jvmId + " given task: " + task.GetTaskID());
                    task.SetEncryptedSpillKey(encryptedSpillKey);
                    jvmTask = new JvmTask(task, false);
                }
            }
            return(jvmTask);
        }
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestGetTask()
        {
            AppContext            appCtx             = Org.Mockito.Mockito.Mock <AppContext>();
            JobTokenSecretManager secret             = Org.Mockito.Mockito.Mock <JobTokenSecretManager>();
            RMHeartbeatHandler    rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler
                                                                                 >();
            TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>();

            TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl listener = new TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl
                                                                                   (appCtx, secret, rmHeartbeatHandler, hbHandler);
            Configuration conf = new Configuration();

            listener.Init(conf);
            listener.Start();
            JVMId        id  = new JVMId("foo", 1, true, 1);
            WrappedJvmID wid = new WrappedJvmID(id.GetJobId(), id.isMap, id.GetId());
            // Verify ask before registration.
            //The JVM ID has not been registered yet so we should kill it.
            JvmContext context = new JvmContext();

            context.jvmId = id;
            JvmTask result = listener.GetTask(context);

            NUnit.Framework.Assert.IsNotNull(result);
            NUnit.Framework.Assert.IsTrue(result.shouldDie);
            // Verify ask after registration but before launch.
            // Don't kill, should be null.
            TaskAttemptId attemptID = Org.Mockito.Mockito.Mock <TaskAttemptId>();
            Task          task      = Org.Mockito.Mockito.Mock <Task>();

            //Now put a task with the ID
            listener.RegisterPendingTask(task, wid);
            result = listener.GetTask(context);
            NUnit.Framework.Assert.IsNull(result);
            // Unregister for more testing.
            listener.Unregister(attemptID, wid);
            // Verify ask after registration and launch
            //Now put a task with the ID
            listener.RegisterPendingTask(task, wid);
            listener.RegisterLaunchedTask(attemptID, wid);
            Org.Mockito.Mockito.Verify(hbHandler).Register(attemptID);
            result = listener.GetTask(context);
            NUnit.Framework.Assert.IsNotNull(result);
            NUnit.Framework.Assert.IsFalse(result.shouldDie);
            // Don't unregister yet for more testing.
            //Verify that if we call it again a second time we are told to die.
            result = listener.GetTask(context);
            NUnit.Framework.Assert.IsNotNull(result);
            NUnit.Framework.Assert.IsTrue(result.shouldDie);
            listener.Unregister(attemptID, wid);
            // Verify after unregistration.
            result = listener.GetTask(context);
            NUnit.Framework.Assert.IsNotNull(result);
            NUnit.Framework.Assert.IsTrue(result.shouldDie);
            listener.Stop();
            // test JVMID
            JVMId jvmid = JVMId.ForName("jvm_001_002_m_004");

            NUnit.Framework.Assert.IsNotNull(jvmid);
            try
            {
                JVMId.ForName("jvm_001_002_m_004_006");
                Assert.Fail();
            }
            catch (ArgumentException e)
            {
                NUnit.Framework.Assert.AreEqual(e.Message, "TaskId string : jvm_001_002_m_004_006 is not properly formed"
                                                );
            }
        }
예제 #3
0
        public static IList <string> GetVMCommand(IPEndPoint taskAttemptListenerAddr, Task
                                                  task, JVMId jvmID)
        {
            TaskAttemptID   attemptID = task.GetTaskID();
            JobConf         conf      = task.conf;
            Vector <string> vargs     = new Vector <string>(8);

            vargs.AddItem(MRApps.CrossPlatformifyMREnv(task.conf, ApplicationConstants.Environment
                                                       .JavaHome) + "/bin/java");
            // Add child (task) java-vm options.
            //
            // The following symbols if present in mapred.{map|reduce}.child.java.opts
            // value are replaced:
            // + @taskid@ is interpolated with value of TaskID.
            // Other occurrences of @ will not be altered.
            //
            // Example with multiple arguments and substitutions, showing
            // jvm GC logging, and start of a passwordless JVM JMX agent so can
            // connect with jconsole and the likes to watch child memory, threads
            // and get thread dumps.
            //
            //  <property>
            //    <name>mapred.map.child.java.opts</name>
            //    <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \
            //           -Dcom.sun.management.jmxremote.authenticate=false \
            //           -Dcom.sun.management.jmxremote.ssl=false \
            //    </value>
            //  </property>
            //
            //  <property>
            //    <name>mapred.reduce.child.java.opts</name>
            //    <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \
            //           -Dcom.sun.management.jmxremote.authenticate=false \
            //           -Dcom.sun.management.jmxremote.ssl=false \
            //    </value>
            //  </property>
            //
            string javaOpts = GetChildJavaOpts(conf, task.IsMapTask());

            javaOpts = javaOpts.Replace("@taskid@", attemptID.ToString());
            string[] javaOptsSplit = javaOpts.Split(" ");
            for (int i = 0; i < javaOptsSplit.Length; i++)
            {
                vargs.AddItem(javaOptsSplit[i]);
            }
            Path childTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
                                                                     .Pwd), YarnConfiguration.DefaultContainerTempDir);

            vargs.AddItem("-Djava.io.tmpdir=" + childTmpDir);
            MRApps.AddLog4jSystemProperties(task, vargs, conf);
            if (conf.GetProfileEnabled())
            {
                if (conf.GetProfileTaskRange(task.IsMapTask()).IsIncluded(task.GetPartition()))
                {
                    string profileParams = conf.Get(task.IsMapTask() ? MRJobConfig.TaskMapProfileParams
                                                 : MRJobConfig.TaskReduceProfileParams, conf.GetProfileParams());
                    vargs.AddItem(string.Format(profileParams, GetTaskLogFile(TaskLog.LogName.Profile
                                                                              )));
                }
            }
            // Add main class and its arguments
            vargs.AddItem(typeof(YarnChild).FullName);
            // main of Child
            // pass TaskAttemptListener's address
            vargs.AddItem(taskAttemptListenerAddr.Address.GetHostAddress());
            vargs.AddItem(Sharpen.Extensions.ToString(taskAttemptListenerAddr.Port));
            vargs.AddItem(attemptID.ToString());
            // pass task identifier
            // Finally add the jvmID
            vargs.AddItem(jvmID.GetId().ToString());
            vargs.AddItem("1>" + GetTaskLogFile(TaskLog.LogName.Stdout));
            vargs.AddItem("2>" + GetTaskLogFile(TaskLog.LogName.Stderr));
            // Final commmand
            StringBuilder mergedCommand = new StringBuilder();

            foreach (CharSequence str in vargs)
            {
                mergedCommand.Append(str).Append(" ");
            }
            Vector <string> vargsFinal = new Vector <string>(1);

            vargsFinal.AddItem(mergedCommand.ToString());
            return(vargsFinal);
        }