示例#1
0
        /// <exception cref="System.IO.IOException"/>
        public virtual JvmTask GetTask(JvmContext context)
        {
            // A rough imitation of code from TaskTracker.
            JVMId jvmId = context.jvmId;

            Log.Info("JVM with ID : " + jvmId + " asked for a task");
            JvmTask jvmTask = null;
            // TODO: Is it an authorized container to get a task? Otherwise return null.
            // TODO: Child.java's firstTaskID isn't really firstTaskID. Ask for update
            // to jobId and task-type.
            WrappedJvmID wJvmID = new WrappedJvmID(jvmId.GetJobId(), jvmId.isMap, jvmId.GetId
                                                       ());

            // Try to look up the task. We remove it directly as we don't give
            // multiple tasks to a JVM
            if (!jvmIDToActiveAttemptMap.Contains(wJvmID))
            {
                Log.Info("JVM with ID: " + jvmId + " is invalid and will be killed.");
                jvmTask = TaskForInvalidJvm;
            }
            else
            {
                if (!launchedJVMs.Contains(wJvmID))
                {
                    jvmTask = null;
                    Log.Info("JVM with ID: " + jvmId + " asking for task before AM launch registered. Given null task"
                             );
                }
                else
                {
                    // remove the task as it is no more needed and free up the memory.
                    // Also we have already told the JVM to process a task, so it is no
                    // longer pending, and further request should ask it to exit.
                    Task task = Sharpen.Collections.Remove(jvmIDToActiveAttemptMap, wJvmID);
                    launchedJVMs.Remove(wJvmID);
                    Log.Info("JVM with ID: " + jvmId + " given task: " + task.GetTaskID());
                    task.SetEncryptedSpillKey(encryptedSpillKey);
                    jvmTask = new JvmTask(task, false);
                }
            }
            return(jvmTask);
        }
示例#2
0
        /// <exception cref="System.Exception"/>
        public static void Main(string[] args)
        {
            Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
                                                                  ());
            Log.Debug("Child starting");
            JobConf job = new JobConf(MRJobConfig.JobConfFile);

            // Initing with our JobConf allows us to avoid loading confs twice
            Limits.Init(job);
            UserGroupInformation.SetConfiguration(job);
            string        host        = args[0];
            int           port        = System.Convert.ToInt32(args[1]);
            IPEndPoint    address     = NetUtils.CreateSocketAddrForHost(host, port);
            TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2]));
            long          jvmIdLong   = long.Parse(args[3]);
            JVMId         jvmId       = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType(
                                                      ) == TaskType.Map, jvmIdLong);

            // initialize metrics
            DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString
                                                                     ()) + "Task");
            // Security framework already loaded the tokens into current ugi
            Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials();

            Log.Info("Executing with tokens:");
            foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens
                         ())
            {
                Log.Info(token);
            }
            // Create TaskUmbilicalProtocol as actual task owner.
            UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid
                                                                                    .GetJobID()).ToString());

            Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken
                                                                                 (credentials);
            SecurityUtil.SetTokenService(jt, address);
            taskOwner.AddToken(jt);
            TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108
                                                                 (address, job));
            // report non-pid to application master
            JvmContext context = new JvmContext(jvmId, "-1000");

            Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]);
            Task task = null;
            UserGroupInformation     childUGI  = null;
            ScheduledExecutorService logSyncer = null;

            try
            {
                int     idleLoopCount = 0;
                JvmTask myTask        = null;
                // poll for new task
                for (int idle = 0; null == myTask; ++idle)
                {
                    long sleepTimeMilliSecs = Math.Min(idle * 500, 1500);
                    Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now."
                             );
                    TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs);
                    myTask = umbilical.GetTask(context);
                }
                if (myTask.ShouldDie())
                {
                    return;
                }
                task             = myTask.GetTask();
                YarnChild.taskid = task.GetTaskID();
                // Create the job-conf and set credentials
                ConfigureTask(job, task, credentials, jt);
                // Initiate Java VM metrics
                JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId());
                childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment
                                                                                .User.ToString()));
                // Add tokens to new user so that it may execute its task correctly.
                childUGI.AddCredentials(credentials);
                // set job classloader if configured before invoking the task
                MRApps.SetJobClassLoader(job);
                logSyncer = TaskLog.CreateLogSyncer();
                // Create a final reference to the task for the doAs block
                Task taskFinal = task;
                childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical));
            }
            catch (FSError e)
            {
                // use job-specified working directory
                // run the task
                Log.Fatal("FSError from child", e);
                if (!ShutdownHookManager.Get().IsShutdownInProgress())
                {
                    umbilical.FsError(taskid, e.Message);
                }
            }
            catch (Exception exception)
            {
                Log.Warn("Exception running child : " + StringUtils.StringifyException(exception)
                         );
                try
                {
                    if (task != null)
                    {
                        // do cleanup for the task
                        if (childUGI == null)
                        {
                            // no need to job into doAs block
                            task.TaskCleanup(umbilical);
                        }
                        else
                        {
                            Task taskFinal = task;
                            childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical));
                        }
                    }
                }
                catch (Exception e)
                {
                    Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e));
                }
                // Report back any failures, for diagnostic purposes
                if (taskid != null)
                {
                    if (!ShutdownHookManager.Get().IsShutdownInProgress())
                    {
                        umbilical.FatalError(taskid, StringUtils.StringifyException(exception));
                    }
                }
            }
            catch (Exception throwable)
            {
                Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable));
                if (taskid != null)
                {
                    if (!ShutdownHookManager.Get().IsShutdownInProgress())
                    {
                        Exception tCause = throwable.InnerException;
                        string    cause  = tCause == null ? throwable.Message : StringUtils.StringifyException
                                               (tCause);
                        umbilical.FatalError(taskid, cause);
                    }
                }
            }
            finally
            {
                RPC.StopProxy(umbilical);
                DefaultMetricsSystem.Shutdown();
                TaskLog.SyncLogsShutdown(logSyncer);
            }
        }
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestGetTask()
        {
            AppContext            appCtx             = Org.Mockito.Mockito.Mock <AppContext>();
            JobTokenSecretManager secret             = Org.Mockito.Mockito.Mock <JobTokenSecretManager>();
            RMHeartbeatHandler    rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler
                                                                                 >();
            TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>();

            TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl listener = new TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl
                                                                                   (appCtx, secret, rmHeartbeatHandler, hbHandler);
            Configuration conf = new Configuration();

            listener.Init(conf);
            listener.Start();
            JVMId        id  = new JVMId("foo", 1, true, 1);
            WrappedJvmID wid = new WrappedJvmID(id.GetJobId(), id.isMap, id.GetId());
            // Verify ask before registration.
            //The JVM ID has not been registered yet so we should kill it.
            JvmContext context = new JvmContext();

            context.jvmId = id;
            JvmTask result = listener.GetTask(context);

            NUnit.Framework.Assert.IsNotNull(result);
            NUnit.Framework.Assert.IsTrue(result.shouldDie);
            // Verify ask after registration but before launch.
            // Don't kill, should be null.
            TaskAttemptId attemptID = Org.Mockito.Mockito.Mock <TaskAttemptId>();
            Task          task      = Org.Mockito.Mockito.Mock <Task>();

            //Now put a task with the ID
            listener.RegisterPendingTask(task, wid);
            result = listener.GetTask(context);
            NUnit.Framework.Assert.IsNull(result);
            // Unregister for more testing.
            listener.Unregister(attemptID, wid);
            // Verify ask after registration and launch
            //Now put a task with the ID
            listener.RegisterPendingTask(task, wid);
            listener.RegisterLaunchedTask(attemptID, wid);
            Org.Mockito.Mockito.Verify(hbHandler).Register(attemptID);
            result = listener.GetTask(context);
            NUnit.Framework.Assert.IsNotNull(result);
            NUnit.Framework.Assert.IsFalse(result.shouldDie);
            // Don't unregister yet for more testing.
            //Verify that if we call it again a second time we are told to die.
            result = listener.GetTask(context);
            NUnit.Framework.Assert.IsNotNull(result);
            NUnit.Framework.Assert.IsTrue(result.shouldDie);
            listener.Unregister(attemptID, wid);
            // Verify after unregistration.
            result = listener.GetTask(context);
            NUnit.Framework.Assert.IsNotNull(result);
            NUnit.Framework.Assert.IsTrue(result.shouldDie);
            listener.Stop();
            // test JVMID
            JVMId jvmid = JVMId.ForName("jvm_001_002_m_004");

            NUnit.Framework.Assert.IsNotNull(jvmid);
            try
            {
                JVMId.ForName("jvm_001_002_m_004_006");
                Assert.Fail();
            }
            catch (ArgumentException e)
            {
                NUnit.Framework.Assert.AreEqual(e.Message, "TaskId string : jvm_001_002_m_004_006 is not properly formed"
                                                );
            }
        }