/// <summary>test without TASK_LOG_DIR</summary> /// <exception cref="System.IO.IOException"/> public virtual void TestTaskLogWithoutTaskLogDir() { // TaskLog tasklog= new TaskLog(); Runtime.ClearProperty(YarnConfiguration.YarnAppContainerLogDir); // test TaskLog NUnit.Framework.Assert.AreEqual(TaskLog.GetMRv2LogDir(), null); TaskAttemptID taid = Org.Mockito.Mockito.Mock <TaskAttemptID>(); JobID jid = new JobID("job", 1); Org.Mockito.Mockito.When(((JobID)taid.GetJobID())).ThenReturn(jid); Org.Mockito.Mockito.When(taid.ToString()).ThenReturn("JobId"); FilePath f = TaskLog.GetTaskLogFile(taid, true, TaskLog.LogName.Stdout); NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("stdout")); }
//taskId should be managed as String rather than TaskID object //so that log4j can configure it from the configuration(log4j.properties). // System properties passed in from JVM runner public override void ActivateOptions() { lock (this) { SetOptionsFromSystemProperties(); if (maxEvents > 0) { tail = new List <LoggingEvent>(); } SetFile(TaskLog.GetTaskLogFile(((TaskAttemptID)TaskAttemptID.ForName(taskId)), isCleanup , TaskLog.LogName.Syslog).ToString()); SetAppend(true); base.ActivateOptions(); } }
/// <summary>test TaskAttemptID</summary> /// <exception cref="System.IO.IOException"/> public virtual void TestTaskLog() { // test TaskLog Runtime.SetProperty(YarnConfiguration.YarnAppContainerLogDir, "testString"); NUnit.Framework.Assert.AreEqual(TaskLog.GetMRv2LogDir(), "testString"); TaskAttemptID taid = Org.Mockito.Mockito.Mock <TaskAttemptID>(); JobID jid = new JobID("job", 1); Org.Mockito.Mockito.When(((JobID)taid.GetJobID())).ThenReturn(jid); Org.Mockito.Mockito.When(taid.ToString()).ThenReturn("JobId"); FilePath f = TaskLog.GetTaskLogFile(taid, true, TaskLog.LogName.Stdout); NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("testString" + FilePath .separatorChar + "stdout")); // test getRealTaskLogFileLocation FilePath indexFile = TaskLog.GetIndexFile(taid, true); if (!indexFile.GetParentFile().Exists()) { indexFile.GetParentFile().Mkdirs(); } indexFile.Delete(); indexFile.CreateNewFile(); TaskLog.SyncLogs("location", taid, true); NUnit.Framework.Assert.IsTrue(indexFile.GetAbsolutePath().EndsWith("userlogs" + FilePath .separatorChar + "job_job_0001" + FilePath.separatorChar + "JobId.cleanup" + FilePath .separatorChar + "log.index")); f = TaskLog.GetRealTaskLogFileLocation(taid, true, TaskLog.LogName.Debugout); if (f != null) { NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("location" + FilePath. separatorChar + "debugout")); FileUtils.CopyFile(indexFile, f); } // test obtainLogDirOwner NUnit.Framework.Assert.IsTrue(TaskLog.ObtainLogDirOwner(taid).Length > 0); // test TaskLog.Reader NUnit.Framework.Assert.IsTrue(ReadTaskLog(TaskLog.LogName.Debugout, taid, true).Length > 0); }
public void Run() { TaskLog.SyncLogs(); }
public void Run() { TaskLog.SyncLogsShutdown(scheduler); }
/// <exception cref="System.Exception"/> public static void Main(string[] args) { Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler ()); Log.Debug("Child starting"); JobConf job = new JobConf(MRJobConfig.JobConfFile); // Initing with our JobConf allows us to avoid loading confs twice Limits.Init(job); UserGroupInformation.SetConfiguration(job); string host = args[0]; int port = System.Convert.ToInt32(args[1]); IPEndPoint address = NetUtils.CreateSocketAddrForHost(host, port); TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2])); long jvmIdLong = long.Parse(args[3]); JVMId jvmId = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType( ) == TaskType.Map, jvmIdLong); // initialize metrics DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString ()) + "Task"); // Security framework already loaded the tokens into current ugi Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials(); Log.Info("Executing with tokens:"); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens ()) { Log.Info(token); } // Create TaskUmbilicalProtocol as actual task owner. UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid .GetJobID()).ToString()); Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken (credentials); SecurityUtil.SetTokenService(jt, address); taskOwner.AddToken(jt); TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108 (address, job)); // report non-pid to application master JvmContext context = new JvmContext(jvmId, "-1000"); Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]); Task task = null; UserGroupInformation childUGI = null; ScheduledExecutorService logSyncer = null; try { int idleLoopCount = 0; JvmTask myTask = null; // poll for new task for (int idle = 0; null == myTask; ++idle) { long sleepTimeMilliSecs = Math.Min(idle * 500, 1500); Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now." ); TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs); myTask = umbilical.GetTask(context); } if (myTask.ShouldDie()) { return; } task = myTask.GetTask(); YarnChild.taskid = task.GetTaskID(); // Create the job-conf and set credentials ConfigureTask(job, task, credentials, jt); // Initiate Java VM metrics JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId()); childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment .User.ToString())); // Add tokens to new user so that it may execute its task correctly. childUGI.AddCredentials(credentials); // set job classloader if configured before invoking the task MRApps.SetJobClassLoader(job); logSyncer = TaskLog.CreateLogSyncer(); // Create a final reference to the task for the doAs block Task taskFinal = task; childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical)); } catch (FSError e) { // use job-specified working directory // run the task Log.Fatal("FSError from child", e); if (!ShutdownHookManager.Get().IsShutdownInProgress()) { umbilical.FsError(taskid, e.Message); } } catch (Exception exception) { Log.Warn("Exception running child : " + StringUtils.StringifyException(exception) ); try { if (task != null) { // do cleanup for the task if (childUGI == null) { // no need to job into doAs block task.TaskCleanup(umbilical); } else { Task taskFinal = task; childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical)); } } } catch (Exception e) { Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e)); } // Report back any failures, for diagnostic purposes if (taskid != null) { if (!ShutdownHookManager.Get().IsShutdownInProgress()) { umbilical.FatalError(taskid, StringUtils.StringifyException(exception)); } } } catch (Exception throwable) { Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable)); if (taskid != null) { if (!ShutdownHookManager.Get().IsShutdownInProgress()) { Exception tCause = throwable.InnerException; string cause = tCause == null ? throwable.Message : StringUtils.StringifyException (tCause); umbilical.FatalError(taskid, cause); } } } finally { RPC.StopProxy(umbilical); DefaultMetricsSystem.Shutdown(); TaskLog.SyncLogsShutdown(logSyncer); } }