/// <exception cref="System.Exception"/>
public virtual Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job Submit(Configuration conf
, bool mapSpeculative, bool reduceSpeculative)
{
string user = conf.Get(MRJobConfig.UserName, UserGroupInformation.GetCurrentUser(
).GetShortUserName());
conf.Set(MRJobConfig.UserName, user);
conf.Set(MRJobConfig.MrAmStagingDir, testAbsPath.ToString());
conf.SetBoolean(MRJobConfig.MrAmCreateJhIntermediateBaseDir, true);
// TODO: fix the bug where the speculator gets events with
// not-fully-constructed objects. For now, disable speculative exec
conf.SetBoolean(MRJobConfig.MapSpeculative, mapSpeculative);
conf.SetBoolean(MRJobConfig.ReduceSpeculative, reduceSpeculative);
Init(conf);
Start();
DefaultMetricsSystem.Shutdown();
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = GetContext().GetAllJobs().Values
.GetEnumerator().Next();
if (assignedQueue != null)
{
job.SetQueueName(assignedQueue);
}
// Write job.xml
string jobFile = MRApps.GetJobFile(conf, user, TypeConverter.FromYarn(job.GetID()
));
Log.Info("Writing job conf to " + jobFile);
new FilePath(jobFile).GetParentFile().Mkdirs();
conf.WriteXml(new FileOutputStream(jobFile));
return(job);
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
if (webApp != null)
{
webApp.Stop();
}
DefaultMetricsSystem.Shutdown();
base.ServiceStop();
}
public virtual void TestInitFirstVerifyStopInvokedImmediately()
{
DefaultMetricsSystem.Shutdown();
new ConfigBuilder().Add("*.period", 8).Add("test.sink.test.class", typeof(TestMetricsSystemImpl.TestSink
).FullName).Add("test.*.source.filter.exclude", "s0").Add("test.source.s1.metric.filter.exclude"
, "X*").Add("test.sink.sink1.metric.filter.exclude", "Y*").Add("test.sink.sink2.metric.filter.exclude"
, "Y*").Save(TestMetricsConfig.GetTestFilename("hadoop-metrics2-test"));
//.add("test.sink.plugin.urls", getPluginUrlsAsString())
MetricsSystemImpl ms = new MetricsSystemImpl("Test");
ms.Start();
ms.Register("s0", "s0 desc", new TestMetricsSystemImpl.TestSource("s0rec"));
TestMetricsSystemImpl.TestSource s1 = ms.Register("s1", "s1 desc", new TestMetricsSystemImpl.TestSource
("s1rec"));
s1.c1.Incr();
s1.xxx.Incr();
s1.g1.Set(2);
s1.yyy.Incr(2);
s1.s1.Add(0);
MetricsSink sink1 = Org.Mockito.Mockito.Mock <MetricsSink>();
MetricsSink sink2 = Org.Mockito.Mockito.Mock <MetricsSink>();
ms.RegisterSink("sink1", "sink1 desc", sink1);
ms.RegisterSink("sink2", "sink2 desc", sink2);
ms.PublishMetricsNow();
// publish the metrics
ms.Stop();
ms.Shutdown();
//When we call stop, at most two sources will be consumed by each sink thread.
Org.Mockito.Mockito.Verify(sink1, Org.Mockito.Mockito.AtMost(2)).PutMetrics(r1.Capture
());
IList <MetricsRecord> mr1 = r1.GetAllValues();
Org.Mockito.Mockito.Verify(sink2, Org.Mockito.Mockito.AtMost(2)).PutMetrics(r2.Capture
());
IList <MetricsRecord> mr2 = r2.GetAllValues();
if (mr1.Count != 0 && mr2.Count != 0)
{
CheckMetricsRecords(mr1);
MoreAsserts.AssertEquals("output", mr1, mr2);
}
else
{
if (mr1.Count != 0)
{
CheckMetricsRecords(mr1);
}
else
{
if (mr2.Count != 0)
{
CheckMetricsRecords(mr2);
}
}
}
}
public virtual void TearDown()
{
ClusterMetrics.Destroy();
MetricsSystem ms = DefaultMetricsSystem.Instance();
if (ms.GetSource("ClusterMetrics") != null)
{
DefaultMetricsSystem.Shutdown();
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
if (isStopping.GetAndSet(true))
{
return;
}
try
{
base.ServiceStop();
DefaultMetricsSystem.Shutdown();
}
finally
{
// YARN-3641: NM's services stop get failed shouldn't block the
// release of NMLevelDBStore.
StopRecoveryStore();
}
}
public virtual void TearDown()
{
if (hostFile != null && hostFile.Exists())
{
hostFile.Delete();
}
ClusterMetrics.Destroy();
if (rm != null)
{
rm.Stop();
}
MetricsSystem ms = DefaultMetricsSystem.Instance();
if (ms.GetSource("ClusterMetrics") != null)
{
DefaultMetricsSystem.Shutdown();
}
}
public virtual void SetUp()
{
configuration = new Configuration();
UserGroupInformation.SetConfiguration(configuration);
configuration.SetBoolean(YarnConfiguration.RmHaEnabled, true);
configuration.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm2NodeId);
foreach (string confKey in YarnConfiguration.GetServiceAddressConfKeys(configuration
))
{
configuration.Set(HAUtil.AddSuffix(confKey, Rm1NodeId), Rm1Address);
configuration.Set(HAUtil.AddSuffix(confKey, Rm2NodeId), Rm2Address);
configuration.Set(HAUtil.AddSuffix(confKey, Rm3NodeId), Rm3Address);
}
// Enable webapp to test web-services also
configuration.SetBoolean(MockRM.EnableWebapp, true);
configuration.SetBoolean(YarnConfiguration.YarnAclEnable, true);
ClusterMetrics.Destroy();
QueueMetrics.ClearQueueMetrics();
DefaultMetricsSystem.Shutdown();
}
public virtual void TestStartStopStart()
{
DefaultMetricsSystem.Shutdown();
// Clear pre-existing source names.
MetricsSystemImpl ms = new MetricsSystemImpl("test");
TestMetricsSystemImpl.TestSource ts = new TestMetricsSystemImpl.TestSource("ts");
ms.Start();
ms.Register("ts", string.Empty, ts);
MetricsSourceAdapter sa = ms.GetSourceAdapter("ts");
NUnit.Framework.Assert.IsNotNull(sa);
NUnit.Framework.Assert.IsNotNull(sa.GetMBeanName());
ms.Stop();
ms.Shutdown();
ms.Start();
sa = ms.GetSourceAdapter("ts");
NUnit.Framework.Assert.IsNotNull(sa);
NUnit.Framework.Assert.IsNotNull(sa.GetMBeanName());
ms.Stop();
ms.Shutdown();
}
public virtual void TearDown()
{
QueueMetrics.ClearQueueMetrics();
DefaultMetricsSystem.Shutdown();
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
());
Log.Debug("Child starting");
JobConf job = new JobConf(MRJobConfig.JobConfFile);
// Initing with our JobConf allows us to avoid loading confs twice
Limits.Init(job);
UserGroupInformation.SetConfiguration(job);
string host = args[0];
int port = System.Convert.ToInt32(args[1]);
IPEndPoint address = NetUtils.CreateSocketAddrForHost(host, port);
TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2]));
long jvmIdLong = long.Parse(args[3]);
JVMId jvmId = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType(
) == TaskType.Map, jvmIdLong);
// initialize metrics
DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString
()) + "Task");
// Security framework already loaded the tokens into current ugi
Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials();
Log.Info("Executing with tokens:");
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens
())
{
Log.Info(token);
}
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid
.GetJobID()).ToString());
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken
(credentials);
SecurityUtil.SetTokenService(jt, address);
taskOwner.AddToken(jt);
TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108
(address, job));
// report non-pid to application master
JvmContext context = new JvmContext(jvmId, "-1000");
Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]);
Task task = null;
UserGroupInformation childUGI = null;
ScheduledExecutorService logSyncer = null;
try
{
int idleLoopCount = 0;
JvmTask myTask = null;
// poll for new task
for (int idle = 0; null == myTask; ++idle)
{
long sleepTimeMilliSecs = Math.Min(idle * 500, 1500);
Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now."
);
TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs);
myTask = umbilical.GetTask(context);
}
if (myTask.ShouldDie())
{
return;
}
task = myTask.GetTask();
YarnChild.taskid = task.GetTaskID();
// Create the job-conf and set credentials
ConfigureTask(job, task, credentials, jt);
// Initiate Java VM metrics
JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId());
childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment
.User.ToString()));
// Add tokens to new user so that it may execute its task correctly.
childUGI.AddCredentials(credentials);
// set job classloader if configured before invoking the task
MRApps.SetJobClassLoader(job);
logSyncer = TaskLog.CreateLogSyncer();
// Create a final reference to the task for the doAs block
Task taskFinal = task;
childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical));
}
catch (FSError e)
{
// use job-specified working directory
// run the task
Log.Fatal("FSError from child", e);
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
umbilical.FsError(taskid, e.Message);
}
}
catch (Exception exception)
{
Log.Warn("Exception running child : " + StringUtils.StringifyException(exception)
);
try
{
if (task != null)
{
// do cleanup for the task
if (childUGI == null)
{
// no need to job into doAs block
task.TaskCleanup(umbilical);
}
else
{
Task taskFinal = task;
childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical));
}
}
}
catch (Exception e)
{
Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e));
}
// Report back any failures, for diagnostic purposes
if (taskid != null)
{
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
umbilical.FatalError(taskid, StringUtils.StringifyException(exception));
}
}
}
catch (Exception throwable)
{
Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable));
if (taskid != null)
{
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
Exception tCause = throwable.InnerException;
string cause = tCause == null ? throwable.Message : StringUtils.StringifyException
(tCause);
umbilical.FatalError(taskid, cause);
}
}
}
finally
{
RPC.StopProxy(umbilical);
DefaultMetricsSystem.Shutdown();
TaskLog.SyncLogsShutdown(logSyncer);
}
}
public virtual void Shutdown()
{
DefaultMetricsSystem.Shutdown();
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
DefaultMetricsSystem.Shutdown();
base.ServiceStop();
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
base.ServiceStart();
DefaultMetricsSystem.Shutdown();
}
public virtual void TearDown()
{
ClusterMetrics.Destroy();
QueueMetrics.ClearQueueMetrics();
DefaultMetricsSystem.Shutdown();
}