public virtual void TestJobTokenRpc()
{
TaskUmbilicalProtocol mockTT = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol>();
Org.Mockito.Mockito.DoReturn(TaskUmbilicalProtocol.versionID).When(mockTT).GetProtocolVersion
(Matchers.AnyString(), Matchers.AnyLong());
Org.Mockito.Mockito.DoReturn(ProtocolSignature.GetProtocolSignature(mockTT, typeof(
TaskUmbilicalProtocol).FullName, TaskUmbilicalProtocol.versionID, 0)).When(mockTT
).GetProtocolSignature(Matchers.AnyString(), Matchers.AnyLong(), Matchers.AnyInt
());
JobTokenSecretManager sm = new JobTokenSecretManager();
Server server = new RPC.Builder(conf).SetProtocol(typeof(TaskUmbilicalProtocol)).
SetInstance(mockTT).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose
(true).SetSecretManager(sm).Build();
server.Start();
UserGroupInformation current = UserGroupInformation.GetCurrentUser();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
string jobId = current.GetUserName();
JobTokenIdentifier tokenId = new JobTokenIdentifier(new Text(jobId));
Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<JobTokenIdentifier>(tokenId, sm);
sm.AddTokenForJob(jobId, token);
SecurityUtil.SetTokenService(token, addr);
Log.Info("Service address for token is " + token.GetService());
current.AddToken(token);
current.DoAs(new _PrivilegedExceptionAction_110(addr, server));
}
public virtual void Setup()
{
// mocked generics
Log.Info(">>>> " + name.GetMethodName());
job = new JobConf();
job.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, false);
jobWithRetry = new JobConf();
jobWithRetry.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, true);
id = TaskAttemptID.ForName("attempt_0_1_r_1_1");
ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>();
mm = Org.Mockito.Mockito.Mock <MergeManagerImpl>();
r = Org.Mockito.Mockito.Mock <Reporter>();
metrics = Org.Mockito.Mockito.Mock <ShuffleClientMetrics>();
except = Org.Mockito.Mockito.Mock <ExceptionReporter>();
key = JobTokenSecretManager.CreateSecretKey(new byte[] { 0, 0, 0, 0 });
connection = Org.Mockito.Mockito.Mock <HttpURLConnection>();
allErrs = Org.Mockito.Mockito.Mock <Counters.Counter>();
Org.Mockito.Mockito.When(r.GetCounter(Matchers.AnyString(), Matchers.AnyString())
).ThenReturn(allErrs);
AList <TaskAttemptID> maps = new AList <TaskAttemptID>(1);
maps.AddItem(map1ID);
maps.AddItem(map2ID);
Org.Mockito.Mockito.When(ss.GetMapsForHost(host)).ThenReturn(maps);
}
// TODO change AbstractService to throw InterruptedException
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
Configuration conf = GetConfig();
userRsrc = new ConcurrentHashMap<string, string>();
secretManager = new JobTokenSecretManager();
RecoverState(conf);
ServerBootstrap bootstrap = new ServerBootstrap(selector);
try
{
pipelineFact = new ShuffleHandler.HttpPipelineFactory(this, conf);
}
catch (Exception ex)
{
throw new RuntimeException(ex);
}
bootstrap.SetOption("child.keepAlive", true);
bootstrap.SetPipelineFactory(pipelineFact);
port = conf.GetInt(ShufflePortConfigKey, DefaultShufflePort);
Org.Jboss.Netty.Channel.Channel ch = bootstrap.Bind(new IPEndPoint(port));
accepted.AddItem(ch);
port = ((IPEndPoint)ch.GetLocalAddress()).Port;
conf.Set(ShufflePortConfigKey, Sharpen.Extensions.ToString(port));
pipelineFact.Shuffle.SetPort(port);
Log.Info(GetName() + " listening on port " + port);
base.ServiceStart();
sslFileBufferSize = conf.GetInt(SuffleSslFileBufferSizeKey, DefaultSuffleSslFileBufferSize
);
connectionKeepAliveEnabled = conf.GetBoolean(ShuffleConnectionKeepAliveEnabled, DefaultShuffleConnectionKeepAliveEnabled
);
connectionKeepAliveTimeOut = Math.Max(1, conf.GetInt(ShuffleConnectionKeepAliveTimeOut
, DefaultShuffleConnectionKeepAliveTimeOut));
mapOutputMetaInfoCacheSize = Math.Max(1, conf.GetInt(ShuffleMapoutputMetaInfoCacheSize
, DefaultShuffleMapoutputMetaInfoCacheSize));
}
public TaskAttemptListenerImpl(AppContext context, JobTokenSecretManager jobTokenSecretManager
, RMHeartbeatHandler rmHeartbeatHandler, byte[] secretShuffleKey)
: base(typeof(Org.Apache.Hadoop.Mapred.TaskAttemptListenerImpl).FullName)
{
this.context = context;
this.jobTokenSecretManager = jobTokenSecretManager;
this.rmHeartbeatHandler = rmHeartbeatHandler;
this.encryptedSpillKey = secretShuffleKey;
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetMapCompletionEvents()
{
TaskAttemptCompletionEvent[] empty = new TaskAttemptCompletionEvent[] { };
TaskAttemptCompletionEvent[] taskEvents = new TaskAttemptCompletionEvent[] { CreateTce
(0, true, TaskAttemptCompletionEventStatus.Obsolete), CreateTce(1, false, TaskAttemptCompletionEventStatus
.Failed), CreateTce(2, true, TaskAttemptCompletionEventStatus.Succeeded), CreateTce
(3, false, TaskAttemptCompletionEventStatus.Failed) };
TaskAttemptCompletionEvent[] mapEvents = new TaskAttemptCompletionEvent[] { taskEvents
[0], taskEvents[2] };
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job mockJob = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(0, 100)).ThenReturn
(taskEvents);
Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(0, 2)).ThenReturn
(Arrays.CopyOfRange(taskEvents, 0, 2));
Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(2, 100)).ThenReturn
(Arrays.CopyOfRange(taskEvents, 2, 4));
Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(0, 100)).ThenReturn
(TypeConverter.FromYarn(mapEvents));
Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(0, 2)).ThenReturn(
TypeConverter.FromYarn(mapEvents));
Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(2, 100)).ThenReturn
(TypeConverter.FromYarn(empty));
AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>();
Org.Mockito.Mockito.When(appCtx.GetJob(Matchers.Any <JobId>())).ThenReturn(mockJob
);
JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>();
RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler
>();
TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>();
TaskAttemptListenerImpl listener = new _MockTaskAttemptListenerImpl_200(hbHandler
, appCtx, secret, rmHeartbeatHandler);
Configuration conf = new Configuration();
listener.Init(conf);
listener.Start();
JobID jid = new JobID("12345", 1);
TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 0);
MapTaskCompletionEventsUpdate update = listener.GetMapCompletionEvents(jid, 0, 100
, tid);
NUnit.Framework.Assert.AreEqual(2, update.events.Length);
update = listener.GetMapCompletionEvents(jid, 0, 2, tid);
NUnit.Framework.Assert.AreEqual(2, update.events.Length);
update = listener.GetMapCompletionEvents(jid, 2, 100, tid);
NUnit.Framework.Assert.AreEqual(0, update.events.Length);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestCommitWindow()
{
SystemClock clock = new SystemClock();
Task mockTask = Org.Mockito.Mockito.Mock <Task>();
Org.Mockito.Mockito.When(mockTask.CanCommit(Matchers.Any <TaskAttemptId>())).ThenReturn
(true);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job mockJob = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
>();
Org.Mockito.Mockito.When(mockJob.GetTask(Matchers.Any <TaskId>())).ThenReturn(mockTask
);
AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>();
Org.Mockito.Mockito.When(appCtx.GetJob(Matchers.Any <JobId>())).ThenReturn(mockJob
);
Org.Mockito.Mockito.When(appCtx.GetClock()).ThenReturn(clock);
JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>();
RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler
>();
TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>();
TaskAttemptListenerImpl listener = new _MockTaskAttemptListenerImpl_254(hbHandler
, appCtx, secret, rmHeartbeatHandler);
Configuration conf = new Configuration();
listener.Init(conf);
listener.Start();
// verify commit not allowed when RM heartbeat has not occurred recently
TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 0);
bool canCommit = listener.CanCommit(tid);
NUnit.Framework.Assert.IsFalse(canCommit);
Org.Mockito.Mockito.Verify(mockTask, Org.Mockito.Mockito.Never()).CanCommit(Matchers.Any
<TaskAttemptId>());
// verify commit allowed when RM heartbeat is recent
Org.Mockito.Mockito.When(rmHeartbeatHandler.GetLastHeartbeatTime()).ThenReturn(clock
.GetTime());
canCommit = listener.CanCommit(tid);
NUnit.Framework.Assert.IsTrue(canCommit);
Org.Mockito.Mockito.Verify(mockTask, Org.Mockito.Mockito.Times(1)).CanCommit(Matchers.Any
<TaskAttemptId>());
listener.Stop();
}
/// <exception cref="System.IO.IOException"/>
private static void ConfigureTask(JobConf job, Task task, Credentials credentials
, Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt)
{
job.SetCredentials(credentials);
ApplicationAttemptId appAttemptId = ConverterUtils.ToContainerId(Runtime.Getenv(ApplicationConstants.Environment
.ContainerId.ToString())).GetApplicationAttemptId();
Log.Debug("APPLICATION_ATTEMPT_ID: " + appAttemptId);
// Set it in conf, so as to be able to be used the the OutputCommitter.
job.SetInt(MRJobConfig.ApplicationAttemptId, appAttemptId.GetAttemptId());
// set tcp nodelay
job.SetBoolean("ipc.client.tcpnodelay", true);
job.SetClass(MRConfig.TaskLocalOutputClass, typeof(YarnOutputFiles), typeof(MapOutputFile
));
// set the jobToken and shuffle secrets into task
task.SetJobTokenSecret(JobTokenSecretManager.CreateSecretKey(jt.GetPassword()));
byte[] shuffleSecret = TokenCache.GetShuffleSecretKey(credentials);
if (shuffleSecret == null)
{
Log.Warn("Shuffle secret missing from task credentials." + " Using job token secret as shuffle secret."
);
shuffleSecret = jt.GetPassword();
}
task.SetShuffleSecret(JobTokenSecretManager.CreateSecretKey(shuffleSecret));
// setup the child's MRConfig.LOCAL_DIR.
ConfigureLocalDirs(task, job);
// setup the child's attempt directories
// Do the task-type specific localization
task.LocalizeConfiguration(job);
// Set up the DistributedCache related configs
MRApps.SetupDistributedCacheLocal(job);
// Overwrite the localized task jobconf which is linked to in the current
// work-dir.
Path localTaskFile = new Path(MRJobConfig.JobConfFile);
WriteLocalJobFile(localTaskFile, job);
task.SetJobFile(localTaskFile.ToString());
task.SetConf(job);
}
/// <exception cref="System.IO.IOException"/>
public static string CreateDigest(byte[] password, string data)
{
SecretKey key = JobTokenSecretManager.CreateSecretKey(password);
return(SecureShuffleUtils.HashFromString(data, key));
}
public _TaskAttemptListenerImpl_256(AppContext baseArg1, JobTokenSecretManager baseArg2
, RMHeartbeatHandler baseArg3, byte[] baseArg4)
: base(baseArg1, baseArg2, baseArg3, baseArg4)
{
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual string CreateDigest(byte[] password, string data)
{
SecretKey key = JobTokenSecretManager.CreateSecretKey(password);
return(SecureShuffleUtils.HashFromString(data, key));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetTask()
{
AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>();
JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>();
RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler
>();
TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>();
TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl listener = new TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl
(appCtx, secret, rmHeartbeatHandler, hbHandler);
Configuration conf = new Configuration();
listener.Init(conf);
listener.Start();
JVMId id = new JVMId("foo", 1, true, 1);
WrappedJvmID wid = new WrappedJvmID(id.GetJobId(), id.isMap, id.GetId());
// Verify ask before registration.
//The JVM ID has not been registered yet so we should kill it.
JvmContext context = new JvmContext();
context.jvmId = id;
JvmTask result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.shouldDie);
// Verify ask after registration but before launch.
// Don't kill, should be null.
TaskAttemptId attemptID = Org.Mockito.Mockito.Mock <TaskAttemptId>();
Task task = Org.Mockito.Mockito.Mock <Task>();
//Now put a task with the ID
listener.RegisterPendingTask(task, wid);
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNull(result);
// Unregister for more testing.
listener.Unregister(attemptID, wid);
// Verify ask after registration and launch
//Now put a task with the ID
listener.RegisterPendingTask(task, wid);
listener.RegisterLaunchedTask(attemptID, wid);
Org.Mockito.Mockito.Verify(hbHandler).Register(attemptID);
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsFalse(result.shouldDie);
// Don't unregister yet for more testing.
//Verify that if we call it again a second time we are told to die.
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.shouldDie);
listener.Unregister(attemptID, wid);
// Verify after unregistration.
result = listener.GetTask(context);
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.shouldDie);
listener.Stop();
// test JVMID
JVMId jvmid = JVMId.ForName("jvm_001_002_m_004");
NUnit.Framework.Assert.IsNotNull(jvmid);
try
{
JVMId.ForName("jvm_001_002_m_004_006");
Assert.Fail();
}
catch (ArgumentException e)
{
NUnit.Framework.Assert.AreEqual(e.Message, "TaskId string : jvm_001_002_m_004_006 is not properly formed"
);
}
}
public MockTaskAttemptListenerImpl(AppContext context, JobTokenSecretManager jobTokenSecretManager
, RMHeartbeatHandler rmHeartbeatHandler, TaskHeartbeatHandler hbHandler)
: base(context, jobTokenSecretManager, rmHeartbeatHandler, null)
{
this.taskHeartbeatHandler = hbHandler;
}
public _MockTaskAttemptListenerImpl_254(TaskHeartbeatHandler hbHandler, AppContext
baseArg1, JobTokenSecretManager baseArg2, RMHeartbeatHandler baseArg3)
: base(baseArg1, baseArg2, baseArg3)
{
this.hbHandler = hbHandler;
}
/// <summary>calculate hash of msg</summary>
/// <param name="msg"/>
/// <returns/>
private static byte[] GenerateByteHash(byte[] msg, SecretKey key)
{
return(JobTokenSecretManager.ComputeHash(msg, key));
}
