public virtual void TestJobTokenRpc() { TaskUmbilicalProtocol mockTT = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol>(); Org.Mockito.Mockito.DoReturn(TaskUmbilicalProtocol.versionID).When(mockTT).GetProtocolVersion (Matchers.AnyString(), Matchers.AnyLong()); Org.Mockito.Mockito.DoReturn(ProtocolSignature.GetProtocolSignature(mockTT, typeof( TaskUmbilicalProtocol).FullName, TaskUmbilicalProtocol.versionID, 0)).When(mockTT ).GetProtocolSignature(Matchers.AnyString(), Matchers.AnyLong(), Matchers.AnyInt ()); JobTokenSecretManager sm = new JobTokenSecretManager(); Server server = new RPC.Builder(conf).SetProtocol(typeof(TaskUmbilicalProtocol)). SetInstance(mockTT).SetBindAddress(Address).SetPort(0).SetNumHandlers(5).SetVerbose (true).SetSecretManager(sm).Build(); server.Start(); UserGroupInformation current = UserGroupInformation.GetCurrentUser(); IPEndPoint addr = NetUtils.GetConnectAddress(server); string jobId = current.GetUserName(); JobTokenIdentifier tokenId = new JobTokenIdentifier(new Text(jobId)); Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier>(tokenId, sm); sm.AddTokenForJob(jobId, token); SecurityUtil.SetTokenService(token, addr); Log.Info("Service address for token is " + token.GetService()); current.AddToken(token); current.DoAs(new _PrivilegedExceptionAction_110(addr, server)); }
public virtual void Setup() { // mocked generics Log.Info(">>>> " + name.GetMethodName()); job = new JobConf(); job.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, false); jobWithRetry = new JobConf(); jobWithRetry.SetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, true); id = TaskAttemptID.ForName("attempt_0_1_r_1_1"); ss = Org.Mockito.Mockito.Mock <ShuffleSchedulerImpl>(); mm = Org.Mockito.Mockito.Mock <MergeManagerImpl>(); r = Org.Mockito.Mockito.Mock <Reporter>(); metrics = Org.Mockito.Mockito.Mock <ShuffleClientMetrics>(); except = Org.Mockito.Mockito.Mock <ExceptionReporter>(); key = JobTokenSecretManager.CreateSecretKey(new byte[] { 0, 0, 0, 0 }); connection = Org.Mockito.Mockito.Mock <HttpURLConnection>(); allErrs = Org.Mockito.Mockito.Mock <Counters.Counter>(); Org.Mockito.Mockito.When(r.GetCounter(Matchers.AnyString(), Matchers.AnyString()) ).ThenReturn(allErrs); AList <TaskAttemptID> maps = new AList <TaskAttemptID>(1); maps.AddItem(map1ID); maps.AddItem(map2ID); Org.Mockito.Mockito.When(ss.GetMapsForHost(host)).ThenReturn(maps); }
// TODO change AbstractService to throw InterruptedException /// <exception cref="System.Exception"/> protected override void ServiceStart() { Configuration conf = GetConfig(); userRsrc = new ConcurrentHashMap<string, string>(); secretManager = new JobTokenSecretManager(); RecoverState(conf); ServerBootstrap bootstrap = new ServerBootstrap(selector); try { pipelineFact = new ShuffleHandler.HttpPipelineFactory(this, conf); } catch (Exception ex) { throw new RuntimeException(ex); } bootstrap.SetOption("child.keepAlive", true); bootstrap.SetPipelineFactory(pipelineFact); port = conf.GetInt(ShufflePortConfigKey, DefaultShufflePort); Org.Jboss.Netty.Channel.Channel ch = bootstrap.Bind(new IPEndPoint(port)); accepted.AddItem(ch); port = ((IPEndPoint)ch.GetLocalAddress()).Port; conf.Set(ShufflePortConfigKey, Sharpen.Extensions.ToString(port)); pipelineFact.Shuffle.SetPort(port); Log.Info(GetName() + " listening on port " + port); base.ServiceStart(); sslFileBufferSize = conf.GetInt(SuffleSslFileBufferSizeKey, DefaultSuffleSslFileBufferSize ); connectionKeepAliveEnabled = conf.GetBoolean(ShuffleConnectionKeepAliveEnabled, DefaultShuffleConnectionKeepAliveEnabled ); connectionKeepAliveTimeOut = Math.Max(1, conf.GetInt(ShuffleConnectionKeepAliveTimeOut , DefaultShuffleConnectionKeepAliveTimeOut)); mapOutputMetaInfoCacheSize = Math.Max(1, conf.GetInt(ShuffleMapoutputMetaInfoCacheSize , DefaultShuffleMapoutputMetaInfoCacheSize)); }
public TaskAttemptListenerImpl(AppContext context, JobTokenSecretManager jobTokenSecretManager , RMHeartbeatHandler rmHeartbeatHandler, byte[] secretShuffleKey) : base(typeof(Org.Apache.Hadoop.Mapred.TaskAttemptListenerImpl).FullName) { this.context = context; this.jobTokenSecretManager = jobTokenSecretManager; this.rmHeartbeatHandler = rmHeartbeatHandler; this.encryptedSpillKey = secretShuffleKey; }
/// <exception cref="System.IO.IOException"/> public virtual void TestGetMapCompletionEvents() { TaskAttemptCompletionEvent[] empty = new TaskAttemptCompletionEvent[] { }; TaskAttemptCompletionEvent[] taskEvents = new TaskAttemptCompletionEvent[] { CreateTce (0, true, TaskAttemptCompletionEventStatus.Obsolete), CreateTce(1, false, TaskAttemptCompletionEventStatus .Failed), CreateTce(2, true, TaskAttemptCompletionEventStatus.Succeeded), CreateTce (3, false, TaskAttemptCompletionEventStatus.Failed) }; TaskAttemptCompletionEvent[] mapEvents = new TaskAttemptCompletionEvent[] { taskEvents [0], taskEvents[2] }; Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job mockJob = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job >(); Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(0, 100)).ThenReturn (taskEvents); Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(0, 2)).ThenReturn (Arrays.CopyOfRange(taskEvents, 0, 2)); Org.Mockito.Mockito.When(mockJob.GetTaskAttemptCompletionEvents(2, 100)).ThenReturn (Arrays.CopyOfRange(taskEvents, 2, 4)); Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(0, 100)).ThenReturn (TypeConverter.FromYarn(mapEvents)); Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(0, 2)).ThenReturn( TypeConverter.FromYarn(mapEvents)); Org.Mockito.Mockito.When(mockJob.GetMapAttemptCompletionEvents(2, 100)).ThenReturn (TypeConverter.FromYarn(empty)); AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>(); Org.Mockito.Mockito.When(appCtx.GetJob(Matchers.Any <JobId>())).ThenReturn(mockJob ); JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>(); RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler >(); TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>(); TaskAttemptListenerImpl listener = new _MockTaskAttemptListenerImpl_200(hbHandler , appCtx, secret, rmHeartbeatHandler); Configuration conf = new Configuration(); listener.Init(conf); listener.Start(); JobID jid = new JobID("12345", 1); TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 0); MapTaskCompletionEventsUpdate update = listener.GetMapCompletionEvents(jid, 0, 100 , tid); NUnit.Framework.Assert.AreEqual(2, update.events.Length); update = listener.GetMapCompletionEvents(jid, 0, 2, tid); NUnit.Framework.Assert.AreEqual(2, update.events.Length); update = listener.GetMapCompletionEvents(jid, 2, 100, tid); NUnit.Framework.Assert.AreEqual(0, update.events.Length); }
/// <exception cref="System.IO.IOException"/> public virtual void TestCommitWindow() { SystemClock clock = new SystemClock(); Task mockTask = Org.Mockito.Mockito.Mock <Task>(); Org.Mockito.Mockito.When(mockTask.CanCommit(Matchers.Any <TaskAttemptId>())).ThenReturn (true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job mockJob = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job >(); Org.Mockito.Mockito.When(mockJob.GetTask(Matchers.Any <TaskId>())).ThenReturn(mockTask ); AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>(); Org.Mockito.Mockito.When(appCtx.GetJob(Matchers.Any <JobId>())).ThenReturn(mockJob ); Org.Mockito.Mockito.When(appCtx.GetClock()).ThenReturn(clock); JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>(); RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler >(); TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>(); TaskAttemptListenerImpl listener = new _MockTaskAttemptListenerImpl_254(hbHandler , appCtx, secret, rmHeartbeatHandler); Configuration conf = new Configuration(); listener.Init(conf); listener.Start(); // verify commit not allowed when RM heartbeat has not occurred recently TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 0); bool canCommit = listener.CanCommit(tid); NUnit.Framework.Assert.IsFalse(canCommit); Org.Mockito.Mockito.Verify(mockTask, Org.Mockito.Mockito.Never()).CanCommit(Matchers.Any <TaskAttemptId>()); // verify commit allowed when RM heartbeat is recent Org.Mockito.Mockito.When(rmHeartbeatHandler.GetLastHeartbeatTime()).ThenReturn(clock .GetTime()); canCommit = listener.CanCommit(tid); NUnit.Framework.Assert.IsTrue(canCommit); Org.Mockito.Mockito.Verify(mockTask, Org.Mockito.Mockito.Times(1)).CanCommit(Matchers.Any <TaskAttemptId>()); listener.Stop(); }
/// <exception cref="System.IO.IOException"/> private static void ConfigureTask(JobConf job, Task task, Credentials credentials , Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt) { job.SetCredentials(credentials); ApplicationAttemptId appAttemptId = ConverterUtils.ToContainerId(Runtime.Getenv(ApplicationConstants.Environment .ContainerId.ToString())).GetApplicationAttemptId(); Log.Debug("APPLICATION_ATTEMPT_ID: " + appAttemptId); // Set it in conf, so as to be able to be used the the OutputCommitter. job.SetInt(MRJobConfig.ApplicationAttemptId, appAttemptId.GetAttemptId()); // set tcp nodelay job.SetBoolean("ipc.client.tcpnodelay", true); job.SetClass(MRConfig.TaskLocalOutputClass, typeof(YarnOutputFiles), typeof(MapOutputFile )); // set the jobToken and shuffle secrets into task task.SetJobTokenSecret(JobTokenSecretManager.CreateSecretKey(jt.GetPassword())); byte[] shuffleSecret = TokenCache.GetShuffleSecretKey(credentials); if (shuffleSecret == null) { Log.Warn("Shuffle secret missing from task credentials." + " Using job token secret as shuffle secret." ); shuffleSecret = jt.GetPassword(); } task.SetShuffleSecret(JobTokenSecretManager.CreateSecretKey(shuffleSecret)); // setup the child's MRConfig.LOCAL_DIR. ConfigureLocalDirs(task, job); // setup the child's attempt directories // Do the task-type specific localization task.LocalizeConfiguration(job); // Set up the DistributedCache related configs MRApps.SetupDistributedCacheLocal(job); // Overwrite the localized task jobconf which is linked to in the current // work-dir. Path localTaskFile = new Path(MRJobConfig.JobConfFile); WriteLocalJobFile(localTaskFile, job); task.SetJobFile(localTaskFile.ToString()); task.SetConf(job); }
/// <exception cref="System.IO.IOException"/> public static string CreateDigest(byte[] password, string data) { SecretKey key = JobTokenSecretManager.CreateSecretKey(password); return(SecureShuffleUtils.HashFromString(data, key)); }
public _TaskAttemptListenerImpl_256(AppContext baseArg1, JobTokenSecretManager baseArg2 , RMHeartbeatHandler baseArg3, byte[] baseArg4) : base(baseArg1, baseArg2, baseArg3, baseArg4) { }
/// <exception cref="System.IO.IOException"/> protected internal virtual string CreateDigest(byte[] password, string data) { SecretKey key = JobTokenSecretManager.CreateSecretKey(password); return(SecureShuffleUtils.HashFromString(data, key)); }
/// <exception cref="System.IO.IOException"/> public virtual void TestGetTask() { AppContext appCtx = Org.Mockito.Mockito.Mock <AppContext>(); JobTokenSecretManager secret = Org.Mockito.Mockito.Mock <JobTokenSecretManager>(); RMHeartbeatHandler rmHeartbeatHandler = Org.Mockito.Mockito.Mock <RMHeartbeatHandler >(); TaskHeartbeatHandler hbHandler = Org.Mockito.Mockito.Mock <TaskHeartbeatHandler>(); TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl listener = new TestTaskAttemptListenerImpl.MockTaskAttemptListenerImpl (appCtx, secret, rmHeartbeatHandler, hbHandler); Configuration conf = new Configuration(); listener.Init(conf); listener.Start(); JVMId id = new JVMId("foo", 1, true, 1); WrappedJvmID wid = new WrappedJvmID(id.GetJobId(), id.isMap, id.GetId()); // Verify ask before registration. //The JVM ID has not been registered yet so we should kill it. JvmContext context = new JvmContext(); context.jvmId = id; JvmTask result = listener.GetTask(context); NUnit.Framework.Assert.IsNotNull(result); NUnit.Framework.Assert.IsTrue(result.shouldDie); // Verify ask after registration but before launch. // Don't kill, should be null. TaskAttemptId attemptID = Org.Mockito.Mockito.Mock <TaskAttemptId>(); Task task = Org.Mockito.Mockito.Mock <Task>(); //Now put a task with the ID listener.RegisterPendingTask(task, wid); result = listener.GetTask(context); NUnit.Framework.Assert.IsNull(result); // Unregister for more testing. listener.Unregister(attemptID, wid); // Verify ask after registration and launch //Now put a task with the ID listener.RegisterPendingTask(task, wid); listener.RegisterLaunchedTask(attemptID, wid); Org.Mockito.Mockito.Verify(hbHandler).Register(attemptID); result = listener.GetTask(context); NUnit.Framework.Assert.IsNotNull(result); NUnit.Framework.Assert.IsFalse(result.shouldDie); // Don't unregister yet for more testing. //Verify that if we call it again a second time we are told to die. result = listener.GetTask(context); NUnit.Framework.Assert.IsNotNull(result); NUnit.Framework.Assert.IsTrue(result.shouldDie); listener.Unregister(attemptID, wid); // Verify after unregistration. result = listener.GetTask(context); NUnit.Framework.Assert.IsNotNull(result); NUnit.Framework.Assert.IsTrue(result.shouldDie); listener.Stop(); // test JVMID JVMId jvmid = JVMId.ForName("jvm_001_002_m_004"); NUnit.Framework.Assert.IsNotNull(jvmid); try { JVMId.ForName("jvm_001_002_m_004_006"); Assert.Fail(); } catch (ArgumentException e) { NUnit.Framework.Assert.AreEqual(e.Message, "TaskId string : jvm_001_002_m_004_006 is not properly formed" ); } }
public MockTaskAttemptListenerImpl(AppContext context, JobTokenSecretManager jobTokenSecretManager , RMHeartbeatHandler rmHeartbeatHandler, TaskHeartbeatHandler hbHandler) : base(context, jobTokenSecretManager, rmHeartbeatHandler, null) { this.taskHeartbeatHandler = hbHandler; }
public _MockTaskAttemptListenerImpl_254(TaskHeartbeatHandler hbHandler, AppContext baseArg1, JobTokenSecretManager baseArg2, RMHeartbeatHandler baseArg3) : base(baseArg1, baseArg2, baseArg3) { this.hbHandler = hbHandler; }
/// <summary>calculate hash of msg</summary> /// <param name="msg"/> /// <returns/> private static byte[] GenerateByteHash(byte[] msg, SecretKey key) { return(JobTokenSecretManager.ComputeHash(msg, key)); }