/// <exception cref="System.Exception"/> public virtual void TestCompletedTask() { HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath); completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager); TaskId mt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map); TaskId rt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Reduce); IDictionary <TaskId, Task> mapTasks = completedJob.GetTasks(TaskType.Map); IDictionary <TaskId, Task> reduceTasks = completedJob.GetTasks(TaskType.Reduce); NUnit.Framework.Assert.AreEqual(10, mapTasks.Count); NUnit.Framework.Assert.AreEqual(2, reduceTasks.Count); Task mt1 = mapTasks[mt1Id]; NUnit.Framework.Assert.AreEqual(1, mt1.GetAttempts().Count); NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mt1.GetState()); TaskReport mt1Report = mt1.GetReport(); NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mt1Report.GetTaskState()); NUnit.Framework.Assert.AreEqual(mt1Id, mt1Report.GetTaskId()); Task rt1 = reduceTasks[rt1Id]; NUnit.Framework.Assert.AreEqual(1, rt1.GetAttempts().Count); NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, rt1.GetState()); TaskReport rt1Report = rt1.GetReport(); NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, rt1Report.GetTaskState()); NUnit.Framework.Assert.AreEqual(rt1Id, rt1Report.GetTaskId()); }
public virtual void TestTimeout() { EventHandler mockHandler = Org.Mockito.Mockito.Mock <EventHandler>(); Clock clock = new SystemClock(); TaskHeartbeatHandler hb = new TaskHeartbeatHandler(mockHandler, clock, 1); Configuration conf = new Configuration(); conf.SetInt(MRJobConfig.TaskTimeout, 10); //10 ms conf.SetInt(MRJobConfig.TaskTimeoutCheckIntervalMs, 10); //10 ms hb.Init(conf); hb.Start(); try { ApplicationId appId = ApplicationId.NewInstance(0l, 5); JobId jobId = MRBuilderUtils.NewJobId(appId, 4); TaskId tid = MRBuilderUtils.NewTaskId(jobId, 3, TaskType.Map); TaskAttemptId taid = MRBuilderUtils.NewTaskAttemptId(tid, 2); hb.Register(taid); Sharpen.Thread.Sleep(100); //Events only happen when the task is canceled Org.Mockito.Mockito.Verify(mockHandler, Org.Mockito.Mockito.Times(2)).Handle(Matchers.Any <Org.Apache.Hadoop.Yarn.Event.Event>()); } finally { hb.Stop(); } }
public virtual IList <AMInfo> GetAMInfos() { IList <AMInfo> amInfos = new List <AMInfo>(); foreach (JobHistoryParser.AMInfo jhAmInfo in jobInfo.GetAMInfos()) { AMInfo amInfo = MRBuilderUtils.NewAMInfo(jhAmInfo.GetAppAttemptId(), jhAmInfo.GetStartTime (), jhAmInfo.GetContainerId(), jhAmInfo.GetNodeManagerHost(), jhAmInfo.GetNodeManagerPort (), jhAmInfo.GetNodeManagerHttpPort()); amInfos.AddItem(amInfo); } return(amInfos); }
public virtual void TestAddExisting() { HistoryFileManager.JobListCache cache = new HistoryFileManager.JobListCache(2, 1000 ); JobId jobId = MRBuilderUtils.NewJobId(1, 1, 1); HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(fileInfo.GetJobId()).ThenReturn(jobId); cache.AddIfAbsent(fileInfo); cache.AddIfAbsent(fileInfo); NUnit.Framework.Assert.AreEqual("Incorrect number of cache entries", 1, cache.Values ().Count); }
public virtual void TestAverageReduceTime() { Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <CompletedJob >(); Task task1 = Org.Mockito.Mockito.Mock <Task>(); Task task2 = Org.Mockito.Mockito.Mock <Task>(); JobId jobId = MRBuilderUtils.NewJobId(1L, 1, 1); TaskId taskId1 = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Reduce); TaskId taskId2 = MRBuilderUtils.NewTaskId(jobId, 2, TaskType.Reduce); TaskAttemptId taskAttemptId1 = MRBuilderUtils.NewTaskAttemptId(taskId1, 1); TaskAttemptId taskAttemptId2 = MRBuilderUtils.NewTaskAttemptId(taskId2, 2); TaskAttempt taskAttempt1 = Org.Mockito.Mockito.Mock <TaskAttempt>(); TaskAttempt taskAttempt2 = Org.Mockito.Mockito.Mock <TaskAttempt>(); JobReport jobReport = Org.Mockito.Mockito.Mock <JobReport>(); Org.Mockito.Mockito.When(taskAttempt1.GetState()).ThenReturn(TaskAttemptState.Succeeded ); Org.Mockito.Mockito.When(taskAttempt1.GetLaunchTime()).ThenReturn(0L); Org.Mockito.Mockito.When(taskAttempt1.GetShuffleFinishTime()).ThenReturn(4L); Org.Mockito.Mockito.When(taskAttempt1.GetSortFinishTime()).ThenReturn(6L); Org.Mockito.Mockito.When(taskAttempt1.GetFinishTime()).ThenReturn(8L); Org.Mockito.Mockito.When(taskAttempt2.GetState()).ThenReturn(TaskAttemptState.Succeeded ); Org.Mockito.Mockito.When(taskAttempt2.GetLaunchTime()).ThenReturn(5L); Org.Mockito.Mockito.When(taskAttempt2.GetShuffleFinishTime()).ThenReturn(10L); Org.Mockito.Mockito.When(taskAttempt2.GetSortFinishTime()).ThenReturn(22L); Org.Mockito.Mockito.When(taskAttempt2.GetFinishTime()).ThenReturn(42L); Org.Mockito.Mockito.When(task1.GetType()).ThenReturn(TaskType.Reduce); Org.Mockito.Mockito.When(task2.GetType()).ThenReturn(TaskType.Reduce); Org.Mockito.Mockito.When(task1.GetAttempts()).ThenReturn(new _Dictionary_120(taskAttemptId1 , taskAttempt1)); Org.Mockito.Mockito.When(task2.GetAttempts()).ThenReturn(new _Dictionary_123(taskAttemptId2 , taskAttempt2)); Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(new _Dictionary_127(taskId1, task1, taskId2, task2)); Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId); Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(jobReport); Org.Mockito.Mockito.When(job.GetName()).ThenReturn("TestJobInfo"); Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded); JobInfo jobInfo = new JobInfo(job); NUnit.Framework.Assert.AreEqual(11L, jobInfo.GetAvgReduceTime()); }
/// <exception cref="System.Exception"/> public virtual void TestEviction() { int maxSize = 2; HistoryFileManager.JobListCache cache = new HistoryFileManager.JobListCache(maxSize , 1000); JobId jobId1 = MRBuilderUtils.NewJobId(1, 1, 1); HistoryFileManager.HistoryFileInfo fileInfo1 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(fileInfo1.GetJobId()).ThenReturn(jobId1); JobId jobId2 = MRBuilderUtils.NewJobId(2, 2, 2); HistoryFileManager.HistoryFileInfo fileInfo2 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(fileInfo2.GetJobId()).ThenReturn(jobId2); JobId jobId3 = MRBuilderUtils.NewJobId(3, 3, 3); HistoryFileManager.HistoryFileInfo fileInfo3 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(fileInfo3.GetJobId()).ThenReturn(jobId3); cache.AddIfAbsent(fileInfo1); cache.AddIfAbsent(fileInfo2); cache.AddIfAbsent(fileInfo3); ICollection <HistoryFileManager.HistoryFileInfo> values; for (int i = 0; i < 9; i++) { values = cache.Values(); if (values.Count > maxSize) { Sharpen.Thread.Sleep(100); } else { NUnit.Framework.Assert.IsFalse("fileInfo1 should have been evicted", values.Contains (fileInfo1)); return; } } NUnit.Framework.Assert.Fail("JobListCache didn't delete the extra entry"); }
/// <exception cref="System.Exception"/> public virtual void TestCompletedTaskAttempt() { HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath); completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager); TaskId mt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map); TaskId rt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Reduce); TaskAttemptId mta1Id = MRBuilderUtils.NewTaskAttemptId(mt1Id, 0); TaskAttemptId rta1Id = MRBuilderUtils.NewTaskAttemptId(rt1Id, 0); Task mt1 = completedJob.GetTask(mt1Id); Task rt1 = completedJob.GetTask(rt1Id); TaskAttempt mta1 = mt1.GetAttempt(mta1Id); NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, mta1.GetState()); NUnit.Framework.Assert.AreEqual("localhost:45454", mta1.GetAssignedContainerMgrAddress ()); NUnit.Framework.Assert.AreEqual("localhost:9999", mta1.GetNodeHttpAddress()); TaskAttemptReport mta1Report = mta1.GetReport(); NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, mta1Report.GetTaskAttemptState ()); NUnit.Framework.Assert.AreEqual("localhost", mta1Report.GetNodeManagerHost()); NUnit.Framework.Assert.AreEqual(45454, mta1Report.GetNodeManagerPort()); NUnit.Framework.Assert.AreEqual(9999, mta1Report.GetNodeManagerHttpPort()); TaskAttempt rta1 = rt1.GetAttempt(rta1Id); NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, rta1.GetState()); NUnit.Framework.Assert.AreEqual("localhost:45454", rta1.GetAssignedContainerMgrAddress ()); NUnit.Framework.Assert.AreEqual("localhost:9999", rta1.GetNodeHttpAddress()); TaskAttemptReport rta1Report = rta1.GetReport(); NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, rta1Report.GetTaskAttemptState ()); NUnit.Framework.Assert.AreEqual("localhost", rta1Report.GetNodeManagerHost()); NUnit.Framework.Assert.AreEqual(45454, rta1Report.GetNodeManagerPort()); NUnit.Framework.Assert.AreEqual(9999, rta1Report.GetNodeManagerHttpPort()); }
/// <exception cref="System.IO.IOException"/> public virtual void TestAverageMergeTime() { string historyFileName = "job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist"; string confFileName = "job_1329348432655_0001_conf.xml"; Configuration conf = new Configuration(); JobACLsManager jobAclsMgr = new JobACLsManager(conf); Path fulleHistoryPath = new Path(typeof(TestJobHistoryEntities).GetClassLoader(). GetResource(historyFileName).GetFile()); Path fullConfPath = new Path(typeof(TestJobHistoryEntities).GetClassLoader().GetResource (confFileName).GetFile()); HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath); JobId jobId = MRBuilderUtils.NewJobId(1329348432655l, 1, 1); CompletedJob completedJob = new CompletedJob(conf, jobId, fulleHistoryPath, true, "user", info, jobAclsMgr); JobInfo jobInfo = new JobInfo(completedJob); // There are 2 tasks with merge time of 45 and 55 respectively. So average // merge time should be 50. NUnit.Framework.Assert.AreEqual(50L, jobInfo.GetAvgMergeTime()); }
private TaskAttemptId CreateTaskAttemptId(long clusterTimestamp, int jobIdInt, int taskIdInt, TaskType taskType, int taskAttemptIdInt) { return(MRBuilderUtils.NewTaskAttemptId(CreateTaskId(clusterTimestamp, jobIdInt, taskIdInt , taskType), taskAttemptIdInt)); }
public virtual void TestAttemptContainerRequest() { Text SecretKeyAlias = new Text("secretkeyalias"); byte[] SecretKey = Sharpen.Runtime.GetBytesForString(("secretkey")); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(1); acls[ApplicationAccessType.ViewApp] = "otheruser"; ApplicationId appId = ApplicationId.NewInstance(1, 1); JobId jobId = MRBuilderUtils.NewJobId(appId, 1); TaskId taskId = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Map); Path jobFile = Org.Mockito.Mockito.Mock <Path>(); EventHandler eventHandler = Org.Mockito.Mockito.Mock <EventHandler>(); TaskAttemptListener taListener = Org.Mockito.Mockito.Mock <TaskAttemptListener>(); Org.Mockito.Mockito.When(taListener.GetAddress()).ThenReturn(new IPEndPoint("localhost" , 0)); JobConf jobConf = new JobConf(); jobConf.SetClass("fs.file.impl", typeof(TestTaskAttemptContainerRequest.StubbedFS ), typeof(FileSystem)); jobConf.SetBoolean("fs.file.impl.disable.cache", true); jobConf.Set(JobConf.MapredMapTaskEnv, string.Empty); // setup UGI for security so tokens and keys are preserved jobConf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos" ); UserGroupInformation.SetConfiguration(jobConf); Credentials credentials = new Credentials(); credentials.AddSecretKey(SecretKeyAlias, SecretKey); Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken = new Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier>(Sharpen.Runtime.GetBytesForString(("tokenid")), Sharpen.Runtime.GetBytesForString (("tokenpw")), new Text("tokenkind"), new Text("tokenservice")); TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1, Org.Mockito.Mockito.Mock <JobSplit.TaskSplitMetaInfo>(), jobConf, taListener, jobToken, credentials, new SystemClock(), null); jobConf.Set(MRJobConfig.ApplicationAttemptId, taImpl.GetID().ToString()); ContainerLaunchContext launchCtx = TaskAttemptImpl.CreateContainerLaunchContext(acls , jobConf, jobToken, taImpl.CreateRemoteTask(), TypeConverter.FromYarn(jobId), Org.Mockito.Mockito.Mock <WrappedJvmID>(), taListener, credentials); NUnit.Framework.Assert.AreEqual("ACLs mismatch", acls, launchCtx.GetApplicationACLs ()); Credentials launchCredentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); dibb.Reset(launchCtx.GetTokens()); launchCredentials.ReadTokenStorageStream(dibb); // verify all tokens specified for the task attempt are in the launch context foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> token in credentials .GetAllTokens()) { Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> launchToken = launchCredentials .GetToken(token.GetService()); NUnit.Framework.Assert.IsNotNull("Token " + token.GetService() + " is missing", launchToken ); NUnit.Framework.Assert.AreEqual("Token " + token.GetService() + " mismatch", token , launchToken); } // verify the secret key is in the launch context NUnit.Framework.Assert.IsNotNull("Secret key missing", launchCredentials.GetSecretKey (SecretKeyAlias)); NUnit.Framework.Assert.IsTrue("Secret key mismatch", Arrays.Equals(SecretKey, launchCredentials .GetSecretKey(SecretKeyAlias))); }
public virtual void TestDelegationToken() { Logger rootLogger = LogManager.GetRootLogger(); rootLogger.SetLevel(Level.Debug); YarnConfiguration conf = new YarnConfiguration(new JobConf()); // Just a random principle conf.Set(JHAdminConfig.MrHistoryPrincipal, "RandomOrc/[email protected]"); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); long initialInterval = 10000l; long maxLifetime = 20000l; long renewInterval = 10000l; JobHistoryServer jobHistoryServer = null; MRClientProtocol clientUsingDT = null; long tokenFetchTime; try { jobHistoryServer = new _JobHistoryServer_87(initialInterval, maxLifetime, renewInterval ); // no keytab based login // Don't need it, skip.; // final JobHistoryServer jobHistoryServer = jhServer; jobHistoryServer.Init(conf); jobHistoryServer.Start(); MRClientProtocol hsService = jobHistoryServer.GetClientService().GetClientHandler (); // Fake the authentication-method UserGroupInformation loggedInUser = UserGroupInformation.CreateRemoteUser("*****@*****.**" ); NUnit.Framework.Assert.AreEqual("testrenewer", loggedInUser.GetShortUserName()); // Default realm is APACHE.ORG loggedInUser.SetAuthenticationMethod(UserGroupInformation.AuthenticationMethod.Kerberos ); Token token = GetDelegationToken(loggedInUser, hsService, loggedInUser.GetShortUserName ()); tokenFetchTime = Runtime.CurrentTimeMillis(); Log.Info("Got delegation token at: " + tokenFetchTime); // Now try talking to JHS using the delegation token clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress (), "TheDarkLord", conf); GetJobReportRequest jobReportRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <GetJobReportRequest>(); jobReportRequest.SetJobId(MRBuilderUtils.NewJobId(123456, 1, 1)); try { clientUsingDT.GetJobReport(jobReportRequest); } catch (IOException e) { NUnit.Framework.Assert.AreEqual("Unknown job job_123456_0001", e.Message); } // Renew after 50% of token age. while (Runtime.CurrentTimeMillis() < tokenFetchTime + initialInterval / 2) { Sharpen.Thread.Sleep(500l); } long nextExpTime = RenewDelegationToken(loggedInUser, hsService, token); long renewalTime = Runtime.CurrentTimeMillis(); Log.Info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + nextExpTime); // Wait for first expiry, but before renewed expiry. while (Runtime.CurrentTimeMillis() > tokenFetchTime + initialInterval && Runtime. CurrentTimeMillis() < nextExpTime) { Sharpen.Thread.Sleep(500l); } Sharpen.Thread.Sleep(50l); // Valid token because of renewal. try { clientUsingDT.GetJobReport(jobReportRequest); } catch (IOException e) { NUnit.Framework.Assert.AreEqual("Unknown job job_123456_0001", e.Message); } // Wait for expiry. while (Runtime.CurrentTimeMillis() < renewalTime + renewInterval) { Sharpen.Thread.Sleep(500l); } Sharpen.Thread.Sleep(50l); Log.Info("At time: " + Runtime.CurrentTimeMillis() + ", token should be invalid"); // Token should have expired. try { clientUsingDT.GetJobReport(jobReportRequest); NUnit.Framework.Assert.Fail("Should not have succeeded with an expired token"); } catch (IOException e) { NUnit.Framework.Assert.IsTrue(e.InnerException.Message.Contains("is expired")); } // Test cancellation // Stop the existing proxy, start another. if (clientUsingDT != null) { // RPC.stopProxy(clientUsingDT); clientUsingDT = null; } token = GetDelegationToken(loggedInUser, hsService, loggedInUser.GetShortUserName ()); tokenFetchTime = Runtime.CurrentTimeMillis(); Log.Info("Got delegation token at: " + tokenFetchTime); // Now try talking to HSService using the delegation token clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress (), "loginuser2", conf); try { clientUsingDT.GetJobReport(jobReportRequest); } catch (IOException e) { NUnit.Framework.Assert.Fail("Unexpected exception" + e); } CancelDelegationToken(loggedInUser, hsService, token); // Testing the token with different renewer to cancel the token Token tokenWithDifferentRenewer = GetDelegationToken(loggedInUser, hsService, "yarn" ); CancelDelegationToken(loggedInUser, hsService, tokenWithDifferentRenewer); if (clientUsingDT != null) { // RPC.stopProxy(clientUsingDT); clientUsingDT = null; } // Creating a new connection. clientUsingDT = GetMRClientProtocol(token, jobHistoryServer.GetClientService().GetBindAddress (), "loginuser2", conf); Log.Info("Cancelled delegation token at: " + Runtime.CurrentTimeMillis()); // Verify cancellation worked. try { clientUsingDT.GetJobReport(jobReportRequest); NUnit.Framework.Assert.Fail("Should not have succeeded with a cancelled delegation token" ); } catch (IOException) { } } finally { jobHistoryServer.Stop(); } }