/// <exception cref="System.Exception"/> public virtual Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job Submit(Configuration conf , bool mapSpeculative, bool reduceSpeculative) { string user = conf.Get(MRJobConfig.UserName, UserGroupInformation.GetCurrentUser( ).GetShortUserName()); conf.Set(MRJobConfig.UserName, user); conf.Set(MRJobConfig.MrAmStagingDir, testAbsPath.ToString()); conf.SetBoolean(MRJobConfig.MrAmCreateJhIntermediateBaseDir, true); // TODO: fix the bug where the speculator gets events with // not-fully-constructed objects. For now, disable speculative exec conf.SetBoolean(MRJobConfig.MapSpeculative, mapSpeculative); conf.SetBoolean(MRJobConfig.ReduceSpeculative, reduceSpeculative); Init(conf); Start(); DefaultMetricsSystem.Shutdown(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = GetContext().GetAllJobs().Values .GetEnumerator().Next(); if (assignedQueue != null) { job.SetQueueName(assignedQueue); } // Write job.xml string jobFile = MRApps.GetJobFile(conf, user, TypeConverter.FromYarn(job.GetID() )); Log.Info("Writing job conf to " + jobFile); new FilePath(jobFile).GetParentFile().Mkdirs(); conf.WriteXml(new FileOutputStream(jobFile)); return(job); }
public virtual void TestKillJob() { CountDownLatch latch = new CountDownLatch(1); MRApp app = new TestKill.BlockingMRApp(1, 0, latch); //this will start the job but job won't complete as task is //blocked Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); //wait and vailidate for Job to become RUNNING app.WaitForState(job, JobState.Running); //send the kill signal to Job app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType. JobKill)); //unblock Task latch.CountDown(); //wait and validate for Job to be KILLED app.WaitForState(job, JobState.Killed); IDictionary <TaskId, Task> tasks = job.GetTasks(); NUnit.Framework.Assert.AreEqual("No of tasks is not correct", 1, tasks.Count); Task task = tasks.Values.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual("Task state not correct", TaskState.Killed, task. GetReport().GetTaskState()); IDictionary <TaskAttemptId, TaskAttempt> attempts = tasks.Values.GetEnumerator().Next ().GetAttempts(); NUnit.Framework.Assert.AreEqual("No of attempts is not correct", 1, attempts.Count ); IEnumerator <TaskAttempt> it = attempts.Values.GetEnumerator(); NUnit.Framework.Assert.AreEqual("Attempt state not correct", TaskAttemptState.Killed , it.Next().GetReport().GetTaskAttemptState()); }
private Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job GetJob() { Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job >(); JobId jobId = new JobIdPBImpl(); ApplicationId appId = ApplicationIdPBImpl.NewInstance(Runtime.CurrentTimeMillis() , 4); jobId.SetAppId(appId); jobId.SetId(1); Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId); JobReport report = Org.Mockito.Mockito.Mock <JobReport>(); Org.Mockito.Mockito.When(report.GetStartTime()).ThenReturn(100010L); Org.Mockito.Mockito.When(report.GetFinishTime()).ThenReturn(100015L); Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(report); Org.Mockito.Mockito.When(job.GetName()).ThenReturn("JobName"); Org.Mockito.Mockito.When(job.GetUserName()).ThenReturn("UserName"); Org.Mockito.Mockito.When(job.GetQueueName()).ThenReturn("QueueName"); Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded); Org.Mockito.Mockito.When(job.GetTotalMaps()).ThenReturn(3); Org.Mockito.Mockito.When(job.GetCompletedMaps()).ThenReturn(2); Org.Mockito.Mockito.When(job.GetTotalReduces()).ThenReturn(2); Org.Mockito.Mockito.When(job.GetCompletedReduces()).ThenReturn(1); Org.Mockito.Mockito.When(job.GetCompletedReduces()).ThenReturn(1); return(job); }
public virtual void TestHistoryEvents() { Configuration conf = new Configuration(); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(2, 1, true, this.GetType(). FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString()); app.WaitForState(job, JobState.Succeeded); //make sure all events are flushed app.WaitForState(Service.STATE.Stopped); /* * Use HistoryContext to read logged events and verify the number of * completed maps */ HistoryContext context = new JobHistory(); // test start and stop states ((JobHistory)context).Init(conf); ((JobHistory)context).Start(); NUnit.Framework.Assert.IsTrue(context.GetStartTime() > 0); NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE .Started); // get job before stopping JobHistory Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job parsedJob = context.GetJob(jobId); // stop JobHistory ((JobHistory)context).Stop(); NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE .Stopped); NUnit.Framework.Assert.AreEqual("CompletedMaps not correct", 2, parsedJob.GetCompletedMaps ()); NUnit.Framework.Assert.AreEqual(Runtime.GetProperty("user.name"), parsedJob.GetUserName ()); IDictionary <TaskId, Task> tasks = parsedJob.GetTasks(); NUnit.Framework.Assert.AreEqual("No of tasks not correct", 3, tasks.Count); foreach (Task task in tasks.Values) { VerifyTask(task); } IDictionary <TaskId, Task> maps = parsedJob.GetTasks(TaskType.Map); NUnit.Framework.Assert.AreEqual("No of maps not correct", 2, maps.Count); IDictionary <TaskId, Task> reduces = parsedJob.GetTasks(TaskType.Reduce); NUnit.Framework.Assert.AreEqual("No of reduces not correct", 1, reduces.Count); NUnit.Framework.Assert.AreEqual("CompletedReduce not correct", 1, parsedJob.GetCompletedReduces ()); NUnit.Framework.Assert.AreEqual("Job state not currect", JobState.Succeeded, parsedJob .GetState()); }
public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) { this.id = MRApps.ToString(job.GetID()); JobReport report = job.GetReport(); this.mapsTotal = job.GetTotalMaps(); this.mapsCompleted = job.GetCompletedMaps(); this.reducesTotal = job.GetTotalReduces(); this.reducesCompleted = job.GetCompletedReduces(); this.submitTime = report.GetSubmitTime(); this.startTime = report.GetStartTime(); this.finishTime = report.GetFinishTime(); this.name = job.GetName().ToString(); this.queue = job.GetQueueName(); this.user = job.GetUserName(); this.state = job.GetState().ToString(); this.acls = new AList <ConfEntryInfo>(); if (job is CompletedJob) { avgMapTime = 0l; avgReduceTime = 0l; avgShuffleTime = 0l; avgMergeTime = 0l; failedReduceAttempts = 0; killedReduceAttempts = 0; successfulReduceAttempts = 0; failedMapAttempts = 0; killedMapAttempts = 0; successfulMapAttempts = 0; CountTasksAndAttempts(job); this.uberized = job.IsUber(); this.diagnostics = string.Empty; IList <string> diagnostics = job.GetDiagnostics(); if (diagnostics != null && !diagnostics.IsEmpty()) { StringBuilder b = new StringBuilder(); foreach (string diag in diagnostics) { b.Append(diag); } this.diagnostics = b.ToString(); } IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs(); if (allacls != null) { foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls) { this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString ())); } } } }
public JobInfo(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, bool hasAccess) { // ok for any user to see // these should only be seen if acls allow this.id = MRApps.ToString(job.GetID()); JobReport report = job.GetReport(); this.startTime = report.GetStartTime(); this.finishTime = report.GetFinishTime(); this.elapsedTime = Times.Elapsed(this.startTime, this.finishTime); if (this.elapsedTime == -1) { this.elapsedTime = 0; } this.name = job.GetName().ToString(); this.user = job.GetUserName(); this.state = job.GetState(); this.mapsTotal = job.GetTotalMaps(); this.mapsCompleted = job.GetCompletedMaps(); this.mapProgress = report.GetMapProgress() * 100; this.mapProgressPercent = StringHelper.Percent(report.GetMapProgress()); this.reducesTotal = job.GetTotalReduces(); this.reducesCompleted = job.GetCompletedReduces(); this.reduceProgress = report.GetReduceProgress() * 100; this.reduceProgressPercent = StringHelper.Percent(report.GetReduceProgress()); this.acls = new AList <ConfEntryInfo>(); if (hasAccess) { this.diagnostics = string.Empty; CountTasksAndAttempts(job); this.uberized = job.IsUber(); IList <string> diagnostics = job.GetDiagnostics(); if (diagnostics != null && !diagnostics.IsEmpty()) { StringBuilder b = new StringBuilder(); foreach (string diag in diagnostics) { b.Append(diag); } this.diagnostics = b.ToString(); } IDictionary <JobACL, AccessControlList> allacls = job.GetJobACLs(); if (allacls != null) { foreach (KeyValuePair <JobACL, AccessControlList> entry in allacls) { this.acls.AddItem(new ConfEntryInfo(entry.Key.GetAclName(), entry.Value.GetAclString ())); } } } }
public static IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> NewJobs (ApplicationId appID, int numJobsPerApp, int numTasksPerJob, int numAttemptsPerTask ) { IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> map = Maps.NewHashMap (); for (int j = 0; j < numJobsPerApp; ++j) { Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = NewJob(appID, j, numTasksPerJob, numAttemptsPerTask); map[job.GetID()] = job; } return(map); }
public virtual AMAttemptsInfo GetJobAttempts(string jid) { Init(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = AMWebServices.GetJobFromJobIdString (jid, ctx); AMAttemptsInfo amAttempts = new AMAttemptsInfo(); foreach (AMInfo amInfo in job.GetAMInfos()) { AMAttemptInfo attempt = new AMAttemptInfo(amInfo, MRApps.ToString(job.GetID()), job .GetUserName(), uriInfo.GetBaseUri().ToString(), webapp.Name()); amAttempts.Add(attempt); } return(amAttempts); }
/// <summary>test clean old history files.</summary> /// <remarks> /// test clean old history files. Files should be deleted after 1 week by /// default. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestDeleteFileInfo() { Log.Info("STARTING testDeleteFileInfo"); try { Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType(). FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); HistoryFileManager hfm = new HistoryFileManager(); hfm.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = hfm.GetFileInfo(jobId); hfm.InitExisting(); // wait for move files form the done_intermediate directory to the gone // directory while (fileInfo.IsMovePending()) { Sharpen.Thread.Sleep(300); } NUnit.Framework.Assert.IsNotNull(hfm.jobListCache.Values()); // try to remove fileInfo hfm.Clean(); // check that fileInfo does not deleted NUnit.Framework.Assert.IsFalse(fileInfo.IsDeleted()); // correct live time hfm.SetMaxHistoryAge(-1); hfm.Clean(); hfm.Stop(); NUnit.Framework.Assert.IsTrue("Thread pool shutdown", hfm.moveToDoneExecutor.IsTerminated ()); // should be deleted ! NUnit.Framework.Assert.IsTrue("file should be deleted ", fileInfo.IsDeleted()); } finally { Log.Info("FINISHED testDeleteFileInfo"); } }
/// <exception cref="System.Exception"/> public virtual void TestScanningOldDirs() { Log.Info("STARTING testScanningOldDirs"); try { Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType(). FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString()); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); TestJobHistoryParsing.HistoryFileManagerForTest hfm = new TestJobHistoryParsing.HistoryFileManagerForTest (); hfm.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = hfm.GetFileInfo(jobId); NUnit.Framework.Assert.IsNotNull("Unable to locate job history", fileInfo); // force the manager to "forget" the job hfm.DeleteJobFromJobListCache(fileInfo); int msecPerSleep = 10; int msecToSleep = 10 * 1000; while (fileInfo.IsMovePending() && msecToSleep > 0) { NUnit.Framework.Assert.IsTrue(!fileInfo.DidMoveFail()); msecToSleep -= msecPerSleep; Sharpen.Thread.Sleep(msecPerSleep); } NUnit.Framework.Assert.IsTrue("Timeout waiting for history move", msecToSleep > 0 ); fileInfo = hfm.GetFileInfo(jobId); hfm.Stop(); NUnit.Framework.Assert.IsNotNull("Unable to locate old job history", fileInfo); NUnit.Framework.Assert.IsTrue("HistoryFileManager not shutdown properly", hfm.moveToDoneExecutor .IsTerminated()); } finally { Log.Info("FINISHED testScanningOldDirs"); } }
public virtual void TestSingleCounterView() { AppContext appContext = new MockAppContext(0, 1, 1, 1); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = appContext.GetAllJobs().Values.GetEnumerator ().Next(); // add a failed task to the job without any counters Task failedTask = MockJobs.NewTask(job.GetID(), 2, 1, true); IDictionary <TaskId, Task> tasks = job.GetTasks(); tasks[failedTask.GetID()] = failedTask; IDictionary <string, string> @params = GetJobParams(appContext); @params[AMParams.CounterGroup] = "org.apache.hadoop.mapreduce.FileSystemCounter"; @params[AMParams.CounterName] = "HDFS_WRITE_OPS"; WebAppTests.TestPage <AppContext>(typeof(SingleCounterPage), appContext, @params); }
public virtual void TestJobRebootNotLastRetryOnUnregistrationFailure() { MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(new Configuration()); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); //send an reboot event app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType. JobAmReboot)); // return exteranl state as RUNNING since otherwise the JobClient will // prematurely exit. app.WaitForState(job, JobState.Running); }
public virtual void TestAverageReduceTime() { Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = Org.Mockito.Mockito.Mock <CompletedJob >(); Task task1 = Org.Mockito.Mockito.Mock <Task>(); Task task2 = Org.Mockito.Mockito.Mock <Task>(); JobId jobId = MRBuilderUtils.NewJobId(1L, 1, 1); TaskId taskId1 = MRBuilderUtils.NewTaskId(jobId, 1, TaskType.Reduce); TaskId taskId2 = MRBuilderUtils.NewTaskId(jobId, 2, TaskType.Reduce); TaskAttemptId taskAttemptId1 = MRBuilderUtils.NewTaskAttemptId(taskId1, 1); TaskAttemptId taskAttemptId2 = MRBuilderUtils.NewTaskAttemptId(taskId2, 2); TaskAttempt taskAttempt1 = Org.Mockito.Mockito.Mock <TaskAttempt>(); TaskAttempt taskAttempt2 = Org.Mockito.Mockito.Mock <TaskAttempt>(); JobReport jobReport = Org.Mockito.Mockito.Mock <JobReport>(); Org.Mockito.Mockito.When(taskAttempt1.GetState()).ThenReturn(TaskAttemptState.Succeeded ); Org.Mockito.Mockito.When(taskAttempt1.GetLaunchTime()).ThenReturn(0L); Org.Mockito.Mockito.When(taskAttempt1.GetShuffleFinishTime()).ThenReturn(4L); Org.Mockito.Mockito.When(taskAttempt1.GetSortFinishTime()).ThenReturn(6L); Org.Mockito.Mockito.When(taskAttempt1.GetFinishTime()).ThenReturn(8L); Org.Mockito.Mockito.When(taskAttempt2.GetState()).ThenReturn(TaskAttemptState.Succeeded ); Org.Mockito.Mockito.When(taskAttempt2.GetLaunchTime()).ThenReturn(5L); Org.Mockito.Mockito.When(taskAttempt2.GetShuffleFinishTime()).ThenReturn(10L); Org.Mockito.Mockito.When(taskAttempt2.GetSortFinishTime()).ThenReturn(22L); Org.Mockito.Mockito.When(taskAttempt2.GetFinishTime()).ThenReturn(42L); Org.Mockito.Mockito.When(task1.GetType()).ThenReturn(TaskType.Reduce); Org.Mockito.Mockito.When(task2.GetType()).ThenReturn(TaskType.Reduce); Org.Mockito.Mockito.When(task1.GetAttempts()).ThenReturn(new _Dictionary_120(taskAttemptId1 , taskAttempt1)); Org.Mockito.Mockito.When(task2.GetAttempts()).ThenReturn(new _Dictionary_123(taskAttemptId2 , taskAttempt2)); Org.Mockito.Mockito.When(job.GetTasks()).ThenReturn(new _Dictionary_127(taskId1, task1, taskId2, task2)); Org.Mockito.Mockito.When(job.GetID()).ThenReturn(jobId); Org.Mockito.Mockito.When(job.GetReport()).ThenReturn(jobReport); Org.Mockito.Mockito.When(job.GetName()).ThenReturn("TestJobInfo"); Org.Mockito.Mockito.When(job.GetState()).ThenReturn(JobState.Succeeded); JobInfo jobInfo = new JobInfo(job); NUnit.Framework.Assert.AreEqual(11L, jobInfo.GetAvgReduceTime()); }
public JobCounterInfo(AppContext ctx, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) { GetCounters(ctx, job); counterGroup = new AList <CounterGroupInfo>(); this.id = MRApps.ToString(job.GetID()); if (total != null) { foreach (CounterGroup g in total) { if (g != null) { CounterGroup mg = map == null ? null : map.GetGroup(g.GetName()); CounterGroup rg = reduce == null ? null : reduce.GetGroup(g.GetName()); CounterGroupInfo cginfo = new CounterGroupInfo(g.GetName(), g, mg, rg); counterGroup.AddItem(cginfo); } } } }
public virtual void Setup() { this.conf = new JobConf(); this.conf.Set(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestHsWebServicesAcls.NullGroupsProvider ).FullName); this.conf.SetBoolean(MRConfig.MrAclsEnabled, true); Groups.GetUserToGroupsMappingService(conf); this.ctx = BuildHistoryContext(this.conf); WebApp webApp = Org.Mockito.Mockito.Mock <HsWebApp>(); Org.Mockito.Mockito.When(webApp.Name()).ThenReturn("hsmockwebapp"); this.hsWebServices = new HsWebServices(ctx, conf, webApp); this.hsWebServices.SetResponse(Org.Mockito.Mockito.Mock <HttpServletResponse>()); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = ctx.GetAllJobs().Values.GetEnumerator ().Next(); this.jobIdStr = job.GetID().ToString(); Task task = job.GetTasks().Values.GetEnumerator().Next(); this.taskIdStr = task.GetID().ToString(); this.taskAttemptIdStr = task.GetAttempts().Keys.GetEnumerator().Next().ToString(); }
public virtual void TestHsJobsBlock() { AppContext ctx = Org.Mockito.Mockito.Mock <AppContext>(); IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = new Dictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = GetJob(); jobs[job.GetID()] = job; Org.Mockito.Mockito.When(ctx.GetAllJobs()).ThenReturn(jobs); HsJobsBlock block = new TestBlocks.HsJobsBlockForTest(this, ctx); PrintWriter pWriter = new PrintWriter(data); HtmlBlock.Block html = new BlockForTest(new TestBlocks.HtmlBlockForTest(this), pWriter , 0, false); block.Render(html); pWriter.Flush(); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("JobName")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("UserName")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("QueueName")); NUnit.Framework.Assert.IsTrue(data.ToString().Contains("SUCCEEDED")); }
public virtual void TestJobRebootOnLastRetryOnUnregistrationFailure() { // make startCount as 2 since this is last retry which equals to // DEFAULT_MAX_AM_RETRY // The last param mocks the unregistration failure MRApp app = new MRApp(1, 0, false, this.GetType().FullName, true, 2, false); Configuration conf = new Configuration(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); //send an reboot event app.GetContext().GetEventHandler().Handle(new JobEvent(job.GetID(), JobEventType. JobAmReboot)); app.WaitForInternalState((JobImpl)job, JobStateInternal.Reboot); // return exteranl state as RUNNING if this is the last retry while // unregistration fails app.WaitForState(job, JobState.Running); }
public virtual void TestAssignedQueue() { Configuration conf = new Configuration(); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(2, 1, true, this.GetType(). FullName, true, "assignedQueue"); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString()); app.WaitForState(job, JobState.Succeeded); //make sure all events are flushed app.WaitForState(Service.STATE.Stopped); /* * Use HistoryContext to read logged events and verify the number of * completed maps */ HistoryContext context = new JobHistory(); // test start and stop states ((JobHistory)context).Init(conf); ((JobHistory)context).Start(); NUnit.Framework.Assert.IsTrue(context.GetStartTime() > 0); NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE .Started); // get job before stopping JobHistory Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job parsedJob = context.GetJob(jobId); // stop JobHistory ((JobHistory)context).Stop(); NUnit.Framework.Assert.AreEqual(((JobHistory)context).GetServiceState(), Service.STATE .Stopped); NUnit.Framework.Assert.AreEqual("QueueName not correct", "assignedQueue", parsedJob .GetQueueName()); }
public virtual void TestEventsFlushOnStop() { Configuration conf = new Configuration(); MRApp app = new TestJobHistoryEvents.MRAppWithSpecialHistoryHandler(1, 0, true, this .GetType().FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString()); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); /* * Use HistoryContext to read logged events and verify the number of * completed maps */ HistoryContext context = new JobHistory(); ((JobHistory)context).Init(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job parsedJob = context.GetJob(jobId); NUnit.Framework.Assert.AreEqual("CompletedMaps not correct", 1, parsedJob.GetCompletedMaps ()); IDictionary <TaskId, Task> tasks = parsedJob.GetTasks(); NUnit.Framework.Assert.AreEqual("No of tasks not correct", 1, tasks.Count); VerifyTask(tasks.Values.GetEnumerator().Next()); IDictionary <TaskId, Task> maps = parsedJob.GetTasks(TaskType.Map); NUnit.Framework.Assert.AreEqual("No of maps not correct", 1, maps.Count); NUnit.Framework.Assert.AreEqual("Job state not currect", JobState.Succeeded, parsedJob .GetState()); }
/// <exception cref="System.Exception"/> private void CheckHistoryParsing(int numMaps, int numReduces, int numSuccessfulMaps ) { Configuration conf = new Configuration(); conf.Set(MRJobConfig.UserName, Runtime.GetProperty("user.name")); long amStartTimeEst = Runtime.CurrentTimeMillis(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(numMaps, numReduces, true, this.GetType().FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString()); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); string jobhistoryDir = JobHistoryUtils.GetHistoryIntermediateDoneDirForUser(conf); FileContext fc = null; try { fc = FileContext.GetFileContext(conf); } catch (IOException ioe) { Log.Info("Can not get FileContext", ioe); throw (new Exception("Can not get File Context")); } if (numMaps == numSuccessfulMaps) { string summaryFileName = JobHistoryUtils.GetIntermediateSummaryFileName(jobId); Path summaryFile = new Path(jobhistoryDir, summaryFileName); string jobSummaryString = GetJobSummary(fc, summaryFile); NUnit.Framework.Assert.IsNotNull(jobSummaryString); NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerMap=100")); NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerReduce=100") ); IDictionary <string, string> jobSummaryElements = new Dictionary <string, string>(); StringTokenizer strToken = new StringTokenizer(jobSummaryString, ","); while (strToken.HasMoreTokens()) { string keypair = strToken.NextToken(); jobSummaryElements[keypair.Split("=")[0]] = keypair.Split("=")[1]; } NUnit.Framework.Assert.AreEqual("JobId does not match", jobId.ToString(), jobSummaryElements ["jobId"]); NUnit.Framework.Assert.AreEqual("JobName does not match", "test", jobSummaryElements ["jobName"]); NUnit.Framework.Assert.IsTrue("submitTime should not be 0", long.Parse(jobSummaryElements ["submitTime"]) != 0); NUnit.Framework.Assert.IsTrue("launchTime should not be 0", long.Parse(jobSummaryElements ["launchTime"]) != 0); NUnit.Framework.Assert.IsTrue("firstMapTaskLaunchTime should not be 0", long.Parse (jobSummaryElements["firstMapTaskLaunchTime"]) != 0); NUnit.Framework.Assert.IsTrue("firstReduceTaskLaunchTime should not be 0", long.Parse (jobSummaryElements["firstReduceTaskLaunchTime"]) != 0); NUnit.Framework.Assert.IsTrue("finishTime should not be 0", long.Parse(jobSummaryElements ["finishTime"]) != 0); NUnit.Framework.Assert.AreEqual("Mismatch in num map slots", numSuccessfulMaps, System.Convert.ToInt32 (jobSummaryElements["numMaps"])); NUnit.Framework.Assert.AreEqual("Mismatch in num reduce slots", numReduces, System.Convert.ToInt32 (jobSummaryElements["numReduces"])); NUnit.Framework.Assert.AreEqual("User does not match", Runtime.GetProperty("user.name" ), jobSummaryElements["user"]); NUnit.Framework.Assert.AreEqual("Queue does not match", "default", jobSummaryElements ["queue"]); NUnit.Framework.Assert.AreEqual("Status does not match", "SUCCEEDED", jobSummaryElements ["status"]); } JobHistory jobHistory = new JobHistory(); jobHistory.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId); JobHistoryParser.JobInfo jobInfo; long numFinishedMaps; lock (fileInfo) { Path historyFilePath = fileInfo.GetHistoryFile(); FSDataInputStream @in = null; Log.Info("JobHistoryFile is: " + historyFilePath); try { @in = fc.Open(fc.MakeQualified(historyFilePath)); } catch (IOException ioe) { Log.Info("Can not open history file: " + historyFilePath, ioe); throw (new Exception("Can not open History File")); } JobHistoryParser parser = new JobHistoryParser(@in); EventReader realReader = new EventReader(@in); EventReader reader = Org.Mockito.Mockito.Mock <EventReader>(); if (numMaps == numSuccessfulMaps) { reader = realReader; } else { AtomicInteger numFinishedEvents = new AtomicInteger(0); // Hack! Org.Mockito.Mockito.When(reader.GetNextEvent()).ThenAnswer(new _Answer_257(realReader , numFinishedEvents, numSuccessfulMaps)); } jobInfo = parser.Parse(reader); numFinishedMaps = ComputeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps); if (numFinishedMaps != numMaps) { Exception parseException = parser.GetParseException(); NUnit.Framework.Assert.IsNotNull("Didn't get expected parse exception", parseException ); } } NUnit.Framework.Assert.AreEqual("Incorrect username ", Runtime.GetProperty("user.name" ), jobInfo.GetUsername()); NUnit.Framework.Assert.AreEqual("Incorrect jobName ", "test", jobInfo.GetJobname( )); NUnit.Framework.Assert.AreEqual("Incorrect queuename ", "default", jobInfo.GetJobQueueName ()); NUnit.Framework.Assert.AreEqual("incorrect conf path", "test", jobInfo.GetJobConfPath ()); NUnit.Framework.Assert.AreEqual("incorrect finishedMap ", numSuccessfulMaps, numFinishedMaps ); NUnit.Framework.Assert.AreEqual("incorrect finishedReduces ", numReduces, jobInfo .GetFinishedReduces()); NUnit.Framework.Assert.AreEqual("incorrect uberized ", job.IsUber(), jobInfo.GetUberized ()); IDictionary <TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.GetAllTasks(); int totalTasks = allTasks.Count; NUnit.Framework.Assert.AreEqual("total number of tasks is incorrect ", (numMaps + numReduces), totalTasks); // Verify aminfo NUnit.Framework.Assert.AreEqual(1, jobInfo.GetAMInfos().Count); NUnit.Framework.Assert.AreEqual(MRApp.NmHost, jobInfo.GetAMInfos()[0].GetNodeManagerHost ()); JobHistoryParser.AMInfo amInfo = jobInfo.GetAMInfos()[0]; NUnit.Framework.Assert.AreEqual(MRApp.NmPort, amInfo.GetNodeManagerPort()); NUnit.Framework.Assert.AreEqual(MRApp.NmHttpPort, amInfo.GetNodeManagerHttpPort() ); NUnit.Framework.Assert.AreEqual(1, amInfo.GetAppAttemptId().GetAttemptId()); NUnit.Framework.Assert.AreEqual(amInfo.GetAppAttemptId(), amInfo.GetContainerId() .GetApplicationAttemptId()); NUnit.Framework.Assert.IsTrue(amInfo.GetStartTime() <= Runtime.CurrentTimeMillis( ) && amInfo.GetStartTime() >= amStartTimeEst); ContainerId fakeCid = MRApp.NewContainerId(-1, -1, -1, -1); // Assert at taskAttempt level foreach (JobHistoryParser.TaskInfo taskInfo in allTasks.Values) { int taskAttemptCount = taskInfo.GetAllTaskAttempts().Count; NUnit.Framework.Assert.AreEqual("total number of task attempts ", 1, taskAttemptCount ); JobHistoryParser.TaskAttemptInfo taInfo = taskInfo.GetAllTaskAttempts().Values.GetEnumerator ().Next(); NUnit.Framework.Assert.IsNotNull(taInfo.GetContainerId()); // Verify the wrong ctor is not being used. Remove after mrv1 is removed. NUnit.Framework.Assert.IsFalse(taInfo.GetContainerId().Equals(fakeCid)); } // Deep compare Job and JobInfo foreach (Task task in job.GetTasks().Values) { JobHistoryParser.TaskInfo taskInfo_1 = allTasks[TypeConverter.FromYarn(task.GetID ())]; NUnit.Framework.Assert.IsNotNull("TaskInfo not found", taskInfo_1); foreach (TaskAttempt taskAttempt in task.GetAttempts().Values) { JobHistoryParser.TaskAttemptInfo taskAttemptInfo = taskInfo_1.GetAllTaskAttempts( )[TypeConverter.FromYarn((taskAttempt.GetID()))]; NUnit.Framework.Assert.IsNotNull("TaskAttemptInfo not found", taskAttemptInfo); NUnit.Framework.Assert.AreEqual("Incorrect shuffle port for task attempt", taskAttempt .GetShufflePort(), taskAttemptInfo.GetShufflePort()); if (numMaps == numSuccessfulMaps) { NUnit.Framework.Assert.AreEqual(MRApp.NmHost, taskAttemptInfo.GetHostname()); NUnit.Framework.Assert.AreEqual(MRApp.NmPort, taskAttemptInfo.GetPort()); // Verify rack-name NUnit.Framework.Assert.AreEqual("rack-name is incorrect", taskAttemptInfo.GetRackname (), RackName); } } } // test output for HistoryViewer TextWriter stdps = System.Console.Out; try { Runtime.SetOut(new TextWriter(outContent)); HistoryViewer viewer; lock (fileInfo) { viewer = new HistoryViewer(fc.MakeQualified(fileInfo.GetHistoryFile()).ToString() , conf, true); } viewer.Print(); foreach (JobHistoryParser.TaskInfo taskInfo_1 in allTasks.Values) { string test = (taskInfo_1.GetTaskStatus() == null ? string.Empty : taskInfo_1.GetTaskStatus ()) + " " + taskInfo_1.GetTaskType() + " task list for " + taskInfo_1.GetTaskId( ).GetJobID(); NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(test) > 0); NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(taskInfo_1.GetTaskId( ).ToString()) > 0); } } finally { Runtime.SetOut(stdps); } }
public virtual void Test() { TestMRClientService.MRAppWithClientService app = new TestMRClientService.MRAppWithClientService (this, 1, 0, false); Configuration conf = new Configuration(); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 1, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task task = it.Next(); app.WaitForState(task, TaskState.Running); TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next(); app.WaitForState(attempt, TaskAttemptState.Running); // send the diagnostic string diagnostic1 = "Diagnostic1"; string diagnostic2 = "Diagnostic2"; app.GetContext().GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attempt .GetID(), diagnostic1)); // send the status update TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus (); taskAttemptStatus.id = attempt.GetID(); taskAttemptStatus.progress = 0.5f; taskAttemptStatus.stateString = "RUNNING"; taskAttemptStatus.taskState = TaskAttemptState.Running; taskAttemptStatus.phase = Phase.Map; // send the status update app.GetContext().GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(attempt .GetID(), taskAttemptStatus)); //verify that all object are fully populated by invoking RPCs. YarnRPC rpc = YarnRPC.Create(conf); MRClientProtocol proxy = (MRClientProtocol)rpc.GetProxy(typeof(MRClientProtocol), app.clientService.GetBindAddress(), conf); GetCountersRequest gcRequest = recordFactory.NewRecordInstance <GetCountersRequest >(); gcRequest.SetJobId(job.GetID()); NUnit.Framework.Assert.IsNotNull("Counters is null", proxy.GetCounters(gcRequest) .GetCounters()); GetJobReportRequest gjrRequest = recordFactory.NewRecordInstance <GetJobReportRequest >(); gjrRequest.SetJobId(job.GetID()); JobReport jr = proxy.GetJobReport(gjrRequest).GetJobReport(); VerifyJobReport(jr); GetTaskAttemptCompletionEventsRequest gtaceRequest = recordFactory.NewRecordInstance <GetTaskAttemptCompletionEventsRequest>(); gtaceRequest.SetJobId(job.GetID()); gtaceRequest.SetFromEventId(0); gtaceRequest.SetMaxEvents(10); NUnit.Framework.Assert.IsNotNull("TaskCompletionEvents is null", proxy.GetTaskAttemptCompletionEvents (gtaceRequest).GetCompletionEventList()); GetDiagnosticsRequest gdRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest >(); gdRequest.SetTaskAttemptId(attempt.GetID()); NUnit.Framework.Assert.IsNotNull("Diagnostics is null", proxy.GetDiagnostics(gdRequest ).GetDiagnosticsList()); GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest >(); gtarRequest.SetTaskAttemptId(attempt.GetID()); TaskAttemptReport tar = proxy.GetTaskAttemptReport(gtarRequest).GetTaskAttemptReport (); VerifyTaskAttemptReport(tar); GetTaskReportRequest gtrRequest = recordFactory.NewRecordInstance <GetTaskReportRequest >(); gtrRequest.SetTaskId(task.GetID()); NUnit.Framework.Assert.IsNotNull("TaskReport is null", proxy.GetTaskReport(gtrRequest ).GetTaskReport()); GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest >(); gtreportsRequest.SetJobId(job.GetID()); gtreportsRequest.SetTaskType(TaskType.Map); NUnit.Framework.Assert.IsNotNull("TaskReports for map is null", proxy.GetTaskReports (gtreportsRequest).GetTaskReportList()); gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>(); gtreportsRequest.SetJobId(job.GetID()); gtreportsRequest.SetTaskType(TaskType.Reduce); NUnit.Framework.Assert.IsNotNull("TaskReports for reduce is null", proxy.GetTaskReports (gtreportsRequest).GetTaskReportList()); IList <string> diag = proxy.GetDiagnostics(gdRequest).GetDiagnosticsList(); NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, diag.Count); NUnit.Framework.Assert.AreEqual("Diag 1 not correct", diagnostic1, diag[0].ToString ()); TaskReport taskReport = proxy.GetTaskReport(gtrRequest).GetTaskReport(); NUnit.Framework.Assert.AreEqual("Num diagnostics not correct", 1, taskReport.GetDiagnosticsCount ()); //send the done signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task.GetAttempts() .Values.GetEnumerator().Next().GetID(), TaskAttemptEventType.TaDone)); app.WaitForState(job, JobState.Succeeded); // For invalid jobid, throw IOException gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest>(); gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001" ))); gtreportsRequest.SetTaskType(TaskType.Reduce); try { proxy.GetTaskReports(gtreportsRequest); NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id"); } catch (IOException) { } }
/// <exception cref="System.Exception"/> public virtual void TestHistoryParsingForFailedAttempts() { Log.Info("STARTING testHistoryParsingForFailedAttempts"); try { Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryParsing.MRAppWithHistoryWithFailedAttempt(2, 1, true , this.GetType().FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); JobHistory jobHistory = new JobHistory(); jobHistory.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId); JobHistoryParser parser; JobHistoryParser.JobInfo jobInfo; lock (fileInfo) { Path historyFilePath = fileInfo.GetHistoryFile(); FSDataInputStream @in = null; FileContext fc = null; try { fc = FileContext.GetFileContext(conf); @in = fc.Open(fc.MakeQualified(historyFilePath)); } catch (IOException ioe) { Log.Info("Can not open history file: " + historyFilePath, ioe); throw (new Exception("Can not open History File")); } parser = new JobHistoryParser(@in); jobInfo = parser.Parse(); } Exception parseException = parser.GetParseException(); NUnit.Framework.Assert.IsNull("Caught an expected exception " + parseException, parseException ); int noOffailedAttempts = 0; IDictionary <TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.GetAllTasks(); foreach (Task task in job.GetTasks().Values) { JobHistoryParser.TaskInfo taskInfo = allTasks[TypeConverter.FromYarn(task.GetID() )]; foreach (TaskAttempt taskAttempt in task.GetAttempts().Values) { JobHistoryParser.TaskAttemptInfo taskAttemptInfo = taskInfo.GetAllTaskAttempts()[ TypeConverter.FromYarn((taskAttempt.GetID()))]; // Verify rack-name for all task attempts NUnit.Framework.Assert.AreEqual("rack-name is incorrect", taskAttemptInfo.GetRackname (), RackName); if (taskAttemptInfo.GetTaskStatus().Equals("FAILED")) { noOffailedAttempts++; } } } NUnit.Framework.Assert.AreEqual("No of Failed tasks doesn't match.", 2, noOffailedAttempts ); } finally { Log.Info("FINISHED testHistoryParsingForFailedAttempts"); } }
/// <exception cref="System.Exception"/> public virtual void TestDiagnosticsForKilledJob() { Log.Info("STARTING testDiagnosticsForKilledJob"); try { Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryParsing.MRAppWithHistoryWithJobKilled(2, 1, true, this .GetType().FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); app.WaitForState(job, JobState.Killed); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); JobHistory jobHistory = new JobHistory(); jobHistory.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId); JobHistoryParser parser; JobHistoryParser.JobInfo jobInfo; lock (fileInfo) { Path historyFilePath = fileInfo.GetHistoryFile(); FSDataInputStream @in = null; FileContext fc = null; try { fc = FileContext.GetFileContext(conf); @in = fc.Open(fc.MakeQualified(historyFilePath)); } catch (IOException ioe) { Log.Info("Can not open history file: " + historyFilePath, ioe); throw (new Exception("Can not open History File")); } parser = new JobHistoryParser(@in); jobInfo = parser.Parse(); } Exception parseException = parser.GetParseException(); NUnit.Framework.Assert.IsNull("Caught an expected exception " + parseException, parseException ); IList <string> originalDiagnostics = job.GetDiagnostics(); string historyError = jobInfo.GetErrorInfo(); NUnit.Framework.Assert.IsTrue("No original diagnostics for a failed job", originalDiagnostics != null && !originalDiagnostics.IsEmpty()); NUnit.Framework.Assert.IsNotNull("No history error info for a failed job ", historyError ); foreach (string diagString in originalDiagnostics) { NUnit.Framework.Assert.IsTrue(historyError.Contains(diagString)); } NUnit.Framework.Assert.IsTrue("No killed message in diagnostics", historyError.Contains (JobImpl.JobKilledDiag)); } finally { Log.Info("FINISHED testDiagnosticsForKilledJob"); } }
//Test reports of JobHistoryServer. History server should get log files from MRApp and read them /// <exception cref="System.Exception"/> public virtual void TestReports() { Configuration config = new Configuration(); config.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey , typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(config); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType(). FullName, true); app.Submit(config); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); app.WaitForState(job, JobState.Succeeded); historyServer = new JobHistoryServer(); historyServer.Init(config); historyServer.Start(); // search JobHistory service JobHistory jobHistory = null; foreach (Org.Apache.Hadoop.Service.Service service in historyServer.GetServices()) { if (service is JobHistory) { jobHistory = (JobHistory)service; } } IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobs = jobHistory. GetAllJobs(); NUnit.Framework.Assert.AreEqual(1, jobs.Count); NUnit.Framework.Assert.AreEqual("job_0_0000", jobs.Keys.GetEnumerator().Next().ToString ()); Task task = job.GetTasks().Values.GetEnumerator().Next(); TaskAttempt attempt = task.GetAttempts().Values.GetEnumerator().Next(); HistoryClientService historyService = historyServer.GetClientService(); MRClientProtocol protocol = historyService.GetClientHandler(); GetTaskAttemptReportRequest gtarRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest >(); // test getTaskAttemptReport TaskAttemptId taId = attempt.GetID(); taId.SetTaskId(task.GetID()); taId.GetTaskId().SetJobId(job.GetID()); gtarRequest.SetTaskAttemptId(taId); GetTaskAttemptReportResponse response = protocol.GetTaskAttemptReport(gtarRequest ); NUnit.Framework.Assert.AreEqual("container_0_0000_01_000000", response.GetTaskAttemptReport ().GetContainerId().ToString()); NUnit.Framework.Assert.IsTrue(response.GetTaskAttemptReport().GetDiagnosticInfo() .IsEmpty()); // counters NUnit.Framework.Assert.IsNotNull(response.GetTaskAttemptReport().GetCounters().GetCounter (TaskCounter.PhysicalMemoryBytes)); NUnit.Framework.Assert.AreEqual(taId.ToString(), response.GetTaskAttemptReport(). GetTaskAttemptId().ToString()); // test getTaskReport GetTaskReportRequest request = recordFactory.NewRecordInstance <GetTaskReportRequest >(); TaskId taskId = task.GetID(); taskId.SetJobId(job.GetID()); request.SetTaskId(taskId); GetTaskReportResponse reportResponse = protocol.GetTaskReport(request); NUnit.Framework.Assert.AreEqual(string.Empty, reportResponse.GetTaskReport().GetDiagnosticsList ().GetEnumerator().Next()); // progress NUnit.Framework.Assert.AreEqual(1.0f, reportResponse.GetTaskReport().GetProgress( ), 0.01); // report has corrected taskId NUnit.Framework.Assert.AreEqual(taskId.ToString(), reportResponse.GetTaskReport() .GetTaskId().ToString()); // Task state should be SUCCEEDED NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, reportResponse.GetTaskReport ().GetTaskState()); // For invalid jobid, throw IOException GetTaskReportsRequest gtreportsRequest = recordFactory.NewRecordInstance <GetTaskReportsRequest >(); gtreportsRequest.SetJobId(TypeConverter.ToYarn(JobID.ForName("job_1415730144495_0001" ))); gtreportsRequest.SetTaskType(TaskType.Reduce); try { protocol.GetTaskReports(gtreportsRequest); NUnit.Framework.Assert.Fail("IOException not thrown for invalid job id"); } catch (IOException) { } // Expected // test getTaskAttemptCompletionEvents GetTaskAttemptCompletionEventsRequest taskAttemptRequest = recordFactory.NewRecordInstance <GetTaskAttemptCompletionEventsRequest>(); taskAttemptRequest.SetJobId(job.GetID()); GetTaskAttemptCompletionEventsResponse taskAttemptCompletionEventsResponse = protocol .GetTaskAttemptCompletionEvents(taskAttemptRequest); NUnit.Framework.Assert.AreEqual(0, taskAttemptCompletionEventsResponse.GetCompletionEventCount ()); // test getDiagnostics GetDiagnosticsRequest diagnosticRequest = recordFactory.NewRecordInstance <GetDiagnosticsRequest >(); diagnosticRequest.SetTaskAttemptId(taId); GetDiagnosticsResponse diagnosticResponse = protocol.GetDiagnostics(diagnosticRequest ); // it is strange : why one empty string ? NUnit.Framework.Assert.AreEqual(1, diagnosticResponse.GetDiagnosticsCount()); NUnit.Framework.Assert.AreEqual(string.Empty, diagnosticResponse.GetDiagnostics(0 )); }
/// <exception cref="System.IO.IOException"/> private void CheckAccess(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job, JobACL jobOperation ) { UserGroupInformation callerUGI; callerUGI = UserGroupInformation.GetCurrentUser(); if (!job.CheckAccess(callerUGI, jobOperation)) { throw new IOException(new AccessControlException("User " + callerUGI.GetShortUserName () + " cannot perform operation " + jobOperation.ToString() + " on " + job.GetID ())); } }
public virtual JobId GetID() { return(mockJob.GetID()); }
/// <exception cref="System.IO.IOException"/> public MockCompletedJob(Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job) : base(new Configuration(), job.GetID(), null, true, job.GetUserName(), null, null ) { this.job = job; }
public override JobId GetID() { return(job.GetID()); }
protected override void Render(HtmlBlock.Block html) { if (job == null) { html.P().("Sorry, no counters for nonexistent", $(AMParams.JobId, "job")).(); return; } if (!$(AMParams.TaskId).IsEmpty() && task == null) { html.P().("Sorry, no counters for nonexistent", $(AMParams.TaskId, "task")).(); return; } if (total == null || total.GetGroupNames() == null || total.CountCounters() == 0) { string type = $(AMParams.TaskId); if (type == null || type.IsEmpty()) { type = $(AMParams.JobId, "the job"); } html.P().("Sorry it looks like ", type, " has no counters.").(); return; } string urlBase; string urlId; if (task != null) { urlBase = "singletaskcounter"; urlId = MRApps.ToString(task.GetID()); } else { urlBase = "singlejobcounter"; urlId = MRApps.ToString(job.GetID()); } int numGroups = 0; Hamlet.TBODY <Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > tbody = html.Div(JQueryUI.InfoWrap).Table("#counters").Thead().Tr().Th(".group.ui-state-default" , "Counter Group").Th(".ui-state-default", "Counters").().().Tbody(); foreach (CounterGroup g in total) { CounterGroup mg = map == null ? null : map.GetGroup(g.GetName()); CounterGroup rg = reduce == null ? null : reduce.GetGroup(g.GetName()); ++numGroups; // This is mostly for demonstration :) Typically we'd introduced // a CounterGroup block to reduce the verbosity. OTOH, this // serves as an indicator of where we're in the tag hierarchy. Hamlet.TR <Hamlet.THEAD <Hamlet.TABLE <Hamlet.TD <Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > > > > > > groupHeadRow = tbody .Tr().Th().$title(g.GetName()).$class("ui-state-default").(FixGroupDisplayName(g .GetDisplayName())).().Td().$class(JQueryUI.CTable).Table(".dt-counters").$id(job .GetID() + "." + g.GetName()).Thead().Tr().Th(".name", "Name"); if (map != null) { groupHeadRow.Th("Map").Th("Reduce"); } // Ditto Hamlet.TBODY <Hamlet.TABLE <Hamlet.TD <Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > > > > > group = groupHeadRow.Th(map == null ? "Value" : "Total").().().Tbody(); foreach (Counter counter in g) { // Ditto Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Hamlet.TD <Hamlet.TR <Hamlet.TBODY <Hamlet.TABLE <Hamlet.DIV <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > > > > > > > groupRow = group .Tr(); if (task == null && mg == null && rg == null) { groupRow.Td().$title(counter.GetName()).(counter.GetDisplayName()).(); } else { groupRow.Td().$title(counter.GetName()).A(Url(urlBase, urlId, g.GetName(), counter .GetName()), counter.GetDisplayName()).(); } if (map != null) { Counter mc = mg == null ? null : mg.FindCounter(counter.GetName()); Counter rc = rg == null ? null : rg.FindCounter(counter.GetName()); groupRow.Td(mc == null ? "0" : string.Format("%,d", mc.GetValue())).Td(rc == null ? "0" : string.Format("%,d", rc.GetValue())); } groupRow.Td(string.Format("%,d", counter.GetValue())).(); } group.().().().(); } tbody.().().(); }
public virtual void TestUpdatedNodes() { int runCount = 0; MRApp app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName , true, ++runCount); Configuration conf = new Configuration(); // after half of the map completion, reduce will start conf.SetFloat(MRJobConfig.CompletedMapsForReduceSlowstart, 0.5f); // uberization forces full slowstart (1.0), so disable that conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("Num tasks not correct", 4, job.GetTasks().Count); IEnumerator <Task> it = job.GetTasks().Values.GetEnumerator(); Task mapTask1 = it.Next(); Task mapTask2 = it.Next(); // all maps must be running app.WaitForState(mapTask1, TaskState.Running); app.WaitForState(mapTask2, TaskState.Running); TaskAttempt task1Attempt = mapTask1.GetAttempts().Values.GetEnumerator().Next(); TaskAttempt task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next(); NodeId node1 = task1Attempt.GetNodeId(); NodeId node2 = task2Attempt.GetNodeId(); NUnit.Framework.Assert.AreEqual(node1, node2); // send the done signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID (), TaskAttemptEventType.TaDone)); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID (), TaskAttemptEventType.TaDone)); // all maps must be succeeded app.WaitForState(mapTask1, TaskState.Succeeded); app.WaitForState(mapTask2, TaskState.Succeeded); TaskAttemptCompletionEvent[] events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for success", 2, events .Length); // send updated nodes info AList <NodeReport> updatedNodes = new AList <NodeReport>(); NodeReport nr = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <NodeReport >(); nr.SetNodeId(node1); nr.SetNodeState(NodeState.Unhealthy); updatedNodes.AddItem(nr); app.GetContext().GetEventHandler().Handle(new JobUpdatedNodesEvent(job.GetID(), updatedNodes )); app.WaitForState(task1Attempt, TaskAttemptState.Killed); app.WaitForState(task2Attempt, TaskAttemptState.Killed); events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for killed", 4, events.Length); // all maps must be back to running app.WaitForState(mapTask1, TaskState.Running); app.WaitForState(mapTask2, TaskState.Running); IEnumerator <TaskAttempt> itr = mapTask1.GetAttempts().Values.GetEnumerator(); itr.Next(); task1Attempt = itr.Next(); // send the done signal to the task app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task1Attempt.GetID (), TaskAttemptEventType.TaDone)); // map1 must be succeeded. map2 must be running app.WaitForState(mapTask1, TaskState.Succeeded); app.WaitForState(mapTask2, TaskState.Running); events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success", 5, events.Length); // Crash the app again. app.Stop(); // rerun // in rerun the 1st map will be recovered from previous run app = new TestMRApp.MRAppWithHistory(this, 2, 2, false, this.GetType().FullName, false, ++runCount); conf = new Configuration(); conf.SetBoolean(MRJobConfig.MrAmJobRecoveryEnable, true); conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); job = app.Submit(conf); app.WaitForState(job, JobState.Running); NUnit.Framework.Assert.AreEqual("No of tasks not correct", 4, job.GetTasks().Count ); it = job.GetTasks().Values.GetEnumerator(); mapTask1 = it.Next(); mapTask2 = it.Next(); Task reduceTask1 = it.Next(); Task reduceTask2 = it.Next(); // map 1 will be recovered, no need to send done app.WaitForState(mapTask1, TaskState.Succeeded); app.WaitForState(mapTask2, TaskState.Running); events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Expecting 2 completion events for killed & success of map1" , 2, events.Length); task2Attempt = mapTask2.GetAttempts().Values.GetEnumerator().Next(); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task2Attempt.GetID (), TaskAttemptEventType.TaDone)); app.WaitForState(mapTask2, TaskState.Succeeded); events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Expecting 1 more completion events for success", 3, events.Length); app.WaitForState(reduceTask1, TaskState.Running); app.WaitForState(reduceTask2, TaskState.Running); TaskAttempt task3Attempt = reduceTask1.GetAttempts().Values.GetEnumerator().Next( ); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID (), TaskAttemptEventType.TaDone)); app.WaitForState(reduceTask1, TaskState.Succeeded); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task3Attempt.GetID (), TaskAttemptEventType.TaKill)); app.WaitForState(reduceTask1, TaskState.Succeeded); TaskAttempt task4Attempt = reduceTask2.GetAttempts().Values.GetEnumerator().Next( ); app.GetContext().GetEventHandler().Handle(new TaskAttemptEvent(task4Attempt.GetID (), TaskAttemptEventType.TaDone)); app.WaitForState(reduceTask2, TaskState.Succeeded); events = job.GetTaskAttemptCompletionEvents(0, 100); NUnit.Framework.Assert.AreEqual("Expecting 2 more completion events for reduce success" , 5, events.Length); // job succeeds app.WaitForState(job, JobState.Succeeded); }