/* Verify some expected values based on the history file */ /// <exception cref="System.Exception"/> public virtual void TestCompletedJob() { HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath); //Re-initialize to verify the delayed load. completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager); //Verify tasks loaded based on loadTask parameter. NUnit.Framework.Assert.AreEqual(loadTasks, completedJob.tasksLoaded.Get()); NUnit.Framework.Assert.AreEqual(1, completedJob.GetAMInfos().Count); NUnit.Framework.Assert.AreEqual(10, completedJob.GetCompletedMaps()); NUnit.Framework.Assert.AreEqual(1, completedJob.GetCompletedReduces()); NUnit.Framework.Assert.AreEqual(12, completedJob.GetTasks().Count); //Verify tasks loaded at this point. NUnit.Framework.Assert.AreEqual(true, completedJob.tasksLoaded.Get()); NUnit.Framework.Assert.AreEqual(10, completedJob.GetTasks(TaskType.Map).Count); NUnit.Framework.Assert.AreEqual(2, completedJob.GetTasks(TaskType.Reduce).Count); NUnit.Framework.Assert.AreEqual("user", completedJob.GetUserName()); NUnit.Framework.Assert.AreEqual(JobState.Succeeded, completedJob.GetState()); JobReport jobReport = completedJob.GetReport(); NUnit.Framework.Assert.AreEqual("user", jobReport.GetUser()); NUnit.Framework.Assert.AreEqual(JobState.Succeeded, jobReport.GetJobState()); }
/// <exception cref="System.Exception"/> public virtual void TestCompletedTask() { HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath); completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager); TaskId mt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map); TaskId rt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Reduce); IDictionary <TaskId, Task> mapTasks = completedJob.GetTasks(TaskType.Map); IDictionary <TaskId, Task> reduceTasks = completedJob.GetTasks(TaskType.Reduce); NUnit.Framework.Assert.AreEqual(10, mapTasks.Count); NUnit.Framework.Assert.AreEqual(2, reduceTasks.Count); Task mt1 = mapTasks[mt1Id]; NUnit.Framework.Assert.AreEqual(1, mt1.GetAttempts().Count); NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mt1.GetState()); TaskReport mt1Report = mt1.GetReport(); NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mt1Report.GetTaskState()); NUnit.Framework.Assert.AreEqual(mt1Id, mt1Report.GetTaskId()); Task rt1 = reduceTasks[rt1Id]; NUnit.Framework.Assert.AreEqual(1, rt1.GetAttempts().Count); NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, rt1.GetState()); TaskReport rt1Report = rt1.GetReport(); NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, rt1Report.GetTaskState()); NUnit.Framework.Assert.AreEqual(rt1Id, rt1Report.GetTaskId()); }
/// <summary> /// Trivial test case that verifies basic functionality of /// <see cref="JobIdHistoryFileInfoMap"/> /// </summary> /// <exception cref="System.Exception"/> public virtual void TestWithSingleElement() { HistoryFileManager.JobIdHistoryFileInfoMap mapWithSize = new HistoryFileManager.JobIdHistoryFileInfoMap (); JobId jobId = MRBuilderUtils.NewJobId(1, 1, 1); HistoryFileManager.HistoryFileInfo fileInfo1 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(fileInfo1.GetJobId()).ThenReturn(jobId); // add it twice NUnit.Framework.Assert.AreEqual("Incorrect return on putIfAbsent()", null, mapWithSize .PutIfAbsent(jobId, fileInfo1)); NUnit.Framework.Assert.AreEqual("Incorrect return on putIfAbsent()", fileInfo1, mapWithSize .PutIfAbsent(jobId, fileInfo1)); // check get() NUnit.Framework.Assert.AreEqual("Incorrect get()", fileInfo1, mapWithSize.Get(jobId )); NUnit.Framework.Assert.IsTrue("Incorrect size()", CheckSize(mapWithSize, 1)); // check navigableKeySet() NavigableSet <JobId> set = mapWithSize.NavigableKeySet(); NUnit.Framework.Assert.AreEqual("Incorrect navigableKeySet()", 1, set.Count); NUnit.Framework.Assert.IsTrue("Incorrect navigableKeySet()", set.Contains(jobId)); // check values() ICollection <HistoryFileManager.HistoryFileInfo> values = mapWithSize.Values(); NUnit.Framework.Assert.AreEqual("Incorrect values()", 1, values.Count); NUnit.Framework.Assert.IsTrue("Incorrect values()", values.Contains(fileInfo1)); }
public virtual void TestAddExisting() { HistoryFileManager.JobListCache cache = new HistoryFileManager.JobListCache(2, 1000 ); JobId jobId = MRBuilderUtils.NewJobId(1, 1, 1); HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(fileInfo.GetJobId()).ThenReturn(jobId); cache.AddIfAbsent(fileInfo); cache.AddIfAbsent(fileInfo); NUnit.Framework.Assert.AreEqual("Incorrect number of cache entries", 1, cache.Values ().Count); }
/// <exception cref="System.IO.IOException"/> public CompletedJob(Configuration conf, JobId jobId, Path historyFile, bool loadTasks , string userName, HistoryFileManager.HistoryFileInfo info, JobACLsManager aclsMgr ) { //Can be picked from JobInfo with a conversion. //Can be picked up from JobInfo Log.Info("Loading job: " + jobId + " from file: " + historyFile); this.conf = conf; this.jobId = jobId; this.user = userName; this.info = info; this.aclsMgr = aclsMgr; LoadFullHistoryData(loadTasks, historyFile); }
/// <exception cref="System.Exception"/> public virtual void TestCopmletedJobReportWithZeroTasks() { HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath); completedJob = new CompletedJob(conf, jobId, fullHistoryPathZeroReduces, loadTasks , "user", info, jobAclsManager); JobReport jobReport = completedJob.GetReport(); // Make sure that the number reduces (completed and total) are equal to zero. NUnit.Framework.Assert.AreEqual(0, completedJob.GetTotalReduces()); NUnit.Framework.Assert.AreEqual(0, completedJob.GetCompletedReduces()); // Verify that the reduce progress is 1.0 (not NaN) NUnit.Framework.Assert.AreEqual(1.0, jobReport.GetReduceProgress(), 0.001); }
/// <summary>test clean old history files.</summary> /// <remarks> /// test clean old history files. Files should be deleted after 1 week by /// default. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestDeleteFileInfo() { Log.Info("STARTING testDeleteFileInfo"); try { Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType(). FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); HistoryFileManager hfm = new HistoryFileManager(); hfm.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = hfm.GetFileInfo(jobId); hfm.InitExisting(); // wait for move files form the done_intermediate directory to the gone // directory while (fileInfo.IsMovePending()) { Sharpen.Thread.Sleep(300); } NUnit.Framework.Assert.IsNotNull(hfm.jobListCache.Values()); // try to remove fileInfo hfm.Clean(); // check that fileInfo does not deleted NUnit.Framework.Assert.IsFalse(fileInfo.IsDeleted()); // correct live time hfm.SetMaxHistoryAge(-1); hfm.Clean(); hfm.Stop(); NUnit.Framework.Assert.IsTrue("Thread pool shutdown", hfm.moveToDoneExecutor.IsTerminated ()); // should be deleted ! NUnit.Framework.Assert.IsTrue("file should be deleted ", fileInfo.IsDeleted()); } finally { Log.Info("FINISHED testDeleteFileInfo"); } }
/// <exception cref="System.Exception"/> public virtual void TestScanningOldDirs() { Log.Info("STARTING testScanningOldDirs"); try { Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType(). FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString()); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); TestJobHistoryParsing.HistoryFileManagerForTest hfm = new TestJobHistoryParsing.HistoryFileManagerForTest (); hfm.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = hfm.GetFileInfo(jobId); NUnit.Framework.Assert.IsNotNull("Unable to locate job history", fileInfo); // force the manager to "forget" the job hfm.DeleteJobFromJobListCache(fileInfo); int msecPerSleep = 10; int msecToSleep = 10 * 1000; while (fileInfo.IsMovePending() && msecToSleep > 0) { NUnit.Framework.Assert.IsTrue(!fileInfo.DidMoveFail()); msecToSleep -= msecPerSleep; Sharpen.Thread.Sleep(msecPerSleep); } NUnit.Framework.Assert.IsTrue("Timeout waiting for history move", msecToSleep > 0 ); fileInfo = hfm.GetFileInfo(jobId); hfm.Stop(); NUnit.Framework.Assert.IsNotNull("Unable to locate old job history", fileInfo); NUnit.Framework.Assert.IsTrue("HistoryFileManager not shutdown properly", hfm.moveToDoneExecutor .IsTerminated()); } finally { Log.Info("FINISHED testScanningOldDirs"); } }
public virtual void TestHistoryFileInfoSummaryFileNotExist() { TestHistoryFileManager.HistoryFileManagerTest hmTest = new TestHistoryFileManager.HistoryFileManagerTest (); string job = "job_1410889000000_123456"; Path summaryFile = new Path(job + ".summary"); JobIndexInfo jobIndexInfo = new JobIndexInfo(); jobIndexInfo.SetJobId(TypeConverter.ToYarn(JobID.ForName(job))); Configuration conf = dfsCluster.GetConfiguration(0); conf.Set(JHAdminConfig.MrHistoryDoneDir, "/" + UUID.RandomUUID()); conf.Set(JHAdminConfig.MrHistoryIntermediateDoneDir, "/" + UUID.RandomUUID()); hmTest.ServiceInit(conf); HistoryFileManager.HistoryFileInfo info = hmTest.GetHistoryFileInfo(null, null, summaryFile , jobIndexInfo, false); info.MoveToDone(); NUnit.Framework.Assert.IsFalse(info.DidMoveFail()); }
/// <exception cref="System.Exception"/> public virtual void TestEviction() { int maxSize = 2; HistoryFileManager.JobListCache cache = new HistoryFileManager.JobListCache(maxSize , 1000); JobId jobId1 = MRBuilderUtils.NewJobId(1, 1, 1); HistoryFileManager.HistoryFileInfo fileInfo1 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(fileInfo1.GetJobId()).ThenReturn(jobId1); JobId jobId2 = MRBuilderUtils.NewJobId(2, 2, 2); HistoryFileManager.HistoryFileInfo fileInfo2 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(fileInfo2.GetJobId()).ThenReturn(jobId2); JobId jobId3 = MRBuilderUtils.NewJobId(3, 3, 3); HistoryFileManager.HistoryFileInfo fileInfo3 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(fileInfo3.GetJobId()).ThenReturn(jobId3); cache.AddIfAbsent(fileInfo1); cache.AddIfAbsent(fileInfo2); cache.AddIfAbsent(fileInfo3); ICollection <HistoryFileManager.HistoryFileInfo> values; for (int i = 0; i < 9; i++) { values = cache.Values(); if (values.Count > maxSize) { Sharpen.Thread.Sleep(100); } else { NUnit.Framework.Assert.IsFalse("fileInfo1 should have been evicted", values.Contains (fileInfo1)); return; } } NUnit.Framework.Assert.Fail("JobListCache didn't delete the extra entry"); }
/// <exception cref="System.Exception"/> public virtual void TestCompletedTaskAttempt() { HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath); completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager); TaskId mt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map); TaskId rt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Reduce); TaskAttemptId mta1Id = MRBuilderUtils.NewTaskAttemptId(mt1Id, 0); TaskAttemptId rta1Id = MRBuilderUtils.NewTaskAttemptId(rt1Id, 0); Task mt1 = completedJob.GetTask(mt1Id); Task rt1 = completedJob.GetTask(rt1Id); TaskAttempt mta1 = mt1.GetAttempt(mta1Id); NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, mta1.GetState()); NUnit.Framework.Assert.AreEqual("localhost:45454", mta1.GetAssignedContainerMgrAddress ()); NUnit.Framework.Assert.AreEqual("localhost:9999", mta1.GetNodeHttpAddress()); TaskAttemptReport mta1Report = mta1.GetReport(); NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, mta1Report.GetTaskAttemptState ()); NUnit.Framework.Assert.AreEqual("localhost", mta1Report.GetNodeManagerHost()); NUnit.Framework.Assert.AreEqual(45454, mta1Report.GetNodeManagerPort()); NUnit.Framework.Assert.AreEqual(9999, mta1Report.GetNodeManagerHttpPort()); TaskAttempt rta1 = rt1.GetAttempt(rta1Id); NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, rta1.GetState()); NUnit.Framework.Assert.AreEqual("localhost:45454", rta1.GetAssignedContainerMgrAddress ()); NUnit.Framework.Assert.AreEqual("localhost:9999", rta1.GetNodeHttpAddress()); TaskAttemptReport rta1Report = rta1.GetReport(); NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, rta1Report.GetTaskAttemptState ()); NUnit.Framework.Assert.AreEqual("localhost", rta1Report.GetNodeManagerHost()); NUnit.Framework.Assert.AreEqual(45454, rta1Report.GetNodeManagerPort()); NUnit.Framework.Assert.AreEqual(9999, rta1Report.GetNodeManagerHttpPort()); }
/// <summary>Simple test of some methods of CompletedJob</summary> /// <exception cref="System.Exception"/> public virtual void TestGetTaskAttemptCompletionEvent() { HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath); completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager); TaskCompletionEvent[] events = completedJob.GetMapAttemptCompletionEvents(0, 1000 ); NUnit.Framework.Assert.AreEqual(10, completedJob.GetMapAttemptCompletionEvents(0, 10).Length); int currentEventId = 0; foreach (TaskCompletionEvent taskAttemptCompletionEvent in events) { int eventId = taskAttemptCompletionEvent.GetEventId(); NUnit.Framework.Assert.IsTrue(eventId >= currentEventId); currentEventId = eventId; } NUnit.Framework.Assert.IsNull(completedJob.LoadConfFile()); // job name NUnit.Framework.Assert.AreEqual("Sleep job", completedJob.GetName()); // queue name NUnit.Framework.Assert.AreEqual("default", completedJob.GetQueueName()); // progress NUnit.Framework.Assert.AreEqual(1.0, completedJob.GetProgress(), 0.001); // 12 rows in answer NUnit.Framework.Assert.AreEqual(12, completedJob.GetTaskAttemptCompletionEvents(0 , 1000).Length); // select first 10 rows NUnit.Framework.Assert.AreEqual(10, completedJob.GetTaskAttemptCompletionEvents(0 , 10).Length); // select 5-10 rows include 5th NUnit.Framework.Assert.AreEqual(7, completedJob.GetTaskAttemptCompletionEvents(5, 10).Length); // without errors NUnit.Framework.Assert.AreEqual(1, completedJob.GetDiagnostics().Count); NUnit.Framework.Assert.AreEqual(string.Empty, completedJob.GetDiagnostics()[0]); NUnit.Framework.Assert.AreEqual(0, completedJob.GetJobACLs().Count); }
private Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job LoadJob(HistoryFileManager.HistoryFileInfo fileInfo) { try { Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = fileInfo.LoadJob(); if (Log.IsDebugEnabled()) { Log.Debug("Adding " + job.GetID() + " to loaded job cache"); } // We can clobber results here, but that should be OK, because it only // means that we may have two identical copies of the same job floating // around for a while. loadedJobCache[job.GetID()] = job; return(job); } catch (IOException e) { throw new YarnRuntimeException("Could not find/load job: " + fileInfo.GetJobId(), e); } }
/// <exception cref="System.IO.IOException"/> public virtual void TestAverageMergeTime() { string historyFileName = "job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist"; string confFileName = "job_1329348432655_0001_conf.xml"; Configuration conf = new Configuration(); JobACLsManager jobAclsMgr = new JobACLsManager(conf); Path fulleHistoryPath = new Path(typeof(TestJobHistoryEntities).GetClassLoader(). GetResource(historyFileName).GetFile()); Path fullConfPath = new Path(typeof(TestJobHistoryEntities).GetClassLoader().GetResource (confFileName).GetFile()); HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath); JobId jobId = MRBuilderUtils.NewJobId(1329348432655l, 1, 1); CompletedJob completedJob = new CompletedJob(conf, jobId, fulleHistoryPath, true, "user", info, jobAclsMgr); JobInfo jobInfo = new JobInfo(completedJob); // There are 2 tasks with merge time of 45 and 55 respectively. So average // merge time should be 50. NUnit.Framework.Assert.AreEqual(50L, jobInfo.GetAvgMergeTime()); }
public virtual Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job GetFullJob(JobId jobId) { if (Log.IsDebugEnabled()) { Log.Debug("Looking for Job " + jobId); } try { HistoryFileManager.HistoryFileInfo fileInfo = hsManager.GetFileInfo(jobId); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job result = null; if (fileInfo != null) { result = loadedJobCache[jobId]; if (result == null) { result = LoadJob(fileInfo); } else { if (fileInfo.IsDeleted()) { Sharpen.Collections.Remove(loadedJobCache, jobId); result = null; } } } else { Sharpen.Collections.Remove(loadedJobCache, jobId); } return(result); } catch (IOException e) { throw new YarnRuntimeException(e); } }
/// <exception cref="System.Exception"/> private void CheckHistoryParsing(int numMaps, int numReduces, int numSuccessfulMaps ) { Configuration conf = new Configuration(); conf.Set(MRJobConfig.UserName, Runtime.GetProperty("user.name")); long amStartTimeEst = Runtime.CurrentTimeMillis(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(numMaps, numReduces, true, this.GetType().FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString()); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); string jobhistoryDir = JobHistoryUtils.GetHistoryIntermediateDoneDirForUser(conf); FileContext fc = null; try { fc = FileContext.GetFileContext(conf); } catch (IOException ioe) { Log.Info("Can not get FileContext", ioe); throw (new Exception("Can not get File Context")); } if (numMaps == numSuccessfulMaps) { string summaryFileName = JobHistoryUtils.GetIntermediateSummaryFileName(jobId); Path summaryFile = new Path(jobhistoryDir, summaryFileName); string jobSummaryString = GetJobSummary(fc, summaryFile); NUnit.Framework.Assert.IsNotNull(jobSummaryString); NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerMap=100")); NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerReduce=100") ); IDictionary <string, string> jobSummaryElements = new Dictionary <string, string>(); StringTokenizer strToken = new StringTokenizer(jobSummaryString, ","); while (strToken.HasMoreTokens()) { string keypair = strToken.NextToken(); jobSummaryElements[keypair.Split("=")[0]] = keypair.Split("=")[1]; } NUnit.Framework.Assert.AreEqual("JobId does not match", jobId.ToString(), jobSummaryElements ["jobId"]); NUnit.Framework.Assert.AreEqual("JobName does not match", "test", jobSummaryElements ["jobName"]); NUnit.Framework.Assert.IsTrue("submitTime should not be 0", long.Parse(jobSummaryElements ["submitTime"]) != 0); NUnit.Framework.Assert.IsTrue("launchTime should not be 0", long.Parse(jobSummaryElements ["launchTime"]) != 0); NUnit.Framework.Assert.IsTrue("firstMapTaskLaunchTime should not be 0", long.Parse (jobSummaryElements["firstMapTaskLaunchTime"]) != 0); NUnit.Framework.Assert.IsTrue("firstReduceTaskLaunchTime should not be 0", long.Parse (jobSummaryElements["firstReduceTaskLaunchTime"]) != 0); NUnit.Framework.Assert.IsTrue("finishTime should not be 0", long.Parse(jobSummaryElements ["finishTime"]) != 0); NUnit.Framework.Assert.AreEqual("Mismatch in num map slots", numSuccessfulMaps, System.Convert.ToInt32 (jobSummaryElements["numMaps"])); NUnit.Framework.Assert.AreEqual("Mismatch in num reduce slots", numReduces, System.Convert.ToInt32 (jobSummaryElements["numReduces"])); NUnit.Framework.Assert.AreEqual("User does not match", Runtime.GetProperty("user.name" ), jobSummaryElements["user"]); NUnit.Framework.Assert.AreEqual("Queue does not match", "default", jobSummaryElements ["queue"]); NUnit.Framework.Assert.AreEqual("Status does not match", "SUCCEEDED", jobSummaryElements ["status"]); } JobHistory jobHistory = new JobHistory(); jobHistory.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId); JobHistoryParser.JobInfo jobInfo; long numFinishedMaps; lock (fileInfo) { Path historyFilePath = fileInfo.GetHistoryFile(); FSDataInputStream @in = null; Log.Info("JobHistoryFile is: " + historyFilePath); try { @in = fc.Open(fc.MakeQualified(historyFilePath)); } catch (IOException ioe) { Log.Info("Can not open history file: " + historyFilePath, ioe); throw (new Exception("Can not open History File")); } JobHistoryParser parser = new JobHistoryParser(@in); EventReader realReader = new EventReader(@in); EventReader reader = Org.Mockito.Mockito.Mock <EventReader>(); if (numMaps == numSuccessfulMaps) { reader = realReader; } else { AtomicInteger numFinishedEvents = new AtomicInteger(0); // Hack! Org.Mockito.Mockito.When(reader.GetNextEvent()).ThenAnswer(new _Answer_257(realReader , numFinishedEvents, numSuccessfulMaps)); } jobInfo = parser.Parse(reader); numFinishedMaps = ComputeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps); if (numFinishedMaps != numMaps) { Exception parseException = parser.GetParseException(); NUnit.Framework.Assert.IsNotNull("Didn't get expected parse exception", parseException ); } } NUnit.Framework.Assert.AreEqual("Incorrect username ", Runtime.GetProperty("user.name" ), jobInfo.GetUsername()); NUnit.Framework.Assert.AreEqual("Incorrect jobName ", "test", jobInfo.GetJobname( )); NUnit.Framework.Assert.AreEqual("Incorrect queuename ", "default", jobInfo.GetJobQueueName ()); NUnit.Framework.Assert.AreEqual("incorrect conf path", "test", jobInfo.GetJobConfPath ()); NUnit.Framework.Assert.AreEqual("incorrect finishedMap ", numSuccessfulMaps, numFinishedMaps ); NUnit.Framework.Assert.AreEqual("incorrect finishedReduces ", numReduces, jobInfo .GetFinishedReduces()); NUnit.Framework.Assert.AreEqual("incorrect uberized ", job.IsUber(), jobInfo.GetUberized ()); IDictionary <TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.GetAllTasks(); int totalTasks = allTasks.Count; NUnit.Framework.Assert.AreEqual("total number of tasks is incorrect ", (numMaps + numReduces), totalTasks); // Verify aminfo NUnit.Framework.Assert.AreEqual(1, jobInfo.GetAMInfos().Count); NUnit.Framework.Assert.AreEqual(MRApp.NmHost, jobInfo.GetAMInfos()[0].GetNodeManagerHost ()); JobHistoryParser.AMInfo amInfo = jobInfo.GetAMInfos()[0]; NUnit.Framework.Assert.AreEqual(MRApp.NmPort, amInfo.GetNodeManagerPort()); NUnit.Framework.Assert.AreEqual(MRApp.NmHttpPort, amInfo.GetNodeManagerHttpPort() ); NUnit.Framework.Assert.AreEqual(1, amInfo.GetAppAttemptId().GetAttemptId()); NUnit.Framework.Assert.AreEqual(amInfo.GetAppAttemptId(), amInfo.GetContainerId() .GetApplicationAttemptId()); NUnit.Framework.Assert.IsTrue(amInfo.GetStartTime() <= Runtime.CurrentTimeMillis( ) && amInfo.GetStartTime() >= amStartTimeEst); ContainerId fakeCid = MRApp.NewContainerId(-1, -1, -1, -1); // Assert at taskAttempt level foreach (JobHistoryParser.TaskInfo taskInfo in allTasks.Values) { int taskAttemptCount = taskInfo.GetAllTaskAttempts().Count; NUnit.Framework.Assert.AreEqual("total number of task attempts ", 1, taskAttemptCount ); JobHistoryParser.TaskAttemptInfo taInfo = taskInfo.GetAllTaskAttempts().Values.GetEnumerator ().Next(); NUnit.Framework.Assert.IsNotNull(taInfo.GetContainerId()); // Verify the wrong ctor is not being used. Remove after mrv1 is removed. NUnit.Framework.Assert.IsFalse(taInfo.GetContainerId().Equals(fakeCid)); } // Deep compare Job and JobInfo foreach (Task task in job.GetTasks().Values) { JobHistoryParser.TaskInfo taskInfo_1 = allTasks[TypeConverter.FromYarn(task.GetID ())]; NUnit.Framework.Assert.IsNotNull("TaskInfo not found", taskInfo_1); foreach (TaskAttempt taskAttempt in task.GetAttempts().Values) { JobHistoryParser.TaskAttemptInfo taskAttemptInfo = taskInfo_1.GetAllTaskAttempts( )[TypeConverter.FromYarn((taskAttempt.GetID()))]; NUnit.Framework.Assert.IsNotNull("TaskAttemptInfo not found", taskAttemptInfo); NUnit.Framework.Assert.AreEqual("Incorrect shuffle port for task attempt", taskAttempt .GetShufflePort(), taskAttemptInfo.GetShufflePort()); if (numMaps == numSuccessfulMaps) { NUnit.Framework.Assert.AreEqual(MRApp.NmHost, taskAttemptInfo.GetHostname()); NUnit.Framework.Assert.AreEqual(MRApp.NmPort, taskAttemptInfo.GetPort()); // Verify rack-name NUnit.Framework.Assert.AreEqual("rack-name is incorrect", taskAttemptInfo.GetRackname (), RackName); } } } // test output for HistoryViewer TextWriter stdps = System.Console.Out; try { Runtime.SetOut(new TextWriter(outContent)); HistoryViewer viewer; lock (fileInfo) { viewer = new HistoryViewer(fc.MakeQualified(fileInfo.GetHistoryFile()).ToString() , conf, true); } viewer.Print(); foreach (JobHistoryParser.TaskInfo taskInfo_1 in allTasks.Values) { string test = (taskInfo_1.GetTaskStatus() == null ? string.Empty : taskInfo_1.GetTaskStatus ()) + " " + taskInfo_1.GetTaskType() + " task list for " + taskInfo_1.GetTaskId( ).GetJobID(); NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(test) > 0); NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(taskInfo_1.GetTaskId( ).ToString()) > 0); } } finally { Runtime.SetOut(stdps); } }
public virtual void TestRefreshJobRetentionSettings() { string root = "mockfs://foo/"; string historyDoneDir = root + "mapred/history/done"; long now = Runtime.CurrentTimeMillis(); long someTimeYesterday = now - (25l * 3600 * 1000); long timeBefore200Secs = now - (200l * 1000); // Get yesterday's date in YY/MM/DD format string timestampComponent = JobHistoryUtils.TimestampDirectoryComponent(someTimeYesterday ); // Create a folder under yesterday's done dir Path donePathYesterday = new Path(historyDoneDir, timestampComponent + "/" + "000000" ); FileStatus dirCreatedYesterdayStatus = new FileStatus(0, true, 0, 0, someTimeYesterday , donePathYesterday); // Get today's date in YY/MM/DD format timestampComponent = JobHistoryUtils.TimestampDirectoryComponent(timeBefore200Secs ); // Create a folder under today's done dir Path donePathToday = new Path(historyDoneDir, timestampComponent + "/" + "000000" ); FileStatus dirCreatedTodayStatus = new FileStatus(0, true, 0, 0, timeBefore200Secs , donePathToday); // Create a jhist file with yesterday's timestamp under yesterday's done dir Path fileUnderYesterdayDir = new Path(donePathYesterday.ToString(), "job_1372363578825_0015-" + someTimeYesterday + "-user-Sleep+job-" + someTimeYesterday + "-1-1-SUCCEEDED-default.jhist" ); FileStatus fileUnderYesterdayDirStatus = new FileStatus(10, false, 0, 0, someTimeYesterday , fileUnderYesterdayDir); // Create a jhist file with today's timestamp under today's done dir Path fileUnderTodayDir = new Path(donePathYesterday.ToString(), "job_1372363578825_0016-" + timeBefore200Secs + "-user-Sleep+job-" + timeBefore200Secs + "-1-1-SUCCEEDED-default.jhist" ); FileStatus fileUnderTodayDirStatus = new FileStatus(10, false, 0, 0, timeBefore200Secs , fileUnderTodayDir); HistoryFileManager historyManager = Org.Mockito.Mockito.Spy(new HistoryFileManager ()); jobHistory = Org.Mockito.Mockito.Spy(new JobHistory()); IList <FileStatus> fileStatusList = new List <FileStatus>(); fileStatusList.AddItem(dirCreatedYesterdayStatus); fileStatusList.AddItem(dirCreatedTodayStatus); // Make the initial delay of history job cleaner as 4 secs Org.Mockito.Mockito.DoReturn(4).When(jobHistory).GetInitDelaySecs(); Org.Mockito.Mockito.DoReturn(historyManager).When(jobHistory).CreateHistoryFileManager (); IList <FileStatus> list1 = new List <FileStatus>(); list1.AddItem(fileUnderYesterdayDirStatus); Org.Mockito.Mockito.DoReturn(list1).When(historyManager).ScanDirectoryForHistoryFiles (Eq(donePathYesterday), Any <FileContext>()); IList <FileStatus> list2 = new List <FileStatus>(); list2.AddItem(fileUnderTodayDirStatus); Org.Mockito.Mockito.DoReturn(list2).When(historyManager).ScanDirectoryForHistoryFiles (Eq(donePathToday), Any <FileContext>()); Org.Mockito.Mockito.DoReturn(fileStatusList).When(historyManager).GetHistoryDirsForCleaning (Org.Mockito.Mockito.AnyLong()); Org.Mockito.Mockito.DoReturn(true).When(historyManager).DeleteDir(Any <FileStatus> ()); HistoryFileManager.JobListCache jobListCache = Org.Mockito.Mockito.Mock <HistoryFileManager.JobListCache >(); HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.DoReturn(jobListCache).When(historyManager).CreateJobListCache (); Org.Mockito.Mockito.When(jobListCache.Get(Any <JobId>())).ThenReturn(fileInfo); Org.Mockito.Mockito.DoNothing().When(fileInfo).Delete(); // Set job retention time to 24 hrs and cleaner interval to 2 secs Configuration conf = new Configuration(); conf.SetLong(JHAdminConfig.MrHistoryMaxAgeMs, 24l * 3600 * 1000); conf.SetLong(JHAdminConfig.MrHistoryCleanerIntervalMs, 2 * 1000); jobHistory.Init(conf); jobHistory.Start(); NUnit.Framework.Assert.AreEqual(2 * 1000l, jobHistory.GetCleanerInterval()); // Only yesterday's jhist file should get deleted Org.Mockito.Mockito.Verify(fileInfo, Org.Mockito.Mockito.Timeout(20000).Times(1)) .Delete(); fileStatusList.Remove(dirCreatedYesterdayStatus); // Now reset job retention time to 10 secs conf.SetLong(JHAdminConfig.MrHistoryMaxAgeMs, 10 * 1000); // Set cleaner interval to 1 sec conf.SetLong(JHAdminConfig.MrHistoryCleanerIntervalMs, 1 * 1000); Org.Mockito.Mockito.DoReturn(conf).When(jobHistory).CreateConf(); // Do refresh job retention settings jobHistory.RefreshJobRetentionSettings(); // Cleaner interval should be updated NUnit.Framework.Assert.AreEqual(1 * 1000l, jobHistory.GetCleanerInterval()); // Today's jhist file will also be deleted now since it falls below the // retention threshold Org.Mockito.Mockito.Verify(fileInfo, Org.Mockito.Mockito.Timeout(20000).Times(2)) .Delete(); }
public virtual void TestRefreshLoadedJobCache() { HistoryFileManager historyManager = Org.Mockito.Mockito.Mock <HistoryFileManager>( ); jobHistory = Org.Mockito.Mockito.Spy(new JobHistory()); Org.Mockito.Mockito.DoReturn(historyManager).When(jobHistory).CreateHistoryFileManager (); Configuration conf = new Configuration(); // Set the cache size to 2 conf.Set(JHAdminConfig.MrHistoryLoadedJobCacheSize, "2"); jobHistory.Init(conf); jobHistory.Start(); CachedHistoryStorage storage = Org.Mockito.Mockito.Spy((CachedHistoryStorage)jobHistory .GetHistoryStorage()); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job[] jobs = new Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job [3]; JobId[] jobIds = new JobId[3]; for (int i = 0; i < 3; i++) { jobs[i] = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>(); jobIds[i] = Org.Mockito.Mockito.Mock <JobId>(); Org.Mockito.Mockito.When(jobs[i].GetID()).ThenReturn(jobIds[i]); } HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.When(historyManager.GetFileInfo(Any <JobId>())).ThenReturn(fileInfo ); Org.Mockito.Mockito.When(fileInfo.LoadJob()).ThenReturn(jobs[0]).ThenReturn(jobs[ 1]).ThenReturn(jobs[2]); // getFullJob will put the job in the cache if it isn't there for (int i_1 = 0; i_1 < 3; i_1++) { storage.GetFullJob(jobs[i_1].GetID()); } IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobCache = storage .GetLoadedJobCache(); // job0 should have been purged since cache size is 2 NUnit.Framework.Assert.IsFalse(jobCache.Contains(jobs[0].GetID())); NUnit.Framework.Assert.IsTrue(jobCache.Contains(jobs[1].GetID()) && jobCache.Contains (jobs[2].GetID())); // Setting cache size to 3 conf.Set(JHAdminConfig.MrHistoryLoadedJobCacheSize, "3"); Org.Mockito.Mockito.DoReturn(conf).When(storage).CreateConf(); Org.Mockito.Mockito.When(fileInfo.LoadJob()).ThenReturn(jobs[0]).ThenReturn(jobs[ 1]).ThenReturn(jobs[2]); jobHistory.RefreshLoadedJobCache(); for (int i_2 = 0; i_2 < 3; i_2++) { storage.GetFullJob(jobs[i_2].GetID()); } jobCache = storage.GetLoadedJobCache(); // All three jobs should be in cache since its size is now 3 for (int i_3 = 0; i_3 < 3; i_3++) { NUnit.Framework.Assert.IsTrue(jobCache.Contains(jobs[i_3].GetID())); } }
/// <exception cref="System.Exception"/> public virtual void TestDiagnosticsForKilledJob() { Log.Info("STARTING testDiagnosticsForKilledJob"); try { Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryParsing.MRAppWithHistoryWithJobKilled(2, 1, true, this .GetType().FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); app.WaitForState(job, JobState.Killed); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); JobHistory jobHistory = new JobHistory(); jobHistory.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId); JobHistoryParser parser; JobHistoryParser.JobInfo jobInfo; lock (fileInfo) { Path historyFilePath = fileInfo.GetHistoryFile(); FSDataInputStream @in = null; FileContext fc = null; try { fc = FileContext.GetFileContext(conf); @in = fc.Open(fc.MakeQualified(historyFilePath)); } catch (IOException ioe) { Log.Info("Can not open history file: " + historyFilePath, ioe); throw (new Exception("Can not open History File")); } parser = new JobHistoryParser(@in); jobInfo = parser.Parse(); } Exception parseException = parser.GetParseException(); NUnit.Framework.Assert.IsNull("Caught an expected exception " + parseException, parseException ); IList <string> originalDiagnostics = job.GetDiagnostics(); string historyError = jobInfo.GetErrorInfo(); NUnit.Framework.Assert.IsTrue("No original diagnostics for a failed job", originalDiagnostics != null && !originalDiagnostics.IsEmpty()); NUnit.Framework.Assert.IsNotNull("No history error info for a failed job ", historyError ); foreach (string diagString in originalDiagnostics) { NUnit.Framework.Assert.IsTrue(historyError.Contains(diagString)); } NUnit.Framework.Assert.IsTrue("No killed message in diagnostics", historyError.Contains (JobImpl.JobKilledDiag)); } finally { Log.Info("FINISHED testDiagnosticsForKilledJob"); } }
internal virtual void DeleteJobFromJobListCache(HistoryFileManager.HistoryFileInfo fileInfo) { jobListCache.Delete(fileInfo); }
/// <exception cref="System.Exception"/> public virtual void TestHistoryParsingForFailedAttempts() { Log.Info("STARTING testHistoryParsingForFailedAttempts"); try { Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryParsing.MRAppWithHistoryWithFailedAttempt(2, 1, true , this.GetType().FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); JobHistory jobHistory = new JobHistory(); jobHistory.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId); JobHistoryParser parser; JobHistoryParser.JobInfo jobInfo; lock (fileInfo) { Path historyFilePath = fileInfo.GetHistoryFile(); FSDataInputStream @in = null; FileContext fc = null; try { fc = FileContext.GetFileContext(conf); @in = fc.Open(fc.MakeQualified(historyFilePath)); } catch (IOException ioe) { Log.Info("Can not open history file: " + historyFilePath, ioe); throw (new Exception("Can not open History File")); } parser = new JobHistoryParser(@in); jobInfo = parser.Parse(); } Exception parseException = parser.GetParseException(); NUnit.Framework.Assert.IsNull("Caught an expected exception " + parseException, parseException ); int noOffailedAttempts = 0; IDictionary <TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.GetAllTasks(); foreach (Task task in job.GetTasks().Values) { JobHistoryParser.TaskInfo taskInfo = allTasks[TypeConverter.FromYarn(task.GetID() )]; foreach (TaskAttempt taskAttempt in task.GetAttempts().Values) { JobHistoryParser.TaskAttemptInfo taskAttemptInfo = taskInfo.GetAllTaskAttempts()[ TypeConverter.FromYarn((taskAttempt.GetID()))]; // Verify rack-name for all task attempts NUnit.Framework.Assert.AreEqual("rack-name is incorrect", taskAttemptInfo.GetRackname (), RackName); if (taskAttemptInfo.GetTaskStatus().Equals("FAILED")) { noOffailedAttempts++; } } } NUnit.Framework.Assert.AreEqual("No of Failed tasks doesn't match.", 2, noOffailedAttempts ); } finally { Log.Info("FINISHED testHistoryParsingForFailedAttempts"); } }