/// <exception cref="System.IO.IOException"/> public static FSDataInputStream GetPreviousJobHistoryFileStream(Configuration conf , ApplicationAttemptId applicationAttemptId) { FSDataInputStream @in = null; Path historyFile = null; string jobId = TypeConverter.FromYarn(applicationAttemptId.GetApplicationId()).ToString (); string jobhistoryDir = JobHistoryUtils.GetConfiguredHistoryStagingDirPrefix(conf, jobId); Path histDirPath = FileContext.GetFileContext(conf).MakeQualified(new Path(jobhistoryDir )); FileContext fc = FileContext.GetFileContext(histDirPath.ToUri(), conf); // read the previous history file historyFile = fc.MakeQualified(JobHistoryUtils.GetStagingJobHistoryFile(histDirPath , jobId, (applicationAttemptId.GetAttemptId() - 1))); Log.Info("History file is at " + historyFile); @in = fc.Open(historyFile); return(@in); }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); if (conf.Get(MRJobConfig.MrAmStagingDir) == null) { conf.Set(MRJobConfig.MrAmStagingDir, new FilePath(GetTestWorkDir(), "apps_staging_dir/" ).GetAbsolutePath()); } // By default, VMEM monitoring disabled, PMEM monitoring enabled. if (!conf.GetBoolean(MRConfig.MapreduceMiniclusterControlResourceMonitoring, MRConfig .DefaultMapreduceMiniclusterControlResourceMonitoring)) { conf.SetBoolean(YarnConfiguration.NmPmemCheckEnabled, false); conf.SetBoolean(YarnConfiguration.NmVmemCheckEnabled, false); } conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "000"); try { Path stagingPath = FileContext.GetFileContext(conf).MakeQualified(new Path(conf.Get (MRJobConfig.MrAmStagingDir))); /* * Re-configure the staging path on Windows if the file system is localFs. * We need to use a absolute path that contains the drive letter. The unit * test could run on a different drive than the AM. We can run into the * issue that job files are localized to the drive where the test runs on, * while the AM starts on a different drive and fails to find the job * metafiles. Using absolute path can avoid this ambiguity. */ if (Path.Windows) { if (typeof(LocalFileSystem).IsInstanceOfType(stagingPath.GetFileSystem(conf))) { conf.Set(MRJobConfig.MrAmStagingDir, new FilePath(conf.Get(MRJobConfig.MrAmStagingDir )).GetAbsolutePath()); } } FileContext fc = FileContext.GetFileContext(stagingPath.ToUri(), conf); if (fc.Util().Exists(stagingPath)) { Log.Info(stagingPath + " exists! deleting..."); fc.Delete(stagingPath, true); } Log.Info("mkdir: " + stagingPath); //mkdir the staging directory so that right permissions are set while running as proxy user fc.Mkdir(stagingPath, null, true); //mkdir done directory as well string doneDir = JobHistoryUtils.GetConfiguredHistoryServerDoneDirPrefix(conf); Path doneDirPath = fc.MakeQualified(new Path(doneDir)); fc.Mkdir(doneDirPath, null, true); } catch (IOException e) { throw new YarnRuntimeException("Could not create staging directory. ", e); } conf.Set(MRConfig.MasterAddress, "test"); // The default is local because of // which shuffle doesn't happen //configure the shuffle service in NM conf.SetStrings(YarnConfiguration.NmAuxServices, new string[] { ShuffleHandler.MapreduceShuffleServiceid }); conf.SetClass(string.Format(YarnConfiguration.NmAuxServiceFmt, ShuffleHandler.MapreduceShuffleServiceid ), typeof(ShuffleHandler), typeof(Org.Apache.Hadoop.Service.Service)); // Non-standard shuffle port conf.SetInt(ShuffleHandler.ShufflePortConfigKey, 0); conf.SetClass(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor ), typeof(ContainerExecutor)); // TestMRJobs is for testing non-uberized operation only; see TestUberAM // for corresponding uberized tests. conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); base.ServiceInit(conf); }
public virtual void TestRefreshJobRetentionSettings() { string root = "mockfs://foo/"; string historyDoneDir = root + "mapred/history/done"; long now = Runtime.CurrentTimeMillis(); long someTimeYesterday = now - (25l * 3600 * 1000); long timeBefore200Secs = now - (200l * 1000); // Get yesterday's date in YY/MM/DD format string timestampComponent = JobHistoryUtils.TimestampDirectoryComponent(someTimeYesterday ); // Create a folder under yesterday's done dir Path donePathYesterday = new Path(historyDoneDir, timestampComponent + "/" + "000000" ); FileStatus dirCreatedYesterdayStatus = new FileStatus(0, true, 0, 0, someTimeYesterday , donePathYesterday); // Get today's date in YY/MM/DD format timestampComponent = JobHistoryUtils.TimestampDirectoryComponent(timeBefore200Secs ); // Create a folder under today's done dir Path donePathToday = new Path(historyDoneDir, timestampComponent + "/" + "000000" ); FileStatus dirCreatedTodayStatus = new FileStatus(0, true, 0, 0, timeBefore200Secs , donePathToday); // Create a jhist file with yesterday's timestamp under yesterday's done dir Path fileUnderYesterdayDir = new Path(donePathYesterday.ToString(), "job_1372363578825_0015-" + someTimeYesterday + "-user-Sleep+job-" + someTimeYesterday + "-1-1-SUCCEEDED-default.jhist" ); FileStatus fileUnderYesterdayDirStatus = new FileStatus(10, false, 0, 0, someTimeYesterday , fileUnderYesterdayDir); // Create a jhist file with today's timestamp under today's done dir Path fileUnderTodayDir = new Path(donePathYesterday.ToString(), "job_1372363578825_0016-" + timeBefore200Secs + "-user-Sleep+job-" + timeBefore200Secs + "-1-1-SUCCEEDED-default.jhist" ); FileStatus fileUnderTodayDirStatus = new FileStatus(10, false, 0, 0, timeBefore200Secs , fileUnderTodayDir); HistoryFileManager historyManager = Org.Mockito.Mockito.Spy(new HistoryFileManager ()); jobHistory = Org.Mockito.Mockito.Spy(new JobHistory()); IList <FileStatus> fileStatusList = new List <FileStatus>(); fileStatusList.AddItem(dirCreatedYesterdayStatus); fileStatusList.AddItem(dirCreatedTodayStatus); // Make the initial delay of history job cleaner as 4 secs Org.Mockito.Mockito.DoReturn(4).When(jobHistory).GetInitDelaySecs(); Org.Mockito.Mockito.DoReturn(historyManager).When(jobHistory).CreateHistoryFileManager (); IList <FileStatus> list1 = new List <FileStatus>(); list1.AddItem(fileUnderYesterdayDirStatus); Org.Mockito.Mockito.DoReturn(list1).When(historyManager).ScanDirectoryForHistoryFiles (Eq(donePathYesterday), Any <FileContext>()); IList <FileStatus> list2 = new List <FileStatus>(); list2.AddItem(fileUnderTodayDirStatus); Org.Mockito.Mockito.DoReturn(list2).When(historyManager).ScanDirectoryForHistoryFiles (Eq(donePathToday), Any <FileContext>()); Org.Mockito.Mockito.DoReturn(fileStatusList).When(historyManager).GetHistoryDirsForCleaning (Org.Mockito.Mockito.AnyLong()); Org.Mockito.Mockito.DoReturn(true).When(historyManager).DeleteDir(Any <FileStatus> ()); HistoryFileManager.JobListCache jobListCache = Org.Mockito.Mockito.Mock <HistoryFileManager.JobListCache >(); HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo >(); Org.Mockito.Mockito.DoReturn(jobListCache).When(historyManager).CreateJobListCache (); Org.Mockito.Mockito.When(jobListCache.Get(Any <JobId>())).ThenReturn(fileInfo); Org.Mockito.Mockito.DoNothing().When(fileInfo).Delete(); // Set job retention time to 24 hrs and cleaner interval to 2 secs Configuration conf = new Configuration(); conf.SetLong(JHAdminConfig.MrHistoryMaxAgeMs, 24l * 3600 * 1000); conf.SetLong(JHAdminConfig.MrHistoryCleanerIntervalMs, 2 * 1000); jobHistory.Init(conf); jobHistory.Start(); NUnit.Framework.Assert.AreEqual(2 * 1000l, jobHistory.GetCleanerInterval()); // Only yesterday's jhist file should get deleted Org.Mockito.Mockito.Verify(fileInfo, Org.Mockito.Mockito.Timeout(20000).Times(1)) .Delete(); fileStatusList.Remove(dirCreatedYesterdayStatus); // Now reset job retention time to 10 secs conf.SetLong(JHAdminConfig.MrHistoryMaxAgeMs, 10 * 1000); // Set cleaner interval to 1 sec conf.SetLong(JHAdminConfig.MrHistoryCleanerIntervalMs, 1 * 1000); Org.Mockito.Mockito.DoReturn(conf).When(jobHistory).CreateConf(); // Do refresh job retention settings jobHistory.RefreshJobRetentionSettings(); // Cleaner interval should be updated NUnit.Framework.Assert.AreEqual(1 * 1000l, jobHistory.GetCleanerInterval()); // Today's jhist file will also be deleted now since it falls below the // retention threshold Org.Mockito.Mockito.Verify(fileInfo, Org.Mockito.Mockito.Timeout(20000).Times(2)) .Delete(); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override string GetJobHistoryDir() { return(JobHistoryUtils.GetConfiguredHistoryServerDoneDirPrefix(conf)); }
/// <exception cref="System.Exception"/> private void CheckHistoryParsing(int numMaps, int numReduces, int numSuccessfulMaps ) { Configuration conf = new Configuration(); conf.Set(MRJobConfig.UserName, Runtime.GetProperty("user.name")); long amStartTimeEst = Runtime.CurrentTimeMillis(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(numMaps, numReduces, true, this.GetType().FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString()); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); string jobhistoryDir = JobHistoryUtils.GetHistoryIntermediateDoneDirForUser(conf); FileContext fc = null; try { fc = FileContext.GetFileContext(conf); } catch (IOException ioe) { Log.Info("Can not get FileContext", ioe); throw (new Exception("Can not get File Context")); } if (numMaps == numSuccessfulMaps) { string summaryFileName = JobHistoryUtils.GetIntermediateSummaryFileName(jobId); Path summaryFile = new Path(jobhistoryDir, summaryFileName); string jobSummaryString = GetJobSummary(fc, summaryFile); NUnit.Framework.Assert.IsNotNull(jobSummaryString); NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerMap=100")); NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerReduce=100") ); IDictionary <string, string> jobSummaryElements = new Dictionary <string, string>(); StringTokenizer strToken = new StringTokenizer(jobSummaryString, ","); while (strToken.HasMoreTokens()) { string keypair = strToken.NextToken(); jobSummaryElements[keypair.Split("=")[0]] = keypair.Split("=")[1]; } NUnit.Framework.Assert.AreEqual("JobId does not match", jobId.ToString(), jobSummaryElements ["jobId"]); NUnit.Framework.Assert.AreEqual("JobName does not match", "test", jobSummaryElements ["jobName"]); NUnit.Framework.Assert.IsTrue("submitTime should not be 0", long.Parse(jobSummaryElements ["submitTime"]) != 0); NUnit.Framework.Assert.IsTrue("launchTime should not be 0", long.Parse(jobSummaryElements ["launchTime"]) != 0); NUnit.Framework.Assert.IsTrue("firstMapTaskLaunchTime should not be 0", long.Parse (jobSummaryElements["firstMapTaskLaunchTime"]) != 0); NUnit.Framework.Assert.IsTrue("firstReduceTaskLaunchTime should not be 0", long.Parse (jobSummaryElements["firstReduceTaskLaunchTime"]) != 0); NUnit.Framework.Assert.IsTrue("finishTime should not be 0", long.Parse(jobSummaryElements ["finishTime"]) != 0); NUnit.Framework.Assert.AreEqual("Mismatch in num map slots", numSuccessfulMaps, System.Convert.ToInt32 (jobSummaryElements["numMaps"])); NUnit.Framework.Assert.AreEqual("Mismatch in num reduce slots", numReduces, System.Convert.ToInt32 (jobSummaryElements["numReduces"])); NUnit.Framework.Assert.AreEqual("User does not match", Runtime.GetProperty("user.name" ), jobSummaryElements["user"]); NUnit.Framework.Assert.AreEqual("Queue does not match", "default", jobSummaryElements ["queue"]); NUnit.Framework.Assert.AreEqual("Status does not match", "SUCCEEDED", jobSummaryElements ["status"]); } JobHistory jobHistory = new JobHistory(); jobHistory.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId); JobHistoryParser.JobInfo jobInfo; long numFinishedMaps; lock (fileInfo) { Path historyFilePath = fileInfo.GetHistoryFile(); FSDataInputStream @in = null; Log.Info("JobHistoryFile is: " + historyFilePath); try { @in = fc.Open(fc.MakeQualified(historyFilePath)); } catch (IOException ioe) { Log.Info("Can not open history file: " + historyFilePath, ioe); throw (new Exception("Can not open History File")); } JobHistoryParser parser = new JobHistoryParser(@in); EventReader realReader = new EventReader(@in); EventReader reader = Org.Mockito.Mockito.Mock <EventReader>(); if (numMaps == numSuccessfulMaps) { reader = realReader; } else { AtomicInteger numFinishedEvents = new AtomicInteger(0); // Hack! Org.Mockito.Mockito.When(reader.GetNextEvent()).ThenAnswer(new _Answer_257(realReader , numFinishedEvents, numSuccessfulMaps)); } jobInfo = parser.Parse(reader); numFinishedMaps = ComputeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps); if (numFinishedMaps != numMaps) { Exception parseException = parser.GetParseException(); NUnit.Framework.Assert.IsNotNull("Didn't get expected parse exception", parseException ); } } NUnit.Framework.Assert.AreEqual("Incorrect username ", Runtime.GetProperty("user.name" ), jobInfo.GetUsername()); NUnit.Framework.Assert.AreEqual("Incorrect jobName ", "test", jobInfo.GetJobname( )); NUnit.Framework.Assert.AreEqual("Incorrect queuename ", "default", jobInfo.GetJobQueueName ()); NUnit.Framework.Assert.AreEqual("incorrect conf path", "test", jobInfo.GetJobConfPath ()); NUnit.Framework.Assert.AreEqual("incorrect finishedMap ", numSuccessfulMaps, numFinishedMaps ); NUnit.Framework.Assert.AreEqual("incorrect finishedReduces ", numReduces, jobInfo .GetFinishedReduces()); NUnit.Framework.Assert.AreEqual("incorrect uberized ", job.IsUber(), jobInfo.GetUberized ()); IDictionary <TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.GetAllTasks(); int totalTasks = allTasks.Count; NUnit.Framework.Assert.AreEqual("total number of tasks is incorrect ", (numMaps + numReduces), totalTasks); // Verify aminfo NUnit.Framework.Assert.AreEqual(1, jobInfo.GetAMInfos().Count); NUnit.Framework.Assert.AreEqual(MRApp.NmHost, jobInfo.GetAMInfos()[0].GetNodeManagerHost ()); JobHistoryParser.AMInfo amInfo = jobInfo.GetAMInfos()[0]; NUnit.Framework.Assert.AreEqual(MRApp.NmPort, amInfo.GetNodeManagerPort()); NUnit.Framework.Assert.AreEqual(MRApp.NmHttpPort, amInfo.GetNodeManagerHttpPort() ); NUnit.Framework.Assert.AreEqual(1, amInfo.GetAppAttemptId().GetAttemptId()); NUnit.Framework.Assert.AreEqual(amInfo.GetAppAttemptId(), amInfo.GetContainerId() .GetApplicationAttemptId()); NUnit.Framework.Assert.IsTrue(amInfo.GetStartTime() <= Runtime.CurrentTimeMillis( ) && amInfo.GetStartTime() >= amStartTimeEst); ContainerId fakeCid = MRApp.NewContainerId(-1, -1, -1, -1); // Assert at taskAttempt level foreach (JobHistoryParser.TaskInfo taskInfo in allTasks.Values) { int taskAttemptCount = taskInfo.GetAllTaskAttempts().Count; NUnit.Framework.Assert.AreEqual("total number of task attempts ", 1, taskAttemptCount ); JobHistoryParser.TaskAttemptInfo taInfo = taskInfo.GetAllTaskAttempts().Values.GetEnumerator ().Next(); NUnit.Framework.Assert.IsNotNull(taInfo.GetContainerId()); // Verify the wrong ctor is not being used. Remove after mrv1 is removed. NUnit.Framework.Assert.IsFalse(taInfo.GetContainerId().Equals(fakeCid)); } // Deep compare Job and JobInfo foreach (Task task in job.GetTasks().Values) { JobHistoryParser.TaskInfo taskInfo_1 = allTasks[TypeConverter.FromYarn(task.GetID ())]; NUnit.Framework.Assert.IsNotNull("TaskInfo not found", taskInfo_1); foreach (TaskAttempt taskAttempt in task.GetAttempts().Values) { JobHistoryParser.TaskAttemptInfo taskAttemptInfo = taskInfo_1.GetAllTaskAttempts( )[TypeConverter.FromYarn((taskAttempt.GetID()))]; NUnit.Framework.Assert.IsNotNull("TaskAttemptInfo not found", taskAttemptInfo); NUnit.Framework.Assert.AreEqual("Incorrect shuffle port for task attempt", taskAttempt .GetShufflePort(), taskAttemptInfo.GetShufflePort()); if (numMaps == numSuccessfulMaps) { NUnit.Framework.Assert.AreEqual(MRApp.NmHost, taskAttemptInfo.GetHostname()); NUnit.Framework.Assert.AreEqual(MRApp.NmPort, taskAttemptInfo.GetPort()); // Verify rack-name NUnit.Framework.Assert.AreEqual("rack-name is incorrect", taskAttemptInfo.GetRackname (), RackName); } } } // test output for HistoryViewer TextWriter stdps = System.Console.Out; try { Runtime.SetOut(new TextWriter(outContent)); HistoryViewer viewer; lock (fileInfo) { viewer = new HistoryViewer(fc.MakeQualified(fileInfo.GetHistoryFile()).ToString() , conf, true); } viewer.Print(); foreach (JobHistoryParser.TaskInfo taskInfo_1 in allTasks.Values) { string test = (taskInfo_1.GetTaskStatus() == null ? string.Empty : taskInfo_1.GetTaskStatus ()) + " " + taskInfo_1.GetTaskType() + " task list for " + taskInfo_1.GetTaskId( ).GetJobID(); NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(test) > 0); NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(taskInfo_1.GetTaskId( ).ToString()) > 0); } } finally { Runtime.SetOut(stdps); } }