/// <exception cref="System.IO.IOException"/>
public static FSDataInputStream GetPreviousJobHistoryFileStream(Configuration conf
, ApplicationAttemptId applicationAttemptId)
{
FSDataInputStream @in = null;
Path historyFile = null;
string jobId = TypeConverter.FromYarn(applicationAttemptId.GetApplicationId()).ToString
();
string jobhistoryDir = JobHistoryUtils.GetConfiguredHistoryStagingDirPrefix(conf,
jobId);
Path histDirPath = FileContext.GetFileContext(conf).MakeQualified(new Path(jobhistoryDir
));
FileContext fc = FileContext.GetFileContext(histDirPath.ToUri(), conf);
// read the previous history file
historyFile = fc.MakeQualified(JobHistoryUtils.GetStagingJobHistoryFile(histDirPath
, jobId, (applicationAttemptId.GetAttemptId() - 1)));
Log.Info("History file is at " + historyFile);
@in = fc.Open(historyFile);
return(@in);
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
if (conf.Get(MRJobConfig.MrAmStagingDir) == null)
{
conf.Set(MRJobConfig.MrAmStagingDir, new FilePath(GetTestWorkDir(), "apps_staging_dir/"
).GetAbsolutePath());
}
// By default, VMEM monitoring disabled, PMEM monitoring enabled.
if (!conf.GetBoolean(MRConfig.MapreduceMiniclusterControlResourceMonitoring, MRConfig
.DefaultMapreduceMiniclusterControlResourceMonitoring))
{
conf.SetBoolean(YarnConfiguration.NmPmemCheckEnabled, false);
conf.SetBoolean(YarnConfiguration.NmVmemCheckEnabled, false);
}
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "000");
try
{
Path stagingPath = FileContext.GetFileContext(conf).MakeQualified(new Path(conf.Get
(MRJobConfig.MrAmStagingDir)));
/*
* Re-configure the staging path on Windows if the file system is localFs.
* We need to use a absolute path that contains the drive letter. The unit
* test could run on a different drive than the AM. We can run into the
* issue that job files are localized to the drive where the test runs on,
* while the AM starts on a different drive and fails to find the job
* metafiles. Using absolute path can avoid this ambiguity.
*/
if (Path.Windows)
{
if (typeof(LocalFileSystem).IsInstanceOfType(stagingPath.GetFileSystem(conf)))
{
conf.Set(MRJobConfig.MrAmStagingDir, new FilePath(conf.Get(MRJobConfig.MrAmStagingDir
)).GetAbsolutePath());
}
}
FileContext fc = FileContext.GetFileContext(stagingPath.ToUri(), conf);
if (fc.Util().Exists(stagingPath))
{
Log.Info(stagingPath + " exists! deleting...");
fc.Delete(stagingPath, true);
}
Log.Info("mkdir: " + stagingPath);
//mkdir the staging directory so that right permissions are set while running as proxy user
fc.Mkdir(stagingPath, null, true);
//mkdir done directory as well
string doneDir = JobHistoryUtils.GetConfiguredHistoryServerDoneDirPrefix(conf);
Path doneDirPath = fc.MakeQualified(new Path(doneDir));
fc.Mkdir(doneDirPath, null, true);
}
catch (IOException e)
{
throw new YarnRuntimeException("Could not create staging directory. ", e);
}
conf.Set(MRConfig.MasterAddress, "test");
// The default is local because of
// which shuffle doesn't happen
//configure the shuffle service in NM
conf.SetStrings(YarnConfiguration.NmAuxServices, new string[] { ShuffleHandler.MapreduceShuffleServiceid });
conf.SetClass(string.Format(YarnConfiguration.NmAuxServiceFmt, ShuffleHandler.MapreduceShuffleServiceid
), typeof(ShuffleHandler), typeof(Org.Apache.Hadoop.Service.Service));
// Non-standard shuffle port
conf.SetInt(ShuffleHandler.ShufflePortConfigKey, 0);
conf.SetClass(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor
), typeof(ContainerExecutor));
// TestMRJobs is for testing non-uberized operation only; see TestUberAM
// for corresponding uberized tests.
conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
base.ServiceInit(conf);
}
public virtual void TestRefreshJobRetentionSettings()
{
string root = "mockfs://foo/";
string historyDoneDir = root + "mapred/history/done";
long now = Runtime.CurrentTimeMillis();
long someTimeYesterday = now - (25l * 3600 * 1000);
long timeBefore200Secs = now - (200l * 1000);
// Get yesterday's date in YY/MM/DD format
string timestampComponent = JobHistoryUtils.TimestampDirectoryComponent(someTimeYesterday
);
// Create a folder under yesterday's done dir
Path donePathYesterday = new Path(historyDoneDir, timestampComponent + "/" + "000000"
);
FileStatus dirCreatedYesterdayStatus = new FileStatus(0, true, 0, 0, someTimeYesterday
, donePathYesterday);
// Get today's date in YY/MM/DD format
timestampComponent = JobHistoryUtils.TimestampDirectoryComponent(timeBefore200Secs
);
// Create a folder under today's done dir
Path donePathToday = new Path(historyDoneDir, timestampComponent + "/" + "000000"
);
FileStatus dirCreatedTodayStatus = new FileStatus(0, true, 0, 0, timeBefore200Secs
, donePathToday);
// Create a jhist file with yesterday's timestamp under yesterday's done dir
Path fileUnderYesterdayDir = new Path(donePathYesterday.ToString(), "job_1372363578825_0015-"
+ someTimeYesterday + "-user-Sleep+job-" + someTimeYesterday + "-1-1-SUCCEEDED-default.jhist"
);
FileStatus fileUnderYesterdayDirStatus = new FileStatus(10, false, 0, 0, someTimeYesterday
, fileUnderYesterdayDir);
// Create a jhist file with today's timestamp under today's done dir
Path fileUnderTodayDir = new Path(donePathYesterday.ToString(), "job_1372363578825_0016-"
+ timeBefore200Secs + "-user-Sleep+job-" + timeBefore200Secs + "-1-1-SUCCEEDED-default.jhist"
);
FileStatus fileUnderTodayDirStatus = new FileStatus(10, false, 0, 0, timeBefore200Secs
, fileUnderTodayDir);
HistoryFileManager historyManager = Org.Mockito.Mockito.Spy(new HistoryFileManager
());
jobHistory = Org.Mockito.Mockito.Spy(new JobHistory());
IList <FileStatus> fileStatusList = new List <FileStatus>();
fileStatusList.AddItem(dirCreatedYesterdayStatus);
fileStatusList.AddItem(dirCreatedTodayStatus);
// Make the initial delay of history job cleaner as 4 secs
Org.Mockito.Mockito.DoReturn(4).When(jobHistory).GetInitDelaySecs();
Org.Mockito.Mockito.DoReturn(historyManager).When(jobHistory).CreateHistoryFileManager
();
IList <FileStatus> list1 = new List <FileStatus>();
list1.AddItem(fileUnderYesterdayDirStatus);
Org.Mockito.Mockito.DoReturn(list1).When(historyManager).ScanDirectoryForHistoryFiles
(Eq(donePathYesterday), Any <FileContext>());
IList <FileStatus> list2 = new List <FileStatus>();
list2.AddItem(fileUnderTodayDirStatus);
Org.Mockito.Mockito.DoReturn(list2).When(historyManager).ScanDirectoryForHistoryFiles
(Eq(donePathToday), Any <FileContext>());
Org.Mockito.Mockito.DoReturn(fileStatusList).When(historyManager).GetHistoryDirsForCleaning
(Org.Mockito.Mockito.AnyLong());
Org.Mockito.Mockito.DoReturn(true).When(historyManager).DeleteDir(Any <FileStatus>
());
HistoryFileManager.JobListCache jobListCache = Org.Mockito.Mockito.Mock <HistoryFileManager.JobListCache
>();
HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.DoReturn(jobListCache).When(historyManager).CreateJobListCache
();
Org.Mockito.Mockito.When(jobListCache.Get(Any <JobId>())).ThenReturn(fileInfo);
Org.Mockito.Mockito.DoNothing().When(fileInfo).Delete();
// Set job retention time to 24 hrs and cleaner interval to 2 secs
Configuration conf = new Configuration();
conf.SetLong(JHAdminConfig.MrHistoryMaxAgeMs, 24l * 3600 * 1000);
conf.SetLong(JHAdminConfig.MrHistoryCleanerIntervalMs, 2 * 1000);
jobHistory.Init(conf);
jobHistory.Start();
NUnit.Framework.Assert.AreEqual(2 * 1000l, jobHistory.GetCleanerInterval());
// Only yesterday's jhist file should get deleted
Org.Mockito.Mockito.Verify(fileInfo, Org.Mockito.Mockito.Timeout(20000).Times(1))
.Delete();
fileStatusList.Remove(dirCreatedYesterdayStatus);
// Now reset job retention time to 10 secs
conf.SetLong(JHAdminConfig.MrHistoryMaxAgeMs, 10 * 1000);
// Set cleaner interval to 1 sec
conf.SetLong(JHAdminConfig.MrHistoryCleanerIntervalMs, 1 * 1000);
Org.Mockito.Mockito.DoReturn(conf).When(jobHistory).CreateConf();
// Do refresh job retention settings
jobHistory.RefreshJobRetentionSettings();
// Cleaner interval should be updated
NUnit.Framework.Assert.AreEqual(1 * 1000l, jobHistory.GetCleanerInterval());
// Today's jhist file will also be deleted now since it falls below the
// retention threshold
Org.Mockito.Mockito.Verify(fileInfo, Org.Mockito.Mockito.Timeout(20000).Times(2))
.Delete();
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override string GetJobHistoryDir()
{
return(JobHistoryUtils.GetConfiguredHistoryServerDoneDirPrefix(conf));
}
/// <exception cref="System.Exception"/>
private void CheckHistoryParsing(int numMaps, int numReduces, int numSuccessfulMaps
)
{
Configuration conf = new Configuration();
conf.Set(MRJobConfig.UserName, Runtime.GetProperty("user.name"));
long amStartTimeEst = Runtime.CurrentTimeMillis();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(numMaps, numReduces, true,
this.GetType().FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
string jobhistoryDir = JobHistoryUtils.GetHistoryIntermediateDoneDirForUser(conf);
FileContext fc = null;
try
{
fc = FileContext.GetFileContext(conf);
}
catch (IOException ioe)
{
Log.Info("Can not get FileContext", ioe);
throw (new Exception("Can not get File Context"));
}
if (numMaps == numSuccessfulMaps)
{
string summaryFileName = JobHistoryUtils.GetIntermediateSummaryFileName(jobId);
Path summaryFile = new Path(jobhistoryDir, summaryFileName);
string jobSummaryString = GetJobSummary(fc, summaryFile);
NUnit.Framework.Assert.IsNotNull(jobSummaryString);
NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerMap=100"));
NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerReduce=100")
);
IDictionary <string, string> jobSummaryElements = new Dictionary <string, string>();
StringTokenizer strToken = new StringTokenizer(jobSummaryString, ",");
while (strToken.HasMoreTokens())
{
string keypair = strToken.NextToken();
jobSummaryElements[keypair.Split("=")[0]] = keypair.Split("=")[1];
}
NUnit.Framework.Assert.AreEqual("JobId does not match", jobId.ToString(), jobSummaryElements
["jobId"]);
NUnit.Framework.Assert.AreEqual("JobName does not match", "test", jobSummaryElements
["jobName"]);
NUnit.Framework.Assert.IsTrue("submitTime should not be 0", long.Parse(jobSummaryElements
["submitTime"]) != 0);
NUnit.Framework.Assert.IsTrue("launchTime should not be 0", long.Parse(jobSummaryElements
["launchTime"]) != 0);
NUnit.Framework.Assert.IsTrue("firstMapTaskLaunchTime should not be 0", long.Parse
(jobSummaryElements["firstMapTaskLaunchTime"]) != 0);
NUnit.Framework.Assert.IsTrue("firstReduceTaskLaunchTime should not be 0", long.Parse
(jobSummaryElements["firstReduceTaskLaunchTime"]) != 0);
NUnit.Framework.Assert.IsTrue("finishTime should not be 0", long.Parse(jobSummaryElements
["finishTime"]) != 0);
NUnit.Framework.Assert.AreEqual("Mismatch in num map slots", numSuccessfulMaps, System.Convert.ToInt32
(jobSummaryElements["numMaps"]));
NUnit.Framework.Assert.AreEqual("Mismatch in num reduce slots", numReduces, System.Convert.ToInt32
(jobSummaryElements["numReduces"]));
NUnit.Framework.Assert.AreEqual("User does not match", Runtime.GetProperty("user.name"
), jobSummaryElements["user"]);
NUnit.Framework.Assert.AreEqual("Queue does not match", "default", jobSummaryElements
["queue"]);
NUnit.Framework.Assert.AreEqual("Status does not match", "SUCCEEDED", jobSummaryElements
["status"]);
}
JobHistory jobHistory = new JobHistory();
jobHistory.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId);
JobHistoryParser.JobInfo jobInfo;
long numFinishedMaps;
lock (fileInfo)
{
Path historyFilePath = fileInfo.GetHistoryFile();
FSDataInputStream @in = null;
Log.Info("JobHistoryFile is: " + historyFilePath);
try
{
@in = fc.Open(fc.MakeQualified(historyFilePath));
}
catch (IOException ioe)
{
Log.Info("Can not open history file: " + historyFilePath, ioe);
throw (new Exception("Can not open History File"));
}
JobHistoryParser parser = new JobHistoryParser(@in);
EventReader realReader = new EventReader(@in);
EventReader reader = Org.Mockito.Mockito.Mock <EventReader>();
if (numMaps == numSuccessfulMaps)
{
reader = realReader;
}
else
{
AtomicInteger numFinishedEvents = new AtomicInteger(0);
// Hack!
Org.Mockito.Mockito.When(reader.GetNextEvent()).ThenAnswer(new _Answer_257(realReader
, numFinishedEvents, numSuccessfulMaps));
}
jobInfo = parser.Parse(reader);
numFinishedMaps = ComputeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps);
if (numFinishedMaps != numMaps)
{
Exception parseException = parser.GetParseException();
NUnit.Framework.Assert.IsNotNull("Didn't get expected parse exception", parseException
);
}
}
NUnit.Framework.Assert.AreEqual("Incorrect username ", Runtime.GetProperty("user.name"
), jobInfo.GetUsername());
NUnit.Framework.Assert.AreEqual("Incorrect jobName ", "test", jobInfo.GetJobname(
));
NUnit.Framework.Assert.AreEqual("Incorrect queuename ", "default", jobInfo.GetJobQueueName
());
NUnit.Framework.Assert.AreEqual("incorrect conf path", "test", jobInfo.GetJobConfPath
());
NUnit.Framework.Assert.AreEqual("incorrect finishedMap ", numSuccessfulMaps, numFinishedMaps
);
NUnit.Framework.Assert.AreEqual("incorrect finishedReduces ", numReduces, jobInfo
.GetFinishedReduces());
NUnit.Framework.Assert.AreEqual("incorrect uberized ", job.IsUber(), jobInfo.GetUberized
());
IDictionary <TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.GetAllTasks();
int totalTasks = allTasks.Count;
NUnit.Framework.Assert.AreEqual("total number of tasks is incorrect ", (numMaps
+ numReduces), totalTasks);
// Verify aminfo
NUnit.Framework.Assert.AreEqual(1, jobInfo.GetAMInfos().Count);
NUnit.Framework.Assert.AreEqual(MRApp.NmHost, jobInfo.GetAMInfos()[0].GetNodeManagerHost
());
JobHistoryParser.AMInfo amInfo = jobInfo.GetAMInfos()[0];
NUnit.Framework.Assert.AreEqual(MRApp.NmPort, amInfo.GetNodeManagerPort());
NUnit.Framework.Assert.AreEqual(MRApp.NmHttpPort, amInfo.GetNodeManagerHttpPort()
);
NUnit.Framework.Assert.AreEqual(1, amInfo.GetAppAttemptId().GetAttemptId());
NUnit.Framework.Assert.AreEqual(amInfo.GetAppAttemptId(), amInfo.GetContainerId()
.GetApplicationAttemptId());
NUnit.Framework.Assert.IsTrue(amInfo.GetStartTime() <= Runtime.CurrentTimeMillis(
) && amInfo.GetStartTime() >= amStartTimeEst);
ContainerId fakeCid = MRApp.NewContainerId(-1, -1, -1, -1);
// Assert at taskAttempt level
foreach (JobHistoryParser.TaskInfo taskInfo in allTasks.Values)
{
int taskAttemptCount = taskInfo.GetAllTaskAttempts().Count;
NUnit.Framework.Assert.AreEqual("total number of task attempts ", 1, taskAttemptCount
);
JobHistoryParser.TaskAttemptInfo taInfo = taskInfo.GetAllTaskAttempts().Values.GetEnumerator
().Next();
NUnit.Framework.Assert.IsNotNull(taInfo.GetContainerId());
// Verify the wrong ctor is not being used. Remove after mrv1 is removed.
NUnit.Framework.Assert.IsFalse(taInfo.GetContainerId().Equals(fakeCid));
}
// Deep compare Job and JobInfo
foreach (Task task in job.GetTasks().Values)
{
JobHistoryParser.TaskInfo taskInfo_1 = allTasks[TypeConverter.FromYarn(task.GetID
())];
NUnit.Framework.Assert.IsNotNull("TaskInfo not found", taskInfo_1);
foreach (TaskAttempt taskAttempt in task.GetAttempts().Values)
{
JobHistoryParser.TaskAttemptInfo taskAttemptInfo = taskInfo_1.GetAllTaskAttempts(
)[TypeConverter.FromYarn((taskAttempt.GetID()))];
NUnit.Framework.Assert.IsNotNull("TaskAttemptInfo not found", taskAttemptInfo);
NUnit.Framework.Assert.AreEqual("Incorrect shuffle port for task attempt", taskAttempt
.GetShufflePort(), taskAttemptInfo.GetShufflePort());
if (numMaps == numSuccessfulMaps)
{
NUnit.Framework.Assert.AreEqual(MRApp.NmHost, taskAttemptInfo.GetHostname());
NUnit.Framework.Assert.AreEqual(MRApp.NmPort, taskAttemptInfo.GetPort());
// Verify rack-name
NUnit.Framework.Assert.AreEqual("rack-name is incorrect", taskAttemptInfo.GetRackname
(), RackName);
}
}
}
// test output for HistoryViewer
TextWriter stdps = System.Console.Out;
try
{
Runtime.SetOut(new TextWriter(outContent));
HistoryViewer viewer;
lock (fileInfo)
{
viewer = new HistoryViewer(fc.MakeQualified(fileInfo.GetHistoryFile()).ToString()
, conf, true);
}
viewer.Print();
foreach (JobHistoryParser.TaskInfo taskInfo_1 in allTasks.Values)
{
string test = (taskInfo_1.GetTaskStatus() == null ? string.Empty : taskInfo_1.GetTaskStatus
()) + " " + taskInfo_1.GetTaskType() + " task list for " + taskInfo_1.GetTaskId(
).GetJobID();
NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(test) > 0);
NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(taskInfo_1.GetTaskId(
).ToString()) > 0);
}
}
finally
{
Runtime.SetOut(stdps);
}
}
