/* Verify some expected values based on the history file */
/// <exception cref="System.Exception"/>
public virtual void TestCompletedJob()
{
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
//Re-initialize to verify the delayed load.
completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
//Verify tasks loaded based on loadTask parameter.
NUnit.Framework.Assert.AreEqual(loadTasks, completedJob.tasksLoaded.Get());
NUnit.Framework.Assert.AreEqual(1, completedJob.GetAMInfos().Count);
NUnit.Framework.Assert.AreEqual(10, completedJob.GetCompletedMaps());
NUnit.Framework.Assert.AreEqual(1, completedJob.GetCompletedReduces());
NUnit.Framework.Assert.AreEqual(12, completedJob.GetTasks().Count);
//Verify tasks loaded at this point.
NUnit.Framework.Assert.AreEqual(true, completedJob.tasksLoaded.Get());
NUnit.Framework.Assert.AreEqual(10, completedJob.GetTasks(TaskType.Map).Count);
NUnit.Framework.Assert.AreEqual(2, completedJob.GetTasks(TaskType.Reduce).Count);
NUnit.Framework.Assert.AreEqual("user", completedJob.GetUserName());
NUnit.Framework.Assert.AreEqual(JobState.Succeeded, completedJob.GetState());
JobReport jobReport = completedJob.GetReport();
NUnit.Framework.Assert.AreEqual("user", jobReport.GetUser());
NUnit.Framework.Assert.AreEqual(JobState.Succeeded, jobReport.GetJobState());
}
/// <exception cref="System.Exception"/>
public virtual void TestCompletedTask()
{
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskId mt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map);
TaskId rt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Reduce);
IDictionary <TaskId, Task> mapTasks = completedJob.GetTasks(TaskType.Map);
IDictionary <TaskId, Task> reduceTasks = completedJob.GetTasks(TaskType.Reduce);
NUnit.Framework.Assert.AreEqual(10, mapTasks.Count);
NUnit.Framework.Assert.AreEqual(2, reduceTasks.Count);
Task mt1 = mapTasks[mt1Id];
NUnit.Framework.Assert.AreEqual(1, mt1.GetAttempts().Count);
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mt1.GetState());
TaskReport mt1Report = mt1.GetReport();
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, mt1Report.GetTaskState());
NUnit.Framework.Assert.AreEqual(mt1Id, mt1Report.GetTaskId());
Task rt1 = reduceTasks[rt1Id];
NUnit.Framework.Assert.AreEqual(1, rt1.GetAttempts().Count);
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, rt1.GetState());
TaskReport rt1Report = rt1.GetReport();
NUnit.Framework.Assert.AreEqual(TaskState.Succeeded, rt1Report.GetTaskState());
NUnit.Framework.Assert.AreEqual(rt1Id, rt1Report.GetTaskId());
}
/// <summary>
/// Trivial test case that verifies basic functionality of
/// <see cref="JobIdHistoryFileInfoMap"/>
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestWithSingleElement()
{
HistoryFileManager.JobIdHistoryFileInfoMap mapWithSize = new HistoryFileManager.JobIdHistoryFileInfoMap
();
JobId jobId = MRBuilderUtils.NewJobId(1, 1, 1);
HistoryFileManager.HistoryFileInfo fileInfo1 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo1.GetJobId()).ThenReturn(jobId);
// add it twice
NUnit.Framework.Assert.AreEqual("Incorrect return on putIfAbsent()", null, mapWithSize
.PutIfAbsent(jobId, fileInfo1));
NUnit.Framework.Assert.AreEqual("Incorrect return on putIfAbsent()", fileInfo1, mapWithSize
.PutIfAbsent(jobId, fileInfo1));
// check get()
NUnit.Framework.Assert.AreEqual("Incorrect get()", fileInfo1, mapWithSize.Get(jobId
));
NUnit.Framework.Assert.IsTrue("Incorrect size()", CheckSize(mapWithSize, 1));
// check navigableKeySet()
NavigableSet <JobId> set = mapWithSize.NavigableKeySet();
NUnit.Framework.Assert.AreEqual("Incorrect navigableKeySet()", 1, set.Count);
NUnit.Framework.Assert.IsTrue("Incorrect navigableKeySet()", set.Contains(jobId));
// check values()
ICollection <HistoryFileManager.HistoryFileInfo> values = mapWithSize.Values();
NUnit.Framework.Assert.AreEqual("Incorrect values()", 1, values.Count);
NUnit.Framework.Assert.IsTrue("Incorrect values()", values.Contains(fileInfo1));
}
public virtual void TestAddExisting()
{
HistoryFileManager.JobListCache cache = new HistoryFileManager.JobListCache(2, 1000
);
JobId jobId = MRBuilderUtils.NewJobId(1, 1, 1);
HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo.GetJobId()).ThenReturn(jobId);
cache.AddIfAbsent(fileInfo);
cache.AddIfAbsent(fileInfo);
NUnit.Framework.Assert.AreEqual("Incorrect number of cache entries", 1, cache.Values
().Count);
}
/// <exception cref="System.IO.IOException"/>
public CompletedJob(Configuration conf, JobId jobId, Path historyFile, bool loadTasks
, string userName, HistoryFileManager.HistoryFileInfo info, JobACLsManager aclsMgr
)
{
//Can be picked from JobInfo with a conversion.
//Can be picked up from JobInfo
Log.Info("Loading job: " + jobId + " from file: " + historyFile);
this.conf = conf;
this.jobId = jobId;
this.user = userName;
this.info = info;
this.aclsMgr = aclsMgr;
LoadFullHistoryData(loadTasks, historyFile);
}
/// <exception cref="System.Exception"/>
public virtual void TestCopmletedJobReportWithZeroTasks()
{
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
completedJob = new CompletedJob(conf, jobId, fullHistoryPathZeroReduces, loadTasks
, "user", info, jobAclsManager);
JobReport jobReport = completedJob.GetReport();
// Make sure that the number reduces (completed and total) are equal to zero.
NUnit.Framework.Assert.AreEqual(0, completedJob.GetTotalReduces());
NUnit.Framework.Assert.AreEqual(0, completedJob.GetCompletedReduces());
// Verify that the reduce progress is 1.0 (not NaN)
NUnit.Framework.Assert.AreEqual(1.0, jobReport.GetReduceProgress(), 0.001);
}
/// <summary>test clean old history files.</summary>
/// <remarks>
/// test clean old history files. Files should be deleted after 1 week by
/// default.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDeleteFileInfo()
{
Log.Info("STARTING testDeleteFileInfo");
try
{
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType().
FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
HistoryFileManager hfm = new HistoryFileManager();
hfm.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = hfm.GetFileInfo(jobId);
hfm.InitExisting();
// wait for move files form the done_intermediate directory to the gone
// directory
while (fileInfo.IsMovePending())
{
Sharpen.Thread.Sleep(300);
}
NUnit.Framework.Assert.IsNotNull(hfm.jobListCache.Values());
// try to remove fileInfo
hfm.Clean();
// check that fileInfo does not deleted
NUnit.Framework.Assert.IsFalse(fileInfo.IsDeleted());
// correct live time
hfm.SetMaxHistoryAge(-1);
hfm.Clean();
hfm.Stop();
NUnit.Framework.Assert.IsTrue("Thread pool shutdown", hfm.moveToDoneExecutor.IsTerminated
());
// should be deleted !
NUnit.Framework.Assert.IsTrue("file should be deleted ", fileInfo.IsDeleted());
}
finally
{
Log.Info("FINISHED testDeleteFileInfo");
}
}
/// <exception cref="System.Exception"/>
public virtual void TestScanningOldDirs()
{
Log.Info("STARTING testScanningOldDirs");
try
{
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType().
FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
TestJobHistoryParsing.HistoryFileManagerForTest hfm = new TestJobHistoryParsing.HistoryFileManagerForTest
();
hfm.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = hfm.GetFileInfo(jobId);
NUnit.Framework.Assert.IsNotNull("Unable to locate job history", fileInfo);
// force the manager to "forget" the job
hfm.DeleteJobFromJobListCache(fileInfo);
int msecPerSleep = 10;
int msecToSleep = 10 * 1000;
while (fileInfo.IsMovePending() && msecToSleep > 0)
{
NUnit.Framework.Assert.IsTrue(!fileInfo.DidMoveFail());
msecToSleep -= msecPerSleep;
Sharpen.Thread.Sleep(msecPerSleep);
}
NUnit.Framework.Assert.IsTrue("Timeout waiting for history move", msecToSleep > 0
);
fileInfo = hfm.GetFileInfo(jobId);
hfm.Stop();
NUnit.Framework.Assert.IsNotNull("Unable to locate old job history", fileInfo);
NUnit.Framework.Assert.IsTrue("HistoryFileManager not shutdown properly", hfm.moveToDoneExecutor
.IsTerminated());
}
finally
{
Log.Info("FINISHED testScanningOldDirs");
}
}
public virtual void TestHistoryFileInfoSummaryFileNotExist()
{
TestHistoryFileManager.HistoryFileManagerTest hmTest = new TestHistoryFileManager.HistoryFileManagerTest
();
string job = "job_1410889000000_123456";
Path summaryFile = new Path(job + ".summary");
JobIndexInfo jobIndexInfo = new JobIndexInfo();
jobIndexInfo.SetJobId(TypeConverter.ToYarn(JobID.ForName(job)));
Configuration conf = dfsCluster.GetConfiguration(0);
conf.Set(JHAdminConfig.MrHistoryDoneDir, "/" + UUID.RandomUUID());
conf.Set(JHAdminConfig.MrHistoryIntermediateDoneDir, "/" + UUID.RandomUUID());
hmTest.ServiceInit(conf);
HistoryFileManager.HistoryFileInfo info = hmTest.GetHistoryFileInfo(null, null, summaryFile
, jobIndexInfo, false);
info.MoveToDone();
NUnit.Framework.Assert.IsFalse(info.DidMoveFail());
}
/// <exception cref="System.Exception"/>
public virtual void TestEviction()
{
int maxSize = 2;
HistoryFileManager.JobListCache cache = new HistoryFileManager.JobListCache(maxSize
, 1000);
JobId jobId1 = MRBuilderUtils.NewJobId(1, 1, 1);
HistoryFileManager.HistoryFileInfo fileInfo1 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo1.GetJobId()).ThenReturn(jobId1);
JobId jobId2 = MRBuilderUtils.NewJobId(2, 2, 2);
HistoryFileManager.HistoryFileInfo fileInfo2 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo2.GetJobId()).ThenReturn(jobId2);
JobId jobId3 = MRBuilderUtils.NewJobId(3, 3, 3);
HistoryFileManager.HistoryFileInfo fileInfo3 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo3.GetJobId()).ThenReturn(jobId3);
cache.AddIfAbsent(fileInfo1);
cache.AddIfAbsent(fileInfo2);
cache.AddIfAbsent(fileInfo3);
ICollection <HistoryFileManager.HistoryFileInfo> values;
for (int i = 0; i < 9; i++)
{
values = cache.Values();
if (values.Count > maxSize)
{
Sharpen.Thread.Sleep(100);
}
else
{
NUnit.Framework.Assert.IsFalse("fileInfo1 should have been evicted", values.Contains
(fileInfo1));
return;
}
}
NUnit.Framework.Assert.Fail("JobListCache didn't delete the extra entry");
}
/// <exception cref="System.Exception"/>
public virtual void TestCompletedTaskAttempt()
{
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskId mt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map);
TaskId rt1Id = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Reduce);
TaskAttemptId mta1Id = MRBuilderUtils.NewTaskAttemptId(mt1Id, 0);
TaskAttemptId rta1Id = MRBuilderUtils.NewTaskAttemptId(rt1Id, 0);
Task mt1 = completedJob.GetTask(mt1Id);
Task rt1 = completedJob.GetTask(rt1Id);
TaskAttempt mta1 = mt1.GetAttempt(mta1Id);
NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, mta1.GetState());
NUnit.Framework.Assert.AreEqual("localhost:45454", mta1.GetAssignedContainerMgrAddress
());
NUnit.Framework.Assert.AreEqual("localhost:9999", mta1.GetNodeHttpAddress());
TaskAttemptReport mta1Report = mta1.GetReport();
NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, mta1Report.GetTaskAttemptState
());
NUnit.Framework.Assert.AreEqual("localhost", mta1Report.GetNodeManagerHost());
NUnit.Framework.Assert.AreEqual(45454, mta1Report.GetNodeManagerPort());
NUnit.Framework.Assert.AreEqual(9999, mta1Report.GetNodeManagerHttpPort());
TaskAttempt rta1 = rt1.GetAttempt(rta1Id);
NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, rta1.GetState());
NUnit.Framework.Assert.AreEqual("localhost:45454", rta1.GetAssignedContainerMgrAddress
());
NUnit.Framework.Assert.AreEqual("localhost:9999", rta1.GetNodeHttpAddress());
TaskAttemptReport rta1Report = rta1.GetReport();
NUnit.Framework.Assert.AreEqual(TaskAttemptState.Succeeded, rta1Report.GetTaskAttemptState
());
NUnit.Framework.Assert.AreEqual("localhost", rta1Report.GetNodeManagerHost());
NUnit.Framework.Assert.AreEqual(45454, rta1Report.GetNodeManagerPort());
NUnit.Framework.Assert.AreEqual(9999, rta1Report.GetNodeManagerHttpPort());
}
/// <summary>Simple test of some methods of CompletedJob</summary>
/// <exception cref="System.Exception"/>
public virtual void TestGetTaskAttemptCompletionEvent()
{
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
info, jobAclsManager);
TaskCompletionEvent[] events = completedJob.GetMapAttemptCompletionEvents(0, 1000
);
NUnit.Framework.Assert.AreEqual(10, completedJob.GetMapAttemptCompletionEvents(0,
10).Length);
int currentEventId = 0;
foreach (TaskCompletionEvent taskAttemptCompletionEvent in events)
{
int eventId = taskAttemptCompletionEvent.GetEventId();
NUnit.Framework.Assert.IsTrue(eventId >= currentEventId);
currentEventId = eventId;
}
NUnit.Framework.Assert.IsNull(completedJob.LoadConfFile());
// job name
NUnit.Framework.Assert.AreEqual("Sleep job", completedJob.GetName());
// queue name
NUnit.Framework.Assert.AreEqual("default", completedJob.GetQueueName());
// progress
NUnit.Framework.Assert.AreEqual(1.0, completedJob.GetProgress(), 0.001);
// 12 rows in answer
NUnit.Framework.Assert.AreEqual(12, completedJob.GetTaskAttemptCompletionEvents(0
, 1000).Length);
// select first 10 rows
NUnit.Framework.Assert.AreEqual(10, completedJob.GetTaskAttemptCompletionEvents(0
, 10).Length);
// select 5-10 rows include 5th
NUnit.Framework.Assert.AreEqual(7, completedJob.GetTaskAttemptCompletionEvents(5,
10).Length);
// without errors
NUnit.Framework.Assert.AreEqual(1, completedJob.GetDiagnostics().Count);
NUnit.Framework.Assert.AreEqual(string.Empty, completedJob.GetDiagnostics()[0]);
NUnit.Framework.Assert.AreEqual(0, completedJob.GetJobACLs().Count);
}
private Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job LoadJob(HistoryFileManager.HistoryFileInfo
fileInfo)
{
try
{
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = fileInfo.LoadJob();
if (Log.IsDebugEnabled())
{
Log.Debug("Adding " + job.GetID() + " to loaded job cache");
}
// We can clobber results here, but that should be OK, because it only
// means that we may have two identical copies of the same job floating
// around for a while.
loadedJobCache[job.GetID()] = job;
return(job);
}
catch (IOException e)
{
throw new YarnRuntimeException("Could not find/load job: " + fileInfo.GetJobId(),
e);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestAverageMergeTime()
{
string historyFileName = "job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist";
string confFileName = "job_1329348432655_0001_conf.xml";
Configuration conf = new Configuration();
JobACLsManager jobAclsMgr = new JobACLsManager(conf);
Path fulleHistoryPath = new Path(typeof(TestJobHistoryEntities).GetClassLoader().
GetResource(historyFileName).GetFile());
Path fullConfPath = new Path(typeof(TestJobHistoryEntities).GetClassLoader().GetResource
(confFileName).GetFile());
HistoryFileManager.HistoryFileInfo info = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(info.GetConfFile()).ThenReturn(fullConfPath);
JobId jobId = MRBuilderUtils.NewJobId(1329348432655l, 1, 1);
CompletedJob completedJob = new CompletedJob(conf, jobId, fulleHistoryPath, true,
"user", info, jobAclsMgr);
JobInfo jobInfo = new JobInfo(completedJob);
// There are 2 tasks with merge time of 45 and 55 respectively. So average
// merge time should be 50.
NUnit.Framework.Assert.AreEqual(50L, jobInfo.GetAvgMergeTime());
}
public virtual Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job GetFullJob(JobId jobId)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Looking for Job " + jobId);
}
try
{
HistoryFileManager.HistoryFileInfo fileInfo = hsManager.GetFileInfo(jobId);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job result = null;
if (fileInfo != null)
{
result = loadedJobCache[jobId];
if (result == null)
{
result = LoadJob(fileInfo);
}
else
{
if (fileInfo.IsDeleted())
{
Sharpen.Collections.Remove(loadedJobCache, jobId);
result = null;
}
}
}
else
{
Sharpen.Collections.Remove(loadedJobCache, jobId);
}
return(result);
}
catch (IOException e)
{
throw new YarnRuntimeException(e);
}
}
/// <exception cref="System.Exception"/>
private void CheckHistoryParsing(int numMaps, int numReduces, int numSuccessfulMaps
)
{
Configuration conf = new Configuration();
conf.Set(MRJobConfig.UserName, Runtime.GetProperty("user.name"));
long amStartTimeEst = Runtime.CurrentTimeMillis();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(numMaps, numReduces, true,
this.GetType().FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
Log.Info("JOBID is " + TypeConverter.FromYarn(jobId).ToString());
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
string jobhistoryDir = JobHistoryUtils.GetHistoryIntermediateDoneDirForUser(conf);
FileContext fc = null;
try
{
fc = FileContext.GetFileContext(conf);
}
catch (IOException ioe)
{
Log.Info("Can not get FileContext", ioe);
throw (new Exception("Can not get File Context"));
}
if (numMaps == numSuccessfulMaps)
{
string summaryFileName = JobHistoryUtils.GetIntermediateSummaryFileName(jobId);
Path summaryFile = new Path(jobhistoryDir, summaryFileName);
string jobSummaryString = GetJobSummary(fc, summaryFile);
NUnit.Framework.Assert.IsNotNull(jobSummaryString);
NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerMap=100"));
NUnit.Framework.Assert.IsTrue(jobSummaryString.Contains("resourcesPerReduce=100")
);
IDictionary <string, string> jobSummaryElements = new Dictionary <string, string>();
StringTokenizer strToken = new StringTokenizer(jobSummaryString, ",");
while (strToken.HasMoreTokens())
{
string keypair = strToken.NextToken();
jobSummaryElements[keypair.Split("=")[0]] = keypair.Split("=")[1];
}
NUnit.Framework.Assert.AreEqual("JobId does not match", jobId.ToString(), jobSummaryElements
["jobId"]);
NUnit.Framework.Assert.AreEqual("JobName does not match", "test", jobSummaryElements
["jobName"]);
NUnit.Framework.Assert.IsTrue("submitTime should not be 0", long.Parse(jobSummaryElements
["submitTime"]) != 0);
NUnit.Framework.Assert.IsTrue("launchTime should not be 0", long.Parse(jobSummaryElements
["launchTime"]) != 0);
NUnit.Framework.Assert.IsTrue("firstMapTaskLaunchTime should not be 0", long.Parse
(jobSummaryElements["firstMapTaskLaunchTime"]) != 0);
NUnit.Framework.Assert.IsTrue("firstReduceTaskLaunchTime should not be 0", long.Parse
(jobSummaryElements["firstReduceTaskLaunchTime"]) != 0);
NUnit.Framework.Assert.IsTrue("finishTime should not be 0", long.Parse(jobSummaryElements
["finishTime"]) != 0);
NUnit.Framework.Assert.AreEqual("Mismatch in num map slots", numSuccessfulMaps, System.Convert.ToInt32
(jobSummaryElements["numMaps"]));
NUnit.Framework.Assert.AreEqual("Mismatch in num reduce slots", numReduces, System.Convert.ToInt32
(jobSummaryElements["numReduces"]));
NUnit.Framework.Assert.AreEqual("User does not match", Runtime.GetProperty("user.name"
), jobSummaryElements["user"]);
NUnit.Framework.Assert.AreEqual("Queue does not match", "default", jobSummaryElements
["queue"]);
NUnit.Framework.Assert.AreEqual("Status does not match", "SUCCEEDED", jobSummaryElements
["status"]);
}
JobHistory jobHistory = new JobHistory();
jobHistory.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId);
JobHistoryParser.JobInfo jobInfo;
long numFinishedMaps;
lock (fileInfo)
{
Path historyFilePath = fileInfo.GetHistoryFile();
FSDataInputStream @in = null;
Log.Info("JobHistoryFile is: " + historyFilePath);
try
{
@in = fc.Open(fc.MakeQualified(historyFilePath));
}
catch (IOException ioe)
{
Log.Info("Can not open history file: " + historyFilePath, ioe);
throw (new Exception("Can not open History File"));
}
JobHistoryParser parser = new JobHistoryParser(@in);
EventReader realReader = new EventReader(@in);
EventReader reader = Org.Mockito.Mockito.Mock <EventReader>();
if (numMaps == numSuccessfulMaps)
{
reader = realReader;
}
else
{
AtomicInteger numFinishedEvents = new AtomicInteger(0);
// Hack!
Org.Mockito.Mockito.When(reader.GetNextEvent()).ThenAnswer(new _Answer_257(realReader
, numFinishedEvents, numSuccessfulMaps));
}
jobInfo = parser.Parse(reader);
numFinishedMaps = ComputeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps);
if (numFinishedMaps != numMaps)
{
Exception parseException = parser.GetParseException();
NUnit.Framework.Assert.IsNotNull("Didn't get expected parse exception", parseException
);
}
}
NUnit.Framework.Assert.AreEqual("Incorrect username ", Runtime.GetProperty("user.name"
), jobInfo.GetUsername());
NUnit.Framework.Assert.AreEqual("Incorrect jobName ", "test", jobInfo.GetJobname(
));
NUnit.Framework.Assert.AreEqual("Incorrect queuename ", "default", jobInfo.GetJobQueueName
());
NUnit.Framework.Assert.AreEqual("incorrect conf path", "test", jobInfo.GetJobConfPath
());
NUnit.Framework.Assert.AreEqual("incorrect finishedMap ", numSuccessfulMaps, numFinishedMaps
);
NUnit.Framework.Assert.AreEqual("incorrect finishedReduces ", numReduces, jobInfo
.GetFinishedReduces());
NUnit.Framework.Assert.AreEqual("incorrect uberized ", job.IsUber(), jobInfo.GetUberized
());
IDictionary <TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.GetAllTasks();
int totalTasks = allTasks.Count;
NUnit.Framework.Assert.AreEqual("total number of tasks is incorrect ", (numMaps
+ numReduces), totalTasks);
// Verify aminfo
NUnit.Framework.Assert.AreEqual(1, jobInfo.GetAMInfos().Count);
NUnit.Framework.Assert.AreEqual(MRApp.NmHost, jobInfo.GetAMInfos()[0].GetNodeManagerHost
());
JobHistoryParser.AMInfo amInfo = jobInfo.GetAMInfos()[0];
NUnit.Framework.Assert.AreEqual(MRApp.NmPort, amInfo.GetNodeManagerPort());
NUnit.Framework.Assert.AreEqual(MRApp.NmHttpPort, amInfo.GetNodeManagerHttpPort()
);
NUnit.Framework.Assert.AreEqual(1, amInfo.GetAppAttemptId().GetAttemptId());
NUnit.Framework.Assert.AreEqual(amInfo.GetAppAttemptId(), amInfo.GetContainerId()
.GetApplicationAttemptId());
NUnit.Framework.Assert.IsTrue(amInfo.GetStartTime() <= Runtime.CurrentTimeMillis(
) && amInfo.GetStartTime() >= amStartTimeEst);
ContainerId fakeCid = MRApp.NewContainerId(-1, -1, -1, -1);
// Assert at taskAttempt level
foreach (JobHistoryParser.TaskInfo taskInfo in allTasks.Values)
{
int taskAttemptCount = taskInfo.GetAllTaskAttempts().Count;
NUnit.Framework.Assert.AreEqual("total number of task attempts ", 1, taskAttemptCount
);
JobHistoryParser.TaskAttemptInfo taInfo = taskInfo.GetAllTaskAttempts().Values.GetEnumerator
().Next();
NUnit.Framework.Assert.IsNotNull(taInfo.GetContainerId());
// Verify the wrong ctor is not being used. Remove after mrv1 is removed.
NUnit.Framework.Assert.IsFalse(taInfo.GetContainerId().Equals(fakeCid));
}
// Deep compare Job and JobInfo
foreach (Task task in job.GetTasks().Values)
{
JobHistoryParser.TaskInfo taskInfo_1 = allTasks[TypeConverter.FromYarn(task.GetID
())];
NUnit.Framework.Assert.IsNotNull("TaskInfo not found", taskInfo_1);
foreach (TaskAttempt taskAttempt in task.GetAttempts().Values)
{
JobHistoryParser.TaskAttemptInfo taskAttemptInfo = taskInfo_1.GetAllTaskAttempts(
)[TypeConverter.FromYarn((taskAttempt.GetID()))];
NUnit.Framework.Assert.IsNotNull("TaskAttemptInfo not found", taskAttemptInfo);
NUnit.Framework.Assert.AreEqual("Incorrect shuffle port for task attempt", taskAttempt
.GetShufflePort(), taskAttemptInfo.GetShufflePort());
if (numMaps == numSuccessfulMaps)
{
NUnit.Framework.Assert.AreEqual(MRApp.NmHost, taskAttemptInfo.GetHostname());
NUnit.Framework.Assert.AreEqual(MRApp.NmPort, taskAttemptInfo.GetPort());
// Verify rack-name
NUnit.Framework.Assert.AreEqual("rack-name is incorrect", taskAttemptInfo.GetRackname
(), RackName);
}
}
}
// test output for HistoryViewer
TextWriter stdps = System.Console.Out;
try
{
Runtime.SetOut(new TextWriter(outContent));
HistoryViewer viewer;
lock (fileInfo)
{
viewer = new HistoryViewer(fc.MakeQualified(fileInfo.GetHistoryFile()).ToString()
, conf, true);
}
viewer.Print();
foreach (JobHistoryParser.TaskInfo taskInfo_1 in allTasks.Values)
{
string test = (taskInfo_1.GetTaskStatus() == null ? string.Empty : taskInfo_1.GetTaskStatus
()) + " " + taskInfo_1.GetTaskType() + " task list for " + taskInfo_1.GetTaskId(
).GetJobID();
NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(test) > 0);
NUnit.Framework.Assert.IsTrue(outContent.ToString().IndexOf(taskInfo_1.GetTaskId(
).ToString()) > 0);
}
}
finally
{
Runtime.SetOut(stdps);
}
}
public virtual void TestRefreshJobRetentionSettings()
{
string root = "mockfs://foo/";
string historyDoneDir = root + "mapred/history/done";
long now = Runtime.CurrentTimeMillis();
long someTimeYesterday = now - (25l * 3600 * 1000);
long timeBefore200Secs = now - (200l * 1000);
// Get yesterday's date in YY/MM/DD format
string timestampComponent = JobHistoryUtils.TimestampDirectoryComponent(someTimeYesterday
);
// Create a folder under yesterday's done dir
Path donePathYesterday = new Path(historyDoneDir, timestampComponent + "/" + "000000"
);
FileStatus dirCreatedYesterdayStatus = new FileStatus(0, true, 0, 0, someTimeYesterday
, donePathYesterday);
// Get today's date in YY/MM/DD format
timestampComponent = JobHistoryUtils.TimestampDirectoryComponent(timeBefore200Secs
);
// Create a folder under today's done dir
Path donePathToday = new Path(historyDoneDir, timestampComponent + "/" + "000000"
);
FileStatus dirCreatedTodayStatus = new FileStatus(0, true, 0, 0, timeBefore200Secs
, donePathToday);
// Create a jhist file with yesterday's timestamp under yesterday's done dir
Path fileUnderYesterdayDir = new Path(donePathYesterday.ToString(), "job_1372363578825_0015-"
+ someTimeYesterday + "-user-Sleep+job-" + someTimeYesterday + "-1-1-SUCCEEDED-default.jhist"
);
FileStatus fileUnderYesterdayDirStatus = new FileStatus(10, false, 0, 0, someTimeYesterday
, fileUnderYesterdayDir);
// Create a jhist file with today's timestamp under today's done dir
Path fileUnderTodayDir = new Path(donePathYesterday.ToString(), "job_1372363578825_0016-"
+ timeBefore200Secs + "-user-Sleep+job-" + timeBefore200Secs + "-1-1-SUCCEEDED-default.jhist"
);
FileStatus fileUnderTodayDirStatus = new FileStatus(10, false, 0, 0, timeBefore200Secs
, fileUnderTodayDir);
HistoryFileManager historyManager = Org.Mockito.Mockito.Spy(new HistoryFileManager
());
jobHistory = Org.Mockito.Mockito.Spy(new JobHistory());
IList <FileStatus> fileStatusList = new List <FileStatus>();
fileStatusList.AddItem(dirCreatedYesterdayStatus);
fileStatusList.AddItem(dirCreatedTodayStatus);
// Make the initial delay of history job cleaner as 4 secs
Org.Mockito.Mockito.DoReturn(4).When(jobHistory).GetInitDelaySecs();
Org.Mockito.Mockito.DoReturn(historyManager).When(jobHistory).CreateHistoryFileManager
();
IList <FileStatus> list1 = new List <FileStatus>();
list1.AddItem(fileUnderYesterdayDirStatus);
Org.Mockito.Mockito.DoReturn(list1).When(historyManager).ScanDirectoryForHistoryFiles
(Eq(donePathYesterday), Any <FileContext>());
IList <FileStatus> list2 = new List <FileStatus>();
list2.AddItem(fileUnderTodayDirStatus);
Org.Mockito.Mockito.DoReturn(list2).When(historyManager).ScanDirectoryForHistoryFiles
(Eq(donePathToday), Any <FileContext>());
Org.Mockito.Mockito.DoReturn(fileStatusList).When(historyManager).GetHistoryDirsForCleaning
(Org.Mockito.Mockito.AnyLong());
Org.Mockito.Mockito.DoReturn(true).When(historyManager).DeleteDir(Any <FileStatus>
());
HistoryFileManager.JobListCache jobListCache = Org.Mockito.Mockito.Mock <HistoryFileManager.JobListCache
>();
HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.DoReturn(jobListCache).When(historyManager).CreateJobListCache
();
Org.Mockito.Mockito.When(jobListCache.Get(Any <JobId>())).ThenReturn(fileInfo);
Org.Mockito.Mockito.DoNothing().When(fileInfo).Delete();
// Set job retention time to 24 hrs and cleaner interval to 2 secs
Configuration conf = new Configuration();
conf.SetLong(JHAdminConfig.MrHistoryMaxAgeMs, 24l * 3600 * 1000);
conf.SetLong(JHAdminConfig.MrHistoryCleanerIntervalMs, 2 * 1000);
jobHistory.Init(conf);
jobHistory.Start();
NUnit.Framework.Assert.AreEqual(2 * 1000l, jobHistory.GetCleanerInterval());
// Only yesterday's jhist file should get deleted
Org.Mockito.Mockito.Verify(fileInfo, Org.Mockito.Mockito.Timeout(20000).Times(1))
.Delete();
fileStatusList.Remove(dirCreatedYesterdayStatus);
// Now reset job retention time to 10 secs
conf.SetLong(JHAdminConfig.MrHistoryMaxAgeMs, 10 * 1000);
// Set cleaner interval to 1 sec
conf.SetLong(JHAdminConfig.MrHistoryCleanerIntervalMs, 1 * 1000);
Org.Mockito.Mockito.DoReturn(conf).When(jobHistory).CreateConf();
// Do refresh job retention settings
jobHistory.RefreshJobRetentionSettings();
// Cleaner interval should be updated
NUnit.Framework.Assert.AreEqual(1 * 1000l, jobHistory.GetCleanerInterval());
// Today's jhist file will also be deleted now since it falls below the
// retention threshold
Org.Mockito.Mockito.Verify(fileInfo, Org.Mockito.Mockito.Timeout(20000).Times(2))
.Delete();
}
public virtual void TestRefreshLoadedJobCache()
{
HistoryFileManager historyManager = Org.Mockito.Mockito.Mock <HistoryFileManager>(
);
jobHistory = Org.Mockito.Mockito.Spy(new JobHistory());
Org.Mockito.Mockito.DoReturn(historyManager).When(jobHistory).CreateHistoryFileManager
();
Configuration conf = new Configuration();
// Set the cache size to 2
conf.Set(JHAdminConfig.MrHistoryLoadedJobCacheSize, "2");
jobHistory.Init(conf);
jobHistory.Start();
CachedHistoryStorage storage = Org.Mockito.Mockito.Spy((CachedHistoryStorage)jobHistory
.GetHistoryStorage());
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job[] jobs = new Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
[3];
JobId[] jobIds = new JobId[3];
for (int i = 0; i < 3; i++)
{
jobs[i] = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>();
jobIds[i] = Org.Mockito.Mockito.Mock <JobId>();
Org.Mockito.Mockito.When(jobs[i].GetID()).ThenReturn(jobIds[i]);
}
HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(historyManager.GetFileInfo(Any <JobId>())).ThenReturn(fileInfo
);
Org.Mockito.Mockito.When(fileInfo.LoadJob()).ThenReturn(jobs[0]).ThenReturn(jobs[
1]).ThenReturn(jobs[2]);
// getFullJob will put the job in the cache if it isn't there
for (int i_1 = 0; i_1 < 3; i_1++)
{
storage.GetFullJob(jobs[i_1].GetID());
}
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobCache = storage
.GetLoadedJobCache();
// job0 should have been purged since cache size is 2
NUnit.Framework.Assert.IsFalse(jobCache.Contains(jobs[0].GetID()));
NUnit.Framework.Assert.IsTrue(jobCache.Contains(jobs[1].GetID()) && jobCache.Contains
(jobs[2].GetID()));
// Setting cache size to 3
conf.Set(JHAdminConfig.MrHistoryLoadedJobCacheSize, "3");
Org.Mockito.Mockito.DoReturn(conf).When(storage).CreateConf();
Org.Mockito.Mockito.When(fileInfo.LoadJob()).ThenReturn(jobs[0]).ThenReturn(jobs[
1]).ThenReturn(jobs[2]);
jobHistory.RefreshLoadedJobCache();
for (int i_2 = 0; i_2 < 3; i_2++)
{
storage.GetFullJob(jobs[i_2].GetID());
}
jobCache = storage.GetLoadedJobCache();
// All three jobs should be in cache since its size is now 3
for (int i_3 = 0; i_3 < 3; i_3++)
{
NUnit.Framework.Assert.IsTrue(jobCache.Contains(jobs[i_3].GetID()));
}
}
/// <exception cref="System.Exception"/>
public virtual void TestDiagnosticsForKilledJob()
{
Log.Info("STARTING testDiagnosticsForKilledJob");
try
{
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryParsing.MRAppWithHistoryWithJobKilled(2, 1, true, this
.GetType().FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
app.WaitForState(job, JobState.Killed);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
JobHistory jobHistory = new JobHistory();
jobHistory.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId);
JobHistoryParser parser;
JobHistoryParser.JobInfo jobInfo;
lock (fileInfo)
{
Path historyFilePath = fileInfo.GetHistoryFile();
FSDataInputStream @in = null;
FileContext fc = null;
try
{
fc = FileContext.GetFileContext(conf);
@in = fc.Open(fc.MakeQualified(historyFilePath));
}
catch (IOException ioe)
{
Log.Info("Can not open history file: " + historyFilePath, ioe);
throw (new Exception("Can not open History File"));
}
parser = new JobHistoryParser(@in);
jobInfo = parser.Parse();
}
Exception parseException = parser.GetParseException();
NUnit.Framework.Assert.IsNull("Caught an expected exception " + parseException, parseException
);
IList <string> originalDiagnostics = job.GetDiagnostics();
string historyError = jobInfo.GetErrorInfo();
NUnit.Framework.Assert.IsTrue("No original diagnostics for a failed job", originalDiagnostics
!= null && !originalDiagnostics.IsEmpty());
NUnit.Framework.Assert.IsNotNull("No history error info for a failed job ", historyError
);
foreach (string diagString in originalDiagnostics)
{
NUnit.Framework.Assert.IsTrue(historyError.Contains(diagString));
}
NUnit.Framework.Assert.IsTrue("No killed message in diagnostics", historyError.Contains
(JobImpl.JobKilledDiag));
}
finally
{
Log.Info("FINISHED testDiagnosticsForKilledJob");
}
}
internal virtual void DeleteJobFromJobListCache(HistoryFileManager.HistoryFileInfo
fileInfo)
{
jobListCache.Delete(fileInfo);
}
/// <exception cref="System.Exception"/>
public virtual void TestHistoryParsingForFailedAttempts()
{
Log.Info("STARTING testHistoryParsingForFailedAttempts");
try
{
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryParsing.MRAppWithHistoryWithFailedAttempt(2, 1, true
, this.GetType().FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
JobHistory jobHistory = new JobHistory();
jobHistory.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = jobHistory.GetJobFileInfo(jobId);
JobHistoryParser parser;
JobHistoryParser.JobInfo jobInfo;
lock (fileInfo)
{
Path historyFilePath = fileInfo.GetHistoryFile();
FSDataInputStream @in = null;
FileContext fc = null;
try
{
fc = FileContext.GetFileContext(conf);
@in = fc.Open(fc.MakeQualified(historyFilePath));
}
catch (IOException ioe)
{
Log.Info("Can not open history file: " + historyFilePath, ioe);
throw (new Exception("Can not open History File"));
}
parser = new JobHistoryParser(@in);
jobInfo = parser.Parse();
}
Exception parseException = parser.GetParseException();
NUnit.Framework.Assert.IsNull("Caught an expected exception " + parseException, parseException
);
int noOffailedAttempts = 0;
IDictionary <TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.GetAllTasks();
foreach (Task task in job.GetTasks().Values)
{
JobHistoryParser.TaskInfo taskInfo = allTasks[TypeConverter.FromYarn(task.GetID()
)];
foreach (TaskAttempt taskAttempt in task.GetAttempts().Values)
{
JobHistoryParser.TaskAttemptInfo taskAttemptInfo = taskInfo.GetAllTaskAttempts()[
TypeConverter.FromYarn((taskAttempt.GetID()))];
// Verify rack-name for all task attempts
NUnit.Framework.Assert.AreEqual("rack-name is incorrect", taskAttemptInfo.GetRackname
(), RackName);
if (taskAttemptInfo.GetTaskStatus().Equals("FAILED"))
{
noOffailedAttempts++;
}
}
}
NUnit.Framework.Assert.AreEqual("No of Failed tasks doesn't match.", 2, noOffailedAttempts
);
}
finally
{
Log.Info("FINISHED testHistoryParsingForFailedAttempts");
}
}
