// Time interval for the move thread.
//History job cleaner interval
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
Log.Info("JobHistory Init");
this.conf = conf;
this.appID = ApplicationId.NewInstance(0, 0);
this.appAttemptID = RecordFactoryProvider.GetRecordFactory(conf).NewRecordInstance
<ApplicationAttemptId>();
moveThreadInterval = conf.GetLong(JHAdminConfig.MrHistoryMoveIntervalMs, JHAdminConfig
.DefaultMrHistoryMoveIntervalMs);
hsManager = CreateHistoryFileManager();
hsManager.Init(conf);
try
{
hsManager.InitExisting();
}
catch (IOException e)
{
throw new YarnRuntimeException("Failed to intialize existing directories", e);
}
storage = CreateHistoryStorage();
if (storage is Org.Apache.Hadoop.Service.Service)
{
((Org.Apache.Hadoop.Service.Service)storage).Init(conf);
}
storage.SetHistoryFileManager(hsManager);
base.ServiceInit(conf);
}
/// <exception cref="System.Exception"/>
private void TestCreateHistoryDirs(Configuration conf, Clock clock)
{
conf.Set(JHAdminConfig.MrHistoryDoneDir, "/" + UUID.RandomUUID());
conf.Set(JHAdminConfig.MrHistoryIntermediateDoneDir, "/" + UUID.RandomUUID());
HistoryFileManager hfm = new HistoryFileManager();
hfm.conf = conf;
hfm.CreateHistoryDirs(clock, 500, 2000);
}
/// <exception cref="System.Exception"/>
private void TestTryCreateHistoryDirs(Configuration conf, bool expected)
{
conf.Set(JHAdminConfig.MrHistoryDoneDir, GetDoneDirNameForTest());
conf.Set(JHAdminConfig.MrHistoryIntermediateDoneDir, GetIntermediateDoneDirNameForTest
());
HistoryFileManager hfm = new HistoryFileManager();
hfm.conf = conf;
NUnit.Framework.Assert.AreEqual(expected, hfm.TryCreatingHistoryDirs(false));
}
public virtual void TestScanDirectory()
{
Path p = new Path("any");
FileContext fc = Org.Mockito.Mockito.Mock <FileContext>();
Org.Mockito.Mockito.When(fc.MakeQualified(p)).ThenReturn(p);
Org.Mockito.Mockito.When(fc.ListStatus(p)).ThenThrow(new FileNotFoundException());
IList <FileStatus> lfs = HistoryFileManager.ScanDirectory(p, fc, null);
//primarily, succcess is that an exception was not thrown. Also nice to
//check this
NUnit.Framework.Assert.IsNotNull(lfs);
}
/// <summary>test clean old history files.</summary>
/// <remarks>
/// test clean old history files. Files should be deleted after 1 week by
/// default.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDeleteFileInfo()
{
Log.Info("STARTING testDeleteFileInfo");
try
{
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey,
typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping));
RackResolver.Init(conf);
MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType().
FullName, true);
app.Submit(conf);
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values
.GetEnumerator().Next();
JobId jobId = job.GetID();
app.WaitForState(job, JobState.Succeeded);
// make sure all events are flushed
app.WaitForState(Service.STATE.Stopped);
HistoryFileManager hfm = new HistoryFileManager();
hfm.Init(conf);
HistoryFileManager.HistoryFileInfo fileInfo = hfm.GetFileInfo(jobId);
hfm.InitExisting();
// wait for move files form the done_intermediate directory to the gone
// directory
while (fileInfo.IsMovePending())
{
Sharpen.Thread.Sleep(300);
}
NUnit.Framework.Assert.IsNotNull(hfm.jobListCache.Values());
// try to remove fileInfo
hfm.Clean();
// check that fileInfo does not deleted
NUnit.Framework.Assert.IsFalse(fileInfo.IsDeleted());
// correct live time
hfm.SetMaxHistoryAge(-1);
hfm.Clean();
hfm.Stop();
NUnit.Framework.Assert.IsTrue("Thread pool shutdown", hfm.moveToDoneExecutor.IsTerminated
());
// should be deleted !
NUnit.Framework.Assert.IsTrue("file should be deleted ", fileInfo.IsDeleted());
}
finally
{
Log.Info("FINISHED testDeleteFileInfo");
}
}
public virtual void TestRefreshJobRetentionSettings()
{
string root = "mockfs://foo/";
string historyDoneDir = root + "mapred/history/done";
long now = Runtime.CurrentTimeMillis();
long someTimeYesterday = now - (25l * 3600 * 1000);
long timeBefore200Secs = now - (200l * 1000);
// Get yesterday's date in YY/MM/DD format
string timestampComponent = JobHistoryUtils.TimestampDirectoryComponent(someTimeYesterday
);
// Create a folder under yesterday's done dir
Path donePathYesterday = new Path(historyDoneDir, timestampComponent + "/" + "000000"
);
FileStatus dirCreatedYesterdayStatus = new FileStatus(0, true, 0, 0, someTimeYesterday
, donePathYesterday);
// Get today's date in YY/MM/DD format
timestampComponent = JobHistoryUtils.TimestampDirectoryComponent(timeBefore200Secs
);
// Create a folder under today's done dir
Path donePathToday = new Path(historyDoneDir, timestampComponent + "/" + "000000"
);
FileStatus dirCreatedTodayStatus = new FileStatus(0, true, 0, 0, timeBefore200Secs
, donePathToday);
// Create a jhist file with yesterday's timestamp under yesterday's done dir
Path fileUnderYesterdayDir = new Path(donePathYesterday.ToString(), "job_1372363578825_0015-"
+ someTimeYesterday + "-user-Sleep+job-" + someTimeYesterday + "-1-1-SUCCEEDED-default.jhist"
);
FileStatus fileUnderYesterdayDirStatus = new FileStatus(10, false, 0, 0, someTimeYesterday
, fileUnderYesterdayDir);
// Create a jhist file with today's timestamp under today's done dir
Path fileUnderTodayDir = new Path(donePathYesterday.ToString(), "job_1372363578825_0016-"
+ timeBefore200Secs + "-user-Sleep+job-" + timeBefore200Secs + "-1-1-SUCCEEDED-default.jhist"
);
FileStatus fileUnderTodayDirStatus = new FileStatus(10, false, 0, 0, timeBefore200Secs
, fileUnderTodayDir);
HistoryFileManager historyManager = Org.Mockito.Mockito.Spy(new HistoryFileManager
());
jobHistory = Org.Mockito.Mockito.Spy(new JobHistory());
IList <FileStatus> fileStatusList = new List <FileStatus>();
fileStatusList.AddItem(dirCreatedYesterdayStatus);
fileStatusList.AddItem(dirCreatedTodayStatus);
// Make the initial delay of history job cleaner as 4 secs
Org.Mockito.Mockito.DoReturn(4).When(jobHistory).GetInitDelaySecs();
Org.Mockito.Mockito.DoReturn(historyManager).When(jobHistory).CreateHistoryFileManager
();
IList <FileStatus> list1 = new List <FileStatus>();
list1.AddItem(fileUnderYesterdayDirStatus);
Org.Mockito.Mockito.DoReturn(list1).When(historyManager).ScanDirectoryForHistoryFiles
(Eq(donePathYesterday), Any <FileContext>());
IList <FileStatus> list2 = new List <FileStatus>();
list2.AddItem(fileUnderTodayDirStatus);
Org.Mockito.Mockito.DoReturn(list2).When(historyManager).ScanDirectoryForHistoryFiles
(Eq(donePathToday), Any <FileContext>());
Org.Mockito.Mockito.DoReturn(fileStatusList).When(historyManager).GetHistoryDirsForCleaning
(Org.Mockito.Mockito.AnyLong());
Org.Mockito.Mockito.DoReturn(true).When(historyManager).DeleteDir(Any <FileStatus>
());
HistoryFileManager.JobListCache jobListCache = Org.Mockito.Mockito.Mock <HistoryFileManager.JobListCache
>();
HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.DoReturn(jobListCache).When(historyManager).CreateJobListCache
();
Org.Mockito.Mockito.When(jobListCache.Get(Any <JobId>())).ThenReturn(fileInfo);
Org.Mockito.Mockito.DoNothing().When(fileInfo).Delete();
// Set job retention time to 24 hrs and cleaner interval to 2 secs
Configuration conf = new Configuration();
conf.SetLong(JHAdminConfig.MrHistoryMaxAgeMs, 24l * 3600 * 1000);
conf.SetLong(JHAdminConfig.MrHistoryCleanerIntervalMs, 2 * 1000);
jobHistory.Init(conf);
jobHistory.Start();
NUnit.Framework.Assert.AreEqual(2 * 1000l, jobHistory.GetCleanerInterval());
// Only yesterday's jhist file should get deleted
Org.Mockito.Mockito.Verify(fileInfo, Org.Mockito.Mockito.Timeout(20000).Times(1))
.Delete();
fileStatusList.Remove(dirCreatedYesterdayStatus);
// Now reset job retention time to 10 secs
conf.SetLong(JHAdminConfig.MrHistoryMaxAgeMs, 10 * 1000);
// Set cleaner interval to 1 sec
conf.SetLong(JHAdminConfig.MrHistoryCleanerIntervalMs, 1 * 1000);
Org.Mockito.Mockito.DoReturn(conf).When(jobHistory).CreateConf();
// Do refresh job retention settings
jobHistory.RefreshJobRetentionSettings();
// Cleaner interval should be updated
NUnit.Framework.Assert.AreEqual(1 * 1000l, jobHistory.GetCleanerInterval());
// Today's jhist file will also be deleted now since it falls below the
// retention threshold
Org.Mockito.Mockito.Verify(fileInfo, Org.Mockito.Mockito.Timeout(20000).Times(2))
.Delete();
}
public void SetHistoryFileManager(HistoryFileManager hsManager)
{
}
public virtual void TestRefreshLoadedJobCache()
{
HistoryFileManager historyManager = Org.Mockito.Mockito.Mock <HistoryFileManager>(
);
jobHistory = Org.Mockito.Mockito.Spy(new JobHistory());
Org.Mockito.Mockito.DoReturn(historyManager).When(jobHistory).CreateHistoryFileManager
();
Configuration conf = new Configuration();
// Set the cache size to 2
conf.Set(JHAdminConfig.MrHistoryLoadedJobCacheSize, "2");
jobHistory.Init(conf);
jobHistory.Start();
CachedHistoryStorage storage = Org.Mockito.Mockito.Spy((CachedHistoryStorage)jobHistory
.GetHistoryStorage());
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job[] jobs = new Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job
[3];
JobId[] jobIds = new JobId[3];
for (int i = 0; i < 3; i++)
{
jobs[i] = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job>();
jobIds[i] = Org.Mockito.Mockito.Mock <JobId>();
Org.Mockito.Mockito.When(jobs[i].GetID()).ThenReturn(jobIds[i]);
}
HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(historyManager.GetFileInfo(Any <JobId>())).ThenReturn(fileInfo
);
Org.Mockito.Mockito.When(fileInfo.LoadJob()).ThenReturn(jobs[0]).ThenReturn(jobs[
1]).ThenReturn(jobs[2]);
// getFullJob will put the job in the cache if it isn't there
for (int i_1 = 0; i_1 < 3; i_1++)
{
storage.GetFullJob(jobs[i_1].GetID());
}
IDictionary <JobId, Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job> jobCache = storage
.GetLoadedJobCache();
// job0 should have been purged since cache size is 2
NUnit.Framework.Assert.IsFalse(jobCache.Contains(jobs[0].GetID()));
NUnit.Framework.Assert.IsTrue(jobCache.Contains(jobs[1].GetID()) && jobCache.Contains
(jobs[2].GetID()));
// Setting cache size to 3
conf.Set(JHAdminConfig.MrHistoryLoadedJobCacheSize, "3");
Org.Mockito.Mockito.DoReturn(conf).When(storage).CreateConf();
Org.Mockito.Mockito.When(fileInfo.LoadJob()).ThenReturn(jobs[0]).ThenReturn(jobs[
1]).ThenReturn(jobs[2]);
jobHistory.RefreshLoadedJobCache();
for (int i_2 = 0; i_2 < 3; i_2++)
{
storage.GetFullJob(jobs[i_2].GetID());
}
jobCache = storage.GetLoadedJobCache();
// All three jobs should be in cache since its size is now 3
for (int i_3 = 0; i_3 < 3; i_3++)
{
NUnit.Framework.Assert.IsTrue(jobCache.Contains(jobs[i_3].GetID()));
}
}
// The number of loaded jobs.
public virtual void SetHistoryFileManager(HistoryFileManager hsManager)
{
this.hsManager = hsManager;
}
