public virtual void TestAddExisting()
{
HistoryFileManager.JobListCache cache = new HistoryFileManager.JobListCache(2, 1000
);
JobId jobId = MRBuilderUtils.NewJobId(1, 1, 1);
HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo.GetJobId()).ThenReturn(jobId);
cache.AddIfAbsent(fileInfo);
cache.AddIfAbsent(fileInfo);
NUnit.Framework.Assert.AreEqual("Incorrect number of cache entries", 1, cache.Values
().Count);
}
/// <exception cref="System.Exception"/>
public virtual void TestEviction()
{
int maxSize = 2;
HistoryFileManager.JobListCache cache = new HistoryFileManager.JobListCache(maxSize
, 1000);
JobId jobId1 = MRBuilderUtils.NewJobId(1, 1, 1);
HistoryFileManager.HistoryFileInfo fileInfo1 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo1.GetJobId()).ThenReturn(jobId1);
JobId jobId2 = MRBuilderUtils.NewJobId(2, 2, 2);
HistoryFileManager.HistoryFileInfo fileInfo2 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo2.GetJobId()).ThenReturn(jobId2);
JobId jobId3 = MRBuilderUtils.NewJobId(3, 3, 3);
HistoryFileManager.HistoryFileInfo fileInfo3 = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.When(fileInfo3.GetJobId()).ThenReturn(jobId3);
cache.AddIfAbsent(fileInfo1);
cache.AddIfAbsent(fileInfo2);
cache.AddIfAbsent(fileInfo3);
ICollection <HistoryFileManager.HistoryFileInfo> values;
for (int i = 0; i < 9; i++)
{
values = cache.Values();
if (values.Count > maxSize)
{
Sharpen.Thread.Sleep(100);
}
else
{
NUnit.Framework.Assert.IsFalse("fileInfo1 should have been evicted", values.Contains
(fileInfo1));
return;
}
}
NUnit.Framework.Assert.Fail("JobListCache didn't delete the extra entry");
}
public virtual void TestRefreshJobRetentionSettings()
{
string root = "mockfs://foo/";
string historyDoneDir = root + "mapred/history/done";
long now = Runtime.CurrentTimeMillis();
long someTimeYesterday = now - (25l * 3600 * 1000);
long timeBefore200Secs = now - (200l * 1000);
// Get yesterday's date in YY/MM/DD format
string timestampComponent = JobHistoryUtils.TimestampDirectoryComponent(someTimeYesterday
);
// Create a folder under yesterday's done dir
Path donePathYesterday = new Path(historyDoneDir, timestampComponent + "/" + "000000"
);
FileStatus dirCreatedYesterdayStatus = new FileStatus(0, true, 0, 0, someTimeYesterday
, donePathYesterday);
// Get today's date in YY/MM/DD format
timestampComponent = JobHistoryUtils.TimestampDirectoryComponent(timeBefore200Secs
);
// Create a folder under today's done dir
Path donePathToday = new Path(historyDoneDir, timestampComponent + "/" + "000000"
);
FileStatus dirCreatedTodayStatus = new FileStatus(0, true, 0, 0, timeBefore200Secs
, donePathToday);
// Create a jhist file with yesterday's timestamp under yesterday's done dir
Path fileUnderYesterdayDir = new Path(donePathYesterday.ToString(), "job_1372363578825_0015-"
+ someTimeYesterday + "-user-Sleep+job-" + someTimeYesterday + "-1-1-SUCCEEDED-default.jhist"
);
FileStatus fileUnderYesterdayDirStatus = new FileStatus(10, false, 0, 0, someTimeYesterday
, fileUnderYesterdayDir);
// Create a jhist file with today's timestamp under today's done dir
Path fileUnderTodayDir = new Path(donePathYesterday.ToString(), "job_1372363578825_0016-"
+ timeBefore200Secs + "-user-Sleep+job-" + timeBefore200Secs + "-1-1-SUCCEEDED-default.jhist"
);
FileStatus fileUnderTodayDirStatus = new FileStatus(10, false, 0, 0, timeBefore200Secs
, fileUnderTodayDir);
HistoryFileManager historyManager = Org.Mockito.Mockito.Spy(new HistoryFileManager
());
jobHistory = Org.Mockito.Mockito.Spy(new JobHistory());
IList <FileStatus> fileStatusList = new List <FileStatus>();
fileStatusList.AddItem(dirCreatedYesterdayStatus);
fileStatusList.AddItem(dirCreatedTodayStatus);
// Make the initial delay of history job cleaner as 4 secs
Org.Mockito.Mockito.DoReturn(4).When(jobHistory).GetInitDelaySecs();
Org.Mockito.Mockito.DoReturn(historyManager).When(jobHistory).CreateHistoryFileManager
();
IList <FileStatus> list1 = new List <FileStatus>();
list1.AddItem(fileUnderYesterdayDirStatus);
Org.Mockito.Mockito.DoReturn(list1).When(historyManager).ScanDirectoryForHistoryFiles
(Eq(donePathYesterday), Any <FileContext>());
IList <FileStatus> list2 = new List <FileStatus>();
list2.AddItem(fileUnderTodayDirStatus);
Org.Mockito.Mockito.DoReturn(list2).When(historyManager).ScanDirectoryForHistoryFiles
(Eq(donePathToday), Any <FileContext>());
Org.Mockito.Mockito.DoReturn(fileStatusList).When(historyManager).GetHistoryDirsForCleaning
(Org.Mockito.Mockito.AnyLong());
Org.Mockito.Mockito.DoReturn(true).When(historyManager).DeleteDir(Any <FileStatus>
());
HistoryFileManager.JobListCache jobListCache = Org.Mockito.Mockito.Mock <HistoryFileManager.JobListCache
>();
HistoryFileManager.HistoryFileInfo fileInfo = Org.Mockito.Mockito.Mock <HistoryFileManager.HistoryFileInfo
>();
Org.Mockito.Mockito.DoReturn(jobListCache).When(historyManager).CreateJobListCache
();
Org.Mockito.Mockito.When(jobListCache.Get(Any <JobId>())).ThenReturn(fileInfo);
Org.Mockito.Mockito.DoNothing().When(fileInfo).Delete();
// Set job retention time to 24 hrs and cleaner interval to 2 secs
Configuration conf = new Configuration();
conf.SetLong(JHAdminConfig.MrHistoryMaxAgeMs, 24l * 3600 * 1000);
conf.SetLong(JHAdminConfig.MrHistoryCleanerIntervalMs, 2 * 1000);
jobHistory.Init(conf);
jobHistory.Start();
NUnit.Framework.Assert.AreEqual(2 * 1000l, jobHistory.GetCleanerInterval());
// Only yesterday's jhist file should get deleted
Org.Mockito.Mockito.Verify(fileInfo, Org.Mockito.Mockito.Timeout(20000).Times(1))
.Delete();
fileStatusList.Remove(dirCreatedYesterdayStatus);
// Now reset job retention time to 10 secs
conf.SetLong(JHAdminConfig.MrHistoryMaxAgeMs, 10 * 1000);
// Set cleaner interval to 1 sec
conf.SetLong(JHAdminConfig.MrHistoryCleanerIntervalMs, 1 * 1000);
Org.Mockito.Mockito.DoReturn(conf).When(jobHistory).CreateConf();
// Do refresh job retention settings
jobHistory.RefreshJobRetentionSettings();
// Cleaner interval should be updated
NUnit.Framework.Assert.AreEqual(1 * 1000l, jobHistory.GetCleanerInterval());
// Today's jhist file will also be deleted now since it falls below the
// retention threshold
Org.Mockito.Mockito.Verify(fileInfo, Org.Mockito.Mockito.Timeout(20000).Times(2))
.Delete();
}
