public virtual void TestProcessEvictableResource()
{
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
CleanerMetrics metrics = Org.Mockito.Mockito.Mock <CleanerMetrics>();
SCMStore store = Org.Mockito.Mockito.Mock <SCMStore>();
CleanerTask task = CreateSpiedTask(fs, store, metrics, new ReentrantLock());
// mock an evictable resource
Org.Mockito.Mockito.When(store.IsResourceEvictable(Matchers.IsA <string>(), Matchers.IsA
<FileStatus>())).ThenReturn(true);
FileStatus status = Org.Mockito.Mockito.Mock <FileStatus>();
Org.Mockito.Mockito.When(status.GetPath()).ThenReturn(new Path(Root + "/a/b/c/abc"
));
Org.Mockito.Mockito.When(store.RemoveResource(Matchers.IsA <string>())).ThenReturn
(true);
// rename succeeds
Org.Mockito.Mockito.When(fs.Rename(Matchers.IsA <Path>(), Matchers.IsA <Path>())).ThenReturn
(true);
// delete returns true
Org.Mockito.Mockito.When(fs.Delete(Matchers.IsA <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
// process the resource
task.ProcessSingleResource(status);
// the directory should be renamed
Org.Mockito.Mockito.Verify(fs).Rename(Matchers.Eq(status.GetPath()), Matchers.IsA
<Path>());
// metrics should record a deleted file
Org.Mockito.Mockito.Verify(metrics).ReportAFileDelete();
Org.Mockito.Mockito.Verify(metrics, Org.Mockito.Mockito.Never()).ReportAFileProcess
();
}
public SCMMetricsInfo(CleanerMetrics cleanerMetrics, ClientSCMMetrics clientSCMMetrics
, SharedCacheUploaderMetrics scmUploaderMetrics)
{
totalDeletedFiles = cleanerMetrics.GetTotalDeletedFiles();
totalProcessedFiles = cleanerMetrics.GetTotalProcessedFiles();
cacheHits = clientSCMMetrics.GetCacheHits();
cacheMisses = clientSCMMetrics.GetCacheMisses();
cacheReleases = clientSCMMetrics.GetCacheReleases();
acceptedUploads = scmUploaderMetrics.GetAcceptedUploads();
rejectedUploads = scmUploaderMetrics.GetRejectUploads();
}
/// <summary>
/// Creates a cleaner task based on the root directory location and the
/// filesystem.
/// </summary>
internal CleanerTask(string location, long sleepTime, int nestedLevel, FileSystem
fs, SCMStore store, CleanerMetrics metrics, Lock cleanerTaskLock)
{
this.location = location;
this.sleepTime = sleepTime;
this.nestedLevel = nestedLevel;
this.root = new Path(location);
this.fs = fs;
this.store = store;
this.metrics = metrics;
this.cleanerTaskLock = cleanerTaskLock;
}
protected override void Render(HtmlBlock.Block html)
{
SCMMetricsInfo metricsInfo = new SCMMetricsInfo(CleanerMetrics.GetInstance(), ClientSCMMetrics
.GetInstance(), SharedCacheUploaderMetrics.GetInstance());
Info("Shared Cache Manager overview").("Started on:", Times.Format(scm.GetStartTime
())).("Cache hits: ", metricsInfo.GetCacheHits()).("Cache misses: ", metricsInfo
.GetCacheMisses()).("Cache releases: ", metricsInfo.GetCacheReleases()).("Accepted uploads: "
, metricsInfo.GetAcceptedUploads()).("Rejected uploads: ", metricsInfo.GetRejectUploads
()).("Deleted files by the cleaner: ", metricsInfo.GetTotalDeletedFiles()).("Processed files by the cleaner: "
, metricsInfo.GetTotalProcessedFiles());
html.(typeof(InfoBlock));
}
public virtual void TestNonExistentRoot()
{
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
CleanerMetrics metrics = Org.Mockito.Mockito.Mock <CleanerMetrics>();
SCMStore store = Org.Mockito.Mockito.Mock <SCMStore>();
CleanerTask task = CreateSpiedTask(fs, store, metrics, new ReentrantLock());
// the shared cache root does not exist
Org.Mockito.Mockito.When(fs.Exists(task.GetRootPath())).ThenReturn(false);
task.Run();
// process() should not be called
Org.Mockito.Mockito.Verify(task, Org.Mockito.Mockito.Never()).Process();
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
if (!WriteGlobalCleanerPidFile())
{
throw new YarnException("The global cleaner pid file already exists! " + "It appears there is another CleanerService running in the cluster"
);
}
this.metrics = CleanerMetrics.GetInstance();
// Start dependent services (i.e. AppChecker)
base.ServiceStart();
Runnable task = CleanerTask.Create(conf, store, metrics, cleanerTaskLock);
long periodInMinutes = GetPeriod(conf);
scheduledExecutor.ScheduleAtFixedRate(task, GetInitialDelay(conf), periodInMinutes
, TimeUnit.Minutes);
Log.Info("Scheduled the shared cache cleaner task to run every " + periodInMinutes
+ " minutes.");
}
/// <summary>Creates a cleaner task based on the configuration.</summary>
/// <remarks>
/// Creates a cleaner task based on the configuration. This is provided for
/// convenience.
/// </remarks>
/// <param name="conf"/>
/// <param name="store"/>
/// <param name="metrics"/>
/// <param name="cleanerTaskLock">
/// lock that ensures a serial execution of cleaner
/// task
/// </param>
/// <returns>an instance of a CleanerTask</returns>
public static Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.CleanerTask Create
(Configuration conf, SCMStore store, CleanerMetrics metrics, Lock cleanerTaskLock
)
{
try
{
// get the root directory for the shared cache
string location = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration.DefaultSharedCacheRoot
);
long sleepTime = conf.GetLong(YarnConfiguration.ScmCleanerResourceSleepMs, YarnConfiguration
.DefaultScmCleanerResourceSleepMs);
int nestedLevel = SharedCacheUtil.GetCacheDepth(conf);
FileSystem fs = FileSystem.Get(conf);
return(new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.CleanerTask(location,
sleepTime, nestedLevel, fs, store, metrics, cleanerTaskLock));
}
catch (IOException e)
{
Log.Error("Unable to obtain the filesystem for the cleaner service", e);
throw new ExceptionInInitializerError(e);
}
}
public virtual void TestProcessFreshResource()
{
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
CleanerMetrics metrics = Org.Mockito.Mockito.Mock <CleanerMetrics>();
SCMStore store = Org.Mockito.Mockito.Mock <SCMStore>();
CleanerTask task = CreateSpiedTask(fs, store, metrics, new ReentrantLock());
// mock a resource that is not evictable
Org.Mockito.Mockito.When(store.IsResourceEvictable(Matchers.IsA <string>(), Matchers.IsA
<FileStatus>())).ThenReturn(false);
FileStatus status = Org.Mockito.Mockito.Mock <FileStatus>();
Org.Mockito.Mockito.When(status.GetPath()).ThenReturn(new Path(Root + "/a/b/c/abc"
));
// process the resource
task.ProcessSingleResource(status);
// the directory should not be renamed
Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).Rename(Matchers.Eq(status
.GetPath()), Matchers.IsA <Path>());
// metrics should record a processed file (but not delete)
Org.Mockito.Mockito.Verify(metrics).ReportAFileProcess();
Org.Mockito.Mockito.Verify(metrics, Org.Mockito.Mockito.Never()).ReportAFileDelete
();
}
public virtual void TestResourceIsInUseHasAnActiveApp()
{
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
CleanerMetrics metrics = Org.Mockito.Mockito.Mock <CleanerMetrics>();
SCMStore store = Org.Mockito.Mockito.Mock <SCMStore>();
FileStatus resource = Org.Mockito.Mockito.Mock <FileStatus>();
Org.Mockito.Mockito.When(resource.GetPath()).ThenReturn(new Path(Root + "/a/b/c/abc"
));
// resource is stale
Org.Mockito.Mockito.When(store.IsResourceEvictable(Matchers.IsA <string>(), Matchers.IsA
<FileStatus>())).ThenReturn(true);
// but still has appIds
Org.Mockito.Mockito.When(store.RemoveResource(Matchers.IsA <string>())).ThenReturn
(false);
CleanerTask task = CreateSpiedTask(fs, store, metrics, new ReentrantLock());
// process the resource
task.ProcessSingleResource(resource);
// metrics should record a processed file (but not delete)
Org.Mockito.Mockito.Verify(metrics).ReportAFileProcess();
Org.Mockito.Mockito.Verify(metrics, Org.Mockito.Mockito.Never()).ReportAFileDelete
();
}
private CleanerTask CreateSpiedTask(FileSystem fs, SCMStore store, CleanerMetrics
metrics, Lock isCleanerRunning)
{
return(Org.Mockito.Mockito.Spy(new CleanerTask(Root, SleepTime, NestedLevel, fs,
store, metrics, isCleanerRunning)));
}