public virtual void TestProcessEvictableResource()
{
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
CleanerMetrics metrics = Org.Mockito.Mockito.Mock <CleanerMetrics>();
SCMStore store = Org.Mockito.Mockito.Mock <SCMStore>();
CleanerTask task = CreateSpiedTask(fs, store, metrics, new ReentrantLock());
// mock an evictable resource
Org.Mockito.Mockito.When(store.IsResourceEvictable(Matchers.IsA <string>(), Matchers.IsA
<FileStatus>())).ThenReturn(true);
FileStatus status = Org.Mockito.Mockito.Mock <FileStatus>();
Org.Mockito.Mockito.When(status.GetPath()).ThenReturn(new Path(Root + "/a/b/c/abc"
));
Org.Mockito.Mockito.When(store.RemoveResource(Matchers.IsA <string>())).ThenReturn
(true);
// rename succeeds
Org.Mockito.Mockito.When(fs.Rename(Matchers.IsA <Path>(), Matchers.IsA <Path>())).ThenReturn
(true);
// delete returns true
Org.Mockito.Mockito.When(fs.Delete(Matchers.IsA <Path>(), Matchers.AnyBoolean())).
ThenReturn(true);
// process the resource
task.ProcessSingleResource(status);
// the directory should be renamed
Org.Mockito.Mockito.Verify(fs).Rename(Matchers.Eq(status.GetPath()), Matchers.IsA
<Path>());
// metrics should record a deleted file
Org.Mockito.Mockito.Verify(metrics).ReportAFileDelete();
Org.Mockito.Mockito.Verify(metrics, Org.Mockito.Mockito.Never()).ReportAFileProcess
();
}
/// <summary>
/// Creates a cleaner task based on the root directory location and the
/// filesystem.
/// </summary>
internal CleanerTask(string location, long sleepTime, int nestedLevel, FileSystem
fs, SCMStore store, CleanerMetrics metrics, Lock cleanerTaskLock)
{
this.location = location;
this.sleepTime = sleepTime;
this.nestedLevel = nestedLevel;
this.root = new Path(location);
this.fs = fs;
this.store = store;
this.metrics = metrics;
this.cleanerTaskLock = cleanerTaskLock;
}
public virtual void TestNonExistentRoot()
{
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
CleanerMetrics metrics = Org.Mockito.Mockito.Mock <CleanerMetrics>();
SCMStore store = Org.Mockito.Mockito.Mock <SCMStore>();
CleanerTask task = CreateSpiedTask(fs, store, metrics, new ReentrantLock());
// the shared cache root does not exist
Org.Mockito.Mockito.When(fs.Exists(task.GetRootPath())).ThenReturn(false);
task.Run();
// process() should not be called
Org.Mockito.Mockito.Verify(task, Org.Mockito.Mockito.Never()).Process();
}
public virtual void CleanUp()
{
if (store != null)
{
store.Stop();
store = null;
}
if (service != null)
{
service.Stop();
service = null;
}
if (clientSCMProxy != null)
{
RPC.StopProxy(clientSCMProxy);
clientSCMProxy = null;
}
}
private static SCMStore CreateSCMStoreService(Configuration conf)
{
Type defaultStoreClass;
try
{
defaultStoreClass = (Type)Sharpen.Runtime.GetType(YarnConfiguration.DefaultScmStoreClass
);
}
catch (Exception e)
{
throw new YarnRuntimeException("Invalid default scm store class" + YarnConfiguration
.DefaultScmStoreClass, e);
}
SCMStore store = ReflectionUtils.NewInstance(conf.GetClass <SCMStore>(YarnConfiguration
.ScmStoreClass, defaultStoreClass), conf);
return(store);
}
/// <summary>Creates a cleaner task based on the configuration.</summary>
/// <remarks>
/// Creates a cleaner task based on the configuration. This is provided for
/// convenience.
/// </remarks>
/// <param name="conf"/>
/// <param name="store"/>
/// <param name="metrics"/>
/// <param name="cleanerTaskLock">
/// lock that ensures a serial execution of cleaner
/// task
/// </param>
/// <returns>an instance of a CleanerTask</returns>
public static Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.CleanerTask Create
(Configuration conf, SCMStore store, CleanerMetrics metrics, Lock cleanerTaskLock
)
{
try
{
// get the root directory for the shared cache
string location = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration.DefaultSharedCacheRoot
);
long sleepTime = conf.GetLong(YarnConfiguration.ScmCleanerResourceSleepMs, YarnConfiguration
.DefaultScmCleanerResourceSleepMs);
int nestedLevel = SharedCacheUtil.GetCacheDepth(conf);
FileSystem fs = FileSystem.Get(conf);
return(new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.CleanerTask(location,
sleepTime, nestedLevel, fs, store, metrics, cleanerTaskLock));
}
catch (IOException e)
{
Log.Error("Unable to obtain the filesystem for the cleaner service", e);
throw new ExceptionInInitializerError(e);
}
}
public virtual void StartUp()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.ScmStoreClass, typeof(InMemorySCMStore).FullName);
conf.Set(YarnConfiguration.SharedCacheRoot, testDir.GetPath());
AppChecker appChecker = Org.Mockito.Mockito.Spy(new DummyAppChecker());
store = new InMemorySCMStore(appChecker);
store.Init(conf);
store.Start();
service = new ClientProtocolService(store);
service.Init(conf);
service.Start();
YarnRPC rpc = YarnRPC.Create(new Configuration());
IPEndPoint scmAddress = conf.GetSocketAddr(YarnConfiguration.ScmClientServerAddress
, YarnConfiguration.DefaultScmClientServerAddress, YarnConfiguration.DefaultScmClientServerPort
);
clientSCMProxy = (ClientSCMProtocol)rpc.GetProxy(typeof(ClientSCMProtocol), scmAddress
, conf);
}
public virtual void TestResourceIsInUseHasAnActiveApp()
{
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
CleanerMetrics metrics = Org.Mockito.Mockito.Mock <CleanerMetrics>();
SCMStore store = Org.Mockito.Mockito.Mock <SCMStore>();
FileStatus resource = Org.Mockito.Mockito.Mock <FileStatus>();
Org.Mockito.Mockito.When(resource.GetPath()).ThenReturn(new Path(Root + "/a/b/c/abc"
));
// resource is stale
Org.Mockito.Mockito.When(store.IsResourceEvictable(Matchers.IsA <string>(), Matchers.IsA
<FileStatus>())).ThenReturn(true);
// but still has appIds
Org.Mockito.Mockito.When(store.RemoveResource(Matchers.IsA <string>())).ThenReturn
(false);
CleanerTask task = CreateSpiedTask(fs, store, metrics, new ReentrantLock());
// process the resource
task.ProcessSingleResource(resource);
// metrics should record a processed file (but not delete)
Org.Mockito.Mockito.Verify(metrics).ReportAFileProcess();
Org.Mockito.Mockito.Verify(metrics, Org.Mockito.Mockito.Never()).ReportAFileDelete
();
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
this.store = CreateSCMStoreService(conf);
AddService(store);
CleanerService cs = CreateCleanerService(store);
AddService(cs);
SharedCacheUploaderService nms = CreateNMCacheUploaderSCMProtocolService(store);
AddService(nms);
ClientProtocolService cps = CreateClientProtocolService(store);
AddService(cps);
SCMAdminProtocolService saps = CreateSCMAdminProtocolService(cs);
AddService(saps);
SCMWebServer webUI = CreateSCMWebServer(this);
AddService(webUI);
// init metrics
DefaultMetricsSystem.Initialize("SharedCacheManager");
JvmMetrics.InitSingleton("SharedCacheManager", null);
base.ServiceInit(conf);
}
public virtual void TestProcessFreshResource()
{
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
CleanerMetrics metrics = Org.Mockito.Mockito.Mock <CleanerMetrics>();
SCMStore store = Org.Mockito.Mockito.Mock <SCMStore>();
CleanerTask task = CreateSpiedTask(fs, store, metrics, new ReentrantLock());
// mock a resource that is not evictable
Org.Mockito.Mockito.When(store.IsResourceEvictable(Matchers.IsA <string>(), Matchers.IsA
<FileStatus>())).ThenReturn(false);
FileStatus status = Org.Mockito.Mockito.Mock <FileStatus>();
Org.Mockito.Mockito.When(status.GetPath()).ThenReturn(new Path(Root + "/a/b/c/abc"
));
// process the resource
task.ProcessSingleResource(status);
// the directory should not be renamed
Org.Mockito.Mockito.Verify(fs, Org.Mockito.Mockito.Never()).Rename(Matchers.Eq(status
.GetPath()), Matchers.IsA <Path>());
// metrics should record a processed file (but not delete)
Org.Mockito.Mockito.Verify(metrics).ReportAFileProcess();
Org.Mockito.Mockito.Verify(metrics, Org.Mockito.Mockito.Never()).ReportAFileDelete
();
}
public CleanerService(SCMStore store)
: base("CleanerService")
{
this.store = store;
this.cleanerTaskLock = new ReentrantLock();
}
public SharedCacheUploaderService(SCMStore store)
: base(typeof(Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.SharedCacheUploaderService
).FullName)
{
this.store = store;
}
private ClientProtocolService CreateClientProtocolService(SCMStore store)
{
return(new ClientProtocolService(store));
}
private SharedCacheUploaderService CreateNMCacheUploaderSCMProtocolService(SCMStore
store)
{
return(new SharedCacheUploaderService(store));
}
private CleanerService CreateCleanerService(SCMStore store)
{
return(new CleanerService(store));
}
public ClientProtocolService(SCMStore store)
: base(typeof(Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.ClientProtocolService
).FullName)
{
this.store = store;
}
private CleanerTask CreateSpiedTask(FileSystem fs, SCMStore store, CleanerMetrics
metrics, Lock isCleanerRunning)
{
return(Org.Mockito.Mockito.Spy(new CleanerTask(Root, SleepTime, NestedLevel, fs,
store, metrics, isCleanerRunning)));
}