public virtual void TestGetActualPath()
{
Configuration conf = new Configuration();
conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true);
LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>();
// give public visibility
Org.Mockito.Mockito.When(resource.GetVisibility()).ThenReturn(LocalResourceVisibility
.Public);
Path localPath = new Path("foo.jar");
string user = "******";
SCMUploaderProtocol scmClient = Org.Mockito.Mockito.Mock <SCMUploaderProtocol>();
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
FileSystem localFs = Org.Mockito.Mockito.Mock <FileSystem>();
// stub it to return a status that indicates a directory
FileStatus status = Org.Mockito.Mockito.Mock <FileStatus>();
Org.Mockito.Mockito.When(status.IsDirectory()).ThenReturn(true);
Org.Mockito.Mockito.When(localFs.GetFileStatus(localPath)).ThenReturn(status);
SharedCacheUploader spied = CreateSpiedUploader(resource, localPath, user, conf,
scmClient, fs, localFs);
Path actualPath = spied.GetActualPath();
NUnit.Framework.Assert.AreEqual(actualPath.GetName(), localPath.GetName());
NUnit.Framework.Assert.AreEqual(actualPath.GetParent().GetName(), localPath.GetName
());
}
/// <exception cref="System.IO.IOException"/>
private SharedCacheUploader CreateSpiedUploader()
{
Configuration conf = new Configuration();
conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true);
LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>();
Path localPath = Org.Mockito.Mockito.Mock <Path>();
string user = "******";
SCMUploaderProtocol scmClient = Org.Mockito.Mockito.Mock <SCMUploaderProtocol>();
FileSystem fs = FileSystem.Get(conf);
FileSystem localFs = FileSystem.GetLocal(conf);
return(CreateSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs
));
}
/// <param name="resource">the local resource that contains the original remote path</param>
/// <param name="localPath">
/// the path in the local filesystem where the resource is
/// localized
/// </param>
/// <param name="fs">the filesystem of the shared cache</param>
/// <param name="localFs">the local filesystem</param>
public SharedCacheUploader(LocalResource resource, Path localPath, string user, Configuration
conf, SCMUploaderProtocol scmClient, FileSystem fs, FileSystem localFs)
{
this.resource = resource;
this.localPath = localPath;
this.user = user;
this.conf = conf;
this.scmClient = scmClient;
this.fs = fs;
this.sharedCacheRootDir = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration
.DefaultSharedCacheRoot);
this.nestedLevel = SharedCacheUtil.GetCacheDepth(conf);
this.checksum = SharedCacheChecksumFactory.GetChecksum(conf);
this.localFs = localFs;
this.recordFactory = RecordFactoryProvider.GetRecordFactory(null);
}
public virtual void TestSuccess()
{
Configuration conf = new Configuration();
conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true);
LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>();
Path localPath = Org.Mockito.Mockito.Mock <Path>();
Org.Mockito.Mockito.When(localPath.GetName()).ThenReturn("foo.jar");
string user = "******";
SCMUploaderProtocol scmClient = Org.Mockito.Mockito.Mock <SCMUploaderProtocol>();
SCMUploaderNotifyResponse response = Org.Mockito.Mockito.Mock <SCMUploaderNotifyResponse
>();
Org.Mockito.Mockito.When(response.GetAccepted()).ThenReturn(true);
Org.Mockito.Mockito.When(scmClient.Notify(Matchers.IsA <SCMUploaderNotifyRequest>(
))).ThenReturn(response);
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
// return false when rename is called
Org.Mockito.Mockito.When(fs.Rename(Matchers.IsA <Path>(), Matchers.IsA <Path>())).ThenReturn
(true);
FileSystem localFs = FileSystem.GetLocal(conf);
SharedCacheUploader spied = CreateSpiedUploader(resource, localPath, user, conf,
scmClient, fs, localFs);
// stub verifyAccess() to return true
Org.Mockito.Mockito.DoReturn(true).When(spied).VerifyAccess();
// stub getActualPath()
Org.Mockito.Mockito.DoReturn(localPath).When(spied).GetActualPath();
// stub computeChecksum()
Org.Mockito.Mockito.DoReturn("abcdef0123456789").When(spied).ComputeChecksum(Matchers.IsA
<Path>());
// stub uploadFile() to return true
Org.Mockito.Mockito.DoReturn(true).When(spied).UploadFile(Matchers.IsA <Path>(), Matchers.IsA
<Path>());
// stub notifySharedCacheManager to return true
Org.Mockito.Mockito.DoReturn(true).When(spied).NotifySharedCacheManager(Matchers.IsA
<string>(), Matchers.IsA <string>());
NUnit.Framework.Assert.IsTrue(spied.Call());
}
public virtual void TestVerifyAccessPublicResource()
{
Configuration conf = new Configuration();
conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true);
LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>();
// give public visibility
Org.Mockito.Mockito.When(resource.GetVisibility()).ThenReturn(LocalResourceVisibility
.Public);
Path localPath = Org.Mockito.Mockito.Mock <Path>();
Org.Mockito.Mockito.When(localPath.GetName()).ThenReturn("foo.jar");
string user = "******";
SCMUploaderProtocol scmClient = Org.Mockito.Mockito.Mock <SCMUploaderProtocol>();
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
FileSystem localFs = FileSystem.GetLocal(conf);
SharedCacheUploader spied = CreateSpiedUploader(resource, localPath, user, conf,
scmClient, fs, localFs);
NUnit.Framework.Assert.IsTrue(spied.VerifyAccess());
}
public virtual void StartUp()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.ScmStoreClass, typeof(InMemorySCMStore).FullName);
conf.Set(YarnConfiguration.SharedCacheRoot, testDir.GetPath());
AppChecker appChecker = Org.Mockito.Mockito.Spy(new DummyAppChecker());
store = new InMemorySCMStore(appChecker);
store.Init(conf);
store.Start();
service = new SharedCacheUploaderService(store);
service.Init(conf);
service.Start();
YarnRPC rpc = YarnRPC.Create(new Configuration());
IPEndPoint scmAddress = conf.GetSocketAddr(YarnConfiguration.ScmUploaderServerAddress
, YarnConfiguration.DefaultScmUploaderServerAddress, YarnConfiguration.DefaultScmUploaderServerPort
);
proxy = (SCMUploaderProtocol)rpc.GetProxy(typeof(SCMUploaderProtocol), scmAddress
, conf);
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
enabled = conf.GetBoolean(YarnConfiguration.SharedCacheEnabled, YarnConfiguration
.DefaultSharedCacheEnabled);
if (enabled)
{
int threadCount = conf.GetInt(YarnConfiguration.SharedCacheNmUploaderThreadCount,
YarnConfiguration.DefaultSharedCacheNmUploaderThreadCount);
uploaderPool = Executors.NewFixedThreadPool(threadCount, new ThreadFactoryBuilder
().SetNameFormat("Shared cache uploader #%d").Build());
scmClient = CreateSCMClient(conf);
try
{
fs = FileSystem.Get(conf);
localFs = FileSystem.GetLocal(conf);
}
catch (IOException e)
{
Log.Error("Unexpected exception in getting the filesystem", e);
throw new RuntimeException(e);
}
}
base.ServiceInit(conf);
}
/// <exception cref="System.IO.IOException"/>
private SharedCacheUploader CreateSpiedUploader(LocalResource resource, Path localPath
, string user, Configuration conf, SCMUploaderProtocol scmClient, FileSystem fs,
FileSystem localFs)
{
SharedCacheUploader uploader = new SharedCacheUploader(resource, localPath, user,
conf, scmClient, fs, localFs);
return(Org.Mockito.Mockito.Spy(uploader));
}
public SCMUploaderProtocolPBServiceImpl(SCMUploaderProtocol impl)
{
this.real = impl;
}
/// <exception cref="System.IO.IOException"/>
public SharedCacheUploader(LocalResource resource, Path localPath, string user, Configuration
conf, SCMUploaderProtocol scmClient)
: this(resource, localPath, user, conf, scmClient, FileSystem.Get(conf), localPath
.GetFileSystem(conf))
{
}
