public virtual void TestGetActualPath() { Configuration conf = new Configuration(); conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true); LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>(); // give public visibility Org.Mockito.Mockito.When(resource.GetVisibility()).ThenReturn(LocalResourceVisibility .Public); Path localPath = new Path("foo.jar"); string user = "******"; SCMUploaderProtocol scmClient = Org.Mockito.Mockito.Mock <SCMUploaderProtocol>(); FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>(); FileSystem localFs = Org.Mockito.Mockito.Mock <FileSystem>(); // stub it to return a status that indicates a directory FileStatus status = Org.Mockito.Mockito.Mock <FileStatus>(); Org.Mockito.Mockito.When(status.IsDirectory()).ThenReturn(true); Org.Mockito.Mockito.When(localFs.GetFileStatus(localPath)).ThenReturn(status); SharedCacheUploader spied = CreateSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs); Path actualPath = spied.GetActualPath(); NUnit.Framework.Assert.AreEqual(actualPath.GetName(), localPath.GetName()); NUnit.Framework.Assert.AreEqual(actualPath.GetParent().GetName(), localPath.GetName ()); }
/// <exception cref="System.IO.IOException"/> private SharedCacheUploader CreateSpiedUploader() { Configuration conf = new Configuration(); conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true); LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>(); Path localPath = Org.Mockito.Mockito.Mock <Path>(); string user = "******"; SCMUploaderProtocol scmClient = Org.Mockito.Mockito.Mock <SCMUploaderProtocol>(); FileSystem fs = FileSystem.Get(conf); FileSystem localFs = FileSystem.GetLocal(conf); return(CreateSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs )); }
/// <param name="resource">the local resource that contains the original remote path</param> /// <param name="localPath"> /// the path in the local filesystem where the resource is /// localized /// </param> /// <param name="fs">the filesystem of the shared cache</param> /// <param name="localFs">the local filesystem</param> public SharedCacheUploader(LocalResource resource, Path localPath, string user, Configuration conf, SCMUploaderProtocol scmClient, FileSystem fs, FileSystem localFs) { this.resource = resource; this.localPath = localPath; this.user = user; this.conf = conf; this.scmClient = scmClient; this.fs = fs; this.sharedCacheRootDir = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration .DefaultSharedCacheRoot); this.nestedLevel = SharedCacheUtil.GetCacheDepth(conf); this.checksum = SharedCacheChecksumFactory.GetChecksum(conf); this.localFs = localFs; this.recordFactory = RecordFactoryProvider.GetRecordFactory(null); }
public virtual void TestSuccess() { Configuration conf = new Configuration(); conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true); LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>(); Path localPath = Org.Mockito.Mockito.Mock <Path>(); Org.Mockito.Mockito.When(localPath.GetName()).ThenReturn("foo.jar"); string user = "******"; SCMUploaderProtocol scmClient = Org.Mockito.Mockito.Mock <SCMUploaderProtocol>(); SCMUploaderNotifyResponse response = Org.Mockito.Mockito.Mock <SCMUploaderNotifyResponse >(); Org.Mockito.Mockito.When(response.GetAccepted()).ThenReturn(true); Org.Mockito.Mockito.When(scmClient.Notify(Matchers.IsA <SCMUploaderNotifyRequest>( ))).ThenReturn(response); FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>(); // return false when rename is called Org.Mockito.Mockito.When(fs.Rename(Matchers.IsA <Path>(), Matchers.IsA <Path>())).ThenReturn (true); FileSystem localFs = FileSystem.GetLocal(conf); SharedCacheUploader spied = CreateSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs); // stub verifyAccess() to return true Org.Mockito.Mockito.DoReturn(true).When(spied).VerifyAccess(); // stub getActualPath() Org.Mockito.Mockito.DoReturn(localPath).When(spied).GetActualPath(); // stub computeChecksum() Org.Mockito.Mockito.DoReturn("abcdef0123456789").When(spied).ComputeChecksum(Matchers.IsA <Path>()); // stub uploadFile() to return true Org.Mockito.Mockito.DoReturn(true).When(spied).UploadFile(Matchers.IsA <Path>(), Matchers.IsA <Path>()); // stub notifySharedCacheManager to return true Org.Mockito.Mockito.DoReturn(true).When(spied).NotifySharedCacheManager(Matchers.IsA <string>(), Matchers.IsA <string>()); NUnit.Framework.Assert.IsTrue(spied.Call()); }
public virtual void TestVerifyAccessPublicResource() { Configuration conf = new Configuration(); conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true); LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>(); // give public visibility Org.Mockito.Mockito.When(resource.GetVisibility()).ThenReturn(LocalResourceVisibility .Public); Path localPath = Org.Mockito.Mockito.Mock <Path>(); Org.Mockito.Mockito.When(localPath.GetName()).ThenReturn("foo.jar"); string user = "******"; SCMUploaderProtocol scmClient = Org.Mockito.Mockito.Mock <SCMUploaderProtocol>(); FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>(); FileSystem localFs = FileSystem.GetLocal(conf); SharedCacheUploader spied = CreateSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs); NUnit.Framework.Assert.IsTrue(spied.VerifyAccess()); }
public virtual void StartUp() { Configuration conf = new Configuration(); conf.Set(YarnConfiguration.ScmStoreClass, typeof(InMemorySCMStore).FullName); conf.Set(YarnConfiguration.SharedCacheRoot, testDir.GetPath()); AppChecker appChecker = Org.Mockito.Mockito.Spy(new DummyAppChecker()); store = new InMemorySCMStore(appChecker); store.Init(conf); store.Start(); service = new SharedCacheUploaderService(store); service.Init(conf); service.Start(); YarnRPC rpc = YarnRPC.Create(new Configuration()); IPEndPoint scmAddress = conf.GetSocketAddr(YarnConfiguration.ScmUploaderServerAddress , YarnConfiguration.DefaultScmUploaderServerAddress, YarnConfiguration.DefaultScmUploaderServerPort ); proxy = (SCMUploaderProtocol)rpc.GetProxy(typeof(SCMUploaderProtocol), scmAddress , conf); }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { enabled = conf.GetBoolean(YarnConfiguration.SharedCacheEnabled, YarnConfiguration .DefaultSharedCacheEnabled); if (enabled) { int threadCount = conf.GetInt(YarnConfiguration.SharedCacheNmUploaderThreadCount, YarnConfiguration.DefaultSharedCacheNmUploaderThreadCount); uploaderPool = Executors.NewFixedThreadPool(threadCount, new ThreadFactoryBuilder ().SetNameFormat("Shared cache uploader #%d").Build()); scmClient = CreateSCMClient(conf); try { fs = FileSystem.Get(conf); localFs = FileSystem.GetLocal(conf); } catch (IOException e) { Log.Error("Unexpected exception in getting the filesystem", e); throw new RuntimeException(e); } } base.ServiceInit(conf); }
/// <exception cref="System.IO.IOException"/> private SharedCacheUploader CreateSpiedUploader(LocalResource resource, Path localPath , string user, Configuration conf, SCMUploaderProtocol scmClient, FileSystem fs, FileSystem localFs) { SharedCacheUploader uploader = new SharedCacheUploader(resource, localPath, user, conf, scmClient, fs, localFs); return(Org.Mockito.Mockito.Spy(uploader)); }
public SCMUploaderProtocolPBServiceImpl(SCMUploaderProtocol impl) { this.real = impl; }
/// <exception cref="System.IO.IOException"/> public SharedCacheUploader(LocalResource resource, Path localPath, string user, Configuration conf, SCMUploaderProtocol scmClient) : this(resource, localPath, user, conf, scmClient, FileSystem.Get(conf), localPath .GetFileSystem(conf)) { }