public virtual void TestGetActualPath() { Configuration conf = new Configuration(); conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true); LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>(); // give public visibility Org.Mockito.Mockito.When(resource.GetVisibility()).ThenReturn(LocalResourceVisibility .Public); Path localPath = new Path("foo.jar"); string user = "******"; SCMUploaderProtocol scmClient = Org.Mockito.Mockito.Mock <SCMUploaderProtocol>(); FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>(); FileSystem localFs = Org.Mockito.Mockito.Mock <FileSystem>(); // stub it to return a status that indicates a directory FileStatus status = Org.Mockito.Mockito.Mock <FileStatus>(); Org.Mockito.Mockito.When(status.IsDirectory()).ThenReturn(true); Org.Mockito.Mockito.When(localFs.GetFileStatus(localPath)).ThenReturn(status); SharedCacheUploader spied = CreateSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs); Path actualPath = spied.GetActualPath(); NUnit.Framework.Assert.AreEqual(actualPath.GetName(), localPath.GetName()); NUnit.Framework.Assert.AreEqual(actualPath.GetParent().GetName(), localPath.GetName ()); }
private YarnProtos.LocalResourceProto BuildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr is LocalResourcePBImpl)) { lr = LocalResource.NewInstance(lr.GetResource(), lr.GetType(), lr.GetVisibility() , lr.GetSize(), lr.GetTimestamp(), lr.GetPattern()); } lrpb = (LocalResourcePBImpl)lr; return(lrpb.GetProto()); }
public virtual void TestVerifyAccessPublicResource() { Configuration conf = new Configuration(); conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true); LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>(); // give public visibility Org.Mockito.Mockito.When(resource.GetVisibility()).ThenReturn(LocalResourceVisibility .Public); Path localPath = Org.Mockito.Mockito.Mock <Path>(); Org.Mockito.Mockito.When(localPath.GetName()).ThenReturn("foo.jar"); string user = "******"; SCMUploaderProtocol scmClient = Org.Mockito.Mockito.Mock <SCMUploaderProtocol>(); FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>(); FileSystem localFs = FileSystem.GetLocal(conf); SharedCacheUploader spied = CreateSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs); NUnit.Framework.Assert.IsTrue(spied.VerifyAccess()); }
internal static ResourceLocalizationSpec GetMockRsrc(Random r, LocalResourceVisibility vis, Path p) { ResourceLocalizationSpec resourceLocalizationSpec = Org.Mockito.Mockito.Mock <ResourceLocalizationSpec >(); LocalResource rsrc = Org.Mockito.Mockito.Mock <LocalResource>(); string name = long.ToHexString(r.NextLong()); URL uri = Org.Mockito.Mockito.Mock <URL>(); Org.Mockito.Mockito.When(uri.GetScheme()).ThenReturn("file"); Org.Mockito.Mockito.When(uri.GetHost()).ThenReturn(null); Org.Mockito.Mockito.When(uri.GetFile()).ThenReturn("/local/" + vis + "/" + name); Org.Mockito.Mockito.When(rsrc.GetResource()).ThenReturn(uri); Org.Mockito.Mockito.When(rsrc.GetSize()).ThenReturn(r.Next(1024) + 1024L); Org.Mockito.Mockito.When(rsrc.GetTimestamp()).ThenReturn(r.Next(1024) + 2048L); Org.Mockito.Mockito.When(rsrc.GetType()).ThenReturn(LocalResourceType.File); Org.Mockito.Mockito.When(rsrc.GetVisibility()).ThenReturn(vis); Org.Mockito.Mockito.When(resourceLocalizationSpec.GetResource()).ThenReturn(rsrc); Org.Mockito.Mockito.When(resourceLocalizationSpec.GetDestinationDirectory()).ThenReturn (ConverterUtils.GetYarnUrlFromPath(p)); return(resourceLocalizationSpec); }
internal virtual bool VerifyAccess() { // if it is in the public cache, it's trivially OK if (resource.GetVisibility() == LocalResourceVisibility.Public) { return(true); } Path remotePath; try { remotePath = ConverterUtils.GetPathFromYarnURL(resource.GetResource()); } catch (URISyntaxException e) { throw new IOException("Invalid resource", e); } // get the file status of the HDFS file FileSystem remoteFs = remotePath.GetFileSystem(conf); FileStatus status = remoteFs.GetFileStatus(remotePath); // check to see if the file has been modified in any way if (status.GetModificationTime() != resource.GetTimestamp()) { Log.Warn("The remote file " + remotePath + " has changed since it's localized; will not consider it for upload" ); return(false); } // check for the user ownership if (status.GetOwner().Equals(user)) { return(true); } // the user owns the file // check if the file is publicly readable otherwise return(FileIsPublic(remotePath, remoteFs, status)); }
/// <summary>Wrap API resource to match against cache of localized resources.</summary> /// <param name="resource">Resource requested by container</param> /// <exception cref="Sharpen.URISyntaxException">If the path is malformed</exception> public LocalResourceRequest(LocalResource resource) : this(ConverterUtils.GetPathFromYarnURL(resource.GetResource()), resource.GetTimestamp (), resource.GetType(), resource.GetVisibility(), resource.GetPattern()) { }