public override bool Matches(object o) { LocalizerStatus status = (LocalizerStatus)o; foreach (LocalResourceStatus localized in status.GetResources()) { switch (localized.GetStatus()) { case ResourceStatusType.FetchSuccess: { if (localized.GetLocalPath().GetFile().Contains(rsrc.GetResource().GetFile())) { return(true); } break; } default: { NUnit.Framework.Assert.Fail("Unexpected: " + localized.GetStatus()); break; } } } return(false); }
// TODO - Move this to MR! // Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[], // long[], boolean[], Path[], FileType) /// <exception cref="System.IO.IOException"/> private static void ParseDistributedCacheArtifacts(Configuration conf, IDictionary <string, LocalResource> localResources, LocalResourceType type, URI[] uris, long [] timestamps, long[] sizes, bool[] visibilities) { if (uris != null) { // Sanity check if ((uris.Length != timestamps.Length) || (uris.Length != sizes.Length) || (uris. Length != visibilities.Length)) { throw new ArgumentException("Invalid specification for " + "distributed-cache artifacts of type " + type + " :" + " #uris=" + uris.Length + " #timestamps=" + timestamps.Length + " #visibilities=" + visibilities.Length); } for (int i = 0; i < uris.Length; ++i) { URI u = uris[i]; Path p = new Path(u); FileSystem remoteFS = p.GetFileSystem(conf); p = remoteFS.ResolvePath(p.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory ())); // Add URI fragment or just the filename Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment()); if (name.IsAbsolute()) { throw new ArgumentException("Resource name must be relative"); } string linkName = name.ToUri().GetPath(); LocalResource orig = localResources[linkName]; URL url = ConverterUtils.GetYarnUrlFromURI(p.ToUri()); if (orig != null && !orig.GetResource().Equals(url)) { Log.Warn(GetResourceDescription(orig.GetType()) + ToString(orig.GetResource()) + " conflicts with " + GetResourceDescription(type) + ToString(url) + " This will be an error in Hadoop 2.0" ); continue; } localResources[linkName] = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI (p.ToUri()), type, visibilities[i] ? LocalResourceVisibility.Public : LocalResourceVisibility .Private, sizes[i], timestamps[i]); } } }
private YarnProtos.LocalResourceProto BuildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr is LocalResourcePBImpl)) { lr = LocalResource.NewInstance(lr.GetResource(), lr.GetType(), lr.GetVisibility() , lr.GetSize(), lr.GetTimestamp(), lr.GetPattern()); } lrpb = (LocalResourcePBImpl)lr; return(lrpb.GetProto()); }
internal static LocalResource CreateMockResource() { // mock rsrc location URL uriA = Org.Mockito.Mockito.Mock <URL>(); Org.Mockito.Mockito.When(uriA.GetScheme()).ThenReturn("file"); Org.Mockito.Mockito.When(uriA.GetHost()).ThenReturn(null); Org.Mockito.Mockito.When(uriA.GetFile()).ThenReturn("/localA/rsrc"); LocalResource apiRsrc = Org.Mockito.Mockito.Mock <LocalResource>(); Org.Mockito.Mockito.When(apiRsrc.GetResource()).ThenReturn(uriA); Org.Mockito.Mockito.When(apiRsrc.GetTimestamp()).ThenReturn(4344L); Org.Mockito.Mockito.When(apiRsrc.GetType()).ThenReturn(LocalResourceType.File); return(apiRsrc); }
internal static ResourceLocalizationSpec GetMockRsrc(Random r, LocalResourceVisibility vis, Path p) { ResourceLocalizationSpec resourceLocalizationSpec = Org.Mockito.Mockito.Mock <ResourceLocalizationSpec >(); LocalResource rsrc = Org.Mockito.Mockito.Mock <LocalResource>(); string name = long.ToHexString(r.NextLong()); URL uri = Org.Mockito.Mockito.Mock <URL>(); Org.Mockito.Mockito.When(uri.GetScheme()).ThenReturn("file"); Org.Mockito.Mockito.When(uri.GetHost()).ThenReturn(null); Org.Mockito.Mockito.When(uri.GetFile()).ThenReturn("/local/" + vis + "/" + name); Org.Mockito.Mockito.When(rsrc.GetResource()).ThenReturn(uri); Org.Mockito.Mockito.When(rsrc.GetSize()).ThenReturn(r.Next(1024) + 1024L); Org.Mockito.Mockito.When(rsrc.GetTimestamp()).ThenReturn(r.Next(1024) + 2048L); Org.Mockito.Mockito.When(rsrc.GetType()).ThenReturn(LocalResourceType.File); Org.Mockito.Mockito.When(rsrc.GetVisibility()).ThenReturn(vis); Org.Mockito.Mockito.When(resourceLocalizationSpec.GetResource()).ThenReturn(rsrc); Org.Mockito.Mockito.When(resourceLocalizationSpec.GetDestinationDirectory()).ThenReturn (ConverterUtils.GetYarnUrlFromPath(p)); return(resourceLocalizationSpec); }
internal virtual bool VerifyAccess() { // if it is in the public cache, it's trivially OK if (resource.GetVisibility() == LocalResourceVisibility.Public) { return(true); } Path remotePath; try { remotePath = ConverterUtils.GetPathFromYarnURL(resource.GetResource()); } catch (URISyntaxException e) { throw new IOException("Invalid resource", e); } // get the file status of the HDFS file FileSystem remoteFs = remotePath.GetFileSystem(conf); FileStatus status = remoteFs.GetFileStatus(remotePath); // check to see if the file has been modified in any way if (status.GetModificationTime() != resource.GetTimestamp()) { Log.Warn("The remote file " + remotePath + " has changed since it's localized; will not consider it for upload" ); return(false); } // check for the user ownership if (status.GetOwner().Equals(user)) { return(true); } // the user owns the file // check if the file is publicly readable otherwise return(FileIsPublic(remotePath, remoteFs, status)); }
/// <summary> /// Set up the distributed cache by localizing the resources, and updating /// the configuration with references to the localized resources. /// </summary> /// <param name="conf"/> /// <exception cref="System.IO.IOException"/> public virtual void Setup(JobConf conf) { FilePath workDir = new FilePath(Runtime.GetProperty("user.dir")); // Generate YARN local resources objects corresponding to the distributed // cache configuration IDictionary <string, LocalResource> localResources = new LinkedHashMap <string, LocalResource >(); MRApps.SetupDistributedCache(conf, localResources); // Generating unique numbers for FSDownload. AtomicLong uniqueNumberGenerator = new AtomicLong(Runtime.CurrentTimeMillis()); // Find which resources are to be put on the local classpath IDictionary <string, Path> classpaths = new Dictionary <string, Path>(); Path[] archiveClassPaths = DistributedCache.GetArchiveClassPaths(conf); if (archiveClassPaths != null) { foreach (Path p in archiveClassPaths) { classpaths[p.ToUri().GetPath().ToString()] = p; } } Path[] fileClassPaths = DistributedCache.GetFileClassPaths(conf); if (fileClassPaths != null) { foreach (Path p in fileClassPaths) { classpaths[p.ToUri().GetPath().ToString()] = p; } } // Localize the resources LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LocalDir); FileContext localFSFileContext = FileContext.GetLocalFSFileContext(); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); ExecutorService exec = null; try { ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("LocalDistributedCacheManager Downloader #%d" ).Build(); exec = Executors.NewCachedThreadPool(tf); Path destPath = localDirAllocator.GetLocalPathForWrite(".", conf); IDictionary <LocalResource, Future <Path> > resourcesToPaths = Maps.NewHashMap(); foreach (LocalResource resource in localResources.Values) { Callable <Path> download = new FSDownload(localFSFileContext, ugi, conf, new Path( destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet())), resource ); Future <Path> future = exec.Submit(download); resourcesToPaths[resource] = future; } foreach (KeyValuePair <string, LocalResource> entry in localResources) { LocalResource resource_1 = entry.Value; Path path; try { path = resourcesToPaths[resource_1].Get(); } catch (Exception e) { throw new IOException(e); } catch (ExecutionException e) { throw new IOException(e); } string pathString = path.ToUri().ToString(); string link = entry.Key; string target = new FilePath(path.ToUri()).GetPath(); Symlink(workDir, target, link); if (resource_1.GetType() == LocalResourceType.Archive) { localArchives.AddItem(pathString); } else { if (resource_1.GetType() == LocalResourceType.File) { localFiles.AddItem(pathString); } else { if (resource_1.GetType() == LocalResourceType.Pattern) { //PATTERN is not currently used in local mode throw new ArgumentException("Resource type PATTERN is not " + "implemented yet. " + resource_1.GetResource()); } } } Path resourcePath; try { resourcePath = ConverterUtils.GetPathFromYarnURL(resource_1.GetResource()); } catch (URISyntaxException e) { throw new IOException(e); } Log.Info(string.Format("Localized %s as %s", resourcePath, path)); string cp = resourcePath.ToUri().GetPath(); if (classpaths.Keys.Contains(cp)) { localClasspaths.AddItem(path.ToUri().GetPath().ToString()); } } } finally { if (exec != null) { exec.Shutdown(); } } // Update the configuration object with localized data. if (!localArchives.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localArchives, new string[localArchives.Count]))); } if (!localFiles.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localFiles, new string[localArchives.Count]))); } setupCalled = true; }
/// <summary>Wrap API resource to match against cache of localized resources.</summary> /// <param name="resource">Resource requested by container</param> /// <exception cref="Sharpen.URISyntaxException">If the path is malformed</exception> public LocalResourceRequest(LocalResource resource) : this(ConverterUtils.GetPathFromYarnURL(resource.GetResource()), resource.GetTimestamp (), resource.GetType(), resource.GetVisibility(), resource.GetPattern()) { }