internal virtual bool VerifyAccess() { // if it is in the public cache, it's trivially OK if (resource.GetVisibility() == LocalResourceVisibility.Public) { return(true); } Path remotePath; try { remotePath = ConverterUtils.GetPathFromYarnURL(resource.GetResource()); } catch (URISyntaxException e) { throw new IOException("Invalid resource", e); } // get the file status of the HDFS file FileSystem remoteFs = remotePath.GetFileSystem(conf); FileStatus status = remoteFs.GetFileStatus(remotePath); // check to see if the file has been modified in any way if (status.GetModificationTime() != resource.GetTimestamp()) { Log.Warn("The remote file " + remotePath + " has changed since it's localized; will not consider it for upload" ); return(false); } // check for the user ownership if (status.GetOwner().Equals(user)) { return(true); } // the user owns the file // check if the file is publicly readable otherwise return(FileIsPublic(remotePath, remoteFs, status)); }
/// <summary> /// Set up the distributed cache by localizing the resources, and updating /// the configuration with references to the localized resources. /// </summary> /// <param name="conf"/> /// <exception cref="System.IO.IOException"/> public virtual void Setup(JobConf conf) { FilePath workDir = new FilePath(Runtime.GetProperty("user.dir")); // Generate YARN local resources objects corresponding to the distributed // cache configuration IDictionary <string, LocalResource> localResources = new LinkedHashMap <string, LocalResource >(); MRApps.SetupDistributedCache(conf, localResources); // Generating unique numbers for FSDownload. AtomicLong uniqueNumberGenerator = new AtomicLong(Runtime.CurrentTimeMillis()); // Find which resources are to be put on the local classpath IDictionary <string, Path> classpaths = new Dictionary <string, Path>(); Path[] archiveClassPaths = DistributedCache.GetArchiveClassPaths(conf); if (archiveClassPaths != null) { foreach (Path p in archiveClassPaths) { classpaths[p.ToUri().GetPath().ToString()] = p; } } Path[] fileClassPaths = DistributedCache.GetFileClassPaths(conf); if (fileClassPaths != null) { foreach (Path p in fileClassPaths) { classpaths[p.ToUri().GetPath().ToString()] = p; } } // Localize the resources LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LocalDir); FileContext localFSFileContext = FileContext.GetLocalFSFileContext(); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); ExecutorService exec = null; try { ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("LocalDistributedCacheManager Downloader #%d" ).Build(); exec = Executors.NewCachedThreadPool(tf); Path destPath = localDirAllocator.GetLocalPathForWrite(".", conf); IDictionary <LocalResource, Future <Path> > resourcesToPaths = Maps.NewHashMap(); foreach (LocalResource resource in localResources.Values) { Callable <Path> download = new FSDownload(localFSFileContext, ugi, conf, new Path( destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet())), resource ); Future <Path> future = exec.Submit(download); resourcesToPaths[resource] = future; } foreach (KeyValuePair <string, LocalResource> entry in localResources) { LocalResource resource_1 = entry.Value; Path path; try { path = resourcesToPaths[resource_1].Get(); } catch (Exception e) { throw new IOException(e); } catch (ExecutionException e) { throw new IOException(e); } string pathString = path.ToUri().ToString(); string link = entry.Key; string target = new FilePath(path.ToUri()).GetPath(); Symlink(workDir, target, link); if (resource_1.GetType() == LocalResourceType.Archive) { localArchives.AddItem(pathString); } else { if (resource_1.GetType() == LocalResourceType.File) { localFiles.AddItem(pathString); } else { if (resource_1.GetType() == LocalResourceType.Pattern) { //PATTERN is not currently used in local mode throw new ArgumentException("Resource type PATTERN is not " + "implemented yet. " + resource_1.GetResource()); } } } Path resourcePath; try { resourcePath = ConverterUtils.GetPathFromYarnURL(resource_1.GetResource()); } catch (URISyntaxException e) { throw new IOException(e); } Log.Info(string.Format("Localized %s as %s", resourcePath, path)); string cp = resourcePath.ToUri().GetPath(); if (classpaths.Keys.Contains(cp)) { localClasspaths.AddItem(path.ToUri().GetPath().ToString()); } } } finally { if (exec != null) { exec.Shutdown(); } } // Update the configuration object with localized data. if (!localArchives.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localArchives, new string[localArchives.Count]))); } if (!localFiles.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localFiles, new string[localArchives.Count]))); } setupCalled = true; }
/// <summary>Wrap API resource to match against cache of localized resources.</summary> /// <param name="resource">Resource requested by container</param> /// <exception cref="Sharpen.URISyntaxException">If the path is malformed</exception> public LocalResourceRequest(LocalResource resource) : this(ConverterUtils.GetPathFromYarnURL(resource.GetResource()), resource.GetTimestamp (), resource.GetType(), resource.GetVisibility(), resource.GetPattern()) { }