/// <exception cref="System.Exception"/> public virtual void TestSetupDistributedCacheConflictsFiles() { Configuration conf = new Configuration(); conf.SetClass("fs.mockfs.impl", typeof(TestMRApps.MockFileSystem), typeof(FileSystem )); URI mockUri = URI.Create("mockfs://mock/"); FileSystem mockFs = ((FilterFileSystem)FileSystem.Get(mockUri, conf)).GetRawFileSystem (); URI file = new URI("mockfs://mock/tmp/something.zip#something"); Path filePath = new Path(file); URI file2 = new URI("mockfs://mock/tmp/something.txt#something"); Path file2Path = new Path(file2); Org.Mockito.Mockito.When(mockFs.ResolvePath(filePath)).ThenReturn(filePath); Org.Mockito.Mockito.When(mockFs.ResolvePath(file2Path)).ThenReturn(file2Path); DistributedCache.AddCacheFile(file, conf); DistributedCache.AddCacheFile(file2, conf); conf.Set(MRJobConfig.CacheFileTimestamps, "10,11"); conf.Set(MRJobConfig.CacheFilesSizes, "10,11"); conf.Set(MRJobConfig.CacheFileVisibilities, "true,true"); IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); MRApps.SetupDistributedCache(conf, localResources); NUnit.Framework.Assert.AreEqual(1, localResources.Count); LocalResource lr = localResources["something"]; //First one wins NUnit.Framework.Assert.IsNotNull(lr); NUnit.Framework.Assert.AreEqual(10l, lr.GetSize()); NUnit.Framework.Assert.AreEqual(10l, lr.GetTimestamp()); NUnit.Framework.Assert.AreEqual(LocalResourceType.File, lr.GetType()); }
private YarnProtos.LocalResourceProto BuildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr is LocalResourcePBImpl)) { lr = LocalResource.NewInstance(lr.GetResource(), lr.GetType(), lr.GetVisibility() , lr.GetSize(), lr.GetTimestamp(), lr.GetPattern()); } lrpb = (LocalResourcePBImpl)lr; return(lrpb.GetProto()); }
internal static LocalResource CreateMockResource() { // mock rsrc location URL uriA = Org.Mockito.Mockito.Mock <URL>(); Org.Mockito.Mockito.When(uriA.GetScheme()).ThenReturn("file"); Org.Mockito.Mockito.When(uriA.GetHost()).ThenReturn(null); Org.Mockito.Mockito.When(uriA.GetFile()).ThenReturn("/localA/rsrc"); LocalResource apiRsrc = Org.Mockito.Mockito.Mock <LocalResource>(); Org.Mockito.Mockito.When(apiRsrc.GetResource()).ThenReturn(uriA); Org.Mockito.Mockito.When(apiRsrc.GetTimestamp()).ThenReturn(4344L); Org.Mockito.Mockito.When(apiRsrc.GetType()).ThenReturn(LocalResourceType.File); return(apiRsrc); }
// TODO - Move this to MR! // Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[], // long[], boolean[], Path[], FileType) /// <exception cref="System.IO.IOException"/> private static void ParseDistributedCacheArtifacts(Configuration conf, IDictionary <string, LocalResource> localResources, LocalResourceType type, URI[] uris, long [] timestamps, long[] sizes, bool[] visibilities) { if (uris != null) { // Sanity check if ((uris.Length != timestamps.Length) || (uris.Length != sizes.Length) || (uris. Length != visibilities.Length)) { throw new ArgumentException("Invalid specification for " + "distributed-cache artifacts of type " + type + " :" + " #uris=" + uris.Length + " #timestamps=" + timestamps.Length + " #visibilities=" + visibilities.Length); } for (int i = 0; i < uris.Length; ++i) { URI u = uris[i]; Path p = new Path(u); FileSystem remoteFS = p.GetFileSystem(conf); p = remoteFS.ResolvePath(p.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory ())); // Add URI fragment or just the filename Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment()); if (name.IsAbsolute()) { throw new ArgumentException("Resource name must be relative"); } string linkName = name.ToUri().GetPath(); LocalResource orig = localResources[linkName]; URL url = ConverterUtils.GetYarnUrlFromURI(p.ToUri()); if (orig != null && !orig.GetResource().Equals(url)) { Log.Warn(GetResourceDescription(orig.GetType()) + ToString(orig.GetResource()) + " conflicts with " + GetResourceDescription(type) + ToString(url) + " This will be an error in Hadoop 2.0" ); continue; } localResources[linkName] = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI (p.ToUri()), type, visibilities[i] ? LocalResourceVisibility.Public : LocalResourceVisibility .Private, sizes[i], timestamps[i]); } } }
internal static long GetEstimatedSize(LocalResource rsrc) { if (rsrc.GetSize() < 0) { return(-1); } switch (rsrc.GetType()) { case LocalResourceType.Archive: case LocalResourceType.Pattern: { return(5 * rsrc.GetSize()); } case LocalResourceType.File: default: { return(rsrc.GetSize()); } } }
internal static ResourceLocalizationSpec GetMockRsrc(Random r, LocalResourceVisibility vis, Path p) { ResourceLocalizationSpec resourceLocalizationSpec = Org.Mockito.Mockito.Mock <ResourceLocalizationSpec >(); LocalResource rsrc = Org.Mockito.Mockito.Mock <LocalResource>(); string name = long.ToHexString(r.NextLong()); URL uri = Org.Mockito.Mockito.Mock <URL>(); Org.Mockito.Mockito.When(uri.GetScheme()).ThenReturn("file"); Org.Mockito.Mockito.When(uri.GetHost()).ThenReturn(null); Org.Mockito.Mockito.When(uri.GetFile()).ThenReturn("/local/" + vis + "/" + name); Org.Mockito.Mockito.When(rsrc.GetResource()).ThenReturn(uri); Org.Mockito.Mockito.When(rsrc.GetSize()).ThenReturn(r.Next(1024) + 1024L); Org.Mockito.Mockito.When(rsrc.GetTimestamp()).ThenReturn(r.Next(1024) + 2048L); Org.Mockito.Mockito.When(rsrc.GetType()).ThenReturn(LocalResourceType.File); Org.Mockito.Mockito.When(rsrc.GetVisibility()).ThenReturn(vis); Org.Mockito.Mockito.When(resourceLocalizationSpec.GetResource()).ThenReturn(rsrc); Org.Mockito.Mockito.When(resourceLocalizationSpec.GetDestinationDirectory()).ThenReturn (ConverterUtils.GetYarnUrlFromPath(p)); return(resourceLocalizationSpec); }
/// <summary> /// Set up the distributed cache by localizing the resources, and updating /// the configuration with references to the localized resources. /// </summary> /// <param name="conf"/> /// <exception cref="System.IO.IOException"/> public virtual void Setup(JobConf conf) { FilePath workDir = new FilePath(Runtime.GetProperty("user.dir")); // Generate YARN local resources objects corresponding to the distributed // cache configuration IDictionary <string, LocalResource> localResources = new LinkedHashMap <string, LocalResource >(); MRApps.SetupDistributedCache(conf, localResources); // Generating unique numbers for FSDownload. AtomicLong uniqueNumberGenerator = new AtomicLong(Runtime.CurrentTimeMillis()); // Find which resources are to be put on the local classpath IDictionary <string, Path> classpaths = new Dictionary <string, Path>(); Path[] archiveClassPaths = DistributedCache.GetArchiveClassPaths(conf); if (archiveClassPaths != null) { foreach (Path p in archiveClassPaths) { classpaths[p.ToUri().GetPath().ToString()] = p; } } Path[] fileClassPaths = DistributedCache.GetFileClassPaths(conf); if (fileClassPaths != null) { foreach (Path p in fileClassPaths) { classpaths[p.ToUri().GetPath().ToString()] = p; } } // Localize the resources LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LocalDir); FileContext localFSFileContext = FileContext.GetLocalFSFileContext(); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); ExecutorService exec = null; try { ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("LocalDistributedCacheManager Downloader #%d" ).Build(); exec = Executors.NewCachedThreadPool(tf); Path destPath = localDirAllocator.GetLocalPathForWrite(".", conf); IDictionary <LocalResource, Future <Path> > resourcesToPaths = Maps.NewHashMap(); foreach (LocalResource resource in localResources.Values) { Callable <Path> download = new FSDownload(localFSFileContext, ugi, conf, new Path( destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet())), resource ); Future <Path> future = exec.Submit(download); resourcesToPaths[resource] = future; } foreach (KeyValuePair <string, LocalResource> entry in localResources) { LocalResource resource_1 = entry.Value; Path path; try { path = resourcesToPaths[resource_1].Get(); } catch (Exception e) { throw new IOException(e); } catch (ExecutionException e) { throw new IOException(e); } string pathString = path.ToUri().ToString(); string link = entry.Key; string target = new FilePath(path.ToUri()).GetPath(); Symlink(workDir, target, link); if (resource_1.GetType() == LocalResourceType.Archive) { localArchives.AddItem(pathString); } else { if (resource_1.GetType() == LocalResourceType.File) { localFiles.AddItem(pathString); } else { if (resource_1.GetType() == LocalResourceType.Pattern) { //PATTERN is not currently used in local mode throw new ArgumentException("Resource type PATTERN is not " + "implemented yet. " + resource_1.GetResource()); } } } Path resourcePath; try { resourcePath = ConverterUtils.GetPathFromYarnURL(resource_1.GetResource()); } catch (URISyntaxException e) { throw new IOException(e); } Log.Info(string.Format("Localized %s as %s", resourcePath, path)); string cp = resourcePath.ToUri().GetPath(); if (classpaths.Keys.Contains(cp)) { localClasspaths.AddItem(path.ToUri().GetPath().ToString()); } } } finally { if (exec != null) { exec.Shutdown(); } } // Update the configuration object with localized data. if (!localArchives.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localArchives, new string[localArchives.Count]))); } if (!localFiles.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localFiles, new string[localArchives.Count]))); } setupCalled = true; }
/// <summary>Wrap API resource to match against cache of localized resources.</summary> /// <param name="resource">Resource requested by container</param> /// <exception cref="Sharpen.URISyntaxException">If the path is malformed</exception> public LocalResourceRequest(LocalResource resource) : this(ConverterUtils.GetPathFromYarnURL(resource.GetResource()), resource.GetTimestamp (), resource.GetType(), resource.GetVisibility(), resource.GetPattern()) { }