/// <exception cref="System.Exception"/>
public virtual void TestSetupDistributedCacheConflictsFiles()
{
Configuration conf = new Configuration();
conf.SetClass("fs.mockfs.impl", typeof(TestMRApps.MockFileSystem), typeof(FileSystem
));
URI mockUri = URI.Create("mockfs://mock/");
FileSystem mockFs = ((FilterFileSystem)FileSystem.Get(mockUri, conf)).GetRawFileSystem
();
URI file = new URI("mockfs://mock/tmp/something.zip#something");
Path filePath = new Path(file);
URI file2 = new URI("mockfs://mock/tmp/something.txt#something");
Path file2Path = new Path(file2);
Org.Mockito.Mockito.When(mockFs.ResolvePath(filePath)).ThenReturn(filePath);
Org.Mockito.Mockito.When(mockFs.ResolvePath(file2Path)).ThenReturn(file2Path);
DistributedCache.AddCacheFile(file, conf);
DistributedCache.AddCacheFile(file2, conf);
conf.Set(MRJobConfig.CacheFileTimestamps, "10,11");
conf.Set(MRJobConfig.CacheFilesSizes, "10,11");
conf.Set(MRJobConfig.CacheFileVisibilities, "true,true");
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
MRApps.SetupDistributedCache(conf, localResources);
NUnit.Framework.Assert.AreEqual(1, localResources.Count);
LocalResource lr = localResources["something"];
//First one wins
NUnit.Framework.Assert.IsNotNull(lr);
NUnit.Framework.Assert.AreEqual(10l, lr.GetSize());
NUnit.Framework.Assert.AreEqual(10l, lr.GetTimestamp());
NUnit.Framework.Assert.AreEqual(LocalResourceType.File, lr.GetType());
}
private YarnProtos.LocalResourceProto BuildLocalResourceProto(LocalResource lr)
{
LocalResourcePBImpl lrpb;
if (!(lr is LocalResourcePBImpl))
{
lr = LocalResource.NewInstance(lr.GetResource(), lr.GetType(), lr.GetVisibility()
, lr.GetSize(), lr.GetTimestamp(), lr.GetPattern());
}
lrpb = (LocalResourcePBImpl)lr;
return(lrpb.GetProto());
}
internal static LocalResource CreateMockResource()
{
// mock rsrc location
URL uriA = Org.Mockito.Mockito.Mock <URL>();
Org.Mockito.Mockito.When(uriA.GetScheme()).ThenReturn("file");
Org.Mockito.Mockito.When(uriA.GetHost()).ThenReturn(null);
Org.Mockito.Mockito.When(uriA.GetFile()).ThenReturn("/localA/rsrc");
LocalResource apiRsrc = Org.Mockito.Mockito.Mock <LocalResource>();
Org.Mockito.Mockito.When(apiRsrc.GetResource()).ThenReturn(uriA);
Org.Mockito.Mockito.When(apiRsrc.GetTimestamp()).ThenReturn(4344L);
Org.Mockito.Mockito.When(apiRsrc.GetType()).ThenReturn(LocalResourceType.File);
return(apiRsrc);
}
// TODO - Move this to MR!
// Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[],
// long[], boolean[], Path[], FileType)
/// <exception cref="System.IO.IOException"/>
private static void ParseDistributedCacheArtifacts(Configuration conf, IDictionary
<string, LocalResource> localResources, LocalResourceType type, URI[] uris, long
[] timestamps, long[] sizes, bool[] visibilities)
{
if (uris != null)
{
// Sanity check
if ((uris.Length != timestamps.Length) || (uris.Length != sizes.Length) || (uris.
Length != visibilities.Length))
{
throw new ArgumentException("Invalid specification for " + "distributed-cache artifacts of type "
+ type + " :" + " #uris=" + uris.Length + " #timestamps=" + timestamps.Length +
" #visibilities=" + visibilities.Length);
}
for (int i = 0; i < uris.Length; ++i)
{
URI u = uris[i];
Path p = new Path(u);
FileSystem remoteFS = p.GetFileSystem(conf);
p = remoteFS.ResolvePath(p.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory
()));
// Add URI fragment or just the filename
Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment());
if (name.IsAbsolute())
{
throw new ArgumentException("Resource name must be relative");
}
string linkName = name.ToUri().GetPath();
LocalResource orig = localResources[linkName];
URL url = ConverterUtils.GetYarnUrlFromURI(p.ToUri());
if (orig != null && !orig.GetResource().Equals(url))
{
Log.Warn(GetResourceDescription(orig.GetType()) + ToString(orig.GetResource()) +
" conflicts with " + GetResourceDescription(type) + ToString(url) + " This will be an error in Hadoop 2.0"
);
continue;
}
localResources[linkName] = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI
(p.ToUri()), type, visibilities[i] ? LocalResourceVisibility.Public : LocalResourceVisibility
.Private, sizes[i], timestamps[i]);
}
}
}
internal static long GetEstimatedSize(LocalResource rsrc)
{
if (rsrc.GetSize() < 0)
{
return(-1);
}
switch (rsrc.GetType())
{
case LocalResourceType.Archive:
case LocalResourceType.Pattern:
{
return(5 * rsrc.GetSize());
}
case LocalResourceType.File:
default:
{
return(rsrc.GetSize());
}
}
}
internal static ResourceLocalizationSpec GetMockRsrc(Random r, LocalResourceVisibility
vis, Path p)
{
ResourceLocalizationSpec resourceLocalizationSpec = Org.Mockito.Mockito.Mock <ResourceLocalizationSpec
>();
LocalResource rsrc = Org.Mockito.Mockito.Mock <LocalResource>();
string name = long.ToHexString(r.NextLong());
URL uri = Org.Mockito.Mockito.Mock <URL>();
Org.Mockito.Mockito.When(uri.GetScheme()).ThenReturn("file");
Org.Mockito.Mockito.When(uri.GetHost()).ThenReturn(null);
Org.Mockito.Mockito.When(uri.GetFile()).ThenReturn("/local/" + vis + "/" + name);
Org.Mockito.Mockito.When(rsrc.GetResource()).ThenReturn(uri);
Org.Mockito.Mockito.When(rsrc.GetSize()).ThenReturn(r.Next(1024) + 1024L);
Org.Mockito.Mockito.When(rsrc.GetTimestamp()).ThenReturn(r.Next(1024) + 2048L);
Org.Mockito.Mockito.When(rsrc.GetType()).ThenReturn(LocalResourceType.File);
Org.Mockito.Mockito.When(rsrc.GetVisibility()).ThenReturn(vis);
Org.Mockito.Mockito.When(resourceLocalizationSpec.GetResource()).ThenReturn(rsrc);
Org.Mockito.Mockito.When(resourceLocalizationSpec.GetDestinationDirectory()).ThenReturn
(ConverterUtils.GetYarnUrlFromPath(p));
return(resourceLocalizationSpec);
}
/// <summary>
/// Set up the distributed cache by localizing the resources, and updating
/// the configuration with references to the localized resources.
/// </summary>
/// <param name="conf"/>
/// <exception cref="System.IO.IOException"/>
public virtual void Setup(JobConf conf)
{
FilePath workDir = new FilePath(Runtime.GetProperty("user.dir"));
// Generate YARN local resources objects corresponding to the distributed
// cache configuration
IDictionary <string, LocalResource> localResources = new LinkedHashMap <string, LocalResource
>();
MRApps.SetupDistributedCache(conf, localResources);
// Generating unique numbers for FSDownload.
AtomicLong uniqueNumberGenerator = new AtomicLong(Runtime.CurrentTimeMillis());
// Find which resources are to be put on the local classpath
IDictionary <string, Path> classpaths = new Dictionary <string, Path>();
Path[] archiveClassPaths = DistributedCache.GetArchiveClassPaths(conf);
if (archiveClassPaths != null)
{
foreach (Path p in archiveClassPaths)
{
classpaths[p.ToUri().GetPath().ToString()] = p;
}
}
Path[] fileClassPaths = DistributedCache.GetFileClassPaths(conf);
if (fileClassPaths != null)
{
foreach (Path p in fileClassPaths)
{
classpaths[p.ToUri().GetPath().ToString()] = p;
}
}
// Localize the resources
LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LocalDir);
FileContext localFSFileContext = FileContext.GetLocalFSFileContext();
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ExecutorService exec = null;
try
{
ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("LocalDistributedCacheManager Downloader #%d"
).Build();
exec = Executors.NewCachedThreadPool(tf);
Path destPath = localDirAllocator.GetLocalPathForWrite(".", conf);
IDictionary <LocalResource, Future <Path> > resourcesToPaths = Maps.NewHashMap();
foreach (LocalResource resource in localResources.Values)
{
Callable <Path> download = new FSDownload(localFSFileContext, ugi, conf, new Path(
destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet())), resource
);
Future <Path> future = exec.Submit(download);
resourcesToPaths[resource] = future;
}
foreach (KeyValuePair <string, LocalResource> entry in localResources)
{
LocalResource resource_1 = entry.Value;
Path path;
try
{
path = resourcesToPaths[resource_1].Get();
}
catch (Exception e)
{
throw new IOException(e);
}
catch (ExecutionException e)
{
throw new IOException(e);
}
string pathString = path.ToUri().ToString();
string link = entry.Key;
string target = new FilePath(path.ToUri()).GetPath();
Symlink(workDir, target, link);
if (resource_1.GetType() == LocalResourceType.Archive)
{
localArchives.AddItem(pathString);
}
else
{
if (resource_1.GetType() == LocalResourceType.File)
{
localFiles.AddItem(pathString);
}
else
{
if (resource_1.GetType() == LocalResourceType.Pattern)
{
//PATTERN is not currently used in local mode
throw new ArgumentException("Resource type PATTERN is not " + "implemented yet. "
+ resource_1.GetResource());
}
}
}
Path resourcePath;
try
{
resourcePath = ConverterUtils.GetPathFromYarnURL(resource_1.GetResource());
}
catch (URISyntaxException e)
{
throw new IOException(e);
}
Log.Info(string.Format("Localized %s as %s", resourcePath, path));
string cp = resourcePath.ToUri().GetPath();
if (classpaths.Keys.Contains(cp))
{
localClasspaths.AddItem(path.ToUri().GetPath().ToString());
}
}
}
finally
{
if (exec != null)
{
exec.Shutdown();
}
}
// Update the configuration object with localized data.
if (!localArchives.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localArchives, new string[localArchives.Count])));
}
if (!localFiles.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localFiles, new string[localArchives.Count])));
}
setupCalled = true;
}
/// <summary>Wrap API resource to match against cache of localized resources.</summary>
/// <param name="resource">Resource requested by container</param>
/// <exception cref="Sharpen.URISyntaxException">If the path is malformed</exception>
public LocalResourceRequest(LocalResource resource)
: this(ConverterUtils.GetPathFromYarnURL(resource.GetResource()), resource.GetTimestamp
(), resource.GetType(), resource.GetVisibility(), resource.GetPattern())
{
}
