/// <exception cref="System.IO.IOException"/>
private void AddToLocalResources(FileSystem fs, string fileSrcPath, string fileDstPath
, string appId, IDictionary <string, LocalResource> localResources, string resources
)
{
string suffix = appName + "/" + appId + "/" + fileDstPath;
Path dst = new Path(fs.GetHomeDirectory(), suffix);
if (fileSrcPath == null)
{
FSDataOutputStream ostream = null;
try
{
ostream = FileSystem.Create(fs, dst, new FsPermission((short)0x1c8));
ostream.WriteUTF(resources);
}
finally
{
IOUtils.CloseQuietly(ostream);
}
}
else
{
fs.CopyFromLocalFile(new Path(fileSrcPath), dst);
}
FileStatus scFileStatus = fs.GetFileStatus(dst);
LocalResource scRsrc = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI
(dst.ToUri()), LocalResourceType.File, LocalResourceVisibility.Application, scFileStatus
.GetLen(), scFileStatus.GetModificationTime());
localResources[fileDstPath] = scRsrc;
}
/// <exception cref="Sharpen.URISyntaxException"/>
internal static LocalResource GetYarnResource(Path p, long size, long timestamp,
LocalResourceType type, LocalResourceVisibility state, string pattern)
{
LocalResource ret = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance
<LocalResource>();
ret.SetResource(ConverterUtils.GetYarnUrlFromURI(p.ToUri()));
ret.SetSize(size);
ret.SetTimestamp(timestamp);
ret.SetType(type);
ret.SetVisibility(state);
ret.SetPattern(pattern);
return(ret);
}
// TODO - Move this to MR!
// Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[],
// long[], boolean[], Path[], FileType)
/// <exception cref="System.IO.IOException"/>
private static void ParseDistributedCacheArtifacts(Configuration conf, IDictionary
<string, LocalResource> localResources, LocalResourceType type, URI[] uris, long
[] timestamps, long[] sizes, bool[] visibilities)
{
if (uris != null)
{
// Sanity check
if ((uris.Length != timestamps.Length) || (uris.Length != sizes.Length) || (uris.
Length != visibilities.Length))
{
throw new ArgumentException("Invalid specification for " + "distributed-cache artifacts of type "
+ type + " :" + " #uris=" + uris.Length + " #timestamps=" + timestamps.Length +
" #visibilities=" + visibilities.Length);
}
for (int i = 0; i < uris.Length; ++i)
{
URI u = uris[i];
Path p = new Path(u);
FileSystem remoteFS = p.GetFileSystem(conf);
p = remoteFS.ResolvePath(p.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory
()));
// Add URI fragment or just the filename
Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment());
if (name.IsAbsolute())
{
throw new ArgumentException("Resource name must be relative");
}
string linkName = name.ToUri().GetPath();
LocalResource orig = localResources[linkName];
URL url = ConverterUtils.GetYarnUrlFromURI(p.ToUri());
if (orig != null && !orig.GetResource().Equals(url))
{
Log.Warn(GetResourceDescription(orig.GetType()) + ToString(orig.GetResource()) +
" conflicts with " + GetResourceDescription(type) + ToString(url) + " This will be an error in Hadoop 2.0"
);
continue;
}
localResources[linkName] = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI
(p.ToUri()), type, visibilities[i] ? LocalResourceVisibility.Public : LocalResourceVisibility
.Private, sizes[i], timestamps[i]);
}
}
}
public static LocalResource NewLocalResource(URI uri, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp, bool shouldBeUploadedToSharedCache)
{
return(NewLocalResource(ConverterUtils.GetYarnUrlFromURI(uri), type, visibility,
size, timestamp, shouldBeUploadedToSharedCache));
}