private static string GetResourceDescription(LocalResourceType type)
{
if (type == LocalResourceType.Archive || type == LocalResourceType.Pattern)
{
return("cache archive (" + MRJobConfig.CacheArchives + ") ");
}
return("cache file (" + MRJobConfig.CacheFiles + ") ");
}
internal LocalResourceRequest(Path loc, long timestamp, LocalResourceType type, LocalResourceVisibility
visibility, string pattern)
{
this.loc = loc;
this.timestamp = timestamp;
this.type = type;
this.visibility = visibility;
this.pattern = pattern;
}
public override void SetType(LocalResourceType type)
{
lock (this)
{
MaybeInitBuilder();
if (type == null)
{
builder.ClearType();
return;
}
builder.SetType(ConvertToProtoFormat(type));
}
}
public static LocalResource NewLocalResource(URL url, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp, bool shouldBeUploadedToSharedCache)
{
LocalResource resource = recordFactory.NewRecordInstance <LocalResource>();
resource.SetResource(url);
resource.SetType(type);
resource.SetVisibility(visibility);
resource.SetSize(size);
resource.SetTimestamp(timestamp);
resource.SetShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
return(resource);
}
/// <exception cref="System.IO.IOException"/>
private LocalResource CreateApplicationResource(FileContext fs, Path p, LocalResourceType
type)
{
LocalResource rsrc = recordFactory.NewRecordInstance <LocalResource>();
FileStatus rsrcStat = fs.GetFileStatus(p);
rsrc.SetResource(ConverterUtils.GetYarnUrlFromPath(fs.GetDefaultFileSystem().ResolvePath
(rsrcStat.GetPath())));
rsrc.SetSize(rsrcStat.GetLen());
rsrc.SetTimestamp(rsrcStat.GetModificationTime());
rsrc.SetType(type);
rsrc.SetVisibility(LocalResourceVisibility.Application);
return(rsrc);
}
/// <exception cref="Sharpen.URISyntaxException"/>
internal static LocalResource GetYarnResource(Path p, long size, long timestamp,
LocalResourceType type, LocalResourceVisibility state, string pattern)
{
LocalResource ret = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance
<LocalResource>();
ret.SetResource(ConverterUtils.GetYarnUrlFromURI(p.ToUri()));
ret.SetSize(size);
ret.SetTimestamp(timestamp);
ret.SetType(type);
ret.SetVisibility(state);
ret.SetPattern(pattern);
return(ret);
}
public static LocalResource NewInstance(URL url, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp, string pattern, bool shouldBeUploadedToSharedCache
)
{
LocalResource resource = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <LocalResource
>();
resource.SetResource(url);
resource.SetType(type);
resource.SetVisibility(visibility);
resource.SetSize(size);
resource.SetTimestamp(timestamp);
resource.SetPattern(pattern);
resource.SetShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
return(resource);
}
public static LocalResourceType ConvertFromProtoFormat(YarnProtos.LocalResourceTypeProto
e)
{
return(LocalResourceType.ValueOf(e.ToString()));
}
public override void SetType(LocalResourceType type)
{
throw new NotSupportedException();
}
public static LocalResource NewInstance(URL url, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp, bool shouldBeUploadedToSharedCache)
{
return(NewInstance(url, type, visibility, size, timestamp, null, shouldBeUploadedToSharedCache
));
}
public static LocalResource NewInstance(URL url, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp)
{
return(NewInstance(url, type, visibility, size, timestamp, null));
}
public static LocalResource NewInstance(URL url, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp, string pattern)
{
return(NewInstance(url, type, visibility, size, timestamp, pattern, false));
}
public abstract void SetType(LocalResourceType type);
private YarnProtos.LocalResourceTypeProto ConvertToProtoFormat(LocalResourceType
e)
{
return(ProtoUtils.ConvertToProtoFormat(e));
}
/*
* LocalResourceType
*/
public static YarnProtos.LocalResourceTypeProto ConvertToProtoFormat(LocalResourceType
e)
{
return(YarnProtos.LocalResourceTypeProto.ValueOf(e.ToString()));
}
public virtual void SetType(LocalResourceType type)
{
this.type = type;
}
public static LocalResource NewLocalResource(URI uri, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp, bool shouldBeUploadedToSharedCache)
{
return(NewLocalResource(ConverterUtils.GetYarnUrlFromURI(uri), type, visibility,
size, timestamp, shouldBeUploadedToSharedCache));
}
// TODO - Move this to MR!
// Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[],
// long[], boolean[], Path[], FileType)
/// <exception cref="System.IO.IOException"/>
private static void ParseDistributedCacheArtifacts(Configuration conf, IDictionary
<string, LocalResource> localResources, LocalResourceType type, URI[] uris, long
[] timestamps, long[] sizes, bool[] visibilities)
{
if (uris != null)
{
// Sanity check
if ((uris.Length != timestamps.Length) || (uris.Length != sizes.Length) || (uris.
Length != visibilities.Length))
{
throw new ArgumentException("Invalid specification for " + "distributed-cache artifacts of type "
+ type + " :" + " #uris=" + uris.Length + " #timestamps=" + timestamps.Length +
" #visibilities=" + visibilities.Length);
}
for (int i = 0; i < uris.Length; ++i)
{
URI u = uris[i];
Path p = new Path(u);
FileSystem remoteFS = p.GetFileSystem(conf);
p = remoteFS.ResolvePath(p.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory
()));
// Add URI fragment or just the filename
Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment());
if (name.IsAbsolute())
{
throw new ArgumentException("Resource name must be relative");
}
string linkName = name.ToUri().GetPath();
LocalResource orig = localResources[linkName];
URL url = ConverterUtils.GetYarnUrlFromURI(p.ToUri());
if (orig != null && !orig.GetResource().Equals(url))
{
Log.Warn(GetResourceDescription(orig.GetType()) + ToString(orig.GetResource()) +
" conflicts with " + GetResourceDescription(type) + ToString(url) + " This will be an error in Hadoop 2.0"
);
continue;
}
localResources[linkName] = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI
(p.ToUri()), type, visibilities[i] ? LocalResourceVisibility.Public : LocalResourceVisibility
.Private, sizes[i], timestamps[i]);
}
}
}