private static string GetResourceDescription(LocalResourceType type) { if (type == LocalResourceType.Archive || type == LocalResourceType.Pattern) { return("cache archive (" + MRJobConfig.CacheArchives + ") "); } return("cache file (" + MRJobConfig.CacheFiles + ") "); }
internal LocalResourceRequest(Path loc, long timestamp, LocalResourceType type, LocalResourceVisibility visibility, string pattern) { this.loc = loc; this.timestamp = timestamp; this.type = type; this.visibility = visibility; this.pattern = pattern; }
public override void SetType(LocalResourceType type) { lock (this) { MaybeInitBuilder(); if (type == null) { builder.ClearType(); return; } builder.SetType(ConvertToProtoFormat(type)); } }
public static LocalResource NewLocalResource(URL url, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, bool shouldBeUploadedToSharedCache) { LocalResource resource = recordFactory.NewRecordInstance <LocalResource>(); resource.SetResource(url); resource.SetType(type); resource.SetVisibility(visibility); resource.SetSize(size); resource.SetTimestamp(timestamp); resource.SetShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache); return(resource); }
/// <exception cref="System.IO.IOException"/> private LocalResource CreateApplicationResource(FileContext fs, Path p, LocalResourceType type) { LocalResource rsrc = recordFactory.NewRecordInstance <LocalResource>(); FileStatus rsrcStat = fs.GetFileStatus(p); rsrc.SetResource(ConverterUtils.GetYarnUrlFromPath(fs.GetDefaultFileSystem().ResolvePath (rsrcStat.GetPath()))); rsrc.SetSize(rsrcStat.GetLen()); rsrc.SetTimestamp(rsrcStat.GetModificationTime()); rsrc.SetType(type); rsrc.SetVisibility(LocalResourceVisibility.Application); return(rsrc); }
/// <exception cref="Sharpen.URISyntaxException"/> internal static LocalResource GetYarnResource(Path p, long size, long timestamp, LocalResourceType type, LocalResourceVisibility state, string pattern) { LocalResource ret = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <LocalResource>(); ret.SetResource(ConverterUtils.GetYarnUrlFromURI(p.ToUri())); ret.SetSize(size); ret.SetTimestamp(timestamp); ret.SetType(type); ret.SetVisibility(state); ret.SetPattern(pattern); return(ret); }
public static LocalResource NewInstance(URL url, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, string pattern, bool shouldBeUploadedToSharedCache ) { LocalResource resource = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <LocalResource >(); resource.SetResource(url); resource.SetType(type); resource.SetVisibility(visibility); resource.SetSize(size); resource.SetTimestamp(timestamp); resource.SetPattern(pattern); resource.SetShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache); return(resource); }
public static LocalResourceType ConvertFromProtoFormat(YarnProtos.LocalResourceTypeProto e) { return(LocalResourceType.ValueOf(e.ToString())); }
public override void SetType(LocalResourceType type) { throw new NotSupportedException(); }
public static LocalResource NewInstance(URL url, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, bool shouldBeUploadedToSharedCache) { return(NewInstance(url, type, visibility, size, timestamp, null, shouldBeUploadedToSharedCache )); }
public static LocalResource NewInstance(URL url, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp) { return(NewInstance(url, type, visibility, size, timestamp, null)); }
public static LocalResource NewInstance(URL url, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, string pattern) { return(NewInstance(url, type, visibility, size, timestamp, pattern, false)); }
public abstract void SetType(LocalResourceType type);
private YarnProtos.LocalResourceTypeProto ConvertToProtoFormat(LocalResourceType e) { return(ProtoUtils.ConvertToProtoFormat(e)); }
/* * LocalResourceType */ public static YarnProtos.LocalResourceTypeProto ConvertToProtoFormat(LocalResourceType e) { return(YarnProtos.LocalResourceTypeProto.ValueOf(e.ToString())); }
public virtual void SetType(LocalResourceType type) { this.type = type; }
public static LocalResource NewLocalResource(URI uri, LocalResourceType type, LocalResourceVisibility visibility, long size, long timestamp, bool shouldBeUploadedToSharedCache) { return(NewLocalResource(ConverterUtils.GetYarnUrlFromURI(uri), type, visibility, size, timestamp, shouldBeUploadedToSharedCache)); }
// TODO - Move this to MR! // Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[], // long[], boolean[], Path[], FileType) /// <exception cref="System.IO.IOException"/> private static void ParseDistributedCacheArtifacts(Configuration conf, IDictionary <string, LocalResource> localResources, LocalResourceType type, URI[] uris, long [] timestamps, long[] sizes, bool[] visibilities) { if (uris != null) { // Sanity check if ((uris.Length != timestamps.Length) || (uris.Length != sizes.Length) || (uris. Length != visibilities.Length)) { throw new ArgumentException("Invalid specification for " + "distributed-cache artifacts of type " + type + " :" + " #uris=" + uris.Length + " #timestamps=" + timestamps.Length + " #visibilities=" + visibilities.Length); } for (int i = 0; i < uris.Length; ++i) { URI u = uris[i]; Path p = new Path(u); FileSystem remoteFS = p.GetFileSystem(conf); p = remoteFS.ResolvePath(p.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory ())); // Add URI fragment or just the filename Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment()); if (name.IsAbsolute()) { throw new ArgumentException("Resource name must be relative"); } string linkName = name.ToUri().GetPath(); LocalResource orig = localResources[linkName]; URL url = ConverterUtils.GetYarnUrlFromURI(p.ToUri()); if (orig != null && !orig.GetResource().Equals(url)) { Log.Warn(GetResourceDescription(orig.GetType()) + ToString(orig.GetResource()) + " conflicts with " + GetResourceDescription(type) + ToString(url) + " This will be an error in Hadoop 2.0" ); continue; } localResources[linkName] = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI (p.ToUri()), type, visibilities[i] ? LocalResourceVisibility.Public : LocalResourceVisibility .Private, sizes[i], timestamps[i]); } } }