// gets the base name of the MapReduce framework or null if no // framework was configured private static string GetMRFrameworkName(Configuration conf) { string frameworkName = null; string framework = conf.Get(MRJobConfig.MapreduceApplicationFrameworkPath, string.Empty ); if (!framework.IsEmpty()) { URI uri; try { uri = new URI(framework); } catch (URISyntaxException e) { throw new ArgumentException("Unable to parse '" + framework + "' as a URI, check the setting for " + MRJobConfig.MapreduceApplicationFrameworkPath, e); } frameworkName = uri.GetFragment(); if (frameworkName == null) { frameworkName = new Path(uri).GetName(); } } return(frameworkName); }
private string AppendOrReplaceParamter(string uri, string newQuery) { if (uri.Contains(YarnWebParams.NextRefreshInterval + "=")) { return(uri.ReplaceAll(YarnWebParams.NextRefreshInterval + "=[^&]+", newQuery)); } try { URI oldUri = new URI(uri); string appendQuery = oldUri.GetQuery(); if (appendQuery == null) { appendQuery = newQuery; } else { appendQuery += "&" + newQuery; } URI newUri = new URI(oldUri.GetScheme(), oldUri.GetAuthority(), oldUri.GetPath(), appendQuery, oldUri.GetFragment()); return(newUri.ToString()); } catch (URISyntaxException) { return(null); } }
/// <summary>Resolve the uri's hostname and add the default port if not in the uri</summary> /// <param name="uri">to resolve</param> /// <param name="defaultPort">if none is given</param> /// <returns>URI</returns> public static URI GetCanonicalUri(URI uri, int defaultPort) { // skip if there is no authority, ie. "file" scheme or relative uri string host = uri.GetHost(); if (host == null) { return(uri); } string fqHost = CanonicalizeHost(host); int port = uri.GetPort(); // short out if already canonical with a port if (host.Equals(fqHost) && port != -1) { return(uri); } // reconstruct the uri with the canonical host and port try { uri = new URI(uri.GetScheme(), uri.GetUserInfo(), fqHost, (port == -1) ? defaultPort : port, uri.GetPath(), uri.GetQuery(), uri.GetFragment()); } catch (URISyntaxException e) { throw new ArgumentException(e); } return(uri); }
/// <summary> /// Derives the interpreted (absolute) URI that was used to generate the last /// request. /// </summary> /// <remarks> /// Derives the interpreted (absolute) URI that was used to generate the last /// request. This is done by extracting the request-uri and target origin for /// the last request and scanning all the redirect locations for the last /// fragment identifier, then combining the result into a /// <see cref="Sharpen.URI">Sharpen.URI</see> /// . /// </remarks> /// <param name="originalURI">original request before any redirects</param> /// <param name="target"> /// if the last URI is relative, it is resolved against this target, /// or <code>null</code> if not available. /// </param> /// <param name="redirects"> /// collection of redirect locations since the original request /// or <code>null</code> if not available. /// </param> /// <returns>interpreted (absolute) URI</returns> /// <exception cref="Sharpen.URISyntaxException"></exception> public static URI Resolve(URI originalURI, HttpHost target, IList <URI> redirects) { Args.NotNull(originalURI, "Request URI"); URIBuilder uribuilder; if (redirects == null || redirects.IsEmpty()) { uribuilder = new URIBuilder(originalURI); } else { uribuilder = new URIBuilder(redirects[redirects.Count - 1]); string frag = uribuilder.GetFragment(); // read interpreted fragment identifier from redirect locations for (int i = redirects.Count - 1; frag == null && i >= 0; i--) { frag = redirects[i].GetFragment(); } uribuilder.SetFragment(frag); } // read interpreted fragment identifier from original request if (uribuilder.GetFragment() == null) { uribuilder.SetFragment(originalURI.GetFragment()); } // last target origin if (target != null && !uribuilder.IsAbsolute()) { uribuilder.SetScheme(target.GetSchemeName()); uribuilder.SetHost(target.GetHostName()); uribuilder.SetPort(target.GetPort()); } return(uribuilder.Build()); }
// TODO - Move this to MR! // Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[], // long[], boolean[], Path[], FileType) /// <exception cref="System.IO.IOException"/> private static void ParseDistributedCacheArtifacts(Configuration conf, IDictionary <string, LocalResource> localResources, LocalResourceType type, URI[] uris, long [] timestamps, long[] sizes, bool[] visibilities) { if (uris != null) { // Sanity check if ((uris.Length != timestamps.Length) || (uris.Length != sizes.Length) || (uris. Length != visibilities.Length)) { throw new ArgumentException("Invalid specification for " + "distributed-cache artifacts of type " + type + " :" + " #uris=" + uris.Length + " #timestamps=" + timestamps.Length + " #visibilities=" + visibilities.Length); } for (int i = 0; i < uris.Length; ++i) { URI u = uris[i]; Path p = new Path(u); FileSystem remoteFS = p.GetFileSystem(conf); p = remoteFS.ResolvePath(p.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory ())); // Add URI fragment or just the filename Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment()); if (name.IsAbsolute()) { throw new ArgumentException("Resource name must be relative"); } string linkName = name.ToUri().GetPath(); LocalResource orig = localResources[linkName]; URL url = ConverterUtils.GetYarnUrlFromURI(p.ToUri()); if (orig != null && !orig.GetResource().Equals(url)) { Log.Warn(GetResourceDescription(orig.GetType()) + ToString(orig.GetResource()) + " conflicts with " + GetResourceDescription(type) + ToString(url) + " This will be an error in Hadoop 2.0" ); continue; } localResources[linkName] = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI (p.ToUri()), type, visibilities[i] ? LocalResourceVisibility.Public : LocalResourceVisibility .Private, sizes[i], timestamps[i]); } } }
/// <summary>Convert a nested URI to decode the underlying path.</summary> /// <remarks> /// Convert a nested URI to decode the underlying path. The translation takes /// the authority and parses it into the underlying scheme and authority. /// For example, "myscheme://hdfs@nn/my/path" is converted to /// "hdfs://nn/my/path". /// </remarks> /// <param name="nestedUri">the URI from the nested URI</param> /// <returns>the unnested path</returns> public static Path UnnestUri(URI nestedUri) { string[] parts = nestedUri.GetAuthority().Split("@", 2); StringBuilder result = new StringBuilder(parts[0]); result.Append("://"); if (parts.Length == 2) { result.Append(parts[1]); } result.Append(nestedUri.GetPath()); if (nestedUri.GetQuery() != null) { result.Append("?"); result.Append(nestedUri.GetQuery()); } if (nestedUri.GetFragment() != null) { result.Append("#"); result.Append(nestedUri.GetFragment()); } return(new Path(result.ToString())); }
public override string ToString() { // we can't use uri.toString(), which escapes everything, because we want // illegal characters unescaped in the string, for glob processing, etc. StringBuilder buffer = new StringBuilder(); if (uri.GetScheme() != null) { buffer.Append(uri.GetScheme()); buffer.Append(":"); } if (uri.GetAuthority() != null) { buffer.Append("//"); buffer.Append(uri.GetAuthority()); } if (uri.GetPath() != null) { string path = uri.GetPath(); if (path.IndexOf('/') == 0 && HasWindowsDrive(path) && uri.GetScheme() == null && uri.GetAuthority() == null) { // has windows drive // but no scheme // or authority path = Runtime.Substring(path, 1); } // remove slash before drive buffer.Append(path); } if (uri.GetFragment() != null) { buffer.Append("#"); buffer.Append(uri.GetFragment()); } return(buffer.ToString()); }
private void DigestURI(URI uri) { this.scheme = uri.GetScheme(); this.encodedSchemeSpecificPart = uri.GetRawSchemeSpecificPart(); this.encodedAuthority = uri.GetRawAuthority(); this.host = uri.GetHost(); this.port = uri.GetPort(); this.encodedUserInfo = uri.GetRawUserInfo(); this.userInfo = uri.GetUserInfo(); this.encodedPath = uri.GetRawPath(); this.path = uri.GetPath(); this.encodedQuery = uri.GetRawQuery(); this.queryParams = ParseQuery(uri.GetRawQuery(), Consts.Utf8); this.encodedFragment = uri.GetRawFragment(); this.fragment = uri.GetFragment(); }
/// <exception cref="Sharpen.URISyntaxException"/> private URI GetPathURI(Path destPath, string fragment) { URI pathURI = destPath.ToUri(); if (pathURI.GetFragment() == null) { if (fragment == null) { pathURI = new URI(pathURI.ToString() + "#" + destPath.GetName()); } else { pathURI = new URI(pathURI.ToString() + "#" + fragment); } } return(pathURI); }
/// <summary> /// Set up the DistributedCache related configs to make /// <see cref="Org.Apache.Hadoop.Mapreduce.Filecache.DistributedCache.GetLocalCacheFiles(Org.Apache.Hadoop.Conf.Configuration) /// "/> /// and /// <see cref="Org.Apache.Hadoop.Mapreduce.Filecache.DistributedCache.GetLocalCacheArchives(Org.Apache.Hadoop.Conf.Configuration) /// "/> /// working. /// </summary> /// <param name="conf"/> /// <exception cref="System.IO.IOException"/> public static void SetupDistributedCacheLocal(Configuration conf) { string localWorkDir = Runtime.Getenv("PWD"); // ^ ^ all symlinks are created in the current work-dir // Update the configuration object with localized archives. URI[] cacheArchives = DistributedCache.GetCacheArchives(conf); if (cacheArchives != null) { IList <string> localArchives = new AList <string>(); for (int i = 0; i < cacheArchives.Length; ++i) { URI u = cacheArchives[i]; Path p = new Path(u); Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment()); string linkName = name.ToUri().GetPath(); localArchives.AddItem(new Path(localWorkDir, linkName).ToUri().GetPath()); } if (!localArchives.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localArchives, new string[localArchives.Count]))); } } // Update the configuration object with localized files. URI[] cacheFiles = DistributedCache.GetCacheFiles(conf); if (cacheFiles != null) { IList <string> localFiles = new AList <string>(); for (int i = 0; i < cacheFiles.Length; ++i) { URI u = cacheFiles[i]; Path p = new Path(u); Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment()); string linkName = name.ToUri().GetPath(); localFiles.AddItem(new Path(localWorkDir, linkName).ToUri().GetPath()); } if (!localFiles.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localFiles, new string[localFiles.Count]))); } } }
/// <summary>Returns a qualified path object.</summary> public virtual Org.Apache.Hadoop.FS.Path MakeQualified(URI defaultUri, Org.Apache.Hadoop.FS.Path workingDir) { Org.Apache.Hadoop.FS.Path path = this; if (!IsAbsolute()) { path = new Org.Apache.Hadoop.FS.Path(workingDir, this); } URI pathUri = path.ToUri(); string scheme = pathUri.GetScheme(); string authority = pathUri.GetAuthority(); string fragment = pathUri.GetFragment(); if (scheme != null && (authority != null || defaultUri.GetAuthority() == null)) { return(path); } if (scheme == null) { scheme = defaultUri.GetScheme(); } if (authority == null) { authority = defaultUri.GetAuthority(); if (authority == null) { authority = string.Empty; } } URI newUri = null; try { newUri = new URI(scheme, authority, NormalizePath(scheme, pathUri.GetPath()), null , fragment); } catch (URISyntaxException e) { throw new ArgumentException(e); } return(new Org.Apache.Hadoop.FS.Path(newUri)); }
/// <exception cref="System.Exception"/> internal static void RunTestCache(int port) { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NameNodePort(port).NumDataNodes(2).Build (); URI uri = cluster.GetFileSystem().GetUri(); Log.Info("uri=" + uri); { FileSystem fs = FileSystem.Get(uri, new Configuration()); CheckPath(cluster, fs); for (int i = 0; i < 100; i++) { NUnit.Framework.Assert.IsTrue(fs == FileSystem.Get(uri, new Configuration())); } } if (port == NameNode.DefaultPort) { //test explicit default port URI uri2 = new URI(uri.GetScheme(), uri.GetUserInfo(), uri.GetHost(), NameNode.DefaultPort , uri.GetPath(), uri.GetQuery(), uri.GetFragment()); Log.Info("uri2=" + uri2); FileSystem fs = FileSystem.Get(uri2, conf); CheckPath(cluster, fs); for (int i = 0; i < 100; i++) { NUnit.Framework.Assert.IsTrue(fs == FileSystem.Get(uri2, new Configuration())); } } } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> private static void AddMRFrameworkToDistributedCache(Configuration conf) { string framework = conf.Get(MRJobConfig.MapreduceApplicationFrameworkPath, string.Empty ); if (!framework.IsEmpty()) { URI uri; try { uri = new URI(framework); } catch (URISyntaxException e) { throw new ArgumentException("Unable to parse '" + framework + "' as a URI, check the setting for " + MRJobConfig.MapreduceApplicationFrameworkPath, e); } string linkedName = uri.GetFragment(); // resolve any symlinks in the URI path so using a "current" symlink // to point to a specific version shows the specific version // in the distributed cache configuration FileSystem fs = FileSystem.Get(conf); Path frameworkPath = fs.MakeQualified(new Path(uri.GetScheme(), uri.GetAuthority( ), uri.GetPath())); FileContext fc = FileContext.GetFileContext(frameworkPath.ToUri(), conf); frameworkPath = fc.ResolvePath(frameworkPath); uri = frameworkPath.ToUri(); try { uri = new URI(uri.GetScheme(), uri.GetAuthority(), uri.GetPath(), null, linkedName ); } catch (URISyntaxException e) { throw new ArgumentException(e); } DistributedCache.AddCacheArchive(uri, conf); } }
/// <summary>Resolve a child path against a parent path.</summary> public Path(Org.Apache.Hadoop.FS.Path parent, Org.Apache.Hadoop.FS.Path child) { // Add a slash to parent's path so resolution is compatible with URI's URI parentUri = parent.uri; string parentPath = parentUri.GetPath(); if (!(parentPath.Equals("/") || parentPath.IsEmpty())) { try { parentUri = new URI(parentUri.GetScheme(), parentUri.GetAuthority(), parentUri.GetPath () + "/", null, parentUri.GetFragment()); } catch (URISyntaxException e) { throw new ArgumentException(e); } } URI resolved = parentUri.Resolve(child.uri); Initialize(resolved.GetScheme(), resolved.GetAuthority(), resolved.GetPath(), resolved .GetFragment()); }
/// <summary> /// Upload and configure files, libjars, jobjars, and archives pertaining to /// the passed job. /// </summary> /// <param name="job">the job containing the files to be uploaded</param> /// <param name="submitJobDir">the submission directory of the job</param> /// <exception cref="System.IO.IOException"/> public virtual void UploadFiles(Job job, Path submitJobDir) { Configuration conf = job.GetConfiguration(); short replication = (short)conf.GetInt(Job.SubmitReplication, Job.DefaultSubmitReplication ); if (!(conf.GetBoolean(Job.UsedGenericParser, false))) { Log.Warn("Hadoop command-line option parsing not performed. " + "Implement the Tool interface and execute your application " + "with ToolRunner to remedy this."); } // get all the command line arguments passed in by the user conf string files = conf.Get("tmpfiles"); string libjars = conf.Get("tmpjars"); string archives = conf.Get("tmparchives"); string jobJar = job.GetJar(); // // Figure out what fs the JobTracker is using. Copy the // job to it, under a temporary name. This allows DFS to work, // and under the local fs also provides UNIX-like object loading // semantics. (that is, if the job file is deleted right after // submission, we can still run the submission to completion) // // Create a number of filenames in the JobTracker's fs namespace Log.Debug("default FileSystem: " + jtFs.GetUri()); if (jtFs.Exists(submitJobDir)) { throw new IOException("Not submitting job. Job directory " + submitJobDir + " already exists!! This is unexpected.Please check what's there in" + " that directory"); } submitJobDir = jtFs.MakeQualified(submitJobDir); submitJobDir = new Path(submitJobDir.ToUri().GetPath()); FsPermission mapredSysPerms = new FsPermission(JobSubmissionFiles.JobDirPermission ); FileSystem.Mkdirs(jtFs, submitJobDir, mapredSysPerms); Path filesDir = JobSubmissionFiles.GetJobDistCacheFiles(submitJobDir); Path archivesDir = JobSubmissionFiles.GetJobDistCacheArchives(submitJobDir); Path libjarsDir = JobSubmissionFiles.GetJobDistCacheLibjars(submitJobDir); // add all the command line files/ jars and archive // first copy them to jobtrackers filesystem if (files != null) { FileSystem.Mkdirs(jtFs, filesDir, mapredSysPerms); string[] fileArr = files.Split(","); foreach (string tmpFile in fileArr) { URI tmpURI = null; try { tmpURI = new URI(tmpFile); } catch (URISyntaxException e) { throw new ArgumentException(e); } Path tmp = new Path(tmpURI); Path newPath = CopyRemoteFiles(filesDir, tmp, conf, replication); try { URI pathURI = GetPathURI(newPath, tmpURI.GetFragment()); DistributedCache.AddCacheFile(pathURI, conf); } catch (URISyntaxException ue) { // should not throw a uri exception throw new IOException("Failed to create uri for " + tmpFile, ue); } } } if (libjars != null) { FileSystem.Mkdirs(jtFs, libjarsDir, mapredSysPerms); string[] libjarsArr = libjars.Split(","); foreach (string tmpjars in libjarsArr) { Path tmp = new Path(tmpjars); Path newPath = CopyRemoteFiles(libjarsDir, tmp, conf, replication); DistributedCache.AddFileToClassPath(new Path(newPath.ToUri().GetPath()), conf, jtFs ); } } if (archives != null) { FileSystem.Mkdirs(jtFs, archivesDir, mapredSysPerms); string[] archivesArr = archives.Split(","); foreach (string tmpArchives in archivesArr) { URI tmpURI; try { tmpURI = new URI(tmpArchives); } catch (URISyntaxException e) { throw new ArgumentException(e); } Path tmp = new Path(tmpURI); Path newPath = CopyRemoteFiles(archivesDir, tmp, conf, replication); try { URI pathURI = GetPathURI(newPath, tmpURI.GetFragment()); DistributedCache.AddCacheArchive(pathURI, conf); } catch (URISyntaxException ue) { // should not throw an uri excpetion throw new IOException("Failed to create uri for " + tmpArchives, ue); } } } if (jobJar != null) { // copy jar to JobTracker's fs // use jar name if job is not named. if (string.Empty.Equals(job.GetJobName())) { job.SetJobName(new Path(jobJar).GetName()); } Path jobJarPath = new Path(jobJar); URI jobJarURI = jobJarPath.ToUri(); // If the job jar is already in a global fs, // we don't need to copy it from local fs if (jobJarURI.GetScheme() == null || jobJarURI.GetScheme().Equals("file")) { CopyJar(jobJarPath, JobSubmissionFiles.GetJobJar(submitJobDir), replication); job.SetJar(JobSubmissionFiles.GetJobJar(submitJobDir).ToString()); } } else { Log.Warn("No job jar file set. User classes may not be found. " + "See Job or Job#setJar(String)." ); } AddLog4jToDistributedCache(job, submitJobDir); // set the timestamps of the archives and files // set the public/private visibility of the archives and files ClientDistributedCacheManager.DetermineTimestampsAndCacheVisibilities(conf); // get DelegationToken for cached file ClientDistributedCacheManager.GetDelegationTokens(conf, job.GetCredentials()); }
private static URI RemoveAuthority(URI uri) { try { uri = new URI(uri.GetScheme(), string.Empty, uri.GetPath(), uri.GetQuery(), uri.GetFragment ()); } catch (URISyntaxException e) { throw new ArgumentException(e.GetLocalizedMessage()); } return(uri); }
/// <summary>Get a proxied URI for the original URI.</summary> /// <param name="originalUri"> /// the original URI to go through the proxy, or null if /// a default path "/" can be used. /// </param> /// <param name="proxyUri">the URI of the proxy itself, scheme, host and port are used. /// </param> /// <param name="id">the id of the application</param> /// <returns>the proxied URI</returns> public static URI GetProxyUri(URI originalUri, URI proxyUri, ApplicationId id) { try { string path = GetPath(id, originalUri == null ? "/" : originalUri.GetPath()); return(new URI(proxyUri.GetScheme(), proxyUri.GetAuthority(), path, originalUri == null ? null : originalUri.GetQuery(), originalUri == null ? null : originalUri. GetFragment())); } catch (URISyntaxException e) { throw new RuntimeException("Could not proxify " + originalUri, e); } }
/// <summary> /// Removes dot segments according to RFC 3986, section 5.2.4 and /// Syntax-Based Normalization according to RFC 3986, section 6.2.2. /// </summary> /// <remarks> /// Removes dot segments according to RFC 3986, section 5.2.4 and /// Syntax-Based Normalization according to RFC 3986, section 6.2.2. /// </remarks> /// <param name="uri">the original URI</param> /// <returns>the URI without dot segments</returns> private static URI NormalizeSyntax(URI uri) { if (uri.IsOpaque() || uri.GetAuthority() == null) { // opaque and file: URIs return(uri); } Args.Check(uri.IsAbsolute(), "Base URI must be absolute"); string path = uri.GetPath() == null ? string.Empty : uri.GetPath(); string[] inputSegments = path.Split("/"); Stack <string> outputSegments = new Stack <string>(); foreach (string inputSegment in inputSegments) { if ((inputSegment.Length == 0) || (".".Equals(inputSegment))) { } else { // Do nothing if ("..".Equals(inputSegment)) { if (!outputSegments.IsEmpty()) { outputSegments.Pop(); } } else { outputSegments.Push(inputSegment); } } } StringBuilder outputBuffer = new StringBuilder(); foreach (string outputSegment in outputSegments) { outputBuffer.Append('/').Append(outputSegment); } if (path.LastIndexOf('/') == path.Length - 1) { // path.endsWith("/") || path.equals("") outputBuffer.Append('/'); } try { string scheme = uri.GetScheme().ToLower(); string auth = uri.GetAuthority().ToLower(); URI @ref = new URI(scheme, auth, outputBuffer.ToString(), null, null); if (uri.GetQuery() == null && uri.GetFragment() == null) { return(@ref); } StringBuilder normalized = new StringBuilder(@ref.ToASCIIString()); if (uri.GetQuery() != null) { // query string passed through unchanged normalized.Append('?').Append(uri.GetRawQuery()); } if (uri.GetFragment() != null) { // fragment passed through unchanged normalized.Append('#').Append(uri.GetRawFragment()); } return(URI.Create(normalized.ToString())); } catch (URISyntaxException e) { throw new ArgumentException(e); } }