protected internal virtual URI GetSchemeAuthorityUri()
{
URI uri = GetFsUri();
string SchemeAuthString = uri.GetScheme() + "://";
if (uri.GetAuthority() == null)
{
SchemeAuthString += "/";
}
else
{
SchemeAuthString += uri.GetAuthority();
}
return(URI.Create(SchemeAuthString));
}
/// <exception cref="System.IO.IOException"/>
private static IList <IPEndPoint> GetLoggerAddresses(URI uri)
{
string authority = uri.GetAuthority();
Preconditions.CheckArgument(authority != null && !authority.IsEmpty(), "URI has no authority: "
+ uri);
string[] parts = StringUtils.Split(authority, ';');
for (int i = 0; i < parts.Length; i++)
{
parts[i] = parts[i].Trim();
}
if (parts.Length % 2 == 0)
{
Log.Warn("Quorum journal URI '" + uri + "' has an even number " + "of Journal Nodes specified. This is not recommended!"
);
}
IList <IPEndPoint> addrs = Lists.NewArrayList();
foreach (string addr in parts)
{
addrs.AddItem(NetUtils.CreateSocketAddr(addr, DFSConfigKeys.DfsJournalnodeRpcPortDefault
));
}
return(addrs);
}
private string AppendOrReplaceParamter(string uri, string newQuery)
{
if (uri.Contains(YarnWebParams.NextRefreshInterval + "="))
{
return(uri.ReplaceAll(YarnWebParams.NextRefreshInterval + "=[^&]+", newQuery));
}
try
{
URI oldUri = new URI(uri);
string appendQuery = oldUri.GetQuery();
if (appendQuery == null)
{
appendQuery = newQuery;
}
else
{
appendQuery += "&" + newQuery;
}
URI newUri = new URI(oldUri.GetScheme(), oldUri.GetAuthority(), oldUri.GetPath(),
appendQuery, oldUri.GetFragment());
return(newUri.ToString());
}
catch (URISyntaxException)
{
return(null);
}
}
public virtual void TestHarFsWithoutAuthority()
{
URI uri = harFileSystem.GetUri();
NUnit.Framework.Assert.IsNull("har uri authority not null: " + uri, uri.GetAuthority
());
FileContext.GetFileContext(uri, conf);
}
/// <summary>Construct a Bookkeeper journal manager.</summary>
/// <exception cref="System.IO.IOException"/>
public BookKeeperJournalManager(Configuration conf, URI uri, NamespaceInfo nsInfo
)
{
this.conf = conf;
this.nsInfo = nsInfo;
string zkConnect = uri.GetAuthority().Replace(";", ",");
basePath = uri.GetPath();
ensembleSize = conf.GetInt(BkjmBookkeeperEnsembleSize, BkjmBookkeeperEnsembleSizeDefault
);
quorumSize = conf.GetInt(BkjmBookkeeperQuorumSize, BkjmBookkeeperQuorumSizeDefault
);
ackQuorumSize = conf.GetInt(BkjmBookkeeperAckQuorumSize, quorumSize);
addEntryTimeout = conf.GetInt(BkjmBookkeeperAddEntryTimeoutSec, BkjmBookkeeperAddEntryTimeoutDefault
);
speculativeReadTimeout = conf.GetInt(BkjmBookkeeperSpeculativeReadTimeoutMs, BkjmBookkeeperSpeculativeReadTimeoutDefault
);
readEntryTimeout = conf.GetInt(BkjmBookkeeperReadEntryTimeoutSec, BkjmBookkeeperReadEntryTimeoutDefault
);
ledgerPath = basePath + "/ledgers";
string maxTxIdPath = basePath + "/maxtxid";
string currentInprogressNodePath = basePath + "/CurrentInprogress";
versionPath = basePath + "/version";
digestpw = conf.Get(BkjmBookkeeperDigestPw, BkjmBookkeeperDigestPwDefault);
try
{
zkConnectLatch = new CountDownLatch(1);
int bkjmZKSessionTimeout = conf.GetInt(BkjmZkSessionTimeout, BkjmZkSessionTimeoutDefault
);
zkc = new ZooKeeper(zkConnect, bkjmZKSessionTimeout, new BookKeeperJournalManager.ZkConnectionWatcher
(this));
// Configured zk session timeout + some extra grace period (here
// BKJM_ZK_SESSION_TIMEOUT_DEFAULT used as grace period)
int zkConnectionLatchTimeout = bkjmZKSessionTimeout + BkjmZkSessionTimeoutDefault;
if (!zkConnectLatch.Await(zkConnectionLatchTimeout, TimeUnit.Milliseconds))
{
throw new IOException("Error connecting to zookeeper");
}
PrepareBookKeeperEnv();
ClientConfiguration clientConf = new ClientConfiguration();
clientConf.SetSpeculativeReadTimeout(speculativeReadTimeout);
clientConf.SetReadEntryTimeout(readEntryTimeout);
clientConf.SetAddEntryTimeout(addEntryTimeout);
bkc = new BookKeeper(clientConf, zkc);
}
catch (KeeperException e)
{
throw new IOException("Error initializing zk", e);
}
catch (Exception ie)
{
Sharpen.Thread.CurrentThread().Interrupt();
throw new IOException("Interrupted while initializing bk journal manager", ie);
}
ci = new CurrentInprogress(zkc, currentInprogressNodePath);
maxTxId = new MaxTxId(zkc, maxTxIdPath);
}
/// <summary>Returns a qualified path object.</summary>
public virtual Org.Apache.Hadoop.FS.Path MakeQualified(URI defaultUri, Org.Apache.Hadoop.FS.Path
workingDir)
{
Org.Apache.Hadoop.FS.Path path = this;
if (!IsAbsolute())
{
path = new Org.Apache.Hadoop.FS.Path(workingDir, this);
}
URI pathUri = path.ToUri();
string scheme = pathUri.GetScheme();
string authority = pathUri.GetAuthority();
string fragment = pathUri.GetFragment();
if (scheme != null && (authority != null || defaultUri.GetAuthority() == null))
{
return(path);
}
if (scheme == null)
{
scheme = defaultUri.GetScheme();
}
if (authority == null)
{
authority = defaultUri.GetAuthority();
if (authority == null)
{
authority = string.Empty;
}
}
URI newUri = null;
try
{
newUri = new URI(scheme, authority, NormalizePath(scheme, pathUri.GetPath()), null
, fragment);
}
catch (URISyntaxException e)
{
throw new ArgumentException(e);
}
return(new Org.Apache.Hadoop.FS.Path(newUri));
}
/// <exception cref="System.IO.IOException"/>
public static Credentials GetDTfromRemote(URLConnectionFactory factory, URI nnUri
, string renewer, string proxyUser)
{
StringBuilder buf = new StringBuilder(nnUri.ToString()).Append(GetDelegationTokenServlet
.PathSpec);
string separator = "?";
if (renewer != null)
{
buf.Append("?").Append(GetDelegationTokenServlet.Renewer).Append("=").Append(renewer
);
separator = "&";
}
if (proxyUser != null)
{
buf.Append(separator).Append("doas=").Append(proxyUser);
}
bool isHttps = nnUri.GetScheme().Equals("https");
HttpURLConnection conn = null;
DataInputStream dis = null;
IPEndPoint serviceAddr = NetUtils.CreateSocketAddr(nnUri.GetAuthority());
try
{
if (Log.IsDebugEnabled())
{
Log.Debug("Retrieving token from: " + buf);
}
conn = Run(factory, new Uri(buf.ToString()));
InputStream @in = conn.GetInputStream();
Credentials ts = new Credentials();
dis = new DataInputStream(@in);
ts.ReadFields(dis);
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in ts.GetAllTokens(
))
{
token.SetKind(isHttps ? HsftpFileSystem.TokenKind : HftpFileSystem.TokenKind);
SecurityUtil.SetTokenService(token, serviceAddr);
}
return(ts);
}
catch (Exception e)
{
throw new IOException("Unable to obtain remote token", e);
}
finally
{
IOUtils.Cleanup(Log, dis);
if (conn != null)
{
conn.Disconnect();
}
}
}
/// <summary>
/// Return a fully-qualified version of the given symlink target if it
/// has no scheme and authority.
/// </summary>
/// <remarks>
/// Return a fully-qualified version of the given symlink target if it
/// has no scheme and authority. Partially and fully-qualified paths
/// are returned unmodified.
/// </remarks>
/// <param name="pathURI">URI of the filesystem of pathWithLink</param>
/// <param name="pathWithLink">Path that contains the symlink</param>
/// <param name="target">The symlink's absolute target</param>
/// <returns>Fully qualified version of the target.</returns>
public static Path QualifySymlinkTarget(URI pathURI, Path pathWithLink, Path target
)
{
// NB: makeQualified uses the target's scheme and authority, if
// specified, and the scheme and authority of pathURI, if not.
URI targetUri = target.ToUri();
string scheme = targetUri.GetScheme();
string auth = targetUri.GetAuthority();
return((scheme == null && auth == null) ? target.MakeQualified(pathURI, pathWithLink
.GetParent()) : target);
}
/// <summary>create the service name for a Delegation token</summary>
/// <param name="uri">of the service</param>
/// <param name="defPort">is used if the uri lacks a port</param>
/// <returns>the token service, or null if no authority</returns>
/// <seealso cref="BuildTokenService(System.Net.IPEndPoint)"/>
public static string BuildDTServiceName(URI uri, int defPort)
{
string authority = uri.GetAuthority();
if (authority == null)
{
return(null);
}
IPEndPoint addr = NetUtils.CreateSocketAddr(authority, defPort);
return(BuildTokenService(addr).ToString());
}
/// <exception cref="System.IO.IOException"/>
public override void Initialize(URI name, Configuration conf)
{
base.Initialize(name, conf);
SetConf(conf);
this.uri = URI.Create(name.GetScheme() + "://" + name.GetAuthority());
tokenAspect = new TokenAspect <TestTokenAspect.DummyFs>(this, SecurityUtil.BuildTokenService
(uri), TokenKind);
if (emulateSecurityEnabled || UserGroupInformation.IsSecurityEnabled())
{
tokenAspect.InitDelegationToken(ugi);
}
}
/// <summary>
/// Convenience method that creates an HTTP <code>URL</code> for the
/// HttpFSServer file system operations.
/// </summary>
/// <remarks>
/// Convenience method that creates an HTTP <code>URL</code> for the
/// HttpFSServer file system operations.
/// <p/>
/// </remarks>
/// <param name="path">the file path.</param>
/// <param name="params">the query string parameters.</param>
/// <param name="multiValuedParams">multi valued parameters of the query string</param>
/// <returns>URL a <code>URL</code> for the HttpFSServer server,</returns>
/// <exception cref="System.IO.IOException">thrown if an IO error occurs.</exception>
internal static Uri CreateURL(Path path, IDictionary <string, string> @params, IDictionary
<string, IList <string> > multiValuedParams)
{
URI uri = path.ToUri();
string realScheme;
if (Sharpen.Runtime.EqualsIgnoreCase(uri.GetScheme(), HttpFSFileSystem.Scheme))
{
realScheme = "http";
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(uri.GetScheme(), HttpsFSFileSystem.Scheme))
{
realScheme = "https";
}
else
{
throw new ArgumentException(MessageFormat.Format("Invalid scheme [{0}] it should be '"
+ HttpFSFileSystem.Scheme + "' " + "or '" + HttpsFSFileSystem.Scheme + "'", uri
));
}
}
StringBuilder sb = new StringBuilder();
sb.Append(realScheme).Append("://").Append(uri.GetAuthority()).Append(ServicePath
).Append(uri.GetPath());
string separator = "?";
foreach (KeyValuePair <string, string> entry in @params)
{
sb.Append(separator).Append(entry.Key).Append("=").Append(URLEncoder.Encode(entry
.Value, "UTF8"));
separator = "&";
}
if (multiValuedParams != null)
{
foreach (KeyValuePair <string, IList <string> > multiValuedEntry in multiValuedParams)
{
string name = URLEncoder.Encode(multiValuedEntry.Key, "UTF8");
IList <string> values = multiValuedEntry.Value;
foreach (string value in values)
{
sb.Append(separator).Append(name).Append("=").Append(URLEncoder.Encode(value, "UTF8"
));
separator = "&";
}
}
}
return(new Uri(sb.ToString()));
}
/// <summary>Get a proxied URI for the original URI.</summary>
/// <param name="originalUri">
/// the original URI to go through the proxy, or null if
/// a default path "/" can be used.
/// </param>
/// <param name="proxyUri">the URI of the proxy itself, scheme, host and port are used.
/// </param>
/// <param name="id">the id of the application</param>
/// <returns>the proxied URI</returns>
public static URI GetProxyUri(URI originalUri, URI proxyUri, ApplicationId id)
{
try
{
string path = GetPath(id, originalUri == null ? "/" : originalUri.GetPath());
return(new URI(proxyUri.GetScheme(), proxyUri.GetAuthority(), path, originalUri ==
null ? null : originalUri.GetQuery(), originalUri == null ? null : originalUri.
GetFragment()));
}
catch (URISyntaxException e)
{
throw new RuntimeException("Could not proxify " + originalUri, e);
}
}
public virtual void TestStartStop()
{
Configuration conf = new Configuration();
MiniJournalCluster c = new MiniJournalCluster.Builder(conf).Build();
try
{
URI uri = c.GetQuorumJournalURI("myjournal");
string[] addrs = uri.GetAuthority().Split(";");
NUnit.Framework.Assert.AreEqual(3, addrs.Length);
JournalNode node = c.GetJournalNode(0);
string dir = node.GetConf().Get(DFSConfigKeys.DfsJournalnodeEditsDirKey);
NUnit.Framework.Assert.AreEqual(new FilePath(MiniDFSCluster.GetBaseDirectory() +
"journalnode-0").GetAbsolutePath(), dir);
}
finally
{
c.Shutdown();
}
}
/// <summary>Called after a new FileSystem instance is constructed.</summary>
/// <param name="theUri">
/// a uri whose authority section names the host, port, etc. for
/// this FileSystem
/// </param>
/// <param name="conf">the configuration</param>
/// <exception cref="System.IO.IOException"/>
public override void Initialize(URI theUri, Configuration conf)
{
base.Initialize(theUri, conf);
SetConf(conf);
config = conf;
// Now build client side view (i.e. client side mount table) from config.
string authority = theUri.GetAuthority();
try
{
myUri = new URI(FsConstants.ViewfsScheme, authority, "/", null, null);
fsState = new _InodeTree_167(this, conf, authority);
// return MergeFs.createMergeFs(mergeFsURIList, config);
workingDir = this.GetHomeDirectory();
}
catch (URISyntaxException)
{
throw new IOException("URISyntax exception: " + theUri);
}
}
/// <exception cref="System.IO.IOException"/>
private WebHdfsFileSystem GetWebHdfsFileSystem(UserGroupInformation ugi, Configuration
conf)
{
if (UserGroupInformation.IsSecurityEnabled())
{
DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text(ugi.GetUserName
()), null, null);
FSNamesystem namesystem = Org.Mockito.Mockito.Mock <FSNamesystem>();
DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager(86400000
, 86400000, 86400000, 86400000, namesystem);
dtSecretManager.StartThreads();
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>(dtId, dtSecretManager);
SecurityUtil.SetTokenService(token, NetUtils.CreateSocketAddr(uri.GetAuthority())
);
token.SetKind(WebHdfsFileSystem.TokenKind);
ugi.AddToken(token);
}
return((WebHdfsFileSystem)FileSystem.Get(uri, conf));
}
public virtual void TestHarUriWithHaUriWithNoPort()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).Build();
cluster.TransitionToActive(0);
HATestUtil.SetFailoverConfigurations(cluster, conf);
CreateEmptyHarArchive(HATestUtil.ConfigureFailoverFs(cluster, conf), TestHarPath);
URI failoverUri = FileSystem.GetDefaultUri(conf);
Path p = new Path("har://hdfs-" + failoverUri.GetAuthority() + TestHarPath);
p.GetFileSystem(conf);
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
private static void AddMRFrameworkToDistributedCache(Configuration conf)
{
string framework = conf.Get(MRJobConfig.MapreduceApplicationFrameworkPath, string.Empty
);
if (!framework.IsEmpty())
{
URI uri;
try
{
uri = new URI(framework);
}
catch (URISyntaxException e)
{
throw new ArgumentException("Unable to parse '" + framework + "' as a URI, check the setting for "
+ MRJobConfig.MapreduceApplicationFrameworkPath, e);
}
string linkedName = uri.GetFragment();
// resolve any symlinks in the URI path so using a "current" symlink
// to point to a specific version shows the specific version
// in the distributed cache configuration
FileSystem fs = FileSystem.Get(conf);
Path frameworkPath = fs.MakeQualified(new Path(uri.GetScheme(), uri.GetAuthority(
), uri.GetPath()));
FileContext fc = FileContext.GetFileContext(frameworkPath.ToUri(), conf);
frameworkPath = fc.ResolvePath(frameworkPath);
uri = frameworkPath.ToUri();
try
{
uri = new URI(uri.GetScheme(), uri.GetAuthority(), uri.GetPath(), null, linkedName
);
}
catch (URISyntaxException e)
{
throw new ArgumentException(e);
}
DistributedCache.AddCacheArchive(uri, conf);
}
}
/// <summary>Resolve a child path against a parent path.</summary>
public Path(Org.Apache.Hadoop.FS.Path parent, Org.Apache.Hadoop.FS.Path child)
{
// Add a slash to parent's path so resolution is compatible with URI's
URI parentUri = parent.uri;
string parentPath = parentUri.GetPath();
if (!(parentPath.Equals("/") || parentPath.IsEmpty()))
{
try
{
parentUri = new URI(parentUri.GetScheme(), parentUri.GetAuthority(), parentUri.GetPath
() + "/", null, parentUri.GetFragment());
}
catch (URISyntaxException e)
{
throw new ArgumentException(e);
}
}
URI resolved = parentUri.Resolve(child.uri);
Initialize(resolved.GetScheme(), resolved.GetAuthority(), resolved.GetPath(), resolved
.GetFragment());
}
/// <summary>Convert a nested URI to decode the underlying path.</summary>
/// <remarks>
/// Convert a nested URI to decode the underlying path. The translation takes
/// the authority and parses it into the underlying scheme and authority.
/// For example, "myscheme://hdfs@nn/my/path" is converted to
/// "hdfs://nn/my/path".
/// </remarks>
/// <param name="nestedUri">the URI from the nested URI</param>
/// <returns>the unnested path</returns>
public static Path UnnestUri(URI nestedUri)
{
string[] parts = nestedUri.GetAuthority().Split("@", 2);
StringBuilder result = new StringBuilder(parts[0]);
result.Append("://");
if (parts.Length == 2)
{
result.Append(parts[1]);
}
result.Append(nestedUri.GetPath());
if (nestedUri.GetQuery() != null)
{
result.Append("?");
result.Append(nestedUri.GetQuery());
}
if (nestedUri.GetFragment() != null)
{
result.Append("#");
result.Append(nestedUri.GetFragment());
}
return(new Path(result.ToString()));
}
/// <exception cref="System.IO.IOException"/>
public override void Initialize(URI name, Configuration conf)
{
base.Initialize(name, conf);
SetConf(conf);
this.connectionFactory = URLConnectionFactory.NewDefaultURLConnectionFactory(conf
);
this.ugi = UserGroupInformation.GetCurrentUser();
this.nnUri = GetNamenodeUri(name);
this.tokenServiceName = SecurityUtil.BuildTokenService(nnUri);
try
{
this.hftpURI = new URI(name.GetScheme(), name.GetAuthority(), null, null, null);
}
catch (URISyntaxException e)
{
throw new ArgumentException(e);
}
InitTokenAspect();
if (UserGroupInformation.IsSecurityEnabled())
{
tokenAspect.InitDelegationToken(ugi);
}
}
/// <summary>Construct the service key for a token</summary>
/// <param name="uri">of remote connection with a token</param>
/// <returns>
/// "ip:port" or "host:port" depending on the value of
/// hadoop.security.token.service.use_ip
/// </returns>
public static Text BuildTokenService(URI uri)
{
return(BuildTokenService(NetUtils.CreateSocketAddr(uri.GetAuthority())));
}
/// <summary>
/// Removes dot segments according to RFC 3986, section 5.2.4 and
/// Syntax-Based Normalization according to RFC 3986, section 6.2.2.
/// </summary>
/// <remarks>
/// Removes dot segments according to RFC 3986, section 5.2.4 and
/// Syntax-Based Normalization according to RFC 3986, section 6.2.2.
/// </remarks>
/// <param name="uri">the original URI</param>
/// <returns>the URI without dot segments</returns>
private static URI NormalizeSyntax(URI uri)
{
if (uri.IsOpaque() || uri.GetAuthority() == null)
{
// opaque and file: URIs
return(uri);
}
Args.Check(uri.IsAbsolute(), "Base URI must be absolute");
string path = uri.GetPath() == null ? string.Empty : uri.GetPath();
string[] inputSegments = path.Split("/");
Stack <string> outputSegments = new Stack <string>();
foreach (string inputSegment in inputSegments)
{
if ((inputSegment.Length == 0) || (".".Equals(inputSegment)))
{
}
else
{
// Do nothing
if ("..".Equals(inputSegment))
{
if (!outputSegments.IsEmpty())
{
outputSegments.Pop();
}
}
else
{
outputSegments.Push(inputSegment);
}
}
}
StringBuilder outputBuffer = new StringBuilder();
foreach (string outputSegment in outputSegments)
{
outputBuffer.Append('/').Append(outputSegment);
}
if (path.LastIndexOf('/') == path.Length - 1)
{
// path.endsWith("/") || path.equals("")
outputBuffer.Append('/');
}
try
{
string scheme = uri.GetScheme().ToLower();
string auth = uri.GetAuthority().ToLower();
URI @ref = new URI(scheme, auth, outputBuffer.ToString(), null, null);
if (uri.GetQuery() == null && uri.GetFragment() == null)
{
return(@ref);
}
StringBuilder normalized = new StringBuilder(@ref.ToASCIIString());
if (uri.GetQuery() != null)
{
// query string passed through unchanged
normalized.Append('?').Append(uri.GetRawQuery());
}
if (uri.GetFragment() != null)
{
// fragment passed through unchanged
normalized.Append('#').Append(uri.GetRawFragment());
}
return(URI.Create(normalized.ToString()));
}
catch (URISyntaxException e)
{
throw new ArgumentException(e);
}
}
/// <summary>
/// Extracts target host from the given
/// <see cref="Sharpen.URI">Sharpen.URI</see>
/// .
/// </summary>
/// <param name="uri"></param>
/// <returns>
/// the target host if the URI is absolute or <code>null</null> if the URI is
/// relative or does not contain a valid host name.
/// </returns>
/// <since>4.1</since>
public static HttpHost ExtractHost(URI uri)
{
if (uri == null)
{
return(null);
}
HttpHost target = null;
if (uri.IsAbsolute())
{
int port = uri.GetPort();
// may be overridden later
string host = uri.GetHost();
if (host == null)
{
// normal parse failed; let's do it ourselves
// authority does not seem to care about the valid character-set for host names
host = uri.GetAuthority();
if (host != null)
{
// Strip off any leading user credentials
int at = host.IndexOf('@');
if (at >= 0)
{
if (host.Length > at + 1)
{
host = Sharpen.Runtime.Substring(host, at + 1);
}
else
{
host = null;
}
}
// @ on its own
// Extract the port suffix, if present
if (host != null)
{
int colon = host.IndexOf(':');
if (colon >= 0)
{
int pos = colon + 1;
int len = 0;
for (int i = pos; i < host.Length; i++)
{
if (char.IsDigit(host[i]))
{
len++;
}
else
{
break;
}
}
if (len > 0)
{
try
{
port = System.Convert.ToInt32(Sharpen.Runtime.Substring(host, pos, pos + len));
}
catch (FormatException)
{
}
}
host = Sharpen.Runtime.Substring(host, 0, colon);
}
}
}
}
string scheme = uri.GetScheme();
if (host != null)
{
target = new HttpHost(host, port, scheme);
}
}
return(target);
}
/// <summary>
/// Is an absolute path (ie a slash relative path part)
/// AND a scheme is null AND authority is null.
/// </summary>
public virtual bool IsAbsoluteAndSchemeAuthorityNull()
{
return(IsUriPathAbsolute() && uri.GetScheme() == null && uri.GetAuthority() == null
);
}
/// <exception cref="System.Exception"/>
private static IDictionary <URI, IList <Path> > GetNameNodePaths(CommandLine line, Configuration
conf)
{
IDictionary <URI, IList <Path> > map = Maps.NewHashMap();
string[] paths = null;
if (line.HasOption("f"))
{
paths = ReadPathFile(line.GetOptionValue("f"));
}
else
{
if (line.HasOption("p"))
{
paths = line.GetOptionValues("p");
}
}
ICollection <URI> namenodes = DFSUtil.GetNsServiceRpcUris(conf);
if (paths == null || paths.Length == 0)
{
foreach (URI namenode in namenodes)
{
map[namenode] = null;
}
return(map);
}
URI singleNs = namenodes.Count == 1 ? namenodes.GetEnumerator().Next() : null;
foreach (string path in paths)
{
Path target = new Path(path);
if (!target.IsUriPathAbsolute())
{
throw new ArgumentException("The path " + target + " is not absolute");
}
URI targetUri = target.ToUri();
if ((targetUri.GetAuthority() == null || targetUri.GetScheme() == null) && singleNs
== null)
{
// each path must contains both scheme and authority information
// unless there is only one name service specified in the
// configuration
throw new ArgumentException("The path " + target + " does not contain scheme and authority thus cannot identify"
+ " its name service");
}
URI key = singleNs;
if (singleNs == null)
{
key = new URI(targetUri.GetScheme(), targetUri.GetAuthority(), null, null, null);
if (!namenodes.Contains(key))
{
throw new ArgumentException("Cannot resolve the path " + target + ". The namenode services specified in the "
+ "configuration: " + namenodes);
}
}
IList <Path> targets = map[key];
if (targets == null)
{
targets = Lists.NewArrayList();
map[key] = targets;
}
targets.AddItem(Path.GetPathWithoutSchemeAndAuthority(target));
}
return(map);
}
/// <summary>Expand the given path as a glob pattern.</summary>
/// <remarks>
/// Expand the given path as a glob pattern. Non-existent paths do not
/// throw an exception because creation commands like touch and mkdir need
/// to create them. The "stat" field will be null if the path does not
/// exist.
/// </remarks>
/// <param name="pattern">the pattern to expand as a glob</param>
/// <param name="conf">the hadoop configuration</param>
/// <returns>
/// list of
/// <see cref="PathData"/>
/// objects. if the pattern is not a glob,
/// and does not exist, the list will contain a single PathData with a null
/// stat
/// </returns>
/// <exception cref="System.IO.IOException">anything else goes wrong...</exception>
public static Org.Apache.Hadoop.FS.Shell.PathData[] ExpandAsGlob(string pattern,
Configuration conf)
{
Path globPath = new Path(pattern);
FileSystem fs = globPath.GetFileSystem(conf);
FileStatus[] stats = fs.GlobStatus(globPath);
Org.Apache.Hadoop.FS.Shell.PathData[] items = null;
if (stats == null)
{
// remove any quoting in the glob pattern
pattern = pattern.ReplaceAll("\\\\(.)", "$1");
// not a glob & file not found, so add the path with a null stat
items = new Org.Apache.Hadoop.FS.Shell.PathData[] { new Org.Apache.Hadoop.FS.Shell.PathData
(fs, pattern, null) };
}
else
{
// figure out what type of glob path was given, will convert globbed
// paths to match the type to preserve relativity
PathData.PathType globType;
URI globUri = globPath.ToUri();
if (globUri.GetScheme() != null)
{
globType = PathData.PathType.HasScheme;
}
else
{
if (!globUri.GetPath().IsEmpty() && new Path(globUri.GetPath()).IsAbsolute())
{
globType = PathData.PathType.SchemelessAbsolute;
}
else
{
globType = PathData.PathType.Relative;
}
}
// convert stats to PathData
items = new Org.Apache.Hadoop.FS.Shell.PathData[stats.Length];
int i = 0;
foreach (FileStatus stat in stats)
{
URI matchUri = stat.GetPath().ToUri();
string globMatch = null;
switch (globType)
{
case PathData.PathType.HasScheme:
{
// use as-is, but remove authority if necessary
if (globUri.GetAuthority() == null)
{
matchUri = RemoveAuthority(matchUri);
}
globMatch = UriToString(matchUri, false);
break;
}
case PathData.PathType.SchemelessAbsolute:
{
// take just the uri's path
globMatch = matchUri.GetPath();
break;
}
case PathData.PathType.Relative:
{
// make it relative to the current working dir
URI cwdUri = fs.GetWorkingDirectory().ToUri();
globMatch = Relativize(cwdUri, matchUri, stat.IsDirectory());
break;
}
}
items[i++] = new Org.Apache.Hadoop.FS.Shell.PathData(fs, globMatch, stat);
}
}
Arrays.Sort(items);
return(items);
}
/// <summary>
/// We generate the address with one of the following ports, in
/// order of preference.
/// </summary>
/// <remarks>
/// We generate the address with one of the following ports, in
/// order of preference.
/// 1. Port from the hftp URI e.g. hftp://namenode:4000/ will return 4000.
/// 2. Port configured via DFS_NAMENODE_HTTP_PORT_KEY
/// 3. DFS_NAMENODE_HTTP_PORT_DEFAULT i.e. 50070.
/// </remarks>
/// <param name="uri"/>
protected internal virtual IPEndPoint GetNamenodeAddr(URI uri)
{
// use authority so user supplied uri can override port
return(NetUtils.CreateSocketAddr(uri.GetAuthority(), GetDefaultPort()));
}
