/// <summary>Resolve the uri's hostname and add the default port if not in the uri</summary>
/// <param name="uri">to resolve</param>
/// <param name="defaultPort">if none is given</param>
/// <returns>URI</returns>
public static URI GetCanonicalUri(URI uri, int defaultPort)
{
// skip if there is no authority, ie. "file" scheme or relative uri
string host = uri.GetHost();
if (host == null)
{
return(uri);
}
string fqHost = CanonicalizeHost(host);
int port = uri.GetPort();
// short out if already canonical with a port
if (host.Equals(fqHost) && port != -1)
{
return(uri);
}
// reconstruct the uri with the canonical host and port
try
{
uri = new URI(uri.GetScheme(), uri.GetUserInfo(), fqHost, (port == -1) ? defaultPort
: port, uri.GetPath(), uri.GetQuery(), uri.GetFragment());
}
catch (URISyntaxException e)
{
throw new ArgumentException(e);
}
return(uri);
}
/// <exception cref="System.IO.IOException"/>
public override void Initialize(URI uri, Configuration conf)
{
// get
base.Initialize(uri, conf);
// get host information from uri (overrides info in conf)
string host = uri.GetHost();
host = (host == null) ? conf.Get(FsFtpHost, null) : host;
if (host == null)
{
throw new IOException("Invalid host specified");
}
conf.Set(FsFtpHost, host);
// get port information from uri, (overrides info in conf)
int port = uri.GetPort();
port = (port == -1) ? FTP.DefaultPort : port;
conf.SetInt("fs.ftp.host.port", port);
// get user/password information from URI (overrides info in conf)
string userAndPassword = uri.GetUserInfo();
if (userAndPassword == null)
{
userAndPassword = (conf.Get("fs.ftp.user." + host, null) + ":" + conf.Get("fs.ftp.password."
+ host, null));
}
string[] userPasswdInfo = userAndPassword.Split(":");
Preconditions.CheckState(userPasswdInfo.Length > 1, "Invalid username / password"
);
conf.Set(FsFtpUserPrefix + host, userPasswdInfo[0]);
conf.Set(FsFtpPasswordPrefix + host, userPasswdInfo[1]);
SetConf(conf);
this.uri = uri;
}
public virtual void Resolve(TaskCompletionEvent @event)
{
switch (@event.GetTaskStatus())
{
case TaskCompletionEvent.Status.Succeeded:
{
URI u = GetBaseURI(reduceId, @event.GetTaskTrackerHttp());
AddKnownMapOutput(u.GetHost() + ":" + u.GetPort(), u.ToString(), ((TaskAttemptID)
@event.GetTaskAttemptId()));
maxMapRuntime = Math.Max(maxMapRuntime, @event.GetTaskRunTime());
break;
}
case TaskCompletionEvent.Status.Failed:
case TaskCompletionEvent.Status.Killed:
case TaskCompletionEvent.Status.Obsolete:
{
ObsoleteMapOutput(((TaskAttemptID)@event.GetTaskAttemptId()));
Log.Info("Ignoring obsolete output of " + @event.GetTaskStatus() + " map-task: '"
+ ((TaskAttemptID)@event.GetTaskAttemptId()) + "'");
break;
}
case TaskCompletionEvent.Status.Tipfailed:
{
TipFailed(((TaskID)((TaskAttemptID)@event.GetTaskAttemptId()).GetTaskID()));
Log.Info("Ignoring output of failed map TIP: '" + ((TaskAttemptID)@event.GetTaskAttemptId
()) + "'");
break;
}
}
}
/*
* see if two file systems are the same or not.
*/
private bool CompareFs(FileSystem srcFs, FileSystem destFs)
{
URI srcUri = srcFs.GetUri();
URI dstUri = destFs.GetUri();
if (srcUri.GetScheme() == null)
{
return(false);
}
if (!srcUri.GetScheme().Equals(dstUri.GetScheme()))
{
return(false);
}
string srcHost = srcUri.GetHost();
string dstHost = dstUri.GetHost();
if ((srcHost != null) && (dstHost != null))
{
try
{
srcHost = Sharpen.Extensions.GetAddressByName(srcHost).ToString();
dstHost = Sharpen.Extensions.GetAddressByName(dstHost).ToString();
}
catch (UnknownHostException)
{
return(false);
}
if (!srcHost.Equals(dstHost))
{
return(false);
}
}
else
{
if (srcHost == null && dstHost != null)
{
return(false);
}
else
{
if (srcHost != null && dstHost == null)
{
return(false);
}
}
}
// check for ports
if (srcUri.GetPort() != dstUri.GetPort())
{
return(false);
}
return(true);
}
private void ConstructHttpServerURI(QJournalProtocolProtos.GetJournalStateResponseProto
ret)
{
if (ret.HasFromURL())
{
URI uri = URI.Create(ret.GetFromURL());
httpServerURL = GetHttpServerURI(uri.GetScheme(), uri.GetPort());
}
else
{
httpServerURL = GetHttpServerURI("http", ret.GetHttpPort());
}
}
internal static IPEndPoint ParseEntry(string type, string fn, string line)
{
try
{
URI uri = new URI("dummy", line, null, null, null);
int port = uri.GetPort() == -1 ? 0 : uri.GetPort();
IPEndPoint addr = new IPEndPoint(uri.GetHost(), port);
if (addr.IsUnresolved())
{
Log.Warn(string.Format("Failed to resolve address `%s` in `%s`. " + "Ignoring in the %s list."
, line, fn, type));
return(null);
}
return(addr);
}
catch (URISyntaxException)
{
Log.Warn(string.Format("Failed to parse `%s` in `%s`. " + "Ignoring in " + "the %s list."
, line, fn, type));
}
return(null);
}
private void DigestURI(URI uri)
{
this.scheme = uri.GetScheme();
this.encodedSchemeSpecificPart = uri.GetRawSchemeSpecificPart();
this.encodedAuthority = uri.GetRawAuthority();
this.host = uri.GetHost();
this.port = uri.GetPort();
this.encodedUserInfo = uri.GetRawUserInfo();
this.userInfo = uri.GetUserInfo();
this.encodedPath = uri.GetRawPath();
this.path = uri.GetPath();
this.encodedQuery = uri.GetRawQuery();
this.queryParams = ParseQuery(uri.GetRawQuery(), Consts.Utf8);
this.encodedFragment = uri.GetRawFragment();
this.fragment = uri.GetFragment();
}
public virtual void TestDfsUrls()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
FileSystem fs = cluster.GetFileSystem();
// Setup our own factory
// setURLSteramHandlerFactor is can be set at most once in the JVM
// the new URLStreamHandler is valid for all tests cases
// in TestStreamHandler
FsUrlStreamHandlerFactory factory = new FsUrlStreamHandlerFactory();
Uri.SetURLStreamHandlerFactory(factory);
Path filePath = new Path("/thefile");
try
{
byte[] fileContent = new byte[1024];
for (int i = 0; i < fileContent.Length; ++i)
{
fileContent[i] = unchecked ((byte)i);
}
// First create the file through the FileSystem API
OutputStream os = fs.Create(filePath);
os.Write(fileContent);
os.Close();
// Second, open and read the file content through the URL API
URI uri = fs.GetUri();
Uri fileURL = new Uri(uri.GetScheme(), uri.GetHost(), uri.GetPort(), filePath.ToString
());
InputStream @is = fileURL.OpenStream();
NUnit.Framework.Assert.IsNotNull(@is);
byte[] bytes = new byte[4096];
NUnit.Framework.Assert.AreEqual(1024, @is.Read(bytes));
@is.Close();
for (int i_1 = 0; i_1 < fileContent.Length; ++i_1)
{
NUnit.Framework.Assert.AreEqual(fileContent[i_1], bytes[i_1]);
}
// Cleanup: delete the file
fs.Delete(filePath, false);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
public static URL GetYarnUrlFromURI(URI uri)
{
URL url = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <URL>();
if (uri.GetHost() != null)
{
url.SetHost(uri.GetHost());
}
if (uri.GetUserInfo() != null)
{
url.SetUserInfo(uri.GetUserInfo());
}
url.SetPort(uri.GetPort());
url.SetScheme(uri.GetScheme());
url.SetFile(uri.GetPath());
return(url);
}
/// <summary>
/// Create an InetSocketAddress from the given target string and
/// default port.
/// </summary>
/// <remarks>
/// Create an InetSocketAddress from the given target string and
/// default port. If the string cannot be parsed correctly, the
/// <code>configName</code> parameter is used as part of the
/// exception message, allowing the user to better diagnose
/// the misconfiguration.
/// </remarks>
/// <param name="target">a string of either "host" or "host:port"</param>
/// <param name="defaultPort">
/// the default port if <code>target</code> does not
/// include a port number
/// </param>
/// <param name="configName">
/// the name of the configuration from which
/// <code>target</code> was loaded. This is used in the
/// exception message in the case that parsing fails.
/// </param>
public static IPEndPoint CreateSocketAddr(string target, int defaultPort, string
configName)
{
string helpText = string.Empty;
if (configName != null)
{
helpText = " (configuration property '" + configName + "')";
}
if (target == null)
{
throw new ArgumentException("Target address cannot be null." + helpText);
}
target = target.Trim();
bool hasScheme = target.Contains("://");
URI uri = null;
try
{
uri = hasScheme ? URI.Create(target) : URI.Create("dummyscheme://" + target);
}
catch (ArgumentException)
{
throw new ArgumentException("Does not contain a valid host:port authority: " + target
+ helpText);
}
string host = uri.GetHost();
int port = uri.GetPort();
if (port == -1)
{
port = defaultPort;
}
string path = uri.GetPath();
if ((host == null) || (port < 0) || (!hasScheme && path != null && !path.IsEmpty(
)))
{
throw new ArgumentException("Does not contain a valid host:port authority: " + target
+ helpText);
}
return(CreateSocketAddrForHost(host, port));
}
/// <exception cref="System.IO.IOException"></exception>
/// <exception cref="Apache.Http.HttpException"></exception>
public virtual CloseableHttpResponse Execute(HttpRoute route, HttpRequestWrapper
request, HttpClientContext context, HttpExecutionAware execAware)
{
Args.NotNull(route, "HTTP route");
Args.NotNull(request, "HTTP request");
Args.NotNull(context, "HTTP context");
IHttpRequest original = request.GetOriginal();
URI uri = null;
if (original is IHttpUriRequest)
{
uri = ((IHttpUriRequest)original).GetURI();
}
else
{
string uriString = original.GetRequestLine().GetUri();
try
{
uri = URI.Create(uriString);
}
catch (ArgumentException ex)
{
if (this.log.IsDebugEnabled())
{
this.log.Debug("Unable to parse '" + uriString + "' as a valid URI; " + "request URI and Host header may be inconsistent"
, ex);
}
}
}
request.SetURI(uri);
// Re-write request URI if needed
RewriteRequestURI(request, route);
HttpParams @params = request.GetParams();
HttpHost virtualHost = (HttpHost)@params.GetParameter(ClientPNames.VirtualHost);
// HTTPCLIENT-1092 - add the port if necessary
if (virtualHost != null && virtualHost.GetPort() == -1)
{
int port = route.GetTargetHost().GetPort();
if (port != -1)
{
virtualHost = new HttpHost(virtualHost.GetHostName(), port, virtualHost.GetSchemeName
());
}
if (this.log.IsDebugEnabled())
{
this.log.Debug("Using virtual host" + virtualHost);
}
}
HttpHost target = null;
if (virtualHost != null)
{
target = virtualHost;
}
else
{
if (uri != null && uri.IsAbsolute() && uri.GetHost() != null)
{
target = new HttpHost(uri.GetHost(), uri.GetPort(), uri.GetScheme());
}
}
if (target == null)
{
target = route.GetTargetHost();
}
// Get user info from the URI
if (uri != null)
{
string userinfo = uri.GetUserInfo();
if (userinfo != null)
{
CredentialsProvider credsProvider = context.GetCredentialsProvider();
if (credsProvider == null)
{
credsProvider = new BasicCredentialsProvider();
context.SetCredentialsProvider(credsProvider);
}
credsProvider.SetCredentials(new AuthScope(target), new UsernamePasswordCredentials
(userinfo));
}
}
// Run request protocol interceptors
context.SetAttribute(HttpClientContext.HttpTargetHost, target);
context.SetAttribute(HttpClientContext.HttpRoute, route);
context.SetAttribute(HttpClientContext.HttpRequest, request);
this.httpProcessor.Process(request, context);
CloseableHttpResponse response = this.requestExecutor.Execute(route, request, context
, execAware);
try
{
// Run response protocol interceptors
context.SetAttribute(HttpClientContext.HttpResponse, response);
this.httpProcessor.Process(response, context);
return(response);
}
catch (RuntimeException ex)
{
response.Close();
throw;
}
catch (IOException ex)
{
response.Close();
throw;
}
catch (HttpException ex)
{
response.Close();
throw;
}
}
public static AbstractNNFailoverProxyProvider <T> CreateFailoverProxyProvider <T>(Configuration
conf, URI nameNodeUri, bool checkPort, AtomicBoolean fallbackToSimpleAuth)
{
System.Type xface = typeof(T);
Type failoverProxyProviderClass = null;
AbstractNNFailoverProxyProvider <T> providerNN;
Preconditions.CheckArgument(xface.IsAssignableFrom(typeof(NamenodeProtocols)), "Interface %s is not a NameNode protocol"
, xface);
try
{
// Obtain the class of the proxy provider
failoverProxyProviderClass = GetFailoverProxyProviderClass(conf, nameNodeUri);
if (failoverProxyProviderClass == null)
{
return(null);
}
// Create a proxy provider instance.
Constructor <FailoverProxyProvider <T> > ctor = failoverProxyProviderClass.GetConstructor
(typeof(Configuration), typeof(URI), typeof(Type));
FailoverProxyProvider <T> provider = ctor.NewInstance(conf, nameNodeUri, xface);
// If the proxy provider is of an old implementation, wrap it.
if (!(provider is AbstractNNFailoverProxyProvider))
{
providerNN = new WrappedFailoverProxyProvider <T>(provider);
}
else
{
providerNN = (AbstractNNFailoverProxyProvider <T>)provider;
}
}
catch (Exception e)
{
string message = "Couldn't create proxy provider " + failoverProxyProviderClass;
if (Log.IsDebugEnabled())
{
Log.Debug(message, e);
}
if (e.InnerException is IOException)
{
throw (IOException)e.InnerException;
}
else
{
throw new IOException(message, e);
}
}
// Check the port in the URI, if it is logical.
if (checkPort && providerNN.UseLogicalURI())
{
int port = nameNodeUri.GetPort();
if (port > 0 && port != NameNode.DefaultPort)
{
// Throwing here without any cleanup is fine since we have not
// actually created the underlying proxies yet.
throw new IOException("Port " + port + " specified in URI " + nameNodeUri + " but host '"
+ nameNodeUri.GetHost() + "' is a logical (HA) namenode" + " and does not use port information."
);
}
}
providerNN.SetFallbackToSimpleAuth(fallbackToSimpleAuth);
return(providerNN);
}
/// <exception cref="System.IO.IOException"></exception>
/// <exception cref="Apache.Http.HttpException"></exception>
public virtual CloseableHttpResponse Execute(HttpRoute route, HttpRequestWrapper
request, HttpClientContext context, HttpExecutionAware execAware)
{
Args.NotNull(route, "HTTP route");
Args.NotNull(request, "HTTP request");
Args.NotNull(context, "HTTP context");
ConnectionRequest connRequest = connManager.RequestConnection(route, null);
if (execAware != null)
{
if (execAware.IsAborted())
{
connRequest.Cancel();
throw new RequestAbortedException("Request aborted");
}
else
{
execAware.SetCancellable(connRequest);
}
}
RequestConfig config = context.GetRequestConfig();
HttpClientConnection managedConn;
try
{
int timeout = config.GetConnectionRequestTimeout();
managedConn = connRequest.Get(timeout > 0 ? timeout : 0, TimeUnit.Milliseconds);
}
catch (Exception interrupted)
{
Sharpen.Thread.CurrentThread().Interrupt();
throw new RequestAbortedException("Request aborted", interrupted);
}
catch (ExecutionException ex)
{
Exception cause = ex.InnerException;
if (cause == null)
{
cause = ex;
}
throw new RequestAbortedException("Request execution failed", cause);
}
ConnectionHolder releaseTrigger = new ConnectionHolder(log, connManager, managedConn
);
try
{
if (execAware != null)
{
if (execAware.IsAborted())
{
releaseTrigger.Close();
throw new RequestAbortedException("Request aborted");
}
else
{
execAware.SetCancellable(releaseTrigger);
}
}
if (!managedConn.IsOpen())
{
int timeout = config.GetConnectTimeout();
this.connManager.Connect(managedConn, route, timeout > 0 ? timeout : 0, context);
this.connManager.RouteComplete(managedConn, route, context);
}
int timeout_1 = config.GetSocketTimeout();
if (timeout_1 >= 0)
{
managedConn.SetSocketTimeout(timeout_1);
}
HttpHost target = null;
IHttpRequest original = request.GetOriginal();
if (original is IHttpUriRequest)
{
URI uri = ((IHttpUriRequest)original).GetURI();
if (uri.IsAbsolute())
{
target = new HttpHost(uri.GetHost(), uri.GetPort(), uri.GetScheme());
}
}
if (target == null)
{
target = route.GetTargetHost();
}
context.SetAttribute(HttpClientContext.HttpTargetHost, target);
context.SetAttribute(HttpClientContext.HttpRequest, request);
context.SetAttribute(HttpClientContext.HttpConnection, managedConn);
context.SetAttribute(HttpClientContext.HttpRoute, route);
httpProcessor.Process(request, context);
HttpResponse response = requestExecutor.Execute(request, managedConn, context);
httpProcessor.Process(response, context);
// The connection is in or can be brought to a re-usable state.
if (reuseStrategy.KeepAlive(response, context))
{
// Set the idle duration of this connection
long duration = keepAliveStrategy.GetKeepAliveDuration(response, context);
releaseTrigger.SetValidFor(duration, TimeUnit.Milliseconds);
releaseTrigger.MarkReusable();
}
else
{
releaseTrigger.MarkNonReusable();
}
// check for entity, release connection if possible
HttpEntity entity = response.GetEntity();
if (entity == null || !entity.IsStreaming())
{
// connection not needed and (assumed to be) in re-usable state
releaseTrigger.ReleaseConnection();
return(Proxies.EnhanceResponse(response, null));
}
else
{
return(Proxies.EnhanceResponse(response, releaseTrigger));
}
}
catch (ConnectionShutdownException ex)
{
ThreadInterruptedException ioex = new ThreadInterruptedException("Connection has been shut down"
);
Sharpen.Extensions.InitCause(ioex, ex);
throw ioex;
}
catch (HttpException ex)
{
releaseTrigger.AbortConnection();
throw;
}
catch (IOException ex)
{
releaseTrigger.AbortConnection();
throw;
}
catch (RuntimeException ex)
{
releaseTrigger.AbortConnection();
throw;
}
}
/// <summary>
/// Extracts target host from the given
/// <see cref="Sharpen.URI">Sharpen.URI</see>
/// .
/// </summary>
/// <param name="uri"></param>
/// <returns>
/// the target host if the URI is absolute or <code>null</null> if the URI is
/// relative or does not contain a valid host name.
/// </returns>
/// <since>4.1</since>
public static HttpHost ExtractHost(URI uri)
{
if (uri == null)
{
return(null);
}
HttpHost target = null;
if (uri.IsAbsolute())
{
int port = uri.GetPort();
// may be overridden later
string host = uri.GetHost();
if (host == null)
{
// normal parse failed; let's do it ourselves
// authority does not seem to care about the valid character-set for host names
host = uri.GetAuthority();
if (host != null)
{
// Strip off any leading user credentials
int at = host.IndexOf('@');
if (at >= 0)
{
if (host.Length > at + 1)
{
host = Sharpen.Runtime.Substring(host, at + 1);
}
else
{
host = null;
}
}
// @ on its own
// Extract the port suffix, if present
if (host != null)
{
int colon = host.IndexOf(':');
if (colon >= 0)
{
int pos = colon + 1;
int len = 0;
for (int i = pos; i < host.Length; i++)
{
if (char.IsDigit(host[i]))
{
len++;
}
else
{
break;
}
}
if (len > 0)
{
try
{
port = System.Convert.ToInt32(Sharpen.Runtime.Substring(host, pos, pos + len));
}
catch (FormatException)
{
}
}
host = Sharpen.Runtime.Substring(host, 0, colon);
}
}
}
}
string scheme = uri.GetScheme();
if (host != null)
{
target = new HttpHost(host, port, scheme);
}
}
return(target);
}
