public static Type GetFailoverProxyProviderClass <T>(Configuration conf, URI nameNodeUri
)
{
if (nameNodeUri == null)
{
return(null);
}
string host = nameNodeUri.GetHost();
string configKey = DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." +
host;
try
{
Type ret = (Type)conf.GetClass <FailoverProxyProvider>(configKey, null);
return(ret);
}
catch (RuntimeException e)
{
if (e.InnerException is TypeLoadException)
{
throw new IOException("Could not load failover proxy provider class " + conf.Get(
configKey) + " which is configured for authority " + nameNodeUri, e);
}
else
{
throw;
}
}
}
/// <returns>true if the given nameNodeUri appears to be a logical URI.</returns>
public static bool IsLogicalUri(Configuration conf, URI nameNodeUri)
{
string host = nameNodeUri.GetHost();
// A logical name must be one of the service IDs.
return(DFSUtil.GetNameServiceIds(conf).Contains(host));
}
public virtual void Resolve(TaskCompletionEvent @event)
{
switch (@event.GetTaskStatus())
{
case TaskCompletionEvent.Status.Succeeded:
{
URI u = GetBaseURI(reduceId, @event.GetTaskTrackerHttp());
AddKnownMapOutput(u.GetHost() + ":" + u.GetPort(), u.ToString(), ((TaskAttemptID)
@event.GetTaskAttemptId()));
maxMapRuntime = Math.Max(maxMapRuntime, @event.GetTaskRunTime());
break;
}
case TaskCompletionEvent.Status.Failed:
case TaskCompletionEvent.Status.Killed:
case TaskCompletionEvent.Status.Obsolete:
{
ObsoleteMapOutput(((TaskAttemptID)@event.GetTaskAttemptId()));
Log.Info("Ignoring obsolete output of " + @event.GetTaskStatus() + " map-task: '"
+ ((TaskAttemptID)@event.GetTaskAttemptId()) + "'");
break;
}
case TaskCompletionEvent.Status.Tipfailed:
{
TipFailed(((TaskID)((TaskAttemptID)@event.GetTaskAttemptId()).GetTaskID()));
Log.Info("Ignoring output of failed map TIP: '" + ((TaskAttemptID)@event.GetTaskAttemptId
()) + "'");
break;
}
}
}
/// <summary>Resolve the uri's hostname and add the default port if not in the uri</summary>
/// <param name="uri">to resolve</param>
/// <param name="defaultPort">if none is given</param>
/// <returns>URI</returns>
public static URI GetCanonicalUri(URI uri, int defaultPort)
{
// skip if there is no authority, ie. "file" scheme or relative uri
string host = uri.GetHost();
if (host == null)
{
return(uri);
}
string fqHost = CanonicalizeHost(host);
int port = uri.GetPort();
// short out if already canonical with a port
if (host.Equals(fqHost) && port != -1)
{
return(uri);
}
// reconstruct the uri with the canonical host and port
try
{
uri = new URI(uri.GetScheme(), uri.GetUserInfo(), fqHost, (port == -1) ? defaultPort
: port, uri.GetPath(), uri.GetQuery(), uri.GetFragment());
}
catch (URISyntaxException e)
{
throw new ArgumentException(e);
}
return(uri);
}
/// <exception cref="System.IO.IOException"/>
public override void Initialize(URI uri, Configuration conf)
{
// get
base.Initialize(uri, conf);
// get host information from uri (overrides info in conf)
string host = uri.GetHost();
host = (host == null) ? conf.Get(FsFtpHost, null) : host;
if (host == null)
{
throw new IOException("Invalid host specified");
}
conf.Set(FsFtpHost, host);
// get port information from uri, (overrides info in conf)
int port = uri.GetPort();
port = (port == -1) ? FTP.DefaultPort : port;
conf.SetInt("fs.ftp.host.port", port);
// get user/password information from URI (overrides info in conf)
string userAndPassword = uri.GetUserInfo();
if (userAndPassword == null)
{
userAndPassword = (conf.Get("fs.ftp.user." + host, null) + ":" + conf.Get("fs.ftp.password."
+ host, null));
}
string[] userPasswdInfo = userAndPassword.Split(":");
Preconditions.CheckState(userPasswdInfo.Length > 1, "Invalid username / password"
);
conf.Set(FsFtpUserPrefix + host, userPasswdInfo[0]);
conf.Set(FsFtpPasswordPrefix + host, userPasswdInfo[1]);
SetConf(conf);
this.uri = uri;
}
/// <summary>
/// Performs a sanity check on the list of hostnames/IPs to verify they at least
/// appear to be valid.
/// </summary>
/// <param name="names">- List of hostnames/IPs</param>
/// <exception cref="UnknownHostException"/>
public static void VerifyHostnames(string[] names)
{
foreach (string name in names)
{
if (name == null)
{
throw new UnknownHostException("null hostname found");
}
// The first check supports URL formats (e.g. hdfs://, etc.).
// java.net.URI requires a schema, so we add a dummy one if it doesn't
// have one already.
URI uri = null;
try
{
uri = new URI(name);
if (uri.GetHost() == null)
{
uri = new URI("http://" + name);
}
}
catch (URISyntaxException)
{
uri = null;
}
if (uri == null || uri.GetHost() == null)
{
throw new UnknownHostException(name + " is not a valid Inet address");
}
}
}
public static URL GetYarnUrlFromURI(URI uri)
{
URL url = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <URL>();
if (uri.GetHost() != null)
{
url.SetHost(uri.GetHost());
}
if (uri.GetUserInfo() != null)
{
url.SetUserInfo(uri.GetUserInfo());
}
url.SetPort(uri.GetPort());
url.SetScheme(uri.GetScheme());
url.SetFile(uri.GetPath());
return(url);
}
/// <summary>
/// Check whether the client has a failover proxy provider configured
/// for the namenode/nameservice.
/// </summary>
/// <param name="conf">Configuration</param>
/// <param name="nameNodeUri">The URI of namenode</param>
/// <returns>true if failover is configured.</returns>
public static bool IsClientFailoverConfigured(Configuration conf, URI nameNodeUri
)
{
string host = nameNodeUri.GetHost();
string configKey = DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." +
host;
return(conf.Get(configKey) != null);
}
public virtual void TestIPFailoverProxyProviderLogicalUri()
{
// setup the config with the IP failover proxy provider class
Configuration config = new HdfsConfiguration(conf);
URI nnUri = cluster.GetURI(0);
config.Set(DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." + nnUri.GetHost
(), typeof(IPFailoverProxyProvider).FullName);
NUnit.Framework.Assert.IsFalse("IPFailoverProxyProvider should not use logical URI."
, HAUtil.UseLogicalUri(config, nnUri));
}
/*
* see if two file systems are the same or not.
*/
private bool CompareFs(FileSystem srcFs, FileSystem destFs)
{
URI srcUri = srcFs.GetUri();
URI dstUri = destFs.GetUri();
if (srcUri.GetScheme() == null)
{
return(false);
}
if (!srcUri.GetScheme().Equals(dstUri.GetScheme()))
{
return(false);
}
string srcHost = srcUri.GetHost();
string dstHost = dstUri.GetHost();
if ((srcHost != null) && (dstHost != null))
{
try
{
srcHost = Sharpen.Extensions.GetAddressByName(srcHost).ToString();
dstHost = Sharpen.Extensions.GetAddressByName(dstHost).ToString();
}
catch (UnknownHostException)
{
return(false);
}
if (!srcHost.Equals(dstHost))
{
return(false);
}
}
else
{
if (srcHost == null && dstHost != null)
{
return(false);
}
else
{
if (srcHost != null && dstHost == null)
{
return(false);
}
}
}
// check for ports
if (srcUri.GetPort() != dstUri.GetPort())
{
return(false);
}
return(true);
}
private void DigestURI(URI uri)
{
this.scheme = uri.GetScheme();
this.encodedSchemeSpecificPart = uri.GetRawSchemeSpecificPart();
this.encodedAuthority = uri.GetRawAuthority();
this.host = uri.GetHost();
this.port = uri.GetPort();
this.encodedUserInfo = uri.GetRawUserInfo();
this.userInfo = uri.GetUserInfo();
this.encodedPath = uri.GetRawPath();
this.path = uri.GetPath();
this.encodedQuery = uri.GetRawQuery();
this.queryParams = ParseQuery(uri.GetRawQuery(), Consts.Utf8);
this.encodedFragment = uri.GetRawFragment();
this.fragment = uri.GetFragment();
}
/// <summary>Returns the Jetty server that the Namenode is listening on.</summary>
/// <exception cref="System.IO.IOException"/>
private Uri GetInfoServer()
{
URI fsName = FileSystem.GetDefaultUri(conf);
if (!Sharpen.Runtime.EqualsIgnoreCase(HdfsConstants.HdfsUriScheme, fsName.GetScheme
()))
{
throw new IOException("This is not a DFS");
}
string scheme = DFSUtil.GetHttpClientScheme(conf);
URI address = DFSUtil.GetInfoServerWithDefaultHost(fsName.GetHost(), conf, scheme
);
Log.Debug("Will connect to NameNode at " + address);
return(address.ToURL());
}
/// <summary>
/// This constructor has the signature needed by
/// <see cref="AbstractFileSystem.CreateFileSystem(Sharpen.URI, Org.Apache.Hadoop.Conf.Configuration)
/// "/>
/// </summary>
/// <param name="theUri">which must be that of Hdfs</param>
/// <param name="conf">configuration</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
internal Hdfs(URI theUri, Configuration conf)
: base(theUri, HdfsConstants.HdfsUriScheme, true, NameNode.DefaultPort)
{
if (!Sharpen.Runtime.EqualsIgnoreCase(theUri.GetScheme(), HdfsConstants.HdfsUriScheme
))
{
throw new ArgumentException("Passed URI's scheme is not for Hdfs");
}
string host = theUri.GetHost();
if (host == null)
{
throw new IOException("Incomplete HDFS URI, no host: " + theUri);
}
this.dfs = new DFSClient(theUri, conf, GetStatistics());
}
public virtual void TestDfsUrls()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
FileSystem fs = cluster.GetFileSystem();
// Setup our own factory
// setURLSteramHandlerFactor is can be set at most once in the JVM
// the new URLStreamHandler is valid for all tests cases
// in TestStreamHandler
FsUrlStreamHandlerFactory factory = new FsUrlStreamHandlerFactory();
Uri.SetURLStreamHandlerFactory(factory);
Path filePath = new Path("/thefile");
try
{
byte[] fileContent = new byte[1024];
for (int i = 0; i < fileContent.Length; ++i)
{
fileContent[i] = unchecked ((byte)i);
}
// First create the file through the FileSystem API
OutputStream os = fs.Create(filePath);
os.Write(fileContent);
os.Close();
// Second, open and read the file content through the URL API
URI uri = fs.GetUri();
Uri fileURL = new Uri(uri.GetScheme(), uri.GetHost(), uri.GetPort(), filePath.ToString
());
InputStream @is = fileURL.OpenStream();
NUnit.Framework.Assert.IsNotNull(@is);
byte[] bytes = new byte[4096];
NUnit.Framework.Assert.AreEqual(1024, @is.Read(bytes));
@is.Close();
for (int i_1 = 0; i_1 < fileContent.Length; ++i_1)
{
NUnit.Framework.Assert.AreEqual(fileContent[i_1], bytes[i_1]);
}
// Cleanup: delete the file
fs.Delete(filePath, false);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
public ConfiguredFailoverProxyProvider(Configuration conf, URI uri, Type xface)
{
Preconditions.CheckArgument(xface.IsAssignableFrom(typeof(NamenodeProtocols)), "Interface class %s is not a valid NameNode protocol!"
);
this.xface = xface;
this.conf = new Configuration(conf);
int maxRetries = this.conf.GetInt(DFSConfigKeys.DfsClientFailoverConnectionRetriesKey
, DFSConfigKeys.DfsClientFailoverConnectionRetriesDefault);
this.conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesKey, maxRetries
);
int maxRetriesOnSocketTimeouts = this.conf.GetInt(DFSConfigKeys.DfsClientFailoverConnectionRetriesOnSocketTimeoutsKey
, DFSConfigKeys.DfsClientFailoverConnectionRetriesOnSocketTimeoutsDefault);
this.conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesOnSocketTimeoutsKey
, maxRetriesOnSocketTimeouts);
try
{
ugi = UserGroupInformation.GetCurrentUser();
IDictionary <string, IDictionary <string, IPEndPoint> > map = DFSUtil.GetHaNnRpcAddresses
(conf);
IDictionary <string, IPEndPoint> addressesInNN = map[uri.GetHost()];
if (addressesInNN == null || addressesInNN.Count == 0)
{
throw new RuntimeException("Could not find any configured addresses " + "for URI "
+ uri);
}
ICollection <IPEndPoint> addressesOfNns = addressesInNN.Values;
foreach (IPEndPoint address in addressesOfNns)
{
proxies.AddItem(new ConfiguredFailoverProxyProvider.AddressRpcProxyPair <T>(address
));
}
// The client may have a delegation token set for the logical
// URI of the cluster. Clone this token to apply to each of the
// underlying IPC addresses so that the IPC code can find it.
HAUtil.CloneDelegationTokenForLogicalUri(ugi, uri, addressesOfNns);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
/// <summary>
/// Create an InetSocketAddress from the given target string and
/// default port.
/// </summary>
/// <remarks>
/// Create an InetSocketAddress from the given target string and
/// default port. If the string cannot be parsed correctly, the
/// <code>configName</code> parameter is used as part of the
/// exception message, allowing the user to better diagnose
/// the misconfiguration.
/// </remarks>
/// <param name="target">a string of either "host" or "host:port"</param>
/// <param name="defaultPort">
/// the default port if <code>target</code> does not
/// include a port number
/// </param>
/// <param name="configName">
/// the name of the configuration from which
/// <code>target</code> was loaded. This is used in the
/// exception message in the case that parsing fails.
/// </param>
public static IPEndPoint CreateSocketAddr(string target, int defaultPort, string
configName)
{
string helpText = string.Empty;
if (configName != null)
{
helpText = " (configuration property '" + configName + "')";
}
if (target == null)
{
throw new ArgumentException("Target address cannot be null." + helpText);
}
target = target.Trim();
bool hasScheme = target.Contains("://");
URI uri = null;
try
{
uri = hasScheme ? URI.Create(target) : URI.Create("dummyscheme://" + target);
}
catch (ArgumentException)
{
throw new ArgumentException("Does not contain a valid host:port authority: " + target
+ helpText);
}
string host = uri.GetHost();
int port = uri.GetPort();
if (port == -1)
{
port = defaultPort;
}
string path = uri.GetPath();
if ((host == null) || (port < 0) || (!hasScheme && path != null && !path.IsEmpty(
)))
{
throw new ArgumentException("Does not contain a valid host:port authority: " + target
+ helpText);
}
return(CreateSocketAddrForHost(host, port));
}
/// <exception cref="System.Exception"/>
internal static void RunTestCache(int port)
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NameNodePort(port).NumDataNodes(2).Build
();
URI uri = cluster.GetFileSystem().GetUri();
Log.Info("uri=" + uri);
{
FileSystem fs = FileSystem.Get(uri, new Configuration());
CheckPath(cluster, fs);
for (int i = 0; i < 100; i++)
{
NUnit.Framework.Assert.IsTrue(fs == FileSystem.Get(uri, new Configuration()));
}
}
if (port == NameNode.DefaultPort)
{
//test explicit default port
URI uri2 = new URI(uri.GetScheme(), uri.GetUserInfo(), uri.GetHost(), NameNode.DefaultPort
, uri.GetPath(), uri.GetQuery(), uri.GetFragment());
Log.Info("uri2=" + uri2);
FileSystem fs = FileSystem.Get(uri2, conf);
CheckPath(cluster, fs);
for (int i = 0; i < 100; i++)
{
NUnit.Framework.Assert.IsTrue(fs == FileSystem.Get(uri2, new Configuration()));
}
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
internal static IPEndPoint ParseEntry(string type, string fn, string line)
{
try
{
URI uri = new URI("dummy", line, null, null, null);
int port = uri.GetPort() == -1 ? 0 : uri.GetPort();
IPEndPoint addr = new IPEndPoint(uri.GetHost(), port);
if (addr.IsUnresolved())
{
Log.Warn(string.Format("Failed to resolve address `%s` in `%s`. " + "Ignoring in the %s list."
, line, fn, type));
return(null);
}
return(addr);
}
catch (URISyntaxException)
{
Log.Warn(string.Format("Failed to parse `%s` in `%s`. " + "Ignoring in " + "the %s list."
, line, fn, type));
}
return(null);
}
SelectToken(URI nnUri, ICollection <Org.Apache.Hadoop.Security.Token.Token <object
> > tokens, Configuration conf)
{
// this guesses the remote cluster's rpc service port.
// the current token design assumes it's the same as the local cluster's
// rpc port unless a config key is set. there should be a way to automatic
// and correctly determine the value
Text serviceName = SecurityUtil.BuildTokenService(nnUri);
string nnServiceName = conf.Get(ServiceNameKey + serviceName);
int nnRpcPort = NameNode.DefaultPort;
if (nnServiceName != null)
{
nnRpcPort = NetUtils.CreateSocketAddr(nnServiceName, nnRpcPort).Port;
}
// use original hostname from the uri to avoid unintentional host resolving
serviceName = SecurityUtil.BuildTokenService(NetUtils.CreateSocketAddrForHost(nnUri
.GetHost(), nnRpcPort));
return(SelectToken(serviceName, tokens));
}
/// <summary>
/// Extracts target host from the given
/// <see cref="Sharpen.URI">Sharpen.URI</see>
/// .
/// </summary>
/// <param name="uri"></param>
/// <returns>
/// the target host if the URI is absolute or <code>null</null> if the URI is
/// relative or does not contain a valid host name.
/// </returns>
/// <since>4.1</since>
public static HttpHost ExtractHost(URI uri)
{
if (uri == null)
{
return(null);
}
HttpHost target = null;
if (uri.IsAbsolute())
{
int port = uri.GetPort();
// may be overridden later
string host = uri.GetHost();
if (host == null)
{
// normal parse failed; let's do it ourselves
// authority does not seem to care about the valid character-set for host names
host = uri.GetAuthority();
if (host != null)
{
// Strip off any leading user credentials
int at = host.IndexOf('@');
if (at >= 0)
{
if (host.Length > at + 1)
{
host = Sharpen.Runtime.Substring(host, at + 1);
}
else
{
host = null;
}
}
// @ on its own
// Extract the port suffix, if present
if (host != null)
{
int colon = host.IndexOf(':');
if (colon >= 0)
{
int pos = colon + 1;
int len = 0;
for (int i = pos; i < host.Length; i++)
{
if (char.IsDigit(host[i]))
{
len++;
}
else
{
break;
}
}
if (len > 0)
{
try
{
port = System.Convert.ToInt32(Sharpen.Runtime.Substring(host, pos, pos + len));
}
catch (FormatException)
{
}
}
host = Sharpen.Runtime.Substring(host, 0, colon);
}
}
}
}
string scheme = uri.GetScheme();
if (host != null)
{
target = new HttpHost(host, port, scheme);
}
}
return(target);
}
/// <exception cref="System.IO.IOException"></exception>
/// <exception cref="Apache.Http.HttpException"></exception>
public virtual CloseableHttpResponse Execute(HttpRoute route, HttpRequestWrapper
request, HttpClientContext context, HttpExecutionAware execAware)
{
Args.NotNull(route, "HTTP route");
Args.NotNull(request, "HTTP request");
Args.NotNull(context, "HTTP context");
ConnectionRequest connRequest = connManager.RequestConnection(route, null);
if (execAware != null)
{
if (execAware.IsAborted())
{
connRequest.Cancel();
throw new RequestAbortedException("Request aborted");
}
else
{
execAware.SetCancellable(connRequest);
}
}
RequestConfig config = context.GetRequestConfig();
HttpClientConnection managedConn;
try
{
int timeout = config.GetConnectionRequestTimeout();
managedConn = connRequest.Get(timeout > 0 ? timeout : 0, TimeUnit.Milliseconds);
}
catch (Exception interrupted)
{
Sharpen.Thread.CurrentThread().Interrupt();
throw new RequestAbortedException("Request aborted", interrupted);
}
catch (ExecutionException ex)
{
Exception cause = ex.InnerException;
if (cause == null)
{
cause = ex;
}
throw new RequestAbortedException("Request execution failed", cause);
}
ConnectionHolder releaseTrigger = new ConnectionHolder(log, connManager, managedConn
);
try
{
if (execAware != null)
{
if (execAware.IsAborted())
{
releaseTrigger.Close();
throw new RequestAbortedException("Request aborted");
}
else
{
execAware.SetCancellable(releaseTrigger);
}
}
if (!managedConn.IsOpen())
{
int timeout = config.GetConnectTimeout();
this.connManager.Connect(managedConn, route, timeout > 0 ? timeout : 0, context);
this.connManager.RouteComplete(managedConn, route, context);
}
int timeout_1 = config.GetSocketTimeout();
if (timeout_1 >= 0)
{
managedConn.SetSocketTimeout(timeout_1);
}
HttpHost target = null;
IHttpRequest original = request.GetOriginal();
if (original is IHttpUriRequest)
{
URI uri = ((IHttpUriRequest)original).GetURI();
if (uri.IsAbsolute())
{
target = new HttpHost(uri.GetHost(), uri.GetPort(), uri.GetScheme());
}
}
if (target == null)
{
target = route.GetTargetHost();
}
context.SetAttribute(HttpClientContext.HttpTargetHost, target);
context.SetAttribute(HttpClientContext.HttpRequest, request);
context.SetAttribute(HttpClientContext.HttpConnection, managedConn);
context.SetAttribute(HttpClientContext.HttpRoute, route);
httpProcessor.Process(request, context);
HttpResponse response = requestExecutor.Execute(request, managedConn, context);
httpProcessor.Process(response, context);
// The connection is in or can be brought to a re-usable state.
if (reuseStrategy.KeepAlive(response, context))
{
// Set the idle duration of this connection
long duration = keepAliveStrategy.GetKeepAliveDuration(response, context);
releaseTrigger.SetValidFor(duration, TimeUnit.Milliseconds);
releaseTrigger.MarkReusable();
}
else
{
releaseTrigger.MarkNonReusable();
}
// check for entity, release connection if possible
HttpEntity entity = response.GetEntity();
if (entity == null || !entity.IsStreaming())
{
// connection not needed and (assumed to be) in re-usable state
releaseTrigger.ReleaseConnection();
return(Proxies.EnhanceResponse(response, null));
}
else
{
return(Proxies.EnhanceResponse(response, releaseTrigger));
}
}
catch (ConnectionShutdownException ex)
{
ThreadInterruptedException ioex = new ThreadInterruptedException("Connection has been shut down"
);
Sharpen.Extensions.InitCause(ioex, ex);
throw ioex;
}
catch (HttpException ex)
{
releaseTrigger.AbortConnection();
throw;
}
catch (IOException ex)
{
releaseTrigger.AbortConnection();
throw;
}
catch (RuntimeException ex)
{
releaseTrigger.AbortConnection();
throw;
}
}
/// <exception cref="System.IO.IOException"></exception>
/// <exception cref="Apache.Http.HttpException"></exception>
public virtual CloseableHttpResponse Execute(HttpRoute route, HttpRequestWrapper
request, HttpClientContext context, HttpExecutionAware execAware)
{
Args.NotNull(route, "HTTP route");
Args.NotNull(request, "HTTP request");
Args.NotNull(context, "HTTP context");
IHttpRequest original = request.GetOriginal();
URI uri = null;
if (original is IHttpUriRequest)
{
uri = ((IHttpUriRequest)original).GetURI();
}
else
{
string uriString = original.GetRequestLine().GetUri();
try
{
uri = URI.Create(uriString);
}
catch (ArgumentException ex)
{
if (this.log.IsDebugEnabled())
{
this.log.Debug("Unable to parse '" + uriString + "' as a valid URI; " + "request URI and Host header may be inconsistent"
, ex);
}
}
}
request.SetURI(uri);
// Re-write request URI if needed
RewriteRequestURI(request, route);
HttpParams @params = request.GetParams();
HttpHost virtualHost = (HttpHost)@params.GetParameter(ClientPNames.VirtualHost);
// HTTPCLIENT-1092 - add the port if necessary
if (virtualHost != null && virtualHost.GetPort() == -1)
{
int port = route.GetTargetHost().GetPort();
if (port != -1)
{
virtualHost = new HttpHost(virtualHost.GetHostName(), port, virtualHost.GetSchemeName
());
}
if (this.log.IsDebugEnabled())
{
this.log.Debug("Using virtual host" + virtualHost);
}
}
HttpHost target = null;
if (virtualHost != null)
{
target = virtualHost;
}
else
{
if (uri != null && uri.IsAbsolute() && uri.GetHost() != null)
{
target = new HttpHost(uri.GetHost(), uri.GetPort(), uri.GetScheme());
}
}
if (target == null)
{
target = route.GetTargetHost();
}
// Get user info from the URI
if (uri != null)
{
string userinfo = uri.GetUserInfo();
if (userinfo != null)
{
CredentialsProvider credsProvider = context.GetCredentialsProvider();
if (credsProvider == null)
{
credsProvider = new BasicCredentialsProvider();
context.SetCredentialsProvider(credsProvider);
}
credsProvider.SetCredentials(new AuthScope(target), new UsernamePasswordCredentials
(userinfo));
}
}
// Run request protocol interceptors
context.SetAttribute(HttpClientContext.HttpTargetHost, target);
context.SetAttribute(HttpClientContext.HttpRoute, route);
context.SetAttribute(HttpClientContext.HttpRequest, request);
this.httpProcessor.Process(request, context);
CloseableHttpResponse response = this.requestExecutor.Execute(route, request, context
, execAware);
try
{
// Run response protocol interceptors
context.SetAttribute(HttpClientContext.HttpResponse, response);
this.httpProcessor.Process(response, context);
return(response);
}
catch (RuntimeException ex)
{
response.Close();
throw;
}
catch (IOException ex)
{
response.Close();
throw;
}
catch (HttpException ex)
{
response.Close();
throw;
}
}
public static AbstractNNFailoverProxyProvider <T> CreateFailoverProxyProvider <T>(Configuration
conf, URI nameNodeUri, bool checkPort, AtomicBoolean fallbackToSimpleAuth)
{
System.Type xface = typeof(T);
Type failoverProxyProviderClass = null;
AbstractNNFailoverProxyProvider <T> providerNN;
Preconditions.CheckArgument(xface.IsAssignableFrom(typeof(NamenodeProtocols)), "Interface %s is not a NameNode protocol"
, xface);
try
{
// Obtain the class of the proxy provider
failoverProxyProviderClass = GetFailoverProxyProviderClass(conf, nameNodeUri);
if (failoverProxyProviderClass == null)
{
return(null);
}
// Create a proxy provider instance.
Constructor <FailoverProxyProvider <T> > ctor = failoverProxyProviderClass.GetConstructor
(typeof(Configuration), typeof(URI), typeof(Type));
FailoverProxyProvider <T> provider = ctor.NewInstance(conf, nameNodeUri, xface);
// If the proxy provider is of an old implementation, wrap it.
if (!(provider is AbstractNNFailoverProxyProvider))
{
providerNN = new WrappedFailoverProxyProvider <T>(provider);
}
else
{
providerNN = (AbstractNNFailoverProxyProvider <T>)provider;
}
}
catch (Exception e)
{
string message = "Couldn't create proxy provider " + failoverProxyProviderClass;
if (Log.IsDebugEnabled())
{
Log.Debug(message, e);
}
if (e.InnerException is IOException)
{
throw (IOException)e.InnerException;
}
else
{
throw new IOException(message, e);
}
}
// Check the port in the URI, if it is logical.
if (checkPort && providerNN.UseLogicalURI())
{
int port = nameNodeUri.GetPort();
if (port > 0 && port != NameNode.DefaultPort)
{
// Throwing here without any cleanup is fine since we have not
// actually created the underlying proxies yet.
throw new IOException("Port " + port + " specified in URI " + nameNodeUri + " but host '"
+ nameNodeUri.GetHost() + "' is a logical (HA) namenode" + " and does not use port information."
);
}
}
providerNN.SetFallbackToSimpleAuth(fallbackToSimpleAuth);
return(providerNN);
}
/// <summary>
/// Get the service name used in the delegation token for the given logical
/// HA service.
/// </summary>
/// <param name="uri">the logical URI of the cluster</param>
/// <param name="scheme">the scheme of the corresponding FileSystem</param>
/// <returns>the service name</returns>
public static Text BuildTokenServiceForLogicalUri(URI uri, string scheme)
{
return(new Text(BuildTokenServicePrefixForLogicalUri(scheme) + uri.GetHost()));
}