/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Security.Authentication.Client.AuthenticationException
/// "/>
private static HttpURLConnection Run(URLConnectionFactory factory, Uri url)
{
HttpURLConnection conn = null;
try
{
conn = (HttpURLConnection)factory.OpenConnection(url, true);
if (conn.GetResponseCode() != HttpURLConnection.HttpOk)
{
string msg = conn.GetResponseMessage();
throw new IOException("Error when dealing remote token: " + msg);
}
}
catch (IOException ie)
{
Log.Info("Error when dealing remote token:", ie);
IOException e = GetExceptionFromResponse(conn);
if (e != null)
{
Log.Info("rethrowing exception from HTTP request: " + e.GetLocalizedMessage());
throw e;
}
throw;
}
return(conn);
}
public virtual void TestReadURL()
{
HttpURLConnection conn = Org.Mockito.Mockito.Mock <HttpURLConnection>();
Org.Mockito.Mockito.DoReturn(new ByteArrayInputStream(FakeLogData)).When(conn).GetInputStream
();
Org.Mockito.Mockito.DoReturn(HttpURLConnection.HttpOk).When(conn).GetResponseCode
();
Org.Mockito.Mockito.DoReturn(Sharpen.Extensions.ToString(FakeLogData.Length)).When
(conn).GetHeaderField("Content-Length");
URLConnectionFactory factory = Org.Mockito.Mockito.Mock <URLConnectionFactory>();
Org.Mockito.Mockito.DoReturn(conn).When(factory).OpenConnection(Org.Mockito.Mockito
.Any <Uri>(), Matchers.AnyBoolean());
Uri url = new Uri("http://localhost/fakeLog");
EditLogInputStream elis = EditLogFileInputStream.FromUrl(factory, url, HdfsConstants
.InvalidTxid, HdfsConstants.InvalidTxid, false);
// Read the edit log and verify that we got all of the data.
EnumMap <FSEditLogOpCodes, Holder <int> > counts = FSImageTestUtil.CountEditLogOpTypes
(elis);
Assert.AssertThat(counts[FSEditLogOpCodes.OpAdd].held, CoreMatchers.Is(1));
Assert.AssertThat(counts[FSEditLogOpCodes.OpSetGenstampV1].held, CoreMatchers.Is(
1));
Assert.AssertThat(counts[FSEditLogOpCodes.OpClose].held, CoreMatchers.Is(1));
// Check that length header was picked up.
NUnit.Framework.Assert.AreEqual(FakeLogData.Length, elis.Length());
elis.Close();
}
/// <exception cref="System.IO.IOException"/>
internal QuorumJournalManager(Configuration conf, URI uri, NamespaceInfo nsInfo,
AsyncLogger.Factory loggerFactory)
{
// Timeouts for which the QJM will wait for each of the following actions.
// Since these don't occur during normal operation, we can
// use rather lengthy timeouts, and don't need to make them
// configurable.
Preconditions.CheckArgument(conf != null, "must be configured");
this.conf = conf;
this.uri = uri;
this.nsInfo = nsInfo;
this.loggers = new AsyncLoggerSet(CreateLoggers(loggerFactory));
this.connectionFactory = URLConnectionFactory.NewDefaultURLConnectionFactory(conf
);
// Configure timeouts.
this.startSegmentTimeoutMs = conf.GetInt(DFSConfigKeys.DfsQjournalStartSegmentTimeoutKey
, DFSConfigKeys.DfsQjournalStartSegmentTimeoutDefault);
this.prepareRecoveryTimeoutMs = conf.GetInt(DFSConfigKeys.DfsQjournalPrepareRecoveryTimeoutKey
, DFSConfigKeys.DfsQjournalPrepareRecoveryTimeoutDefault);
this.acceptRecoveryTimeoutMs = conf.GetInt(DFSConfigKeys.DfsQjournalAcceptRecoveryTimeoutKey
, DFSConfigKeys.DfsQjournalAcceptRecoveryTimeoutDefault);
this.finalizeSegmentTimeoutMs = conf.GetInt(DFSConfigKeys.DfsQjournalFinalizeSegmentTimeoutKey
, DFSConfigKeys.DfsQjournalFinalizeSegmentTimeoutDefault);
this.selectInputStreamsTimeoutMs = conf.GetInt(DFSConfigKeys.DfsQjournalSelectInputStreamsTimeoutKey
, DFSConfigKeys.DfsQjournalSelectInputStreamsTimeoutDefault);
this.getJournalStateTimeoutMs = conf.GetInt(DFSConfigKeys.DfsQjournalGetJournalStateTimeoutKey
, DFSConfigKeys.DfsQjournalGetJournalStateTimeoutDefault);
this.newEpochTimeoutMs = conf.GetInt(DFSConfigKeys.DfsQjournalNewEpochTimeoutKey,
DFSConfigKeys.DfsQjournalNewEpochTimeoutDefault);
this.writeTxnsTimeoutMs = conf.GetInt(DFSConfigKeys.DfsQjournalWriteTxnsTimeoutKey
, DFSConfigKeys.DfsQjournalWriteTxnsTimeoutDefault);
}
static TransferFsImage()
{
Configuration conf = new Configuration();
connectionFactory = URLConnectionFactory.NewDefaultURLConnectionFactory(conf);
isSpnegoEnabled = UserGroupInformation.IsSecurityEnabled();
}
/// <exception cref="System.IO.IOException"/>
public DFSck(Configuration conf, TextWriter @out)
: base(conf)
{
this.ugi = UserGroupInformation.GetCurrentUser();
this.@out = @out;
this.connectionFactory = URLConnectionFactory.NewDefaultURLConnectionFactory(conf
);
this.isSpnegoEnabled = UserGroupInformation.IsSecurityEnabled();
}
/// <summary>Cancel a Delegation Token.</summary>
/// <param name="nnAddr">the NameNode's address</param>
/// <param name="tok">the token to cancel</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Security.Authentication.Client.AuthenticationException
/// "/>
public static void CancelDelegationToken(URLConnectionFactory factory, URI nnAddr
, Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> tok)
{
StringBuilder buf = new StringBuilder(nnAddr.ToString()).Append(CancelDelegationTokenServlet
.PathSpec).Append("?").Append(CancelDelegationTokenServlet.Token).Append("=").Append
(tok.EncodeToUrlString());
HttpURLConnection conn = Run(factory, new Uri(buf.ToString()));
conn.Disconnect();
}
/// <exception cref="System.IO.IOException"/>
public static Credentials GetDTfromRemote(URLConnectionFactory factory, URI nnUri
, string renewer, string proxyUser)
{
StringBuilder buf = new StringBuilder(nnUri.ToString()).Append(GetDelegationTokenServlet
.PathSpec);
string separator = "?";
if (renewer != null)
{
buf.Append("?").Append(GetDelegationTokenServlet.Renewer).Append("=").Append(renewer
);
separator = "&";
}
if (proxyUser != null)
{
buf.Append(separator).Append("doas=").Append(proxyUser);
}
bool isHttps = nnUri.GetScheme().Equals("https");
HttpURLConnection conn = null;
DataInputStream dis = null;
IPEndPoint serviceAddr = NetUtils.CreateSocketAddr(nnUri.GetAuthority());
try
{
if (Log.IsDebugEnabled())
{
Log.Debug("Retrieving token from: " + buf);
}
conn = Run(factory, new Uri(buf.ToString()));
InputStream @in = conn.GetInputStream();
Credentials ts = new Credentials();
dis = new DataInputStream(@in);
ts.ReadFields(dis);
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in ts.GetAllTokens(
))
{
token.SetKind(isHttps ? HsftpFileSystem.TokenKind : HftpFileSystem.TokenKind);
SecurityUtil.SetTokenService(token, serviceAddr);
}
return(ts);
}
catch (Exception e)
{
throw new IOException("Unable to obtain remote token", e);
}
finally
{
IOUtils.Cleanup(Log, dis);
if (conn != null)
{
conn.Disconnect();
}
}
}
public static void SetUp()
{
FilePath @base = new FilePath(Basedir);
FileUtil.FullyDelete(@base);
@base.Mkdirs();
conf = new Configuration();
keystoresDir = new FilePath(Basedir).GetAbsolutePath();
sslConfDir = KeyStoreTestUtil.GetClasspathDir(typeof(Org.Apache.Hadoop.Hdfs.Server.Namenode.TestNameNodeHttpServer
));
KeyStoreTestUtil.SetupSSLConfig(keystoresDir, sslConfDir, conf, false);
connectionFactory = URLConnectionFactory.NewDefaultURLConnectionFactory(conf);
}
/// <summary>Command-line interface</summary>
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
Configuration conf = new HdfsConfiguration();
Options fetcherOptions = new Options();
fetcherOptions.AddOption(Webservice, true, "HTTP url to reach the NameNode at");
fetcherOptions.AddOption(Renewer, true, "Name of the delegation token renewer");
fetcherOptions.AddOption(Cancel, false, "cancel the token");
fetcherOptions.AddOption(Renew, false, "renew the token");
fetcherOptions.AddOption(Print, false, "print the token");
fetcherOptions.AddOption(HelpShort, Help, false, "print out help information");
GenericOptionsParser parser = new GenericOptionsParser(conf, fetcherOptions, args
);
CommandLine cmd = parser.GetCommandLine();
// get options
string webUrl = cmd.HasOption(Webservice) ? cmd.GetOptionValue(Webservice) : null;
string renewer = cmd.HasOption(Renewer) ? cmd.GetOptionValue(Renewer) : null;
bool cancel = cmd.HasOption(Cancel);
bool renew = cmd.HasOption(Renew);
bool print = cmd.HasOption(Print);
bool help = cmd.HasOption(Help);
string[] remaining = parser.GetRemainingArgs();
// check option validity
if (help)
{
PrintUsage(System.Console.Out);
System.Environment.Exit(0);
}
if (cancel && renew || cancel && print || renew && print || cancel && renew && print)
{
System.Console.Error.WriteLine("ERROR: Only specify cancel, renew or print.");
PrintUsage(System.Console.Error);
}
if (remaining.Length != 1 || remaining[0][0] == '-')
{
System.Console.Error.WriteLine("ERROR: Must specify exacltly one token file");
PrintUsage(System.Console.Error);
}
// default to using the local file system
FileSystem local = FileSystem.GetLocal(conf);
Path tokenFile = new Path(local.GetWorkingDirectory(), remaining[0]);
URLConnectionFactory connectionFactory = URLConnectionFactory.DefaultSystemConnectionFactory;
// Login the current user
UserGroupInformation.GetCurrentUser().DoAs(new _PrivilegedExceptionAction_152(print
, tokenFile, conf, renew, cancel, webUrl, connectionFactory, renewer));
}
/// <summary>Renew a Delegation Token.</summary>
/// <param name="nnAddr">the NameNode's address</param>
/// <param name="tok">the token to renew</param>
/// <returns>the Date that the token will expire next.</returns>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Security.Authentication.Client.AuthenticationException
/// "/>
public static long RenewDelegationToken(URLConnectionFactory factory, URI nnAddr,
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> tok)
{
StringBuilder buf = new StringBuilder(nnAddr.ToString()).Append(RenewDelegationTokenServlet
.PathSpec).Append("?").Append(RenewDelegationTokenServlet.Token).Append("=").Append
(tok.EncodeToUrlString());
HttpURLConnection connection = null;
BufferedReader @in = null;
try
{
connection = Run(factory, new Uri(buf.ToString()));
@in = new BufferedReader(new InputStreamReader(connection.GetInputStream(), Charsets
.Utf8));
long result = long.Parse(@in.ReadLine());
return(result);
}
catch (IOException ie)
{
Log.Info("error in renew over HTTP", ie);
IOException e = GetExceptionFromResponse(connection);
if (e != null)
{
Log.Info("rethrowing exception from HTTP request: " + e.GetLocalizedMessage());
throw e;
}
throw;
}
finally
{
IOUtils.Cleanup(Log, @in);
if (connection != null)
{
connection.Disconnect();
}
}
}
public _PrivilegedExceptionAction_152(bool print, Path tokenFile, Configuration conf
, bool renew, bool cancel, string webUrl, URLConnectionFactory connectionFactory
, string renewer)
{
this.print = print;
this.tokenFile = tokenFile;
this.conf = conf;
this.renew = renew;
this.cancel = cancel;
this.webUrl = webUrl;
this.connectionFactory = connectionFactory;
this.renewer = renewer;
}
/// <summary>Open an EditLogInputStream for the given URL.</summary>
/// <param name="connectionFactory">the URLConnectionFactory used to create the connection.
/// </param>
/// <param name="url">the url hosting the log</param>
/// <param name="startTxId">the expected starting txid</param>
/// <param name="endTxId">the expected ending txid</param>
/// <param name="inProgress">whether the log is in-progress</param>
/// <returns>a stream from which edits may be read</returns>
public static EditLogInputStream FromUrl(URLConnectionFactory connectionFactory,
Uri url, long startTxId, long endTxId, bool inProgress)
{
return(new Org.Apache.Hadoop.Hdfs.Server.Namenode.EditLogFileInputStream(new EditLogFileInputStream.URLLog
(connectionFactory, url), startTxId, endTxId, inProgress));
}
public URLLog(URLConnectionFactory connectionFactory, Uri url)
{
this.connectionFactory = connectionFactory;
this.isSpnegoEnabled = UserGroupInformation.IsSecurityEnabled();
this.url = url;
}
