public virtual void TestHttpPolicy()
{
conf.Set(DFSConfigKeys.DfsHttpPolicyKey, policy.ToString());
conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, "localhost:0");
IPEndPoint addr = IPEndPoint.CreateUnresolved("localhost", 0);
NameNodeHttpServer server = null;
try
{
server = new NameNodeHttpServer(conf, null, addr);
server.Start();
NUnit.Framework.Assert.IsTrue(Implies(policy.IsHttpEnabled(), CanAccess("http", server
.GetHttpAddress())));
NUnit.Framework.Assert.IsTrue(Implies(!policy.IsHttpEnabled(), server.GetHttpAddress
() == null));
NUnit.Framework.Assert.IsTrue(Implies(policy.IsHttpsEnabled(), CanAccess("https",
server.GetHttpsAddress())));
NUnit.Framework.Assert.IsTrue(Implies(!policy.IsHttpsEnabled(), server.GetHttpsAddress
() == null));
}
finally
{
if (server != null)
{
server.Stop();
}
}
}
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoPut(HttpServletRequest request, HttpServletResponse response
)
{
try
{
ServletContext context = GetServletContext();
FSImage nnImage = NameNodeHttpServer.GetFsImageFromContext(context);
Configuration conf = (Configuration)GetServletContext().GetAttribute(JspHelper.CurrentConf
);
ImageServlet.PutImageParams parsedParams = new ImageServlet.PutImageParams(request
, response, conf);
NameNodeMetrics metrics = NameNode.GetNameNodeMetrics();
ValidateRequest(context, conf, request, response, nnImage, parsedParams.GetStorageInfoString
());
UserGroupInformation.GetCurrentUser().DoAs(new _PrivilegedExceptionAction_458(parsedParams
, nnImage, response, request, conf, metrics));
}
catch (Exception t)
{
// Metrics non-null only when used inside name node
// Now that we have a new checkpoint, we might be able to
// remove some old ones.
string errMsg = "PutImage failed. " + StringUtils.StringifyException(t);
response.SendError(HttpServletResponse.ScGone, errMsg);
throw new IOException(errMsg);
}
}
/// <summary>Create a redirection URL</summary>
/// <exception cref="System.IO.IOException"/>
private Uri CreateRedirectURL(string path, string encodedPath, HdfsFileStatus status
, UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request,
string dt)
{
string scheme = request.GetScheme();
LocatedBlocks blks = nnproxy.GetBlockLocations(status.GetFullPath(new Path(path))
.ToUri().GetPath(), 0, 1);
Configuration conf = NameNodeHttpServer.GetConfFromContext(GetServletContext());
DatanodeID host = PickSrcDatanode(blks, status, conf);
string hostname;
if (host is DatanodeInfo)
{
hostname = host.GetHostName();
}
else
{
hostname = host.GetIpAddr();
}
int port = "https".Equals(scheme) ? host.GetInfoSecurePort() : host.GetInfoPort();
string dtParam = string.Empty;
if (dt != null)
{
dtParam = JspHelper.GetDelegationTokenUrlParam(dt);
}
// Add namenode address to the url params
NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext());
string addr = nn.GetNameNodeAddressHostPortString();
string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr);
return(new Uri(scheme, hostname, port, "/streamFile" + encodedPath + '?' + "ugi="
+ ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam));
}
/// <summary>Select a datanode to service this request.</summary>
/// <remarks>
/// Select a datanode to service this request.
/// Currently, this looks at no more than the first five blocks of a file,
/// selecting a datanode randomly from the most represented.
/// </remarks>
/// <param name="conf"></param>
/// <exception cref="System.IO.IOException"/>
private DatanodeID PickSrcDatanode(LocatedBlocks blks, HdfsFileStatus i, Configuration
conf)
{
if (i.GetLen() == 0 || blks.GetLocatedBlocks().Count <= 0)
{
// pick a random datanode
NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext());
return(NamenodeJspHelper.GetRandomDatanode(nn));
}
return(JspHelper.BestNode(blks, conf));
}
/// <exception cref="System.Exception"/>
public object Run()
{
NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context);
FSNamesystem namesystem = nn.GetNamesystem();
BlockManager bm = namesystem.GetBlockManager();
int totalDatanodes = namesystem.GetNumberOfDatanodes(HdfsConstants.DatanodeReportType
.Live);
new NamenodeFsck(conf, nn, bm.GetDatanodeManager().GetNetworkTopology(), pmap, @out
, totalDatanodes, remoteAddress).Fsck();
return(null);
}
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest req, HttpServletResponse resp)
{
resp.SetContentType("application/json; charset=UTF-8");
StartupProgress prog = NameNodeHttpServer.GetStartupProgressFromContext(GetServletContext
());
StartupProgressView view = prog.CreateView();
JsonGenerator json = new JsonFactory().CreateJsonGenerator(resp.GetWriter());
try
{
json.WriteStartObject();
json.WriteNumberField(ElapsedTime, view.GetElapsedTime());
json.WriteNumberField(PercentComplete, view.GetPercentComplete());
json.WriteArrayFieldStart(Phases);
foreach (Phase phase in view.GetPhases())
{
json.WriteStartObject();
json.WriteStringField(Name, phase.GetName());
json.WriteStringField(Desc, phase.GetDescription());
json.WriteStringField(Status, view.GetStatus(phase).ToString());
json.WriteNumberField(PercentComplete, view.GetPercentComplete(phase));
json.WriteNumberField(ElapsedTime, view.GetElapsedTime(phase));
WriteStringFieldIfNotNull(json, File, view.GetFile(phase));
WriteNumberFieldIfDefined(json, Size, view.GetSize(phase));
json.WriteArrayFieldStart(Steps);
foreach (Step step in view.GetSteps(phase))
{
json.WriteStartObject();
StepType type = step.GetType();
if (type != null)
{
json.WriteStringField(Name, type.GetName());
json.WriteStringField(Desc, type.GetDescription());
}
json.WriteNumberField(Count, view.GetCount(phase, step));
WriteStringFieldIfNotNull(json, File, step.GetFile());
WriteNumberFieldIfDefined(json, Size, step.GetSize());
json.WriteNumberField(Total, view.GetTotal(phase, step));
json.WriteNumberField(PercentComplete, view.GetPercentComplete(phase, step));
json.WriteNumberField(ElapsedTime, view.GetElapsedTime(phase, step));
json.WriteEndObject();
}
json.WriteEndArray();
json.WriteEndObject();
}
json.WriteEndArray();
json.WriteEndObject();
}
finally
{
IOUtils.Cleanup(Log, json);
}
}
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest req, HttpServletResponse resp)
{
UserGroupInformation ugi;
ServletContext context = GetServletContext();
Configuration conf = NameNodeHttpServer.GetConfFromContext(context);
try
{
ugi = GetUGI(req, conf);
}
catch (IOException ioe)
{
Log.Info("Request for token received with no authentication from " + req.GetRemoteAddr
(), ioe);
resp.SendError(HttpServletResponse.ScForbidden, "Unable to identify or authenticate user"
);
return;
}
NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context);
string tokenString = req.GetParameter(Token);
if (tokenString == null)
{
resp.SendError(HttpServletResponse.ScMultipleChoices, "Token to renew not specified"
);
}
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>();
token.DecodeFromUrlString(tokenString);
try
{
long result = ugi.DoAs(new _PrivilegedExceptionAction_73(nn, token));
PrintWriter os = new PrintWriter(new OutputStreamWriter(resp.GetOutputStream(), Charsets
.Utf8));
os.WriteLine(result);
os.Close();
}
catch (Exception e)
{
// transfer exception over the http
string exceptionClass = e.GetType().FullName;
string exceptionMsg = e.GetLocalizedMessage();
string strException = exceptionClass + ";" + exceptionMsg;
Log.Info("Exception while renewing token. Re-throwing. s=" + strException, e);
resp.SendError(HttpServletResponse.ScInternalServerError, strException);
}
}
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest req, HttpServletResponse resp)
{
UserGroupInformation ugi;
ServletContext context = GetServletContext();
Configuration conf = NameNodeHttpServer.GetConfFromContext(context);
try
{
ugi = GetUGI(req, conf);
}
catch (IOException ioe)
{
Log.Info("Request for token received with no authentication from " + req.GetRemoteAddr
(), ioe);
resp.SendError(HttpServletResponse.ScForbidden, "Unable to identify or authenticate user"
);
return;
}
Log.Info("Sending token: {" + ugi.GetUserName() + "," + req.GetRemoteAddr() + "}"
);
NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context);
string renewer = req.GetParameter(Renewer);
string renewerFinal = (renewer == null) ? req.GetUserPrincipal().GetName() : renewer;
DataOutputStream dos = null;
try
{
dos = new DataOutputStream(resp.GetOutputStream());
DataOutputStream dosFinal = dos;
// for doAs block
ugi.DoAs(new _PrivilegedExceptionAction_69(nn, ugi, renewerFinal, dosFinal));
}
catch (Exception e)
{
Log.Info("Exception while sending token. Re-throwing ", e);
resp.SendError(HttpServletResponse.ScInternalServerError);
}
finally
{
if (dos != null)
{
dos.Close();
}
}
}
/// <summary>
/// Create a
/// <see cref="NameNode"/>
/// proxy from the current
/// <see cref="Javax.Servlet.ServletContext"/>
/// .
/// </summary>
/// <exception cref="System.IO.IOException"/>
protected internal virtual ClientProtocol CreateNameNodeProxy()
{
ServletContext context = GetServletContext();
// if we are running in the Name Node, use it directly rather than via
// rpc
NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context);
if (nn != null)
{
return(nn.GetRpcServer());
}
IPEndPoint nnAddr = NameNodeHttpServer.GetNameNodeAddressFromContext(context);
Configuration conf = new HdfsConfiguration(NameNodeHttpServer.GetConfFromContext(
context));
return(NameNodeProxies.CreateProxy <ClientProtocol>(conf, NameNode.GetUri(nnAddr))
.GetProxy());
}
/// <summary>Service a GET request as described below.</summary>
/// <remarks>
/// Service a GET request as described below.
/// Request:
/// <c>GET http://<nn>:<port>/data[/<path>] HTTP/1.1</c>
/// </remarks>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
Configuration conf = NameNodeHttpServer.GetConfFromContext(GetServletContext());
UserGroupInformation ugi = GetUGI(request, conf);
try
{
ugi.DoAs(new _PrivilegedExceptionAction_116(this, request, response, ugi));
}
catch (IOException e)
{
response.SendError(400, e.Message);
}
catch (Exception e)
{
response.SendError(400, e.Message);
}
}
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
ServletContext context = GetServletContext();
Configuration conf = NameNodeHttpServer.GetConfFromContext(context);
UserGroupInformation ugi = GetUGI(request, conf);
NameNode namenode = NameNodeHttpServer.GetNameNodeFromContext(context);
DatanodeID datanode = NamenodeJspHelper.GetRandomDatanode(namenode);
try
{
response.SendRedirect(CreateRedirectURL(ugi, datanode, request, namenode).ToString
());
}
catch (IOException e)
{
response.SendError(400, e.Message);
}
}
/// <summary>Handle fsck request</summary>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
IDictionary <string, string[]> pmap = request.GetParameterMap();
PrintWriter @out = response.GetWriter();
IPAddress remoteAddress = Sharpen.Extensions.GetAddressByName(request.GetRemoteAddr
());
ServletContext context = GetServletContext();
Configuration conf = NameNodeHttpServer.GetConfFromContext(context);
UserGroupInformation ugi = GetUGI(request, conf);
try
{
ugi.DoAs(new _PrivilegedExceptionAction_58(context, conf, pmap, @out, remoteAddress
));
}
catch (Exception e)
{
response.SendError(400, e.Message);
}
}
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest req, HttpServletResponse resp)
{
UserGroupInformation ugi;
ServletContext context = GetServletContext();
Configuration conf = NameNodeHttpServer.GetConfFromContext(context);
try
{
ugi = GetUGI(req, conf);
}
catch (IOException ioe)
{
Log.Info("Request for token received with no authentication from " + req.GetRemoteAddr
(), ioe);
resp.SendError(HttpServletResponse.ScForbidden, "Unable to identify or authenticate user"
);
return;
}
NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context);
string tokenString = req.GetParameter(Token);
if (tokenString == null)
{
resp.SendError(HttpServletResponse.ScMultipleChoices, "Token to renew not specified"
);
}
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token
<DelegationTokenIdentifier>();
token.DecodeFromUrlString(tokenString);
try
{
ugi.DoAs(new _PrivilegedExceptionAction_70(nn, token));
}
catch (Exception e)
{
Log.Info("Exception while cancelling token. Re-throwing. ", e);
resp.SendError(HttpServletResponse.ScInternalServerError, e.Message);
}
}
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
try
{
ServletContext context = GetServletContext();
FSImage nnImage = NameNodeHttpServer.GetFsImageFromContext(context);
ImageServlet.GetImageParams parsedParams = new ImageServlet.GetImageParams(request
, response);
Configuration conf = (Configuration)context.GetAttribute(JspHelper.CurrentConf);
NameNodeMetrics metrics = NameNode.GetNameNodeMetrics();
ValidateRequest(context, conf, request, response, nnImage, parsedParams.GetStorageInfoString
());
UserGroupInformation.GetCurrentUser().DoAs(new _PrivilegedExceptionAction_98(parsedParams
, nnImage, metrics, response, conf));
}
catch (Exception t)
{
// Metrics non-null only when used inside name node
// Metrics non-null only when used inside name node
// Potential race where the file was deleted while we were in the
// process of setting headers!
// It's possible the file could be deleted after this point, but
// we've already opened the 'fis' stream.
// It's also possible length could change, but this would be
// detected by the client side as an inaccurate length header.
// send file
string errMsg = "GetImage failed. " + StringUtils.StringifyException(t);
response.SendError(HttpServletResponse.ScGone, errMsg);
throw new IOException(errMsg);
}
finally
{
response.GetOutputStream().Close();
}
}
