public virtual void TestHttpPolicy() { conf.Set(DFSConfigKeys.DfsHttpPolicyKey, policy.ToString()); conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, "localhost:0"); IPEndPoint addr = IPEndPoint.CreateUnresolved("localhost", 0); NameNodeHttpServer server = null; try { server = new NameNodeHttpServer(conf, null, addr); server.Start(); NUnit.Framework.Assert.IsTrue(Implies(policy.IsHttpEnabled(), CanAccess("http", server .GetHttpAddress()))); NUnit.Framework.Assert.IsTrue(Implies(!policy.IsHttpEnabled(), server.GetHttpAddress () == null)); NUnit.Framework.Assert.IsTrue(Implies(policy.IsHttpsEnabled(), CanAccess("https", server.GetHttpsAddress()))); NUnit.Framework.Assert.IsTrue(Implies(!policy.IsHttpsEnabled(), server.GetHttpsAddress () == null)); } finally { if (server != null) { server.Stop(); } } }
/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoPut(HttpServletRequest request, HttpServletResponse response ) { try { ServletContext context = GetServletContext(); FSImage nnImage = NameNodeHttpServer.GetFsImageFromContext(context); Configuration conf = (Configuration)GetServletContext().GetAttribute(JspHelper.CurrentConf ); ImageServlet.PutImageParams parsedParams = new ImageServlet.PutImageParams(request , response, conf); NameNodeMetrics metrics = NameNode.GetNameNodeMetrics(); ValidateRequest(context, conf, request, response, nnImage, parsedParams.GetStorageInfoString ()); UserGroupInformation.GetCurrentUser().DoAs(new _PrivilegedExceptionAction_458(parsedParams , nnImage, response, request, conf, metrics)); } catch (Exception t) { // Metrics non-null only when used inside name node // Now that we have a new checkpoint, we might be able to // remove some old ones. string errMsg = "PutImage failed. " + StringUtils.StringifyException(t); response.SendError(HttpServletResponse.ScGone, errMsg); throw new IOException(errMsg); } }
/// <summary>Create a redirection URL</summary> /// <exception cref="System.IO.IOException"/> private Uri CreateRedirectURL(string path, string encodedPath, HdfsFileStatus status , UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request, string dt) { string scheme = request.GetScheme(); LocatedBlocks blks = nnproxy.GetBlockLocations(status.GetFullPath(new Path(path)) .ToUri().GetPath(), 0, 1); Configuration conf = NameNodeHttpServer.GetConfFromContext(GetServletContext()); DatanodeID host = PickSrcDatanode(blks, status, conf); string hostname; if (host is DatanodeInfo) { hostname = host.GetHostName(); } else { hostname = host.GetIpAddr(); } int port = "https".Equals(scheme) ? host.GetInfoSecurePort() : host.GetInfoPort(); string dtParam = string.Empty; if (dt != null) { dtParam = JspHelper.GetDelegationTokenUrlParam(dt); } // Add namenode address to the url params NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext()); string addr = nn.GetNameNodeAddressHostPortString(); string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr); return(new Uri(scheme, hostname, port, "/streamFile" + encodedPath + '?' + "ugi=" + ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam)); }
/// <summary>Select a datanode to service this request.</summary> /// <remarks> /// Select a datanode to service this request. /// Currently, this looks at no more than the first five blocks of a file, /// selecting a datanode randomly from the most represented. /// </remarks> /// <param name="conf"></param> /// <exception cref="System.IO.IOException"/> private DatanodeID PickSrcDatanode(LocatedBlocks blks, HdfsFileStatus i, Configuration conf) { if (i.GetLen() == 0 || blks.GetLocatedBlocks().Count <= 0) { // pick a random datanode NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext()); return(NamenodeJspHelper.GetRandomDatanode(nn)); } return(JspHelper.BestNode(blks, conf)); }
/// <exception cref="System.Exception"/> public object Run() { NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context); FSNamesystem namesystem = nn.GetNamesystem(); BlockManager bm = namesystem.GetBlockManager(); int totalDatanodes = namesystem.GetNumberOfDatanodes(HdfsConstants.DatanodeReportType .Live); new NamenodeFsck(conf, nn, bm.GetDatanodeManager().GetNetworkTopology(), pmap, @out , totalDatanodes, remoteAddress).Fsck(); return(null); }
/// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest req, HttpServletResponse resp) { resp.SetContentType("application/json; charset=UTF-8"); StartupProgress prog = NameNodeHttpServer.GetStartupProgressFromContext(GetServletContext ()); StartupProgressView view = prog.CreateView(); JsonGenerator json = new JsonFactory().CreateJsonGenerator(resp.GetWriter()); try { json.WriteStartObject(); json.WriteNumberField(ElapsedTime, view.GetElapsedTime()); json.WriteNumberField(PercentComplete, view.GetPercentComplete()); json.WriteArrayFieldStart(Phases); foreach (Phase phase in view.GetPhases()) { json.WriteStartObject(); json.WriteStringField(Name, phase.GetName()); json.WriteStringField(Desc, phase.GetDescription()); json.WriteStringField(Status, view.GetStatus(phase).ToString()); json.WriteNumberField(PercentComplete, view.GetPercentComplete(phase)); json.WriteNumberField(ElapsedTime, view.GetElapsedTime(phase)); WriteStringFieldIfNotNull(json, File, view.GetFile(phase)); WriteNumberFieldIfDefined(json, Size, view.GetSize(phase)); json.WriteArrayFieldStart(Steps); foreach (Step step in view.GetSteps(phase)) { json.WriteStartObject(); StepType type = step.GetType(); if (type != null) { json.WriteStringField(Name, type.GetName()); json.WriteStringField(Desc, type.GetDescription()); } json.WriteNumberField(Count, view.GetCount(phase, step)); WriteStringFieldIfNotNull(json, File, step.GetFile()); WriteNumberFieldIfDefined(json, Size, step.GetSize()); json.WriteNumberField(Total, view.GetTotal(phase, step)); json.WriteNumberField(PercentComplete, view.GetPercentComplete(phase, step)); json.WriteNumberField(ElapsedTime, view.GetElapsedTime(phase, step)); json.WriteEndObject(); } json.WriteEndArray(); json.WriteEndObject(); } json.WriteEndArray(); json.WriteEndObject(); } finally { IOUtils.Cleanup(Log, json); } }
/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest req, HttpServletResponse resp) { UserGroupInformation ugi; ServletContext context = GetServletContext(); Configuration conf = NameNodeHttpServer.GetConfFromContext(context); try { ugi = GetUGI(req, conf); } catch (IOException ioe) { Log.Info("Request for token received with no authentication from " + req.GetRemoteAddr (), ioe); resp.SendError(HttpServletResponse.ScForbidden, "Unable to identify or authenticate user" ); return; } NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context); string tokenString = req.GetParameter(Token); if (tokenString == null) { resp.SendError(HttpServletResponse.ScMultipleChoices, "Token to renew not specified" ); } Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>(); token.DecodeFromUrlString(tokenString); try { long result = ugi.DoAs(new _PrivilegedExceptionAction_73(nn, token)); PrintWriter os = new PrintWriter(new OutputStreamWriter(resp.GetOutputStream(), Charsets .Utf8)); os.WriteLine(result); os.Close(); } catch (Exception e) { // transfer exception over the http string exceptionClass = e.GetType().FullName; string exceptionMsg = e.GetLocalizedMessage(); string strException = exceptionClass + ";" + exceptionMsg; Log.Info("Exception while renewing token. Re-throwing. s=" + strException, e); resp.SendError(HttpServletResponse.ScInternalServerError, strException); } }
/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest req, HttpServletResponse resp) { UserGroupInformation ugi; ServletContext context = GetServletContext(); Configuration conf = NameNodeHttpServer.GetConfFromContext(context); try { ugi = GetUGI(req, conf); } catch (IOException ioe) { Log.Info("Request for token received with no authentication from " + req.GetRemoteAddr (), ioe); resp.SendError(HttpServletResponse.ScForbidden, "Unable to identify or authenticate user" ); return; } Log.Info("Sending token: {" + ugi.GetUserName() + "," + req.GetRemoteAddr() + "}" ); NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context); string renewer = req.GetParameter(Renewer); string renewerFinal = (renewer == null) ? req.GetUserPrincipal().GetName() : renewer; DataOutputStream dos = null; try { dos = new DataOutputStream(resp.GetOutputStream()); DataOutputStream dosFinal = dos; // for doAs block ugi.DoAs(new _PrivilegedExceptionAction_69(nn, ugi, renewerFinal, dosFinal)); } catch (Exception e) { Log.Info("Exception while sending token. Re-throwing ", e); resp.SendError(HttpServletResponse.ScInternalServerError); } finally { if (dos != null) { dos.Close(); } } }
/// <summary> /// Create a /// <see cref="NameNode"/> /// proxy from the current /// <see cref="Javax.Servlet.ServletContext"/> /// . /// </summary> /// <exception cref="System.IO.IOException"/> protected internal virtual ClientProtocol CreateNameNodeProxy() { ServletContext context = GetServletContext(); // if we are running in the Name Node, use it directly rather than via // rpc NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context); if (nn != null) { return(nn.GetRpcServer()); } IPEndPoint nnAddr = NameNodeHttpServer.GetNameNodeAddressFromContext(context); Configuration conf = new HdfsConfiguration(NameNodeHttpServer.GetConfFromContext( context)); return(NameNodeProxies.CreateProxy <ClientProtocol>(conf, NameNode.GetUri(nnAddr)) .GetProxy()); }
/// <summary>Service a GET request as described below.</summary> /// <remarks> /// Service a GET request as described below. /// Request: /// <c>GET http://<nn>:<port>/data[/<path>] HTTP/1.1</c> /// </remarks> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest request, HttpServletResponse response ) { Configuration conf = NameNodeHttpServer.GetConfFromContext(GetServletContext()); UserGroupInformation ugi = GetUGI(request, conf); try { ugi.DoAs(new _PrivilegedExceptionAction_116(this, request, response, ugi)); } catch (IOException e) { response.SendError(400, e.Message); } catch (Exception e) { response.SendError(400, e.Message); } }
/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest request, HttpServletResponse response ) { ServletContext context = GetServletContext(); Configuration conf = NameNodeHttpServer.GetConfFromContext(context); UserGroupInformation ugi = GetUGI(request, conf); NameNode namenode = NameNodeHttpServer.GetNameNodeFromContext(context); DatanodeID datanode = NamenodeJspHelper.GetRandomDatanode(namenode); try { response.SendRedirect(CreateRedirectURL(ugi, datanode, request, namenode).ToString ()); } catch (IOException e) { response.SendError(400, e.Message); } }
/// <summary>Handle fsck request</summary> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest request, HttpServletResponse response ) { IDictionary <string, string[]> pmap = request.GetParameterMap(); PrintWriter @out = response.GetWriter(); IPAddress remoteAddress = Sharpen.Extensions.GetAddressByName(request.GetRemoteAddr ()); ServletContext context = GetServletContext(); Configuration conf = NameNodeHttpServer.GetConfFromContext(context); UserGroupInformation ugi = GetUGI(request, conf); try { ugi.DoAs(new _PrivilegedExceptionAction_58(context, conf, pmap, @out, remoteAddress )); } catch (Exception e) { response.SendError(400, e.Message); } }
/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest req, HttpServletResponse resp) { UserGroupInformation ugi; ServletContext context = GetServletContext(); Configuration conf = NameNodeHttpServer.GetConfFromContext(context); try { ugi = GetUGI(req, conf); } catch (IOException ioe) { Log.Info("Request for token received with no authentication from " + req.GetRemoteAddr (), ioe); resp.SendError(HttpServletResponse.ScForbidden, "Unable to identify or authenticate user" ); return; } NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(context); string tokenString = req.GetParameter(Token); if (tokenString == null) { resp.SendError(HttpServletResponse.ScMultipleChoices, "Token to renew not specified" ); } Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = new Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier>(); token.DecodeFromUrlString(tokenString); try { ugi.DoAs(new _PrivilegedExceptionAction_70(nn, token)); } catch (Exception e) { Log.Info("Exception while cancelling token. Re-throwing. ", e); resp.SendError(HttpServletResponse.ScInternalServerError, e.Message); } }
/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest request, HttpServletResponse response ) { try { ServletContext context = GetServletContext(); FSImage nnImage = NameNodeHttpServer.GetFsImageFromContext(context); ImageServlet.GetImageParams parsedParams = new ImageServlet.GetImageParams(request , response); Configuration conf = (Configuration)context.GetAttribute(JspHelper.CurrentConf); NameNodeMetrics metrics = NameNode.GetNameNodeMetrics(); ValidateRequest(context, conf, request, response, nnImage, parsedParams.GetStorageInfoString ()); UserGroupInformation.GetCurrentUser().DoAs(new _PrivilegedExceptionAction_98(parsedParams , nnImage, metrics, response, conf)); } catch (Exception t) { // Metrics non-null only when used inside name node // Metrics non-null only when used inside name node // Potential race where the file was deleted while we were in the // process of setting headers! // It's possible the file could be deleted after this point, but // we've already opened the 'fis' stream. // It's also possible length could change, but this would be // detected by the client side as an inaccurate length header. // send file string errMsg = "GetImage failed. " + StringUtils.StringifyException(t); response.SendError(HttpServletResponse.ScGone, errMsg); throw new IOException(errMsg); } finally { response.GetOutputStream().Close(); } }