/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest request, HttpServletResponse response ) { FileInputStream editFileIn = null; try { ServletContext context = GetServletContext(); Configuration conf = (Configuration)GetServletContext().GetAttribute(JspHelper.CurrentConf ); string journalId = request.GetParameter(JournalIdParam); QuorumJournalManager.CheckJournalId(journalId); JNStorage storage = JournalNodeHttpServer.GetJournalFromContext(context, journalId ).GetStorage(); // Check security if (!CheckRequestorOrSendError(conf, request, response)) { return; } // Check that the namespace info is correct if (!CheckStorageInfoOrSendError(storage, request, response)) { return; } long segmentTxId = ServletUtil.ParseLongParam(request, SegmentTxidParam); FileJournalManager fjm = storage.GetJournalManager(); FilePath editFile; lock (fjm) { // Synchronize on the FJM so that the file doesn't get finalized // out from underneath us while we're in the process of opening // it up. FileJournalManager.EditLogFile elf = fjm.GetLogFile(segmentTxId); if (elf == null) { response.SendError(HttpServletResponse.ScNotFound, "No edit log found starting at txid " + segmentTxId); return; } editFile = elf.GetFile(); ImageServlet.SetVerificationHeadersForGet(response, editFile); ImageServlet.SetFileNameHeaders(response, editFile); editFileIn = new FileInputStream(editFile); } DataTransferThrottler throttler = ImageServlet.GetThrottler(conf); // send edits TransferFsImage.CopyFileToStream(response.GetOutputStream(), editFile, editFileIn , throttler); } catch (Exception t) { string errMsg = "getedit failed. " + StringUtils.StringifyException(t); response.SendError(HttpServletResponse.ScInternalServerError, errMsg); throw new IOException(errMsg); } finally { IOUtils.CloseStream(editFileIn); } }
/// <summary>Create a redirection URL</summary> /// <exception cref="System.IO.IOException"/> private Uri CreateRedirectURL(UserGroupInformation ugi, DatanodeID host, HttpServletRequest request, NameNode nn) { string hostname = host is DatanodeInfo?host.GetHostName() : host.GetIpAddr(); string scheme = request.GetScheme(); int port = host.GetInfoPort(); if ("https".Equals(scheme)) { int portObject = (int)GetServletContext().GetAttribute(DFSConfigKeys.DfsDatanodeHttpsPortKey ); if (portObject != null) { port = portObject; } } string encodedPath = ServletUtil.GetRawPath(request, "/fileChecksum"); string dtParam = string.Empty; if (UserGroupInformation.IsSecurityEnabled()) { string tokenString = ugi.GetTokens().GetEnumerator().Next().EncodeToUrlString(); dtParam = JspHelper.GetDelegationTokenUrlParam(tokenString); } string addr = nn.GetNameNodeAddressHostPortString(); string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr); return(new Uri(scheme, hostname, port, "/getFileChecksum" + encodedPath + '?' + "ugi=" + ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam)); }
/// <exception cref="System.IO.IOException"/> public Void Run() { ClientProtocol nn = this._enclosing.CreateNameNodeProxy(); string path = ServletUtil.GetDecodedPath(request, "/data"); string encodedPath = ServletUtil.GetRawPath(request, "/data"); string delegationToken = request.GetParameter(JspHelper.DelegationParameterName); HdfsFileStatus info = nn.GetFileInfo(path); if (info != null && !info.IsDir()) { response.SendRedirect(this._enclosing.CreateRedirectURL(path, encodedPath, info, ugi, nn, request, delegationToken).ToString()); } else { if (info == null) { response.SendError(400, "File not found " + path); } else { response.SendError(400, path + ": is a directory"); } } return(null); }
/// <exception cref="System.Exception"/> public Void Run() { string path = ServletUtil.GetDecodedPath(request, "/contentSummary"); PrintWriter @out = response.GetWriter(); XMLOutputter xml = new XMLOutputter(@out, "UTF-8"); xml.Declaration(); try { ClientProtocol nnproxy = this._enclosing.CreateNameNodeProxy(); ContentSummary cs = nnproxy.GetContentSummary(path); xml.StartTag(typeof(ContentSummary).FullName); if (cs != null) { xml.Attribute("length", string.Empty + cs.GetLength()); xml.Attribute("fileCount", string.Empty + cs.GetFileCount()); xml.Attribute("directoryCount", string.Empty + cs.GetDirectoryCount()); xml.Attribute("quota", string.Empty + cs.GetQuota()); xml.Attribute("spaceConsumed", string.Empty + cs.GetSpaceConsumed()); xml.Attribute("spaceQuota", string.Empty + cs.GetSpaceQuota()); } xml.EndTag(); } catch (IOException ioe) { this._enclosing.WriteXml(ioe, path, xml); } xml.EndDocument(); return(null); }
/// <summary>Create a redirection URL</summary> /// <exception cref="System.IO.IOException"/> private Uri CreateRedirectURL(string path, string encodedPath, HdfsFileStatus status , UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request, string dt) { string scheme = request.GetScheme(); LocatedBlocks blks = nnproxy.GetBlockLocations(status.GetFullPath(new Path(path)) .ToUri().GetPath(), 0, 1); Configuration conf = NameNodeHttpServer.GetConfFromContext(GetServletContext()); DatanodeID host = PickSrcDatanode(blks, status, conf); string hostname; if (host is DatanodeInfo) { hostname = host.GetHostName(); } else { hostname = host.GetIpAddr(); } int port = "https".Equals(scheme) ? host.GetInfoSecurePort() : host.GetInfoPort(); string dtParam = string.Empty; if (dt != null) { dtParam = JspHelper.GetDelegationTokenUrlParam(dt); } // Add namenode address to the url params NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext()); string addr = nn.GetNameNodeAddressHostPortString(); string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr); return(new Uri(scheme, hostname, port, "/streamFile" + encodedPath + '?' + "ugi=" + ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam)); }
/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest request, HttpServletResponse response ) { PrintWriter @out = response.GetWriter(); string path = ServletUtil.GetDecodedPath(request, "/getFileChecksum"); XMLOutputter xml = new XMLOutputter(@out, "UTF-8"); xml.Declaration(); ServletContext context = GetServletContext(); DataNode datanode = (DataNode)context.GetAttribute("datanode"); Configuration conf = new HdfsConfiguration(datanode.GetConf()); try { DFSClient dfs = DatanodeJspHelper.GetDFSClient(request, datanode, conf, GetUGI(request , conf)); MD5MD5CRC32FileChecksum checksum = dfs.GetFileChecksum(path, long.MaxValue); MD5MD5CRC32FileChecksum.Write(xml, checksum); } catch (IOException ioe) { WriteXml(ioe, path, xml); } catch (Exception e) { WriteXml(e, path, xml); } xml.EndDocument(); }
/// <exception cref="System.IO.IOException"/> public override FSDataInputStream Open(Path f, int buffersize) { f = f.MakeQualified(GetUri(), GetWorkingDirectory()); string path = "/data" + ServletUtil.EncodePath(f.ToUri().GetPath()); string query = AddDelegationTokenParam("ugi=" + GetEncodedUgiParameter()); Uri u = GetNamenodeURL(path, query); return(new FSDataInputStream(new HftpFileSystem.RangeHeaderInputStream(connectionFactory , u))); }
/// <param name="request">the object from which this servlet reads the url contents</param> /// <param name="response">the object into which this servlet writes the url contents /// </param> /// <exception cref="System.IO.IOException">if the request is bad</exception> public GetImageParams(HttpServletRequest request, HttpServletResponse response) { IDictionary <string, string[]> pmap = request.GetParameterMap(); isGetImage = isGetEdit = fetchLatest = false; foreach (KeyValuePair <string, string[]> entry in pmap) { string key = entry.Key; string[] val = entry.Value; if (key.Equals("getimage")) { isGetImage = true; try { txId = ServletUtil.ParseLongParam(request, TxidParam); string imageType = ServletUtil.GetParameter(request, ImageFileType); nnf = imageType == null ? NNStorage.NameNodeFile.Image : NNStorage.NameNodeFile.ValueOf (imageType); } catch (FormatException nfe) { if (request.GetParameter(TxidParam).Equals(LatestFsimageValue)) { fetchLatest = true; } else { throw; } } } else { if (key.Equals("getedit")) { isGetEdit = true; startTxId = ServletUtil.ParseLongParam(request, StartTxidParam); endTxId = ServletUtil.ParseLongParam(request, EndTxidParam); } else { if (key.Equals(StorageinfoParam)) { storageInfoString = val[0]; } } } } int numGets = (isGetImage ? 1 : 0) + (isGetEdit ? 1 : 0); if ((numGets > 1) || (numGets == 0)) { throw new IOException("Illegal parameters to TransferFsImage"); } }
/// <summary>Get encoded UGI parameter string for a URL.</summary> /// <returns>user_shortname,group1,group2...</returns> private string GetEncodedUgiParameter() { StringBuilder ugiParameter = new StringBuilder(ServletUtil.EncodeQueryValue(ugi.GetShortUserName ())); foreach (string g in ugi.GetGroupNames()) { ugiParameter.Append(","); ugiParameter.Append(ServletUtil.EncodeQueryValue(g)); } return(ugiParameter.ToString()); }
/// <exception cref="System.IO.IOException"/> public PutImageParams(HttpServletRequest request, HttpServletResponse response, Configuration conf) { /* * Params required to handle put image request */ txId = ServletUtil.ParseLongParam(request, TxidParam); storageInfoString = ServletUtil.GetParameter(request, StorageinfoParam); fileSize = ServletUtil.ParseLongParam(request, TransferFsImage.FileLength); string imageType = ServletUtil.GetParameter(request, ImageFileType); nnf = imageType == null ? NNStorage.NameNodeFile.Image : NNStorage.NameNodeFile.ValueOf (imageType); if (fileSize == 0 || txId == -1 || storageInfoString == null || storageInfoString .IsEmpty()) { throw new IOException("Illegal parameters to TransferFsImage"); } }
/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest request, HttpServletResponse response ) { // Do the authorization if (!HttpServer2.HasAdministratorAccess(GetServletContext(), request, response)) { return; } PrintWriter @out = ServletUtil.InitHTML(response, "Log Level"); string logName = ServletUtil.GetParameter(request, "log"); string level = ServletUtil.GetParameter(request, "level"); if (logName != null) { @out.WriteLine("<br /><hr /><h3>Results</h3>"); @out.WriteLine(Marker + "Submitted Log Name: <b>" + logName + "</b><br />"); Org.Apache.Commons.Logging.Log log = LogFactory.GetLog(logName); @out.WriteLine(Marker + "Log Class: <b>" + log.GetType().FullName + "</b><br />"); if (level != null) { @out.WriteLine(Marker + "Submitted Level: <b>" + level + "</b><br />"); } if (log is Log4JLogger) { Process(((Log4JLogger)log).GetLogger(), level, @out); } else { if (log is Jdk14Logger) { Process(((Jdk14Logger)log).GetLogger(), level, @out); } else { @out.WriteLine("Sorry, " + log.GetType() + " not supported.<br />"); } } } @out.WriteLine(Forms); @out.WriteLine(ServletUtil.HtmlTail); }
/// <exception cref="System.IO.IOException"/> private void TestDataNodeRedirect(Path path) { // Create the file if (hdfs.Exists(path)) { hdfs.Delete(path, true); } FSDataOutputStream @out = hdfs.Create(path, (short)1); @out.WriteBytes("0123456789"); @out.Close(); // Get the path's block location so we can determine // if we were redirected to the right DN. BlockLocation[] locations = hdfs.GetFileBlockLocations(path, 0, 10); string xferAddr = locations[0].GetNames()[0]; // Connect to the NN to get redirected Uri u = hftpFs.GetNamenodeURL("/data" + ServletUtil.EncodePath(path.ToUri().GetPath ()), "ugi=userx,groupy"); HttpURLConnection conn = (HttpURLConnection)u.OpenConnection(); HttpURLConnection.SetFollowRedirects(true); conn.Connect(); conn.GetInputStream(); bool @checked = false; // Find the datanode that has the block according to locations // and check that the URL was redirected to this DN's info port foreach (DataNode node in cluster.GetDataNodes()) { DatanodeRegistration dnR = DataNodeTestUtils.GetDNRegistrationForBP(node, blockPoolId ); if (dnR.GetXferAddr().Equals(xferAddr)) { @checked = true; NUnit.Framework.Assert.AreEqual(dnR.GetInfoPort(), conn.GetURL().Port); } } NUnit.Framework.Assert.IsTrue("The test never checked that location of " + "the block and hftp desitnation are the same" , @checked); }
/// <summary>Service a GET request as described below.</summary> /// <remarks> /// Service a GET request as described below. /// Request: /// <c>GET http://<nn>:<port>/listPaths[/<path>][<?option>[&option]*] HTTP/1.1 /// </c> /// Where <i>option</i> (default) in: /// recursive ("no") /// filter (".*") /// exclude ("\..*\.crc") /// Response: A flat list of files/directories in the following format: /// <c> /// <listing path="..." recursive="(yes|no)" filter="..." /// time="yyyy-MM-dd hh:mm:ss UTC" version="..."> /// <directory path="..." modified="yyyy-MM-dd hh:mm:ss"/> /// <file path="..." modified="yyyy-MM-dd'T'hh:mm:ssZ" accesstime="yyyy-MM-dd'T'hh:mm:ssZ" /// blocksize="..." /// replication="..." size="..."/> /// </listing> /// </c> /// </remarks> /// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest request, HttpServletResponse response ) { PrintWriter @out = response.GetWriter(); XMLOutputter doc = new XMLOutputter(@out, "UTF-8"); IDictionary <string, string> root = BuildRoot(request, doc); string path = root["path"]; string filePath = ServletUtil.GetDecodedPath(request, "/listPaths"); try { bool recur = "yes".Equals(root["recursive"]); Sharpen.Pattern filter = Sharpen.Pattern.Compile(root["filter"]); Sharpen.Pattern exclude = Sharpen.Pattern.Compile(root["exclude"]); Configuration conf = (Configuration)GetServletContext().GetAttribute(JspHelper.CurrentConf ); GetUGI(request, conf).DoAs(new _PrivilegedExceptionAction_149(this, doc, root, filePath , path, exclude, filter, recur)); } catch (IOException ioe) { WriteXml(ioe, path, doc); } catch (Exception e) { Log.Warn("ListPathServlet encountered InterruptedException", e); response.SendError(400, e.Message); } finally { if (doc != null) { doc.EndDocument(); } if (@out != null) { @out.Close(); } } }
/// <summary>Build a map from the query string, setting values and defaults.</summary> protected internal virtual IDictionary <string, string> BuildRoot(HttpServletRequest request, XMLOutputter doc) { string path = ServletUtil.GetDecodedPath(request, "/listPaths"); string exclude = request.GetParameter("exclude") != null?request.GetParameter("exclude" ) : string.Empty; string filter = request.GetParameter("filter") != null?request.GetParameter("filter" ) : ".*"; bool recur = request.GetParameter("recursive") != null && "yes".Equals(request.GetParameter ("recursive")); IDictionary <string, string> root = new Dictionary <string, string>(); root["path"] = path; root["recursive"] = recur ? "yes" : "no"; root["filter"] = filter; root["exclude"] = exclude; root["time"] = df.Get().Format(new DateTime()); root["version"] = VersionInfo.GetVersion(); return(root); }
/// <summary>Connect to the name node and get content summary.</summary> /// <param name="path">The path</param> /// <returns>The content summary for the path.</returns> /// <exception cref="System.IO.IOException"/> private ContentSummary GetContentSummary(string path) { HttpURLConnection connection = this._enclosing.OpenConnection("/contentSummary" + ServletUtil.EncodePath(path), "ugi=" + this._enclosing.GetEncodedUgiParameter() ); InputStream @in = null; try { @in = connection.GetInputStream(); XMLReader xr = XMLReaderFactory.CreateXMLReader(); xr.SetContentHandler(this); xr.Parse(new InputSource(@in)); } catch (FileNotFoundException) { //the server may not support getContentSummary return(null); } catch (SAXException saxe) { Exception embedded = saxe.GetException(); if (embedded != null && embedded is IOException) { throw (IOException)embedded; } throw new IOException("Invalid xml format", saxe); } finally { if (@in != null) { @in.Close(); } connection.Disconnect(); } return(this.contentsummary); }
/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest request, HttpServletResponse response ) { string path = ServletUtil.GetDecodedPath(request, "/streamFile"); string rawPath = ServletUtil.GetRawPath(request, "/streamFile"); string filename = JspHelper.ValidatePath(path); string rawFilename = JspHelper.ValidatePath(rawPath); if (filename == null) { response.SetContentType("text/plain"); PrintWriter @out = response.GetWriter(); @out.Write("Invalid input"); return; } Enumeration <string> reqRanges = request.GetHeaders("Range"); if (reqRanges != null && !reqRanges.MoveNext()) { reqRanges = null; } DFSClient dfs; try { dfs = GetDFSClient(request); } catch (Exception e) { response.SendError(400, e.Message); return; } HdfsDataInputStream @in = null; OutputStream out_1 = null; try { @in = dfs.CreateWrappedInputStream(dfs.Open(filename)); out_1 = response.GetOutputStream(); long fileLen = @in.GetVisibleLength(); if (reqRanges != null) { IList <InclusiveByteRange> ranges = InclusiveByteRange.SatisfiableRanges(reqRanges , fileLen); StreamFile.SendPartialData(@in, out_1, response, fileLen, ranges); } else { // No ranges, so send entire file response.SetHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\""); response.SetContentType("application/octet-stream"); response.SetHeader(ContentLength, string.Empty + fileLen); StreamFile.CopyFromOffset(@in, out_1, 0L, fileLen); } @in.Close(); @in = null; out_1.Close(); out_1 = null; dfs.Close(); dfs = null; } catch (IOException ioe) { if (Log.IsDebugEnabled()) { Log.Debug("response.isCommitted()=" + response.IsCommitted(), ioe); } throw; } finally { IOUtils.Cleanup(Log, @in); IOUtils.Cleanup(Log, out_1); IOUtils.Cleanup(Log, dfs); } }
/// <exception cref="System.IO.IOException"/> private FileChecksum GetFileChecksum(string f) { HttpURLConnection connection = this._enclosing.OpenConnection("/fileChecksum" + ServletUtil .EncodePath(f), "ugi=" + this._enclosing.GetEncodedUgiParameter()); try { XMLReader xr = XMLReaderFactory.CreateXMLReader(); xr.SetContentHandler(this); xr.Parse(new InputSource(connection.GetInputStream())); } catch (SAXException e) { Exception embedded = e.GetException(); if (embedded != null && embedded is IOException) { throw (IOException)embedded; } throw new IOException("invalid xml directory content", e); } finally { connection.Disconnect(); } return(this.filechecksum); }
/// <exception cref="System.IO.IOException"/> private void FetchList(string path, bool recur) { try { XMLReader xr = XMLReaderFactory.CreateXMLReader(); xr.SetContentHandler(this); HttpURLConnection connection = this._enclosing.OpenConnection("/listPaths" + ServletUtil .EncodePath(path), "ugi=" + this._enclosing.GetEncodedUgiParameter() + (recur ? "&recursive=yes" : string.Empty)); InputStream resp = connection.GetInputStream(); xr.Parse(new InputSource(resp)); } catch (SAXException e) { Exception embedded = e.GetException(); if (embedded != null && embedded is IOException) { throw (IOException)embedded; } throw new IOException("invalid xml directory content", e); } }