/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
FileInputStream editFileIn = null;
try
{
ServletContext context = GetServletContext();
Configuration conf = (Configuration)GetServletContext().GetAttribute(JspHelper.CurrentConf
);
string journalId = request.GetParameter(JournalIdParam);
QuorumJournalManager.CheckJournalId(journalId);
JNStorage storage = JournalNodeHttpServer.GetJournalFromContext(context, journalId
).GetStorage();
// Check security
if (!CheckRequestorOrSendError(conf, request, response))
{
return;
}
// Check that the namespace info is correct
if (!CheckStorageInfoOrSendError(storage, request, response))
{
return;
}
long segmentTxId = ServletUtil.ParseLongParam(request, SegmentTxidParam);
FileJournalManager fjm = storage.GetJournalManager();
FilePath editFile;
lock (fjm)
{
// Synchronize on the FJM so that the file doesn't get finalized
// out from underneath us while we're in the process of opening
// it up.
FileJournalManager.EditLogFile elf = fjm.GetLogFile(segmentTxId);
if (elf == null)
{
response.SendError(HttpServletResponse.ScNotFound, "No edit log found starting at txid "
+ segmentTxId);
return;
}
editFile = elf.GetFile();
ImageServlet.SetVerificationHeadersForGet(response, editFile);
ImageServlet.SetFileNameHeaders(response, editFile);
editFileIn = new FileInputStream(editFile);
}
DataTransferThrottler throttler = ImageServlet.GetThrottler(conf);
// send edits
TransferFsImage.CopyFileToStream(response.GetOutputStream(), editFile, editFileIn
, throttler);
}
catch (Exception t)
{
string errMsg = "getedit failed. " + StringUtils.StringifyException(t);
response.SendError(HttpServletResponse.ScInternalServerError, errMsg);
throw new IOException(errMsg);
}
finally
{
IOUtils.CloseStream(editFileIn);
}
}
/// <summary>Create a redirection URL</summary>
/// <exception cref="System.IO.IOException"/>
private Uri CreateRedirectURL(UserGroupInformation ugi, DatanodeID host, HttpServletRequest
request, NameNode nn)
{
string hostname = host is DatanodeInfo?host.GetHostName() : host.GetIpAddr();
string scheme = request.GetScheme();
int port = host.GetInfoPort();
if ("https".Equals(scheme))
{
int portObject = (int)GetServletContext().GetAttribute(DFSConfigKeys.DfsDatanodeHttpsPortKey
);
if (portObject != null)
{
port = portObject;
}
}
string encodedPath = ServletUtil.GetRawPath(request, "/fileChecksum");
string dtParam = string.Empty;
if (UserGroupInformation.IsSecurityEnabled())
{
string tokenString = ugi.GetTokens().GetEnumerator().Next().EncodeToUrlString();
dtParam = JspHelper.GetDelegationTokenUrlParam(tokenString);
}
string addr = nn.GetNameNodeAddressHostPortString();
string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr);
return(new Uri(scheme, hostname, port, "/getFileChecksum" + encodedPath + '?' + "ugi="
+ ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam));
}
/// <exception cref="System.IO.IOException"/>
public Void Run()
{
ClientProtocol nn = this._enclosing.CreateNameNodeProxy();
string path = ServletUtil.GetDecodedPath(request, "/data");
string encodedPath = ServletUtil.GetRawPath(request, "/data");
string delegationToken = request.GetParameter(JspHelper.DelegationParameterName);
HdfsFileStatus info = nn.GetFileInfo(path);
if (info != null && !info.IsDir())
{
response.SendRedirect(this._enclosing.CreateRedirectURL(path, encodedPath, info,
ugi, nn, request, delegationToken).ToString());
}
else
{
if (info == null)
{
response.SendError(400, "File not found " + path);
}
else
{
response.SendError(400, path + ": is a directory");
}
}
return(null);
}
/// <exception cref="System.Exception"/>
public Void Run()
{
string path = ServletUtil.GetDecodedPath(request, "/contentSummary");
PrintWriter @out = response.GetWriter();
XMLOutputter xml = new XMLOutputter(@out, "UTF-8");
xml.Declaration();
try
{
ClientProtocol nnproxy = this._enclosing.CreateNameNodeProxy();
ContentSummary cs = nnproxy.GetContentSummary(path);
xml.StartTag(typeof(ContentSummary).FullName);
if (cs != null)
{
xml.Attribute("length", string.Empty + cs.GetLength());
xml.Attribute("fileCount", string.Empty + cs.GetFileCount());
xml.Attribute("directoryCount", string.Empty + cs.GetDirectoryCount());
xml.Attribute("quota", string.Empty + cs.GetQuota());
xml.Attribute("spaceConsumed", string.Empty + cs.GetSpaceConsumed());
xml.Attribute("spaceQuota", string.Empty + cs.GetSpaceQuota());
}
xml.EndTag();
}
catch (IOException ioe)
{
this._enclosing.WriteXml(ioe, path, xml);
}
xml.EndDocument();
return(null);
}
/// <summary>Create a redirection URL</summary>
/// <exception cref="System.IO.IOException"/>
private Uri CreateRedirectURL(string path, string encodedPath, HdfsFileStatus status
, UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request,
string dt)
{
string scheme = request.GetScheme();
LocatedBlocks blks = nnproxy.GetBlockLocations(status.GetFullPath(new Path(path))
.ToUri().GetPath(), 0, 1);
Configuration conf = NameNodeHttpServer.GetConfFromContext(GetServletContext());
DatanodeID host = PickSrcDatanode(blks, status, conf);
string hostname;
if (host is DatanodeInfo)
{
hostname = host.GetHostName();
}
else
{
hostname = host.GetIpAddr();
}
int port = "https".Equals(scheme) ? host.GetInfoSecurePort() : host.GetInfoPort();
string dtParam = string.Empty;
if (dt != null)
{
dtParam = JspHelper.GetDelegationTokenUrlParam(dt);
}
// Add namenode address to the url params
NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext());
string addr = nn.GetNameNodeAddressHostPortString();
string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr);
return(new Uri(scheme, hostname, port, "/streamFile" + encodedPath + '?' + "ugi="
+ ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam));
}
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
PrintWriter @out = response.GetWriter();
string path = ServletUtil.GetDecodedPath(request, "/getFileChecksum");
XMLOutputter xml = new XMLOutputter(@out, "UTF-8");
xml.Declaration();
ServletContext context = GetServletContext();
DataNode datanode = (DataNode)context.GetAttribute("datanode");
Configuration conf = new HdfsConfiguration(datanode.GetConf());
try
{
DFSClient dfs = DatanodeJspHelper.GetDFSClient(request, datanode, conf, GetUGI(request
, conf));
MD5MD5CRC32FileChecksum checksum = dfs.GetFileChecksum(path, long.MaxValue);
MD5MD5CRC32FileChecksum.Write(xml, checksum);
}
catch (IOException ioe)
{
WriteXml(ioe, path, xml);
}
catch (Exception e)
{
WriteXml(e, path, xml);
}
xml.EndDocument();
}
/// <exception cref="System.IO.IOException"/>
public override FSDataInputStream Open(Path f, int buffersize)
{
f = f.MakeQualified(GetUri(), GetWorkingDirectory());
string path = "/data" + ServletUtil.EncodePath(f.ToUri().GetPath());
string query = AddDelegationTokenParam("ugi=" + GetEncodedUgiParameter());
Uri u = GetNamenodeURL(path, query);
return(new FSDataInputStream(new HftpFileSystem.RangeHeaderInputStream(connectionFactory
, u)));
}
/// <param name="request">the object from which this servlet reads the url contents</param>
/// <param name="response">the object into which this servlet writes the url contents
/// </param>
/// <exception cref="System.IO.IOException">if the request is bad</exception>
public GetImageParams(HttpServletRequest request, HttpServletResponse response)
{
IDictionary <string, string[]> pmap = request.GetParameterMap();
isGetImage = isGetEdit = fetchLatest = false;
foreach (KeyValuePair <string, string[]> entry in pmap)
{
string key = entry.Key;
string[] val = entry.Value;
if (key.Equals("getimage"))
{
isGetImage = true;
try
{
txId = ServletUtil.ParseLongParam(request, TxidParam);
string imageType = ServletUtil.GetParameter(request, ImageFileType);
nnf = imageType == null ? NNStorage.NameNodeFile.Image : NNStorage.NameNodeFile.ValueOf
(imageType);
}
catch (FormatException nfe)
{
if (request.GetParameter(TxidParam).Equals(LatestFsimageValue))
{
fetchLatest = true;
}
else
{
throw;
}
}
}
else
{
if (key.Equals("getedit"))
{
isGetEdit = true;
startTxId = ServletUtil.ParseLongParam(request, StartTxidParam);
endTxId = ServletUtil.ParseLongParam(request, EndTxidParam);
}
else
{
if (key.Equals(StorageinfoParam))
{
storageInfoString = val[0];
}
}
}
}
int numGets = (isGetImage ? 1 : 0) + (isGetEdit ? 1 : 0);
if ((numGets > 1) || (numGets == 0))
{
throw new IOException("Illegal parameters to TransferFsImage");
}
}
/// <summary>Get encoded UGI parameter string for a URL.</summary>
/// <returns>user_shortname,group1,group2...</returns>
private string GetEncodedUgiParameter()
{
StringBuilder ugiParameter = new StringBuilder(ServletUtil.EncodeQueryValue(ugi.GetShortUserName
()));
foreach (string g in ugi.GetGroupNames())
{
ugiParameter.Append(",");
ugiParameter.Append(ServletUtil.EncodeQueryValue(g));
}
return(ugiParameter.ToString());
}
/// <exception cref="System.IO.IOException"/>
public PutImageParams(HttpServletRequest request, HttpServletResponse response, Configuration
conf)
{
/*
* Params required to handle put image request
*/
txId = ServletUtil.ParseLongParam(request, TxidParam);
storageInfoString = ServletUtil.GetParameter(request, StorageinfoParam);
fileSize = ServletUtil.ParseLongParam(request, TransferFsImage.FileLength);
string imageType = ServletUtil.GetParameter(request, ImageFileType);
nnf = imageType == null ? NNStorage.NameNodeFile.Image : NNStorage.NameNodeFile.ValueOf
(imageType);
if (fileSize == 0 || txId == -1 || storageInfoString == null || storageInfoString
.IsEmpty())
{
throw new IOException("Illegal parameters to TransferFsImage");
}
}
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
// Do the authorization
if (!HttpServer2.HasAdministratorAccess(GetServletContext(), request, response))
{
return;
}
PrintWriter @out = ServletUtil.InitHTML(response, "Log Level");
string logName = ServletUtil.GetParameter(request, "log");
string level = ServletUtil.GetParameter(request, "level");
if (logName != null)
{
@out.WriteLine("<br /><hr /><h3>Results</h3>");
@out.WriteLine(Marker + "Submitted Log Name: <b>" + logName + "</b><br />");
Org.Apache.Commons.Logging.Log log = LogFactory.GetLog(logName);
@out.WriteLine(Marker + "Log Class: <b>" + log.GetType().FullName + "</b><br />");
if (level != null)
{
@out.WriteLine(Marker + "Submitted Level: <b>" + level + "</b><br />");
}
if (log is Log4JLogger)
{
Process(((Log4JLogger)log).GetLogger(), level, @out);
}
else
{
if (log is Jdk14Logger)
{
Process(((Jdk14Logger)log).GetLogger(), level, @out);
}
else
{
@out.WriteLine("Sorry, " + log.GetType() + " not supported.<br />");
}
}
}
@out.WriteLine(Forms);
@out.WriteLine(ServletUtil.HtmlTail);
}
/// <exception cref="System.IO.IOException"/>
private void TestDataNodeRedirect(Path path)
{
// Create the file
if (hdfs.Exists(path))
{
hdfs.Delete(path, true);
}
FSDataOutputStream @out = hdfs.Create(path, (short)1);
@out.WriteBytes("0123456789");
@out.Close();
// Get the path's block location so we can determine
// if we were redirected to the right DN.
BlockLocation[] locations = hdfs.GetFileBlockLocations(path, 0, 10);
string xferAddr = locations[0].GetNames()[0];
// Connect to the NN to get redirected
Uri u = hftpFs.GetNamenodeURL("/data" + ServletUtil.EncodePath(path.ToUri().GetPath
()), "ugi=userx,groupy");
HttpURLConnection conn = (HttpURLConnection)u.OpenConnection();
HttpURLConnection.SetFollowRedirects(true);
conn.Connect();
conn.GetInputStream();
bool @checked = false;
// Find the datanode that has the block according to locations
// and check that the URL was redirected to this DN's info port
foreach (DataNode node in cluster.GetDataNodes())
{
DatanodeRegistration dnR = DataNodeTestUtils.GetDNRegistrationForBP(node, blockPoolId
);
if (dnR.GetXferAddr().Equals(xferAddr))
{
@checked = true;
NUnit.Framework.Assert.AreEqual(dnR.GetInfoPort(), conn.GetURL().Port);
}
}
NUnit.Framework.Assert.IsTrue("The test never checked that location of " + "the block and hftp desitnation are the same"
, @checked);
}
/// <summary>Service a GET request as described below.</summary>
/// <remarks>
/// Service a GET request as described below.
/// Request:
/// <c>GET http://<nn>:<port>/listPaths[/<path>][<?option>[&option]*] HTTP/1.1
/// </c>
/// Where <i>option</i> (default) in:
/// recursive ("no")
/// filter (".*")
/// exclude ("\..*\.crc")
/// Response: A flat list of files/directories in the following format:
/// <c>
/// <listing path="..." recursive="(yes|no)" filter="..."
/// time="yyyy-MM-dd hh:mm:ss UTC" version="...">
/// <directory path="..." modified="yyyy-MM-dd hh:mm:ss"/>
/// <file path="..." modified="yyyy-MM-dd'T'hh:mm:ssZ" accesstime="yyyy-MM-dd'T'hh:mm:ssZ"
/// blocksize="..."
/// replication="..." size="..."/>
/// </listing>
/// </c>
/// </remarks>
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
PrintWriter @out = response.GetWriter();
XMLOutputter doc = new XMLOutputter(@out, "UTF-8");
IDictionary <string, string> root = BuildRoot(request, doc);
string path = root["path"];
string filePath = ServletUtil.GetDecodedPath(request, "/listPaths");
try
{
bool recur = "yes".Equals(root["recursive"]);
Sharpen.Pattern filter = Sharpen.Pattern.Compile(root["filter"]);
Sharpen.Pattern exclude = Sharpen.Pattern.Compile(root["exclude"]);
Configuration conf = (Configuration)GetServletContext().GetAttribute(JspHelper.CurrentConf
);
GetUGI(request, conf).DoAs(new _PrivilegedExceptionAction_149(this, doc, root, filePath
, path, exclude, filter, recur));
}
catch (IOException ioe)
{
WriteXml(ioe, path, doc);
}
catch (Exception e)
{
Log.Warn("ListPathServlet encountered InterruptedException", e);
response.SendError(400, e.Message);
}
finally
{
if (doc != null)
{
doc.EndDocument();
}
if (@out != null)
{
@out.Close();
}
}
}
/// <summary>Build a map from the query string, setting values and defaults.</summary>
protected internal virtual IDictionary <string, string> BuildRoot(HttpServletRequest
request, XMLOutputter doc)
{
string path = ServletUtil.GetDecodedPath(request, "/listPaths");
string exclude = request.GetParameter("exclude") != null?request.GetParameter("exclude"
) : string.Empty;
string filter = request.GetParameter("filter") != null?request.GetParameter("filter"
) : ".*";
bool recur = request.GetParameter("recursive") != null && "yes".Equals(request.GetParameter
("recursive"));
IDictionary <string, string> root = new Dictionary <string, string>();
root["path"] = path;
root["recursive"] = recur ? "yes" : "no";
root["filter"] = filter;
root["exclude"] = exclude;
root["time"] = df.Get().Format(new DateTime());
root["version"] = VersionInfo.GetVersion();
return(root);
}
/// <summary>Connect to the name node and get content summary.</summary>
/// <param name="path">The path</param>
/// <returns>The content summary for the path.</returns>
/// <exception cref="System.IO.IOException"/>
private ContentSummary GetContentSummary(string path)
{
HttpURLConnection connection = this._enclosing.OpenConnection("/contentSummary" +
ServletUtil.EncodePath(path), "ugi=" + this._enclosing.GetEncodedUgiParameter()
);
InputStream @in = null;
try
{
@in = connection.GetInputStream();
XMLReader xr = XMLReaderFactory.CreateXMLReader();
xr.SetContentHandler(this);
xr.Parse(new InputSource(@in));
}
catch (FileNotFoundException)
{
//the server may not support getContentSummary
return(null);
}
catch (SAXException saxe)
{
Exception embedded = saxe.GetException();
if (embedded != null && embedded is IOException)
{
throw (IOException)embedded;
}
throw new IOException("Invalid xml format", saxe);
}
finally
{
if (@in != null)
{
@in.Close();
}
connection.Disconnect();
}
return(this.contentsummary);
}
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
string path = ServletUtil.GetDecodedPath(request, "/streamFile");
string rawPath = ServletUtil.GetRawPath(request, "/streamFile");
string filename = JspHelper.ValidatePath(path);
string rawFilename = JspHelper.ValidatePath(rawPath);
if (filename == null)
{
response.SetContentType("text/plain");
PrintWriter @out = response.GetWriter();
@out.Write("Invalid input");
return;
}
Enumeration <string> reqRanges = request.GetHeaders("Range");
if (reqRanges != null && !reqRanges.MoveNext())
{
reqRanges = null;
}
DFSClient dfs;
try
{
dfs = GetDFSClient(request);
}
catch (Exception e)
{
response.SendError(400, e.Message);
return;
}
HdfsDataInputStream @in = null;
OutputStream out_1 = null;
try
{
@in = dfs.CreateWrappedInputStream(dfs.Open(filename));
out_1 = response.GetOutputStream();
long fileLen = @in.GetVisibleLength();
if (reqRanges != null)
{
IList <InclusiveByteRange> ranges = InclusiveByteRange.SatisfiableRanges(reqRanges
, fileLen);
StreamFile.SendPartialData(@in, out_1, response, fileLen, ranges);
}
else
{
// No ranges, so send entire file
response.SetHeader("Content-Disposition", "attachment; filename=\"" + rawFilename
+ "\"");
response.SetContentType("application/octet-stream");
response.SetHeader(ContentLength, string.Empty + fileLen);
StreamFile.CopyFromOffset(@in, out_1, 0L, fileLen);
}
@in.Close();
@in = null;
out_1.Close();
out_1 = null;
dfs.Close();
dfs = null;
}
catch (IOException ioe)
{
if (Log.IsDebugEnabled())
{
Log.Debug("response.isCommitted()=" + response.IsCommitted(), ioe);
}
throw;
}
finally
{
IOUtils.Cleanup(Log, @in);
IOUtils.Cleanup(Log, out_1);
IOUtils.Cleanup(Log, dfs);
}
}
/// <exception cref="System.IO.IOException"/>
private FileChecksum GetFileChecksum(string f)
{
HttpURLConnection connection = this._enclosing.OpenConnection("/fileChecksum" + ServletUtil
.EncodePath(f), "ugi=" + this._enclosing.GetEncodedUgiParameter());
try
{
XMLReader xr = XMLReaderFactory.CreateXMLReader();
xr.SetContentHandler(this);
xr.Parse(new InputSource(connection.GetInputStream()));
}
catch (SAXException e)
{
Exception embedded = e.GetException();
if (embedded != null && embedded is IOException)
{
throw (IOException)embedded;
}
throw new IOException("invalid xml directory content", e);
}
finally
{
connection.Disconnect();
}
return(this.filechecksum);
}
/// <exception cref="System.IO.IOException"/>
private void FetchList(string path, bool recur)
{
try
{
XMLReader xr = XMLReaderFactory.CreateXMLReader();
xr.SetContentHandler(this);
HttpURLConnection connection = this._enclosing.OpenConnection("/listPaths" + ServletUtil
.EncodePath(path), "ugi=" + this._enclosing.GetEncodedUgiParameter() + (recur ?
"&recursive=yes" : string.Empty));
InputStream resp = connection.GetInputStream();
xr.Parse(new InputSource(resp));
}
catch (SAXException e)
{
Exception embedded = e.GetException();
if (embedded != null && embedded is IOException)
{
throw (IOException)embedded;
}
throw new IOException("invalid xml directory content", e);
}
}
