/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
PrintWriter @out = response.GetWriter();
string path = ServletUtil.GetDecodedPath(request, "/getFileChecksum");
XMLOutputter xml = new XMLOutputter(@out, "UTF-8");
xml.Declaration();
ServletContext context = GetServletContext();
DataNode datanode = (DataNode)context.GetAttribute("datanode");
Configuration conf = new HdfsConfiguration(datanode.GetConf());
try
{
DFSClient dfs = DatanodeJspHelper.GetDFSClient(request, datanode, conf, GetUGI(request
, conf));
MD5MD5CRC32FileChecksum checksum = dfs.GetFileChecksum(path, long.MaxValue);
MD5MD5CRC32FileChecksum.Write(xml, checksum);
}
catch (IOException ioe)
{
WriteXml(ioe, path, xml);
}
catch (Exception e)
{
WriteXml(e, path, xml);
}
xml.EndDocument();
}
/// <exception cref="System.Exception"/>
public Void Run()
{
string path = ServletUtil.GetDecodedPath(request, "/contentSummary");
PrintWriter @out = response.GetWriter();
XMLOutputter xml = new XMLOutputter(@out, "UTF-8");
xml.Declaration();
try
{
ClientProtocol nnproxy = this._enclosing.CreateNameNodeProxy();
ContentSummary cs = nnproxy.GetContentSummary(path);
xml.StartTag(typeof(ContentSummary).FullName);
if (cs != null)
{
xml.Attribute("length", string.Empty + cs.GetLength());
xml.Attribute("fileCount", string.Empty + cs.GetFileCount());
xml.Attribute("directoryCount", string.Empty + cs.GetDirectoryCount());
xml.Attribute("quota", string.Empty + cs.GetQuota());
xml.Attribute("spaceConsumed", string.Empty + cs.GetSpaceConsumed());
xml.Attribute("spaceQuota", string.Empty + cs.GetSpaceQuota());
}
xml.EndTag();
}
catch (IOException ioe)
{
this._enclosing.WriteXml(ioe, path, xml);
}
xml.EndDocument();
return(null);
}
/// <summary>Generate a XML block as such, <item label=key value=value/></summary>
/// <exception cref="System.IO.IOException"/>
private static void ToXmlItemBlock(XMLOutputter doc, string key, string value)
{
doc.StartTag("item");
doc.Attribute("label", key);
doc.Attribute("value", value);
doc.EndTag();
}
/// <summary>
/// Generate a XML block as such, <item label="Node" value="hostname"
/// link="http://hostname:50070" />
/// </summary>
/// <exception cref="System.IO.IOException"/>
private static void ToXmlItemBlockWithLink(XMLOutputter doc, string value, Uri url
, string label)
{
doc.StartTag("item");
doc.Attribute("label", label);
doc.Attribute("value", value);
doc.Attribute("link", url.ToString());
doc.EndTag();
}
/// <summary>Write that object to xml output.</summary>
/// <exception cref="System.IO.IOException"/>
public static void Write(XMLOutputter xml, Org.Apache.Hadoop.FS.MD5MD5CRC32FileChecksum
that)
{
xml.StartTag(typeof(Org.Apache.Hadoop.FS.MD5MD5CRC32FileChecksum).FullName);
if (that != null)
{
xml.Attribute("bytesPerCRC", string.Empty + that.bytesPerCRC);
xml.Attribute("crcPerBlock", string.Empty + that.crcPerBlock);
xml.Attribute("crcType", string.Empty + that.GetCrcType().ToString());
xml.Attribute("md5", string.Empty + that.md5);
}
xml.EndTag();
}
public _PrivilegedExceptionAction_149(ListPathsServlet _enclosing, XMLOutputter doc
, IDictionary <string, string> root, string filePath, string path, Sharpen.Pattern
exclude, Sharpen.Pattern filter, bool recur)
{
this._enclosing = _enclosing;
this.doc = doc;
this.root = root;
this.filePath = filePath;
this.path = path;
this.exclude = exclude;
this.filter = filter;
this.recur = recur;
}
// unreportedNamnodes
/// <summary>create XML block from general exception.</summary>
/// <exception cref="System.IO.IOException"/>
private static void CreateGeneralException(XMLOutputter doc, string clusterid, string
eMsg)
{
doc.StartTag("cluster");
doc.Attribute("clusterId", clusterid);
doc.StartTag("message");
doc.StartTag("item");
doc.Attribute("msg", eMsg);
doc.EndTag();
// item
doc.EndTag();
// message
doc.EndTag();
}
// item
/// <summary>
/// create the XML for exceptions that we encountered when connecting to
/// namenode.
/// </summary>
/// <exception cref="System.IO.IOException"/>
private static void CreateNamenodeExceptionMsg(XMLOutputter doc, IDictionary <string
, Exception> exceptionMsg)
{
if (exceptionMsg.Count > 0)
{
doc.StartTag("unreportedNamenodes");
foreach (KeyValuePair <string, Exception> m in exceptionMsg)
{
doc.StartTag("node");
doc.Attribute("name", m.Key);
doc.Attribute("exception", StringUtils.StringifyException(m.Value));
doc.EndTag();
}
// node
doc.EndTag();
}
}
/// <exception cref="Javax.Servlet.ServletException"/>
public virtual void OutputPretty(PrintWriter @out, Annotation annotation)
{
try
{
Document input = XMLOutputter.AnnotationToDoc(annotation, pipeline);
Nodes output = corenlpTransformer.Transform(input);
for (int i = 0; i < output.Size(); i++)
{
@out.Print(output.Get(i).ToXML());
}
}
catch (Exception e)
{
throw;
}
catch (Exception e)
{
throw new ServletException(e);
}
}
/// <summary>Write a node to output.</summary>
/// <remarks>
/// Write a node to output.
/// Node information includes path, modification, permission, owner and group.
/// For files, it also includes size, replication and block-size.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal static void WriteInfo(Path fullpath, HdfsFileStatus i, XMLOutputter doc)
{
SimpleDateFormat ldf = df.Get();
doc.StartTag(i.IsDir() ? "directory" : "file");
doc.Attribute("path", fullpath.ToUri().GetPath());
doc.Attribute("modified", ldf.Format(Sharpen.Extensions.CreateDate(i.GetModificationTime
())));
doc.Attribute("accesstime", ldf.Format(Sharpen.Extensions.CreateDate(i.GetAccessTime
())));
if (!i.IsDir())
{
doc.Attribute("size", i.GetLen().ToString());
doc.Attribute("replication", i.GetReplication().ToString());
doc.Attribute("blocksize", i.GetBlockSize().ToString());
}
doc.Attribute("permission", (i.IsDir() ? "d" : "-") + i.GetPermission());
doc.Attribute("owner", i.GetOwner());
doc.Attribute("group", i.GetGroup());
doc.EndTag();
}
/// <summary>Service a GET request as described below.</summary>
/// <remarks>
/// Service a GET request as described below.
/// Request:
/// <c>GET http://<nn>:<port>/listPaths[/<path>][<?option>[&option]*] HTTP/1.1
/// </c>
/// Where <i>option</i> (default) in:
/// recursive ("no")
/// filter (".*")
/// exclude ("\..*\.crc")
/// Response: A flat list of files/directories in the following format:
/// <c>
/// <listing path="..." recursive="(yes|no)" filter="..."
/// time="yyyy-MM-dd hh:mm:ss UTC" version="...">
/// <directory path="..." modified="yyyy-MM-dd hh:mm:ss"/>
/// <file path="..." modified="yyyy-MM-dd'T'hh:mm:ssZ" accesstime="yyyy-MM-dd'T'hh:mm:ssZ"
/// blocksize="..."
/// replication="..." size="..."/>
/// </listing>
/// </c>
/// </remarks>
/// <exception cref="Javax.Servlet.ServletException"/>
/// <exception cref="System.IO.IOException"/>
protected override void DoGet(HttpServletRequest request, HttpServletResponse response
)
{
PrintWriter @out = response.GetWriter();
XMLOutputter doc = new XMLOutputter(@out, "UTF-8");
IDictionary <string, string> root = BuildRoot(request, doc);
string path = root["path"];
string filePath = ServletUtil.GetDecodedPath(request, "/listPaths");
try
{
bool recur = "yes".Equals(root["recursive"]);
Sharpen.Pattern filter = Sharpen.Pattern.Compile(root["filter"]);
Sharpen.Pattern exclude = Sharpen.Pattern.Compile(root["exclude"]);
Configuration conf = (Configuration)GetServletContext().GetAttribute(JspHelper.CurrentConf
);
GetUGI(request, conf).DoAs(new _PrivilegedExceptionAction_149(this, doc, root, filePath
, path, exclude, filter, recur));
}
catch (IOException ioe)
{
WriteXml(ioe, path, doc);
}
catch (Exception e)
{
Log.Warn("ListPathServlet encountered InterruptedException", e);
response.SendError(400, e.Message);
}
finally
{
if (doc != null)
{
doc.EndDocument();
}
if (@out != null)
{
@out.Close();
}
}
}
/// <summary>Build a map from the query string, setting values and defaults.</summary>
protected internal virtual IDictionary <string, string> BuildRoot(HttpServletRequest
request, XMLOutputter doc)
{
string path = ServletUtil.GetDecodedPath(request, "/listPaths");
string exclude = request.GetParameter("exclude") != null?request.GetParameter("exclude"
) : string.Empty;
string filter = request.GetParameter("filter") != null?request.GetParameter("filter"
) : ".*";
bool recur = request.GetParameter("recursive") != null && "yes".Equals(request.GetParameter
("recursive"));
IDictionary <string, string> root = new Dictionary <string, string>();
root["path"] = path;
root["recursive"] = recur ? "yes" : "no";
root["filter"] = filter;
root["exclude"] = exclude;
root["time"] = df.Get().Format(new DateTime());
root["version"] = VersionInfo.GetVersion();
return(root);
}
/// <summary>Write the object to XML format</summary>
/// <exception cref="System.IO.IOException"/>
protected internal virtual void WriteXml(Exception except, string path, XMLOutputter
doc)
{
doc.StartTag(typeof(RemoteException).Name);
doc.Attribute("path", path);
if (except is RemoteException)
{
doc.Attribute("class", ((RemoteException)except).GetClassName());
}
else
{
doc.Attribute("class", except.GetType().FullName);
}
string msg = except.GetLocalizedMessage();
int i = msg.IndexOf("\n");
if (i >= 0)
{
msg = Sharpen.Runtime.Substring(msg, 0, i);
}
doc.Attribute("message", Sharpen.Runtime.Substring(msg, msg.IndexOf(":") + 1).Trim
());
doc.EndTag();
}
/// <summary>Generate decommissioning datanode report in XML format</summary>
/// <param name="doc">, xmloutputter</param>
/// <exception cref="System.IO.IOException"/>
public virtual void ToXML(XMLOutputter doc)
{
if (error != null)
{
CreateGeneralException(doc, clusterid, StringUtils.StringifyException(error));
doc.GetWriter().Flush();
return;
}
if (statusMap == null || statusMap.IsEmpty())
{
// none of the namenodes has reported, print exceptions from each nn.
doc.StartTag("cluster");
CreateNamenodeExceptionMsg(doc, exceptions);
doc.EndTag();
doc.GetWriter().Flush();
return;
}
doc.StartTag("cluster");
doc.Attribute("clusterId", clusterid);
doc.StartTag("decommissioningReport");
CountDecommissionDatanodes();
ToXmlItemBlock(doc, ClusterJspHelper.DecommissionStates.Decommissioned.ToString()
, Sharpen.Extensions.ToString(decommissioned));
ToXmlItemBlock(doc, ClusterJspHelper.DecommissionStates.DecommissionInprogress.ToString
(), Sharpen.Extensions.ToString(decommissioning));
ToXmlItemBlock(doc, ClusterJspHelper.DecommissionStates.PartiallyDecommissioned.ToString
(), Sharpen.Extensions.ToString(partial));
doc.EndTag();
// decommissioningReport
doc.StartTag("datanodes");
ICollection <string> dnSet = statusMap.Keys;
foreach (string dnhost in dnSet)
{
IDictionary <string, string> nnStatus = statusMap[dnhost];
if (nnStatus == null || nnStatus.IsEmpty())
{
continue;
}
string overallStatus = nnStatus[OverallStatus];
// check if datanode is in decommission states
if (overallStatus != null && (overallStatus.Equals(DatanodeInfo.AdminStates.DecommissionInprogress
.ToString()) || overallStatus.Equals(DatanodeInfo.AdminStates.Decommissioned.ToString
()) || overallStatus.Equals(ClusterJspHelper.DecommissionStates.PartiallyDecommissioned
.ToString()) || overallStatus.Equals(ClusterJspHelper.DecommissionStates.Unknown
.ToString())))
{
doc.StartTag("node");
// dn
ToXmlItemBlockWithLink(doc, dnhost, new Uri("http", dnhost, httpPort, string.Empty
), "DataNode");
// overall status first
ToXmlItemBlock(doc, OverallStatus, overallStatus);
foreach (KeyValuePair <string, string> m in nnStatus)
{
string nn = m.Key;
if (nn.Equals(OverallStatus))
{
continue;
}
// xml
ToXmlItemBlock(doc, nn, nnStatus[nn]);
}
doc.EndTag();
}
}
// node
doc.EndTag();
// datanodes
CreateNamenodeExceptionMsg(doc, exceptions);
doc.EndTag();
}
/// <exception cref="System.IO.IOException"/>
public virtual void ToXML(XMLOutputter doc)
{
if (error != null)
{
// general exception, only print exception message onto web page.
CreateGeneralException(doc, clusterid, StringUtils.StringifyException(error));
doc.GetWriter().Flush();
return;
}
int size = nnList.Count;
long total = 0L;
long free = 0L;
long nonDfsUsed = 0l;
float dfsUsedPercent = 0.0f;
float dfsRemainingPercent = 0.0f;
if (size > 0)
{
total = total_sum / size;
free = free_sum / size;
nonDfsUsed = nonDfsUsed_sum / size;
dfsUsedPercent = DFSUtil.GetPercentUsed(clusterDfsUsed, total);
dfsRemainingPercent = DFSUtil.GetPercentRemaining(free, total);
}
doc.StartTag("cluster");
doc.Attribute("clusterId", clusterid);
doc.StartTag("storage");
ToXmlItemBlock(doc, "Total Files And Directories", System.Convert.ToString(totalFilesAndDirectories
));
ToXmlItemBlock(doc, "Configured Capacity", StringUtils.ByteDesc(total));
ToXmlItemBlock(doc, "DFS Used", StringUtils.ByteDesc(clusterDfsUsed));
ToXmlItemBlock(doc, "Non DFS Used", StringUtils.ByteDesc(nonDfsUsed));
ToXmlItemBlock(doc, "DFS Remaining", StringUtils.ByteDesc(free));
// dfsUsedPercent
ToXmlItemBlock(doc, "DFS Used%", DFSUtil.Percent2String(dfsUsedPercent));
// dfsRemainingPercent
ToXmlItemBlock(doc, "DFS Remaining%", DFSUtil.Percent2String(dfsRemainingPercent)
);
doc.EndTag();
// storage
doc.StartTag("namenodes");
// number of namenodes
ToXmlItemBlock(doc, "NamenodesCount", Sharpen.Extensions.ToString(size));
foreach (ClusterJspHelper.NamenodeStatus nn in nnList)
{
doc.StartTag("node");
ToXmlItemBlockWithLink(doc, nn.host, nn.httpAddress, "NameNode");
ToXmlItemBlock(doc, "Blockpool Used", StringUtils.ByteDesc(nn.bpUsed));
ToXmlItemBlock(doc, "Blockpool Used%", DFSUtil.Percent2String(DFSUtil.GetPercentUsed
(nn.bpUsed, total)));
ToXmlItemBlock(doc, "Files And Directories", System.Convert.ToString(nn.filesAndDirectories
));
ToXmlItemBlock(doc, "Blocks", System.Convert.ToString(nn.blocksCount));
ToXmlItemBlock(doc, "Missing Blocks", System.Convert.ToString(nn.missingBlocksCount
));
ToXmlItemBlockWithLink(doc, nn.liveDatanodeCount + " (" + nn.liveDecomCount + ")"
, new Uri(nn.httpAddress, "/dfsnodelist.jsp?whatNodes=LIVE"), "Live Datanode (Decommissioned)"
);
ToXmlItemBlockWithLink(doc, nn.deadDatanodeCount + " (" + nn.deadDecomCount + ")"
, new Uri(nn.httpAddress, "/dfsnodelist.jsp?whatNodes=DEAD"), "Dead Datanode (Decommissioned)"
);
ToXmlItemBlock(doc, "Software Version", nn.softwareVersion);
doc.EndTag();
}
// node
doc.EndTag();
// namenodes
CreateNamenodeExceptionMsg(doc, nnExceptions);
doc.EndTag();
// cluster
doc.GetWriter().Flush();
}
