public static HttpResult ReadFilePart(string file, long offset, long size)
{
try
{
if (string.IsNullOrWhiteSpace(file))
{
throw new ArgumentNullException("file");
}
if (null == GetStatus(file))
{
return new HttpResult()
{
Code = -1,
ErrorLog = "File Not Exist."
}
}
;
string args = string.Format("&offset={0}&length={1}", offset, size);
string url = BuildUrl(file, WebHdfsAPI.OPEN, "user.name=hadoop&noredirect=false" + args);
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.GET);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
}
return(HttpClientHelperV45.Request(url, new TimeSpan(0, 0, 6)));
}
catch (Exception ex)
{
DebugHelper.OutLog("获取HDFS文件失败：" + ex.Message);
DebugHelper.Error(ex, ex.Message);
return(null);
}
}
/// <summary>
/// 上传文件（自动生成上级目录）
/// </summary>
/// <param name="srcFile">本地文件路径</param>
/// <param name="hdfsPath">HDFS目标路径</param>
/// <param name="errlog">日志信息</param>
/// <returns></returns>
public static bool UploadFile(string srcFile, string hdfsPath, out string errlog)
{
try
{
// 确保目录存在
MkDir(hdfsPath);
// 自动拼接文件名
string fileName = new FileInfo(srcFile).Name;
if (!hdfsPath.Contains(fileName))
{
hdfsPath += "/" + fileName;
}
// Step 1. redirect to datanode
// CAUSE: avoid sending file data before the server redirection. and Jetty 6 server didn't implement "Expect: 100-continue" protocol.
string url = BuildUrl(hdfsPath, WebHdfsAPI.UPLOAD,
"user.name=hadoop&overwrite=true&noredirect=true");
/*string result = HttpClientHelperV45.PutResponse(url);
* if (!string.IsNullOrWhiteSpace(result) && result.Contains("Location"))
* {
* url = result.Substring(result.IndexOf("http:", StringComparison.OrdinalIgnoreCase));
* url = url.Remove(url.IndexOf("\""));
* }
**/
//Hadoop 2.7.1 服务端总是会重定向（307 TEMPORARY_REDIRECT），noredirect=true参数根本不管用
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.PUT);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
DebugHelper.OutLog("Redirect to " + url);
}
// Step 2. upload file to datanode
using (FileStream fs = new FileStream(srcFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
string response = HttpClientHelperV45.HttpUpload(url, fs, HttpMethod.PUT);
fs.Close();
}
errlog = "succeed";
return(true);
}
catch (System.Exception ex)
{
string err = ex.Message;
if (ex is WebException)
{
err = HttpClientHelperV45.ParseHdfsWebException(ex as WebException);
}
DebugHelper.OutLog("上传文件到HDFS失败：" + err);
DebugHelper.Error(ex, err);
errlog = err;
return(false);
}
}
/// <summary>
/// 获取文件或路径的详细信息
/// </summary>
/// <param name="file">文件或路径名</param>
/// <returns></returns>
public static HdfsFileInfo GetStatus(string file)
{
try
{
string url = BuildUrl(file, WebHdfsAPI.FILESTATUS);
string result = HttpClientHelperV45.Get(url);
return(HdfsFileInfo.FromJson(result));
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(null);
}
}
/// <summary>
/// 列举当前目录内容
/// </summary>
/// <param name="path">HDFS路径</param>
/// <returns></returns>
public static List <HdfsFileInfo> LsDir(string path)
{
try
{
string url = BuildUrl(path, WebHdfsAPI.LIST);
string result = HttpClientHelperV45.Get(url);
return(HdfsFileInfo.ParseJsonArray(result));
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(null);
}
}
/// <summary>
/// 删除路径
/// </summary>
/// <param name="path">文件或路径名</param>
/// <param name="recursively">是否删除所有下级子目录和文件</param>
/// <returns></returns>
public static bool RmDir(string path, bool recursively = true)
{
try
{
string url = BuildUrl(path, WebHdfsAPI.RM, "recursive=true");
HttpClientHelperV45.Delete(url);
return(true);
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(false);
}
}
/// <summary>
/// 创建目录（自动创建上级目录）
/// </summary>
/// <param name="path">HDFS路径</param>
/// <returns></returns>
public static bool MkDir(string path)
{
try
{
string url = BuildUrl(path, WebHdfsAPI.MKDIR, "user.name=hadoop");
HttpClientHelperV45.Put(url);
return(true);
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(false);
}
}
public static bool CombineFiles(string destHdfsFile, params string[] srcHdfsFiles)
{
try
{
string srcs = string.Join(",", srcHdfsFiles);
string url = BuildUrl(destHdfsFile, WebHdfsAPI.CONCAT, "user.name=hadoop&sources=" + srcs);
string statcode = "";
HttpClientHelperV45.Post(url, "", out statcode);
return("OK".Equals(statcode.ToUpper()));
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(false);
}
}
/// <summary>
/// 下载HDFS文件
/// </summary>
/// <param name="file">文件名</param>
/// <returns></returns>
public static byte[] GetFile(string file)
{
try
{
if (string.IsNullOrWhiteSpace(file))
{
throw new ArgumentNullException("file");
}
if (null == GetStatus(file))
{
return(Encoding.UTF8.GetBytes("File Not Exist."));
}
string url = BuildUrl(file, WebHdfsAPI.OPEN, "user.name=atlas&noredirect=false");
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.GET);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
}
HttpResult res = HttpClientHelperV45.Request(url, new TimeSpan(1, 0, 0));
if (200 == res.Code)
{
return(res.Content);
}
else
{
throw new Exception(res.ErrorLog);
}
}
catch (Exception ex)
{
string err = ex.Message;
if (ex is WebException)
{
err = HttpClientHelperV45.ParseHdfsWebException(ex as WebException);
}
DebugHelper.OutLog("获取HDFS文件失败：" + err);
DebugHelper.Error(ex, err);
return(null);
}
}
public static bool UploadStream(Stream ins, long offset, long length, string hdfsDir, string fileName)
{
try
{
// 确保目录存在
MkDir(hdfsDir);
// 拼接文件名
string hdfsFile = hdfsDir + "/" + fileName;
if (hdfsDir.EndsWith("/"))
{
hdfsFile = hdfsDir + fileName;
}
// Step 1. redirect to datanode
string url = BuildUrl(hdfsFile, WebHdfsAPI.UPLOAD,
"user.name=hadoop&overwrite=true&noredirect=true");
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.PUT);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
DebugHelper.OutLog("Redirect to " + url);
}
// Step 2. upload file to datanode
string response = HttpClientHelperV45.HttpUpload(url, ins, offset, length, HttpMethod.PUT);
return(null != response);
}
catch (System.Exception ex)
{
string err = ex.Message;
if (ex is WebException)
{
err = HttpClientHelperV45.ParseHdfsWebException(ex as WebException);
}
DebugHelper.OutLog("上传文件到HDFS失败：" + err);
DebugHelper.Error(ex, err);
return(false);
}
}
public static Dictionary <string, string> GetDataNodes()
{
Dictionary <string, string> dict = new Dictionary <string, string>();
try
{
string url = "HTTP://" + WebHdfsHost + WebHdfsAPI.HDFS_LIVENODES;
string jsonstr = HttpClientHelperV45.Get(url);
if (string.IsNullOrWhiteSpace(jsonstr))
{
return(null);
}
var jObj = JsonConvert.DeserializeObject(jsonstr);
if (null == jObj)
{
throw new Exception("HDFS LiveNodes Json解析错误");
}
var liveNodes = (jObj as JObject)["beans"][0]["LiveNodes"] as JValue;
var jobjNodes = JsonConvert.DeserializeObject(liveNodes.ToString()) as JObject;
foreach (var jprop in jobjNodes.Properties())
{
string hname = jprop.Name.Split(':')[0];
string addr = (jprop.Value as JObject)["infoAddr"].ToString();
string ip = addr.Split(':')[0];
dict.Add(hname, ip);
}
return(dict);
}
catch (Exception ex)
{
DebugHelper.Error(ex);
return(null);
}
}
