public static HttpResult ReadFilePart(string file, long offset, long size)
{
try
{
if (string.IsNullOrWhiteSpace(file))
{
throw new ArgumentNullException("file");
}
if (null == GetStatus(file))
{
return new HttpResult()
{
Code = -1,
ErrorLog = "File Not Exist."
}
}
;
string args = string.Format("&offset={0}&length={1}", offset, size);
string url = BuildUrl(file, WebHdfsAPI.OPEN, "user.name=hadoop&noredirect=false" + args);
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.GET);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
}
return(HttpClientHelperV45.Request(url, new TimeSpan(0, 0, 6)));
}
catch (Exception ex)
{
DebugHelper.OutLog("获取HDFS文件失败：" + ex.Message);
DebugHelper.Error(ex, ex.Message);
return(null);
}
}
/// <summary>
/// 上传文件（自动生成上级目录）
/// </summary>
/// <param name="srcFile">本地文件路径</param>
/// <param name="hdfsPath">HDFS目标路径</param>
/// <param name="errlog">日志信息</param>
/// <returns></returns>
public static bool UploadFile(string srcFile, string hdfsPath, out string errlog)
{
try
{
// 确保目录存在
MkDir(hdfsPath);
// 自动拼接文件名
string fileName = new FileInfo(srcFile).Name;
if (!hdfsPath.Contains(fileName))
{
hdfsPath += "/" + fileName;
}
// Step 1. redirect to datanode
// CAUSE: avoid sending file data before the server redirection. and Jetty 6 server didn't implement "Expect: 100-continue" protocol.
string url = BuildUrl(hdfsPath, WebHdfsAPI.UPLOAD,
"user.name=hadoop&overwrite=true&noredirect=true");
/*string result = HttpClientHelperV45.PutResponse(url);
* if (!string.IsNullOrWhiteSpace(result) && result.Contains("Location"))
* {
* url = result.Substring(result.IndexOf("http:", StringComparison.OrdinalIgnoreCase));
* url = url.Remove(url.IndexOf("\""));
* }
**/
//Hadoop 2.7.1 服务端总是会重定向（307 TEMPORARY_REDIRECT），noredirect=true参数根本不管用
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.PUT);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
DebugHelper.OutLog("Redirect to " + url);
}
// Step 2. upload file to datanode
using (FileStream fs = new FileStream(srcFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
string response = HttpClientHelperV45.HttpUpload(url, fs, HttpMethod.PUT);
fs.Close();
}
errlog = "succeed";
return(true);
}
catch (System.Exception ex)
{
string err = ex.Message;
if (ex is WebException)
{
err = HttpClientHelperV45.ParseHdfsWebException(ex as WebException);
}
DebugHelper.OutLog("上传文件到HDFS失败：" + err);
DebugHelper.Error(ex, err);
errlog = err;
return(false);
}
}
/// <summary>
/// 下载HDFS文件
/// </summary>
/// <param name="file">文件名</param>
/// <returns></returns>
public static byte[] GetFile(string file)
{
try
{
if (string.IsNullOrWhiteSpace(file))
{
throw new ArgumentNullException("file");
}
if (null == GetStatus(file))
{
return(Encoding.UTF8.GetBytes("File Not Exist."));
}
string url = BuildUrl(file, WebHdfsAPI.OPEN, "user.name=atlas&noredirect=false");
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.GET);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
}
HttpResult res = HttpClientHelperV45.Request(url, new TimeSpan(1, 0, 0));
if (200 == res.Code)
{
return(res.Content);
}
else
{
throw new Exception(res.ErrorLog);
}
}
catch (Exception ex)
{
string err = ex.Message;
if (ex is WebException)
{
err = HttpClientHelperV45.ParseHdfsWebException(ex as WebException);
}
DebugHelper.OutLog("获取HDFS文件失败：" + err);
DebugHelper.Error(ex, err);
return(null);
}
}
public static bool UploadStream(Stream ins, long offset, long length, string hdfsDir, string fileName)
{
try
{
// 确保目录存在
MkDir(hdfsDir);
// 拼接文件名
string hdfsFile = hdfsDir + "/" + fileName;
if (hdfsDir.EndsWith("/"))
{
hdfsFile = hdfsDir + fileName;
}
// Step 1. redirect to datanode
string url = BuildUrl(hdfsFile, WebHdfsAPI.UPLOAD,
"user.name=hadoop&overwrite=true&noredirect=true");
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.PUT);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
DebugHelper.OutLog("Redirect to " + url);
}
// Step 2. upload file to datanode
string response = HttpClientHelperV45.HttpUpload(url, ins, offset, length, HttpMethod.PUT);
return(null != response);
}
catch (System.Exception ex)
{
string err = ex.Message;
if (ex is WebException)
{
err = HttpClientHelperV45.ParseHdfsWebException(ex as WebException);
}
DebugHelper.OutLog("上传文件到HDFS失败：" + err);
DebugHelper.Error(ex, err);
return(false);
}
}
