/// <summary>
/// 上传文件（自动生成上级目录）
/// </summary>
/// <param name="srcFile">本地文件路径</param>
/// <param name="hdfsPath">HDFS目标路径</param>
/// <param name="errlog">日志信息</param>
/// <returns></returns>
public static bool UploadFile(string srcFile, string hdfsPath, out string errlog)
{
try
{
// 确保目录存在
MkDir(hdfsPath);
// 自动拼接文件名
string fileName = new FileInfo(srcFile).Name;
if (!hdfsPath.Contains(fileName))
{
hdfsPath += "/" + fileName;
}
// Step 1. redirect to datanode
// CAUSE: avoid sending file data before the server redirection. and Jetty 6 server didn't implement "Expect: 100-continue" protocol.
string url = BuildUrl(hdfsPath, WebHdfsAPI.UPLOAD,
"user.name=hadoop&overwrite=true&noredirect=true");
/*string result = HttpClientHelperV45.PutResponse(url);
* if (!string.IsNullOrWhiteSpace(result) && result.Contains("Location"))
* {
* url = result.Substring(result.IndexOf("http:", StringComparison.OrdinalIgnoreCase));
* url = url.Remove(url.IndexOf("\""));
* }
**/
//Hadoop 2.7.1 服务端总是会重定向（307 TEMPORARY_REDIRECT），noredirect=true参数根本不管用
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.PUT);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
DebugHelper.OutLog("Redirect to " + url);
}
// Step 2. upload file to datanode
using (FileStream fs = new FileStream(srcFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
string response = HttpClientHelperV45.HttpUpload(url, fs, HttpMethod.PUT);
fs.Close();
}
errlog = "succeed";
return(true);
}
catch (System.Exception ex)
{
string err = ex.Message;
if (ex is WebException)
{
err = HttpClientHelperV45.ParseHdfsWebException(ex as WebException);
}
DebugHelper.OutLog("上传文件到HDFS失败：" + err);
DebugHelper.Error(ex, err);
errlog = err;
return(false);
}
}
public static bool UploadStream(Stream ins, long offset, long length, string hdfsDir, string fileName)
{
try
{
// 确保目录存在
MkDir(hdfsDir);
// 拼接文件名
string hdfsFile = hdfsDir + "/" + fileName;
if (hdfsDir.EndsWith("/"))
{
hdfsFile = hdfsDir + fileName;
}
// Step 1. redirect to datanode
string url = BuildUrl(hdfsFile, WebHdfsAPI.UPLOAD,
"user.name=hadoop&overwrite=true&noredirect=true");
string redirUrl = HttpClientHelperV45.GetRedirectedURL(url, HttpMethod.PUT);
if (!string.IsNullOrWhiteSpace(redirUrl))
{
url = TranslateHostname(redirUrl);
DebugHelper.OutLog("Redirect to " + url);
}
// Step 2. upload file to datanode
string response = HttpClientHelperV45.HttpUpload(url, ins, offset, length, HttpMethod.PUT);
return(null != response);
}
catch (System.Exception ex)
{
string err = ex.Message;
if (ex is WebException)
{
err = HttpClientHelperV45.ParseHdfsWebException(ex as WebException);
}
DebugHelper.OutLog("上传文件到HDFS失败：" + err);
DebugHelper.Error(ex, err);
return(false);
}
}
