//重命名文件或文件夹
public bool ReName(string oldPath, string newPath)
{
TBufferedTransport tsport = null;
ThriftHadoopFileSystem.Client client = Connect(out tsport);
bool result = false;
if (client != null)
{
Pathname pn = new Pathname()
{
pathname = oldPath
};
if (client.exists(pn))//如果存在才执行
{
result = client.rename(pn, new Pathname()
{
pathname = newPath
});
}
tsport.Close();
}
return(result);
}
//剪切
public List <string> MoveFile(string[] sourcePath, string dectPath)
{
TBufferedTransport tsport = null;
ThriftHadoopFileSystem.Client client = Connect(out tsport);
List <string> result = new List <string>();
if (client != null)
{
foreach (string itemSource in sourcePath)
{
Pathname pn = new Pathname()
{
pathname = itemSource
};
string fileName = itemSource.Substring(itemSource.LastIndexOf('/') + 1);
if (client.exists(pn))//如果存在才执行
{
bool thResult = client.rename(pn, new Pathname()
{
pathname = dectPath + "/" + fileName
});
if (!thResult)
{
result.Add(fileName);
}
}
}
tsport.Close();
}
return(result);
}
//下载
public bool Open(ThriftHadoopFileSystem.Client client, string path, string savePath, long fileLength)
{
bool result = false;
if (client != null)
{
ThriftHandle th = client.open(new Pathname()
{
pathname = path
});
// 创建文件流
FileStream fs = new FileStream(savePath, FileMode.Create, FileAccess.Write);
long totalBytes = 0;
int readLength = 1024 * 1024;
try
{
UTF8Encoding utf8 = new UTF8Encoding(false, true);
while (true)
{
int needRead = readLength;
if (fileLength - totalBytes < readLength)
{
needRead = (int)(fileLength - totalBytes);
}
if (needRead <= 0)
{
break;
}
byte[] fileBuffer = client.read(th, totalBytes, readLength);
byte[] myfileBuffer = Encoding.Convert(utf8, Encoding.GetEncoding("iso-8859-1"), fileBuffer);
totalBytes += readLength;
fs.Write(myfileBuffer, 0, myfileBuffer.Length);
}
result = true;
}
catch (Exception ee)
{
throw ee;
}
finally
{
fs.Dispose();
if (client != null)
{
client.close(th);
}
}
}
return(result);
}
public static void test()
{
var hadoop_socket = new TSocket("hserver", 59256);
hadoop_socket.Timeout = 10000;// Ten seconds
//$hadoop_socket -> setRecvTimeout(20000); // Twenty seconds
var hadoop_transport = new TBufferedTransport(hadoop_socket);
var hadoop_protocol = new TBinaryProtocol(hadoop_transport);
var hadoopClient = new ThriftHadoopFileSystem.Client(hadoop_protocol);
hadoop_transport.Open();
//if (hadoopClient.exists(new Pathname { pathname = "/user/root/input" }))
//{
// System.Console.WriteLine("exists");
//}
//else
//{
// Console.WriteLine("no exists");
//}
//hadoopClient.mkdirs(new Pathname { pathname = "/zbw911_user" });
//hadoopClient.rm(new Pathname { pathname = "/zbw911_user" }, true);
//var h = hadoopClient.createFile(new Pathname { pathname = "user_zbw911" }, 1, true, 1000, 1, 64000);
var list = hadoopClient.listStatus(new Pathname {
pathname = "/user/root"
});
foreach (var item in list)
{
System.Console.WriteLine(item);
//if (item.Isdir)
//{
// System.Console.WriteLine("文件夹：" + item.Path + "；权限：" + item.Permission);
//}
//else
//{
// System.Console.WriteLine("文件：" + item.Path + "；权限：" + item.Permission);
//}
}
//hadoopClient.close(h);
//hadoopClient.open(new Pathname { pathname = "/user/root" });
hadoop_transport.Close();
}
//上传文件
public bool Create(ThriftHadoopFileSystem.Client client, string localPath, string path)
{
bool result = false;
if (client != null)
{
ThriftHandle th = null;
FileStream fs = null;
try
{
//创建一个文件
th = client.createFile(new Pathname()
{
pathname = path
}, 1, true, 1024, ConfigHelper.HDFSREPLICATION, 1024 * 1024 * 64);
UTF8Encoding utf8 = new UTF8Encoding(false, true);
fs = new FileStream(localPath, FileMode.Open, FileAccess.Read);
byte[] fileBuffer = new byte[1024 * 1024]; // 每次传1MB
int bytesRead;
while ((bytesRead = fs.Read(fileBuffer, 0, fileBuffer.Length)) > 0)
{
byte[] realBuffer = new byte[bytesRead];
Array.Copy(fileBuffer, realBuffer, bytesRead);
//将utf8转为可存储编码
byte[] buf = Encoding.Convert(Encoding.GetEncoding("iso-8859-1"), utf8, realBuffer);
//发送
client.write(th, buf);
//清仓缓存
Array.Clear(fileBuffer, 0, fileBuffer.Length);
}
result = true;
}
catch (Exception ee)
{
throw ee;
}
finally
{
if (th != null)
{
client.close(th);
}
if (fs != null)
{
fs.Close();
}
}
}
return(result);
}
//文件状态
public FileStatus GetFileStatus(string path)
{
TBufferedTransport tsport = null;
ThriftHadoopFileSystem.Client client = Connect(out tsport);
FileStatus result = null; //new List<FileStatus>();
//客户端可连接 且目录存在
if (client != null && client.exists(new Pathname {
pathname = path
}))
{
result = client.stat(new Pathname()
{
pathname = path
});
tsport.Close();
}
return(result);
}
/// <summary>
/// 删除文件或文件夹
/// </summary>
/// <param name="path"></param>
/// <param name="recursive">是否删除子文件夹</param>
/// <returns></returns>
public bool Delete(string path, bool recursive)
{
TBufferedTransport tsport = null;
ThriftHadoopFileSystem.Client client = Connect(out tsport);
bool result = false;
if (client != null)
{
Pathname pn = new Pathname()
{
pathname = path
};
if (client.exists(pn))//如果不存在才执行
{
result = client.rm(pn, recursive);
}
tsport.Close();
}
return(result);
}
//新建文件夹
public bool MakeDir(string path)
{
TBufferedTransport tsport = null;
ThriftHadoopFileSystem.Client client = Connect(out tsport);
bool result = false;
if (client != null)
{
Pathname pn = new Pathname()
{
pathname = path
};
if (!client.exists(pn))//如果不存在才执行
{
result = client.mkdirs(pn);
}
tsport.Close();
}
return(result);
}
//获取连接
public ThriftHadoopFileSystem.Client Connect(out TBufferedTransport tsport)
{
TSocket hadoop_socket = new TSocket(HostIP, HostPort);
//hadoop_socket.Timeout = 10000;// Ten seconds
tsport = new TBufferedTransport(hadoop_socket);
TBinaryProtocol hadoop_protocol = new TBinaryProtocol(tsport, false, false);
ThriftHadoopFileSystem.Client client = new ThriftHadoopFileSystem.Client(hadoop_protocol);
try
{
tsport.Open();
return(client);
}
catch (Exception ee)
{
logger.Error("打开连接失败！", ee);
tsport = null;
return(null);
}
}
//获取连接
public ThriftHadoopFileSystem.Client Connect(out TBufferedTransport tsport)
{
TSocket hadoop_socket = new TSocket(HostIP, HostPort);
//hadoop_socket.Timeout = 10000;// Ten seconds
tsport = new TBufferedTransport(hadoop_socket);
TBinaryProtocol hadoop_protocol = new TBinaryProtocol(tsport, false, false);
ThriftHadoopFileSystem.Client client = new ThriftHadoopFileSystem.Client(hadoop_protocol);
try
{
tsport.Open();
return client;
}
catch (Exception ee)
{
logger.Error("打开连接失败！", ee);
tsport = null;
return null;
}
}
//批量上传
public void MutUpload(BackgroundWorker worker, List <string> localPath, string remoteRootPath, string localRootPath)
{
//相同操作
bool sameOp = false;
//是否覆盖
bool IsOver = false;
//准备创建连接
Thrift.Transport.TBufferedTransport btsport = null;
ThriftHadoopFileSystem.Client thriftClient = Connect(out btsport);
List <string> NoSuccessList = new List <string>();
List <string> skipList = new List <string>();
if (thriftClient != null)//连接成功
{
int totalCount = localPath.Count;
int currentCount = 0;
//开始上传
worker.ReportProgress(0, new ProgressState()
{
ListBoxMsg = "开始上传！", totalCount = totalCount, CurrentCount = 0
});
//循环
foreach (string localFilePath in localPath)
{
string fileName = Path.GetFileName(localFilePath);
if (!string.IsNullOrEmpty(localRootPath))//如果是文件夹，则包含原有路径
{
fileName = localFilePath.Replace(localRootPath, "");
}
string remoteFilePath = remoteRootPath + "/" + fileName;
currentCount++;
int pgPresent = (int)((double)currentCount / totalCount * 100);
//显示总进度
worker.ReportProgress(pgPresent, new ProgressState()
{
CurrentTitle = fileName, CurrentCount = currentCount
});
#region 是否存在
if (!sameOp)
{
bool exsitFile = thriftClient.exists(new Pathname()
{
pathname = remoteFilePath
});
if (exsitFile)
{
SureDialog myDialog = new SureDialog();
MyShowDialogResult myDR = new MyShowDialogResult();
myDialog.ShowDialog(fileName, myDR);
sameOp = myDR.IsCheck;
IsOver = myDR.Result;
if (!myDR.Result)
{
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = fileName + " 跳过 "
});
skipList.Add(fileName);
continue;
}
}
}
else
{
bool exsitFile = thriftClient.exists(new Pathname()
{
pathname = remoteFilePath
});
if (exsitFile)
{
if (!IsOver)
{
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = fileName + "跳过 "
});
skipList.Add(fileName);
continue;
}
}
}
#endregion
#region 单个文件
bool singleResult = false;
ThriftHandle th = null;
FileStream fs = null;
try
{
Pathname myNewFile = new Pathname()
{
pathname = remoteFilePath
};
//创建一个文件
th = thriftClient.createFile(myNewFile, 1, true, 1024 * 1024 * 10, ConfigHelper.HDFSREPLICATION, 1024 * 1024 * 512);
UTF8Encoding utf8 = new UTF8Encoding(false, true);
fs = new FileStream(localFilePath, FileMode.Open, FileAccess.Read);
byte[] fileBuffer = new byte[1024 * 1024 * 10]; // 每次传1MB
int bytesRead;
long bytesTotal = 0;
while ((bytesRead = fs.Read(fileBuffer, 0, fileBuffer.Length)) > 0)
{
bytesTotal += bytesRead;
byte[] realBuffer = new byte[bytesRead];
Array.Copy(fileBuffer, realBuffer, bytesRead);
//将utf8转为可存储编码
realBuffer = Encoding.Convert(Encoding.GetEncoding("iso-8859-1"), utf8, realBuffer);
//发送
thriftClient.write(th, realBuffer);
//清仓缓存
Array.Clear(fileBuffer, 0, fileBuffer.Length);
realBuffer = null;
//显示单个上传进度
int mypresent = (int)((double)bytesTotal / fs.Length * 100);
worker.ReportProgress(pgPresent, new ProgressState()
{
CurrentTitle = mypresent + "% " + fileName
});
}
singleResult = true;
}
catch (Exception ee)
{
//显示上传错误
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = ee.Message
});
}
finally
{
if (th != null)
{
thriftClient.close(th);
}
if (fs != null)
{
fs.Close();
}
}
#endregion
//显示单个上传结果
string msg = string.Format("{0} 上传{1}", fileName, singleResult ? "成功" : "失败");
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = msg
});
if (!singleResult)
{
NoSuccessList.Add(fileName);
}
}
}
//释放连接
if (btsport != null)
{
btsport.Close();
}
//输出没有上传成功的
if (NoSuccessList.Count > 0)
{
try
{
File.WriteAllText("c:/UploadNoSuccess.txt", string.Join("\r\n", NoSuccessList.ToArray()));
worker.ReportProgress(100, new ProgressState()
{
ListBoxMsg = NoSuccessList.Count + "个上传错误！请查看c:/UploadNoSuccess.txt"
});
}
catch (Exception ee)
{
worker.ReportProgress(100, new ProgressState()
{
ListBoxMsg = ee.Message
});
}
}
//输出跳过的
if (skipList.Count > 0)
{
try
{
File.WriteAllText("c:/UploadSkip.txt", string.Join("\r\n", skipList.ToArray()));
worker.ReportProgress(100, new ProgressState()
{
ListBoxMsg = "跳过" + skipList.Count + "个文件！请查看c:/UploadSkip.txt"
});
}
catch (Exception ee)
{
worker.ReportProgress(100, new ProgressState()
{
ListBoxMsg = ee.Message
});
}
}
}
//批量下载
public void MutDownLoad(BackgroundWorker worker, string localRootPath, List <FileStatus> fileList, int fileType)
{
//相同操作
bool sameOp = false;
//是否覆盖
bool IsOver = false;
//准备创建连接
Thrift.Transport.TBufferedTransport btsport = null;
ThriftHadoopFileSystem.Client thriftClient = Connect(out btsport);
if (thriftClient != null)//连接成功
{
int totalCount = fileList.Count;
int currentCount = 0;
//开始上传
worker.ReportProgress(0, new ProgressState()
{
ListBoxMsg = "开始下载！", totalCount = totalCount, CurrentCount = 0
});
//循环
foreach (FileStatus myfile in fileList)
{
currentCount++;
int pgPresent = (int)((double)currentCount / totalCount * 100);
string fileName = Path.GetFileName(myfile.Path);
if (myfile.Isdir == false)
{
string savePath = localRootPath + "/" + fileName;
if (fileType == 1 && myfile.FileName != null)
{
savePath = localRootPath + "/" + myfile.FileName;
}
if (fileType == 2 && myfile.FileName != null)
{
savePath = localRootPath + "/" + myfile.FileName + "/" + fileName;
}
//显示总进度
worker.ReportProgress(pgPresent, new ProgressState()
{
CurrentTitle = fileName, CurrentCount = currentCount
});
#region 是否存在
if (!sameOp)
{
bool exsitFile = File.Exists(savePath);
if (exsitFile)
{
SureDialog myDialog = new SureDialog();
MyShowDialogResult myDR = new MyShowDialogResult();
myDialog.ShowDialog(fileName, myDR);
sameOp = myDR.IsCheck;
IsOver = myDR.Result;
if (!myDR.Result)
{
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = fileName + " 跳过 "
});
continue;
}
}
}
else
{
bool exsitFile = File.Exists(savePath);
if (exsitFile)
{
if (!IsOver)
{
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = fileName + "跳过 "
});
continue;
}
}
}
#endregion
#region 单个下载
bool result = false;
ThriftHandle th = thriftClient.open(new Pathname()
{
pathname = myfile.Path
});
// 创建文件流
FileStream fs = new FileStream(savePath, FileMode.Create, FileAccess.Write);
long totalBytes = 0;
int readLength = 1024 * 1024;
try
{
UTF8Encoding utf8 = new UTF8Encoding(false, true);
while (true)
{
int needRead = readLength;
if (myfile.Length - totalBytes < readLength)
{
needRead = (int)(myfile.Length - totalBytes);
}
if (needRead <= 0)
{
break;
}
byte[] fileBuffer = thriftClient.read(th, totalBytes, needRead);
byte[] myfileBuffer = Encoding.Convert(utf8, Encoding.GetEncoding("iso-8859-1"), fileBuffer);
totalBytes += needRead;
fs.Write(myfileBuffer, 0, myfileBuffer.Length);
//显示单个上传进度
int mypresent = (int)((double)totalBytes / myfile.Length * 100);
worker.ReportProgress(pgPresent, new ProgressState()
{
CurrentTitle = mypresent + "% " + fileName
});
}
result = true;
}
catch (Exception ee)
{
throw ee;
}
finally
{
fs.Dispose();
if (thriftClient != null)
{
thriftClient.close(th);
}
}
#endregion
string msg = string.Format("{0} 下载{1}", Path.GetFileName(fileName), result ? "成功" : "失败");
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = msg
});
}
else
{
worker.ReportProgress(pgPresent, new ProgressState()
{
ListBoxMsg = fileName + "不是文件！"
});
}
}
}
//释放连接
if (btsport != null)
{
btsport.Close();
}
}
