private void ConnectWebHDFS()
{
DebugHelper.OutLog("Hello HDFS Window");
// 加载顶级目录
Task.Run(() =>
{
ShowDirLoading(true);
try
{
HdfsHelper.SetHDFSNameNode(_server);
List <HdfsFileInfo> files = LoadFileInfos("/");
List <HdfsFileInfo> dirs = files.Where((fi) => fi.IsDirectory).ToList();
List <HdfsFileInfo> docs = files.Where((fi) => !fi.IsDirectory).ToList();
/*files.ForEach(fi =>
* {
* DebugHelper.OutLog(fi.ToString());
* });*/
LoadDirs(dirs);
LoadDocs(docs);
}
catch (System.Exception ex)
{
DebugHelper.OutLog(ex.Message);
}
ShowDirLoading(false);
});
}
// 下载文件
public void DownloadFile(string hdfsFile, string localFilename)
{
string hdfsPath = ConstructDfsFullPath(hdfsFile);
string msg = string.Format("Downloading {0} to {1} ", hdfsPath, localFilename);
DebugHelper.OutLog(msg);
string errlog = "";
HdfsFileInfo fi = HdfsHelper.GetStatus(hdfsPath); //TODO: 大文件多线程下载
ShowFileTransmission(true, msg);
Task.Run(() =>
{
Stopwatch wt = Stopwatch.StartNew();
byte[] bytes = HdfsHelper.GetFile(hdfsPath);
bool isOk = bytes.Length == fi.Size;
if (isOk)
{
using (FileStream fs = new FileStream(localFilename, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read))
{
fs.Write(bytes, 0, bytes.Length);
fs.Close();
}
}
wt.Stop();
msg = isOk ? "下载成功 " : "下载失败 " + hdfsPath;
DebugHelper.OutLog(string.Format("下载\"{0}\"{1}，耗时 {2}",
hdfsPath, isOk ? "成功" : "失败", wt.Elapsed.ToString()));
ShowFileTransmission(true, msg);
Thread.Sleep(1000);
ShowFileTransmission(false, "");
});
}
// 读取文件最后一页
public string ReadFileTail(string filepath, long pageSize)
{
try
{
string hdfsPath = ConstructDfsFullPath(filepath);
HdfsFileInfo fi = HdfsHelper.GetStatus(hdfsPath);
long offset = 0;
long size = fi.Size;
if (size > pageSize)
{
offset = size - pageSize;
size = pageSize;
}
var result = HdfsHelper.ReadFilePart(hdfsPath, offset, size);
if (-1 != result.Code)
{
return(Encoding.UTF8.GetString(result.Content));
}
else
{
return(result.ErrorLog);
}
}
catch (Exception ex)
{
return(ex.ToString());
}
}
// 上传文件
public void UploadFile(string localFile, string hdfsDir)
{
string hdfsPath = ConstructDfsFullPath(hdfsDir);
string msg = string.Format("Uploading {0} to {1} ", localFile, hdfsPath);
DebugHelper.OutLog(msg);
string errlog = "";
ShowFileTransmission(true, msg);
Task.Run(() =>
{
Stopwatch wt = Stopwatch.StartNew();
bool bok = HdfsHelper.UploadFile(localFile, hdfsPath, out errlog);
msg = bok ? "上传成功 " : "上传失败 " + localFile;
DebugHelper.OutLog(string.Format("上传\"{0}\"{1}，耗时 {2}",
localFile, bok ? "成功" : "失败", wt.Elapsed.ToString()));
ShowFileTransmission(true, msg);
ReloadDocsInDir(hdfsDir);
Thread.Sleep(500);
ShowFileTransmission(false, "");
});
}
// 查询文件属性
public List <HdfsFileInfo> LoadFileInfos(string dfsPath)
{
if (string.IsNullOrWhiteSpace(dfsPath))
{
dfsPath = "/";
}
List <HdfsFileInfo> files = null;
if (_fileinfoCache.ContainsKey(dfsPath))
{
_fileinfoCache.TryGetValue(dfsPath, out files);
}
else
{
files = HdfsHelper.LsDir(ConstructDfsFullPath(dfsPath));
if (null != files && files.Count > 0)
{
_fileinfoCache.TryAdd(dfsPath, files);
}
}
return(files);
}
// 新建目录
private void menuMkdir_Click(object sender, EventArgs e)
{
try
{
TreeNode currNod = this.treeView1.SelectedNode;
if (null == currNod)
{
return;
}
FormInput fm = new FormInput();
fm.Content = "newdirectory";
if (DialogResult.OK == fm.ShowDialog())
{
string name = fm.Content;
if (string.IsNullOrWhiteSpace(name))
{
MessageBox.Show("目录名称不能为空！");
return;
}
string dirname = name.Replace(":", "").Replace("/", "")
.Replace(" ", "").Replace("*", "");
if (string.IsNullOrWhiteSpace(dirname))
{
return;
}
string parentDir = currNod.Name;
string hdfsDir = (parentDir.Length > 1) ? parentDir + "/" : parentDir;
hdfsDir = hdfsDir + dirname;
bool bok = HdfsHelper.MkDir(hdfsDir);
if (bok)
{
// update cache
if (_fileinfoCache.ContainsKey(parentDir))
{
List <HdfsFileInfo> fis = _fileinfoCache[parentDir];
if (null != fis)
{
fis.Add(new HdfsFileInfo()
{
Name = dirname,
PathName = hdfsDir,
IsDirectory = true,
});
}
}
// Refresh TreeNodes
TreeNode nod = currNod.Nodes.Add(dirname, dirname, 0);
nod.Name = hdfsDir;
}
else
{
MessageBox.Show("目录创建失败");
}
}
}
catch (System.Exception ex)
{
DebugHelper.Error(ex);
MessageBox.Show(ex.Message);
}
}
// 删除文件或目录
private void menuDelete_Click(object sender, EventArgs e)
{
try
{
if (_popInGrid) // 删除文件
{
if (this.dataGridView1.SelectedRows.Count > 1)
{
int n = this.dataGridView1.SelectedRows.Count;
if (DialogResult.OK == MessageBox.Show("确定要删除选中的 " + n + " 个文件 ？",
"确认提示", MessageBoxButtons.OKCancel))
{
string dfspath = this.treeView1.SelectedNode.Name;
foreach (DataGridViewRow row in this.dataGridView1.SelectedRows)
{
string fileName = row.Cells[_fileNameColIndex].Value.ToString();
string hdfsFile = dfspath + "/" + fileName;
bool bok = HdfsHelper.Rm(hdfsFile);
}
ReloadDocsInDir(dfspath);
}
}
else
{
var row = this.dataGridView1.CurrentRow;
string fileName = row.Cells[_fileNameColIndex].Value.ToString();
string dfspath = this.treeView1.SelectedNode.Name;
string hdfsFile = dfspath + "/" + fileName;
if (DialogResult.OK == MessageBox.Show("确定要删除文件：\"" + hdfsFile + "\" ？",
"确认提示", MessageBoxButtons.OKCancel))
{
bool bok = HdfsHelper.Rm(hdfsFile);
if (bok)
{
ReloadDocsInDir(dfspath);
}
}
}
}
else //删除目录
{
TreeNode currNod = this.treeView1.SelectedNode;
string hdfsDir = currNod.Name;
if ("/".Equals(hdfsDir))
{
MessageBox.Show("根目录不能删除");
return;
}
if (DialogResult.OK == MessageBox.Show("确定要删除目录：\"" + currNod.Text + "\" ？",
"确认提示", MessageBoxButtons.OKCancel))
{
bool bok = HdfsHelper.Rm(hdfsDir);
if (bok)
{
// update cache
List <HdfsFileInfo> fis;
_fileinfoCache.TryRemove(hdfsDir, out fis);
string parentDir = hdfsDir.Substring(0, hdfsDir.LastIndexOf('/'));
if (string.IsNullOrWhiteSpace(parentDir))
{
parentDir = "/";
}
if (_fileinfoCache.ContainsKey(parentDir))
{
fis = _fileinfoCache[parentDir];
var currFi = fis.FirstOrDefault(fi => fi.IsDirectory && fi.Name == currNod.Text);
if (null != currFi)
{
fis.Remove(currFi);
}
}
// refresh TreeView
this.treeView1.Nodes.Remove(currNod);
}
}
}
}
catch (System.Exception ex)
{
DebugHelper.Error(ex);
MessageBox.Show(ex.Message);
}
}
/// <summary>
/// 文件上传，多线程并行
/// </summary>
/// <param name="localFile">本地文件名（绝对路径）</param>
/// <param name="hdfsDir">目标HDFS目录</param>
public void UploadFileMT(string localFile, string hdfsDir)
{
FileInfo fi = new FileInfo(localFile);
string fileName = fi.Name;
if (fi.Length < _fileTransBlockSize)
{
UploadFile(localFile, hdfsDir);
return;
}
string msg = string.Format("Uploading {0} to {1} ", localFile, hdfsDir);
ShowFileTransmission(true, msg);
Stopwatch allwt = Stopwatch.StartNew();
// 分块
int n = (int)(fi.Length / _fileTransBlockSize);
if (fi.Length % _fileTransBlockSize > 0)
{
n++;
}
AtomicInt clatch = new AtomicInt();
int parallelNum = 8;
SemaphoreSlim sema = new SemaphoreSlim(parallelNum, parallelNum);
Task[] subtasks = new Task[n];
for (int i = 0; i < n; i++)
{
int pid = i;
subtasks[i] = Task.Run(() =>
{
sema.Wait();
clatch.Increment();
string partName = string.Format("{0}_p_{1:D5}", fileName, pid);
ShowFileTransmission(true, "正在传输\"" + partName + "\"");
Stopwatch wt = Stopwatch.StartNew();
bool bok = false;
using (FileStream fs = new FileStream(localFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
long offset = pid * _fileTransBlockSize;
bok = HdfsHelper.UploadStream(fs, offset, _fileTransBlockSize, hdfsDir, partName);
fs.Close();
}
wt.Stop();
msg = string.Format("上传\"{0}\" part-{1} {2}，耗时 {3}",
localFile, pid, bok ? "成功" : "失败", wt.Elapsed.ToString());
DebugHelper.OutLog(msg);
ShowFileTransmission(true, msg);
clatch.Decrement();
sema.Release();
ReloadDocsInDir(hdfsDir);
});
}
//Task.WaitAll(subtasks);
Task.Run(() =>
{
while (!clatch.IsEmpty)
{
Thread.Sleep(500);
}
allwt.Stop();
string msgtotal = string.Format("文件\"{0}\"已上传完成，共{1}个分区, 耗时：{2}",
localFile, n, allwt.Elapsed.ToString());
ShowFileTransmission(true, msgtotal);
DebugHelper.OutLog(msgtotal);
// 拼接文件名
string wholeFile = hdfsDir + (hdfsDir.EndsWith("/") ? "" : "/") + fileName;
string[] partFiles = new string[n];
for (int i = 0; i < n; i++)
{
partFiles[i] = string.Format("{0}_p_{1:D5}", wholeFile, i);
}
HdfsHelper.NewFile(wholeFile);
HdfsHelper.CombineFiles(wholeFile, partFiles);
ShowFileTransmission(true, "文件已合并完整");
ReloadDocsInDir(hdfsDir);
Thread.Sleep(500);
ShowFileTransmission(false, "");
});
}