public override IEnumerable <HdfsFile> ExpandFileOrDirectoryToFile(Uri uri)
{
HdfsFileInfo fileInfo = this.Instance(uri).GetFileInfo(uri.AbsolutePath, false);
if (fileInfo.IsDirectory)
{
foreach (string subFile in fileInfo.fileNameArray)
{
UriBuilder builder = new UriBuilder(uri);
builder.Path = subFile;
foreach (HdfsFile file in ExpandFileOrDirectoryToFile(builder.Uri))
{
yield return(file);
}
}
}
else
{
yield return(new HdfsFile
{
path = uri,
length = fileInfo.Size,
blockSize = fileInfo.BlockSize
});
}
}
// 下载文件
public void DownloadFile(string hdfsFile, string localFilename)
{
string hdfsPath = ConstructDfsFullPath(hdfsFile);
string msg = string.Format("Downloading {0} to {1} ", hdfsPath, localFilename);
DebugHelper.OutLog(msg);
string errlog = "";
HdfsFileInfo fi = HdfsHelper.GetStatus(hdfsPath); //TODO: 大文件多线程下载
ShowFileTransmission(true, msg);
Task.Run(() =>
{
Stopwatch wt = Stopwatch.StartNew();
byte[] bytes = HdfsHelper.GetFile(hdfsPath);
bool isOk = bytes.Length == fi.Size;
if (isOk)
{
using (FileStream fs = new FileStream(localFilename, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read))
{
fs.Write(bytes, 0, bytes.Length);
fs.Close();
}
}
wt.Stop();
msg = isOk ? "下载成功 " : "下载失败 " + hdfsPath;
DebugHelper.OutLog(string.Format("下载\"{0}\"{1}，耗时 {2}",
hdfsPath, isOk ? "成功" : "失败", wt.Elapsed.ToString()));
ShowFileTransmission(true, msg);
Thread.Sleep(1000);
ShowFileTransmission(false, "");
});
}
// 读取文件最后一页
public string ReadFileTail(string filepath, long pageSize)
{
try
{
string hdfsPath = ConstructDfsFullPath(filepath);
HdfsFileInfo fi = HdfsHelper.GetStatus(hdfsPath);
long offset = 0;
long size = fi.Size;
if (size > pageSize)
{
offset = size - pageSize;
size = pageSize;
}
var result = HdfsHelper.ReadFilePart(hdfsPath, offset, size);
if (-1 != result.Code)
{
return(Encoding.UTF8.GetString(result.Content));
}
else
{
return(result.ErrorLog);
}
}
catch (Exception ex)
{
return(ex.ToString());
}
}
public override void GetDirectoryContentSummary(Uri dfsPath, bool expandBlocks, ref long totalSize, ref int numberOfParts)
{
if (expandBlocks)
{
HdfsFileInfo info = this.Instance(dfsPath).GetFileInfo(dfsPath.AbsolutePath, true);
totalSize = info.totalSize;
numberOfParts = info.blockArray.Length;
}
else
{
long numberOfFiles;
this.Instance(dfsPath).GetContentSummary(dfsPath.AbsolutePath, out totalSize, out numberOfFiles);
numberOfParts = (int)numberOfFiles;
}
}
public override IEnumerable <IPEndPoint[]> GetBlockLocations(HdfsFile file)
{
HdfsFileInfo fileInfo = this.Instance(file.path).GetFileInfo(file.path.AbsolutePath, true);
foreach (HdfsBlockInfo info in fileInfo.blockArray)
{
IPEndPoint[] endpoints = new IPEndPoint[info.Endpoints.Length];
for (int i = 0; i < endpoints.Length; ++i)
{
string[] parts = info.Endpoints[i].Split(':');
endpoints[i] = new IPEndPoint(IPAddress.Parse(parts[0]), Int32.Parse(parts[1]));
}
yield return(endpoints);
}
}
public override bool GetFileStatus(Uri dfsPath, out long modificationTime, out long size)
{
if (this.Instance(dfsPath).IsFileExists(dfsPath.AbsolutePath))
{
HdfsFileInfo info = this.Instance(dfsPath).GetFileInfo(dfsPath.AbsolutePath, false);
modificationTime = info.LastModified;
size = info.Size;
return(true);
}
else
{
modificationTime = -1;
size = -1;
return(false);
}
}
public override IEnumerable <Uri> ExpandFileOrDirectory(Uri dfsPath)
{
HdfsFileInfo info = this.Instance(dfsPath).GetFileInfo(dfsPath.AbsolutePath, false);
if (info.IsDirectory)
{
foreach (string path in info.fileNameArray)
{
UriBuilder builder = new UriBuilder(dfsPath);
builder.Path = path;
foreach (Uri entry in this.ExpandFileOrDirectory(builder.Uri))
{
yield return(entry);
}
}
}
else
{
UriBuilder builder = new UriBuilder(dfsPath);
builder.Path = info.fileNameArray[0];
yield return(builder.Uri);
}
}