//上传文件
public bool Create(ThriftHadoopFileSystem.Client client, string localPath, string path)
{
bool result = false;
if (client != null)
{
ThriftHandle th = null;
FileStream fs = null;
try
{
//创建一个文件
//th = client.createFile(new Pathname() { pathname = path }, 1, true, 1024, ConfigHelper.HDFSREPLICATION, 1024*1024*64);
th = client.createFile(new Pathname() { pathname = path }, 1, true, 1024, 1, 1024 * 1024 * 64);
UTF8Encoding utf8 = new UTF8Encoding(false,true);
fs = new FileStream(localPath, FileMode.Open, FileAccess.Read);
byte[] fileBuffer = new byte[1024 * 1024]; // 每次传1MB
int bytesRead;
while ((bytesRead = fs.Read(fileBuffer, 0, fileBuffer.Length)) > 0)
{
byte[] realBuffer = new byte[bytesRead];
Array.Copy(fileBuffer, realBuffer, bytesRead);
//将utf8转为可存储编码
byte[] buf = Encoding.Convert(Encoding.GetEncoding("iso-8859-1"), utf8, realBuffer);
//发送
client.write(th,buf);
//清仓缓存
Array.Clear(fileBuffer, 0, fileBuffer.Length);
}
result = true;
}
catch (Exception ex)
{
throw ex;
}
finally
{
if (th != null)
client.close(th);
if (fs != null)
fs.Close();
}
}
return result;
}
//下载
public bool Open(ThriftHadoopFileSystem.Client client,string path,string savePath,long fileLength)
{
bool result = false;
if (client != null)
{
ThriftHandle th = client.open(new Pathname() { pathname = path });
// 创建文件流
FileStream fs = new FileStream(savePath, FileMode.Create, FileAccess.Write);
long totalBytes = 0;
int readLength=1024*1024;
try
{
UTF8Encoding utf8 = new UTF8Encoding(false,true);
while (true)
{
int needRead = readLength;
if (fileLength - totalBytes < readLength)
{
needRead = (int)(fileLength - totalBytes);
}
if (needRead <= 0)
break;
byte[] fileBuffer = client.read(th, totalBytes, readLength);
byte[] myfileBuffer = Encoding.Convert(utf8, Encoding.GetEncoding("iso-8859-1"), fileBuffer);
totalBytes += readLength;
fs.Write(myfileBuffer, 0, myfileBuffer.Length);
}
result = true;
}
catch (Exception ex)
{
throw ex;
}
finally
{
fs.Dispose();
if (client != null)
client.close(th);
}
}
return result;
}
public byte[] Open(ThriftHadoopFileSystem.Client client, string path)
{
long fileLength = client.stat(new Pathname() { pathname = path }).Length;
byte[] result = new byte[fileLength];
List<byte> lt = new List<byte>();
if (client != null)
{
ThriftHandle th = client.open(new Pathname() { pathname = path });
long totalBytes = 0;
int readLength = 1024 * 1024;
try
{
UTF8Encoding utf8 = new UTF8Encoding(false, true);
while (true)
{
int needRead = readLength;
if (fileLength - totalBytes < readLength)
{
needRead = (int)(fileLength - totalBytes);
}
if (needRead <= 0)
break;
byte[] fileBuffer = client.read(th, totalBytes, readLength);
byte[] myfileBuffer = Encoding.Convert(utf8, Encoding.GetEncoding("iso-8859-1"), fileBuffer);
foreach (byte b in myfileBuffer)
{
lt.Add(b);
}
totalBytes += readLength;
}
}
catch (Exception ex)
{
throw ex;
}
finally
{
if (client != null)
client.close(th);
}
}
result = lt.ToArray();
return result;
}
//上传文件
public bool Create(ThriftHadoopFileSystem.Client client, byte[] data, string path)
{
bool result = false;
if (client != null)
{
ThriftHandle th = null;
try
{
//创建一个文件
//th = client.createFile(new Pathname() { pathname = path }, 1, true, 1024, ConfigHelper.HDFSREPLICATION, 1024*1024*64);
th = client.createFile(new Pathname() { pathname = path }, 1, true, 1024, 1, 1024 * 1024 * 64);
UTF8Encoding utf8 = new UTF8Encoding(false, true);
byte[] buf = Encoding.Convert(Encoding.GetEncoding("iso-8859-1"), utf8, data);
//发送
client.write(th, buf);
result = true;
}
catch (Exception ex)
{
throw ex;
}
finally
{
if (th != null)
client.close(th);
}
}
return result;
}