Exemple #1
0
        //上传文件
        public bool Create(ThriftHadoopFileSystem.Client client, string localPath, string path)
        {
            bool result = false;
               if (client != null)
               {
               ThriftHandle th = null;
               FileStream fs = null;
               try
               {
                   //创建一个文件
                   //th = client.createFile(new Pathname() { pathname = path }, 1, true, 1024, ConfigHelper.HDFSREPLICATION, 1024*1024*64);
                   th = client.createFile(new Pathname() { pathname = path }, 1, true, 1024, 1, 1024 * 1024 * 64);

                   UTF8Encoding utf8 = new UTF8Encoding(false,true);

                   fs = new FileStream(localPath, FileMode.Open, FileAccess.Read);

                   byte[] fileBuffer = new byte[1024 * 1024];	// 每次传1MB
                   int bytesRead;
                   while ((bytesRead = fs.Read(fileBuffer, 0, fileBuffer.Length)) > 0)
                   {
                       byte[] realBuffer = new byte[bytesRead];
                       Array.Copy(fileBuffer, realBuffer, bytesRead);
                       //将utf8转为可存储编码
                       byte[] buf = Encoding.Convert(Encoding.GetEncoding("iso-8859-1"), utf8, realBuffer);
                       //发送
                       client.write(th,buf);
                       //清仓缓存
                       Array.Clear(fileBuffer, 0, fileBuffer.Length);
                   }
                   result = true;
               }
               catch (Exception ex)
               {
                   throw ex;
               }
               finally
               {
                   if (th != null)
                       client.close(th);
                   if (fs != null)
                       fs.Close();
               }
               }

               return result;
        }
Exemple #2
0
        //下载
        public bool Open(ThriftHadoopFileSystem.Client client,string path,string savePath,long fileLength)
        {
            bool result = false;
               if (client != null)
               {
               ThriftHandle th = client.open(new Pathname() { pathname = path });

               // 创建文件流
               FileStream fs = new FileStream(savePath, FileMode.Create, FileAccess.Write);
               long totalBytes = 0;
               int readLength=1024*1024;
               try
               {
                   UTF8Encoding utf8 = new UTF8Encoding(false,true);

                   while (true)
                   {
                       int needRead = readLength;
                       if (fileLength - totalBytes < readLength)
                       {
                           needRead = (int)(fileLength - totalBytes);
                       }
                       if (needRead <= 0)
                           break;

                       byte[] fileBuffer = client.read(th, totalBytes, readLength);

                       byte[] myfileBuffer =  Encoding.Convert(utf8, Encoding.GetEncoding("iso-8859-1"), fileBuffer);

                       totalBytes += readLength;

                       fs.Write(myfileBuffer, 0, myfileBuffer.Length);

                   }
                   result = true;

               }
               catch (Exception ex)
               {
                   throw ex;
               }
               finally
               {
                   fs.Dispose();
                   if (client != null)
                       client.close(th);
               }
               }
               return result;
        }
Exemple #3
0
        public byte[] Open(ThriftHadoopFileSystem.Client client, string path)
        {
            long fileLength = client.stat(new Pathname() { pathname = path }).Length;
               byte[] result = new byte[fileLength];

               List<byte> lt = new List<byte>();

               if (client != null)
               {
               ThriftHandle th = client.open(new Pathname() { pathname = path });

               long totalBytes = 0;
               int readLength = 1024 * 1024;
               try
               {
                   UTF8Encoding utf8 = new UTF8Encoding(false, true);

                   while (true)
                   {
                       int needRead = readLength;
                       if (fileLength - totalBytes < readLength)
                       {
                           needRead = (int)(fileLength - totalBytes);
                       }
                       if (needRead <= 0)
                           break;

                       byte[] fileBuffer = client.read(th, totalBytes, readLength);

                       byte[] myfileBuffer = Encoding.Convert(utf8, Encoding.GetEncoding("iso-8859-1"), fileBuffer);

                       foreach (byte b in myfileBuffer)
                       {
                           lt.Add(b);
                       }

                       totalBytes += readLength;

                   }
               }
               catch (Exception ex)
               {
                   throw ex;
               }
               finally
               {

                   if (client != null)
                       client.close(th);
               }

               }
               result = lt.ToArray();
               return result;
        }
Exemple #4
0
        //上传文件
        public bool Create(ThriftHadoopFileSystem.Client client, byte[] data, string path)
        {
            bool result = false;
               if (client != null)
               {
               ThriftHandle th = null;
               try
               {
                   //创建一个文件
                   //th = client.createFile(new Pathname() { pathname = path }, 1, true, 1024, ConfigHelper.HDFSREPLICATION, 1024*1024*64);
                   th = client.createFile(new Pathname() { pathname = path }, 1, true, 1024, 1, 1024 * 1024 * 64);

                   UTF8Encoding utf8 = new UTF8Encoding(false, true);

                   byte[] buf = Encoding.Convert(Encoding.GetEncoding("iso-8859-1"), utf8, data);
                   //发送
                   client.write(th, buf);
                   result = true;
               }
               catch (Exception ex)
               {
                   throw ex;
               }
               finally
               {
                   if (th != null)
                       client.close(th);
               }
               }

               return result;
        }