コード例 #1
0
ファイル: FSClient.cs プロジェクト: yaowenhua/NetHadoop
        //上传文件
        public bool Create(ThriftHadoopFileSystem.Client client, string localPath, string path)
        {
            bool result = false;

            if (client != null)
            {
                ThriftHandle th = null;
                FileStream   fs = null;
                try
                {
                    //创建一个文件
                    th = client.createFile(new Pathname()
                    {
                        pathname = path
                    }, 1, true, 1024, ConfigHelper.HDFSREPLICATION, 1024 * 1024 * 64);

                    UTF8Encoding utf8 = new UTF8Encoding(false, true);

                    fs = new FileStream(localPath, FileMode.Open, FileAccess.Read);

                    byte[] fileBuffer = new byte[1024 * 1024];  // 每次传1MB
                    int    bytesRead;
                    while ((bytesRead = fs.Read(fileBuffer, 0, fileBuffer.Length)) > 0)
                    {
                        byte[] realBuffer = new byte[bytesRead];
                        Array.Copy(fileBuffer, realBuffer, bytesRead);
                        //将utf8转为可存储编码
                        byte[] buf = Encoding.Convert(Encoding.GetEncoding("iso-8859-1"), utf8, realBuffer);
                        //发送
                        client.write(th, buf);
                        //清仓缓存
                        Array.Clear(fileBuffer, 0, fileBuffer.Length);
                    }
                    result = true;
                }
                catch (Exception ee)
                {
                    throw ee;
                }
                finally
                {
                    if (th != null)
                    {
                        client.close(th);
                    }
                    if (fs != null)
                    {
                        fs.Close();
                    }
                }
            }

            return(result);
        }
コード例 #2
0
ファイル: FSClient.cs プロジェクト: yaowenhua/NetHadoop
        //批量上传
        public void MutUpload(BackgroundWorker worker, List <string> localPath, string remoteRootPath, string localRootPath)
        {
            //相同操作
            bool sameOp = false;
            //是否覆盖
            bool IsOver = false;

            //准备创建连接
            Thrift.Transport.TBufferedTransport btsport      = null;
            ThriftHadoopFileSystem.Client       thriftClient = Connect(out btsport);
            List <string> NoSuccessList = new List <string>();
            List <string> skipList      = new List <string>();

            if (thriftClient != null)//连接成功
            {
                int totalCount   = localPath.Count;
                int currentCount = 0;
                //开始上传
                worker.ReportProgress(0, new ProgressState()
                {
                    ListBoxMsg = "开始上传!", totalCount = totalCount, CurrentCount = 0
                });
                //循环
                foreach (string localFilePath in localPath)
                {
                    string fileName = Path.GetFileName(localFilePath);
                    if (!string.IsNullOrEmpty(localRootPath))//如果是文件夹,则包含原有路径
                    {
                        fileName = localFilePath.Replace(localRootPath, "");
                    }

                    string remoteFilePath = remoteRootPath + "/" + fileName;
                    currentCount++;
                    int pgPresent = (int)((double)currentCount / totalCount * 100);
                    //显示总进度
                    worker.ReportProgress(pgPresent, new ProgressState()
                    {
                        CurrentTitle = fileName, CurrentCount = currentCount
                    });

                    #region 是否存在
                    if (!sameOp)
                    {
                        bool exsitFile = thriftClient.exists(new Pathname()
                        {
                            pathname = remoteFilePath
                        });
                        if (exsitFile)
                        {
                            SureDialog         myDialog = new SureDialog();
                            MyShowDialogResult myDR     = new MyShowDialogResult();
                            myDialog.ShowDialog(fileName, myDR);
                            sameOp = myDR.IsCheck;
                            IsOver = myDR.Result;
                            if (!myDR.Result)
                            {
                                worker.ReportProgress(pgPresent, new ProgressState()
                                {
                                    ListBoxMsg = fileName + " 跳过 "
                                });
                                skipList.Add(fileName);
                                continue;
                            }
                        }
                    }
                    else
                    {
                        bool exsitFile = thriftClient.exists(new Pathname()
                        {
                            pathname = remoteFilePath
                        });
                        if (exsitFile)
                        {
                            if (!IsOver)
                            {
                                worker.ReportProgress(pgPresent, new ProgressState()
                                {
                                    ListBoxMsg = fileName + "跳过 "
                                });
                                skipList.Add(fileName);
                                continue;
                            }
                        }
                    }
                    #endregion


                    #region   单个文件
                    bool         singleResult = false;
                    ThriftHandle th           = null;
                    FileStream   fs           = null;
                    try
                    {
                        Pathname myNewFile = new Pathname()
                        {
                            pathname = remoteFilePath
                        };

                        //创建一个文件
                        th = thriftClient.createFile(myNewFile, 1, true, 1024 * 1024 * 10, ConfigHelper.HDFSREPLICATION, 1024 * 1024 * 512);

                        UTF8Encoding utf8 = new UTF8Encoding(false, true);

                        fs = new FileStream(localFilePath, FileMode.Open, FileAccess.Read);

                        byte[] fileBuffer = new byte[1024 * 1024 * 10]; // 每次传1MB
                        int    bytesRead;
                        long   bytesTotal = 0;

                        while ((bytesRead = fs.Read(fileBuffer, 0, fileBuffer.Length)) > 0)
                        {
                            bytesTotal += bytesRead;
                            byte[] realBuffer = new byte[bytesRead];
                            Array.Copy(fileBuffer, realBuffer, bytesRead);
                            //将utf8转为可存储编码
                            realBuffer = Encoding.Convert(Encoding.GetEncoding("iso-8859-1"), utf8, realBuffer);
                            //发送
                            thriftClient.write(th, realBuffer);
                            //清仓缓存
                            Array.Clear(fileBuffer, 0, fileBuffer.Length);
                            realBuffer = null;
                            //显示单个上传进度
                            int mypresent = (int)((double)bytesTotal / fs.Length * 100);
                            worker.ReportProgress(pgPresent, new ProgressState()
                            {
                                CurrentTitle = mypresent + "% " + fileName
                            });
                        }
                        singleResult = true;
                    }
                    catch (Exception ee)
                    {
                        //显示上传错误
                        worker.ReportProgress(pgPresent, new ProgressState()
                        {
                            ListBoxMsg = ee.Message
                        });
                    }
                    finally
                    {
                        if (th != null)
                        {
                            thriftClient.close(th);
                        }
                        if (fs != null)
                        {
                            fs.Close();
                        }
                    }
                    #endregion
                    //显示单个上传结果
                    string msg = string.Format("{0} 上传{1}", fileName, singleResult ? "成功" : "失败");
                    worker.ReportProgress(pgPresent, new ProgressState()
                    {
                        ListBoxMsg = msg
                    });
                    if (!singleResult)
                    {
                        NoSuccessList.Add(fileName);
                    }
                }
            }

            //释放连接
            if (btsport != null)
            {
                btsport.Close();
            }
            //输出没有上传成功的
            if (NoSuccessList.Count > 0)
            {
                try
                {
                    File.WriteAllText("c:/UploadNoSuccess.txt", string.Join("\r\n", NoSuccessList.ToArray()));
                    worker.ReportProgress(100, new ProgressState()
                    {
                        ListBoxMsg = NoSuccessList.Count + "个上传错误!请查看c:/UploadNoSuccess.txt"
                    });
                }
                catch (Exception ee)
                {
                    worker.ReportProgress(100, new ProgressState()
                    {
                        ListBoxMsg = ee.Message
                    });
                }
            }
            //输出跳过的
            if (skipList.Count > 0)
            {
                try
                {
                    File.WriteAllText("c:/UploadSkip.txt", string.Join("\r\n", skipList.ToArray()));
                    worker.ReportProgress(100, new ProgressState()
                    {
                        ListBoxMsg = "跳过" + skipList.Count + "个文件!请查看c:/UploadSkip.txt"
                    });
                }
                catch (Exception ee)
                {
                    worker.ReportProgress(100, new ProgressState()
                    {
                        ListBoxMsg = ee.Message
                    });
                }
            }
        }