Example #1
0
        /// <summary>
        /// 添加下载任务到列表
        /// </summary>
        /// <param name="ossPath">OSS路径</param>
        /// <param name="ossName">OSS名称</param>
        /// <param name="downloadPath">下载保存路径</param>
        /// <returns>任务信息</returns>
        public OssTaskDO AddDownloadTask(string ossPath, string ossName, string downloadPath)
        {
            var t = this.taskList.Find(r => r.DownloadOssName == ossName);

            if (t != null)
            {
                throw new Exception("文件已经在任务列表");
            }


            var task = new OssTaskDO()
            {
                Id     = Guid.NewGuid().ToString(),
                Type   = OssTaskType.DOWNLOAD,
                Status = OssTaskStatus.WAIT,

                DownloadOssPath = ossPath == null || ossPath == "" ? "/" : ossPath,
                DownloadOssName = ossName,

                DownloadName = Path.GetFileName(downloadPath),
                DownloadPath = Path.GetDirectoryName(downloadPath),
            };

            taskList.Add(task);
            SaveOssTaskConfig(taskList);
            return(task);
        }
Example #2
0
 /// <summary>
 /// 上传失败回滚
 /// </summary>
 /// <param name="task"></param>
 /// <param name="i"></param>
 private void UploadReadReduction(OssTaskDO task, int i)
 {
     lock (task)
     {
         Console.WriteLine("回滚任务:" + i);
         if (task != null && task.DebrisProgress != null && task.DebrisProgress.Length > i)
         {
             task.DebrisProgress[i] = false;
         }
     }
 }
Example #3
0
 /// <summary>
 /// 停止任务
 /// </summary>
 /// <param name="task"></param>
 private void StopDownload(OssTaskDO task)
 {
     if (task.ThreadTokenList == null || task.ThreadTokenStatusList == null)
     {
         return;
     }
     task.ThreadTokenStatusList.ForEach(it =>
     {
         it.Status = true;
     });
 }
Example #4
0
 /// <summary>
 /// 读取碎片获取需要下载的碎片
 /// </summary>
 /// <param name="task"></param>
 /// <returns></returns>
 private int DownloadReadAsync(OssTaskDO task)
 {
     lock (task)
     {
         var index = task.DebrisProgress.ToList().FindIndex(r => r == false);
         if (index < 0)
         {
             return(-1);
         }
         task.DebrisProgress[index] = null;
         return(index);
     }
 }
Example #5
0
        /// <summary>
        /// 上传完成
        /// </summary>
        /// <param name="task"></param>
        private void UploadComplete(OssTaskDO task)
        {
            //如果是暂停就跳出
            if (task.Progress < task.TotalProgress - 1)
            {
                //关闭内存流
                this.Close(task);
                //清除配置文件,只有删除状态删除配置
                if (task.Status == OssTaskStatus.DELETE)
                {
                    this.ClearConfig(task);
                }
                return;
            }

            //线上检查
            var ossPath          = CheckOssPath(task.UploadOssPath + @"/" + task.UploadOssName);
            var listPartsRequest = new ListPartsRequest(aliyunOSSConfig.BucketName, ossPath, task.UploadOssId);
            var listPartsResult  = client.ListParts(listPartsRequest);

            //生成Tag
            var             completeMultipartUploadRequest = new CompleteMultipartUploadRequest(aliyunOSSConfig.BucketName, ossPath, task.UploadOssId);
            List <PartETag> eTags = new List <PartETag>();

            foreach (var item in task.UploadETag)
            {
                var values = item.Split(',');
                eTags.Add(new PartETag(Convert.ToInt32(values[0]), values[1], values[2], Convert.ToInt64(values[3])));
            }
            eTags.Sort((x, y) => x.PartNumber.CompareTo(y.PartNumber));
            eTags.ForEach(it => completeMultipartUploadRequest.PartETags.Add(it));


            //合并文件上传完成
            client.CompleteMultipartUpload(completeMultipartUploadRequest);


            //保存配置
            task.Status         = OssTaskStatus.COMPLETE;
            task.ActualProgress = 100;
            SaveOssTaskConfig(taskList);

            //关闭内存流
            this.Close(task);
            //清除配置文件
            this.ClearConfig(task);

            //执行回执
            downloadCompleteCallBack?.Invoke(task);
        }
Example #6
0
        /// <summary>
        /// 上传任务的详细
        /// </summary>
        /// <param name="task"></param>
        /// <returns></returns>
        private object UploadItem(OssTaskDO task)
        {
            //获取索引
            var p = UploadReadAsync(task);

            if (p == null)
            {
                return(null);
            }

            try
            {
                //读取文件
                byte[] data = (byte[])p[1];

                //生成文件
                var ossPath = CheckOssPath(task.UploadOssPath + @"/" + task.UploadOssName);
                var request = new UploadPartRequest(aliyunOSSConfig.BucketName, ossPath, task.UploadOssId)
                {
                    InputStream = new MemoryStream(data),
                    PartSize    = data.Length,
                    PartNumber  = ((int)p[0]) + 1
                };

                //上传记录下来
                var result = client.UploadPart(request);
                if (result.HttpStatusCode.ToString() != "OK")
                {
                    throw new Exception("Response NOT OK");
                }

                //保存进度配置
                lock (task.ConfigStream)
                {
                    var resultRow = result.PartNumber + "," + result.ETag + "," + result.Crc64 + "," + result.Length;
                    task.UploadETag.Add(resultRow);
                    task.Progress += data.Length;
                    task.ConfigStream.WriteLine(resultRow);
                    task.ConfigStream.Flush();
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.StackTrace);
                //如果执行失败还原
                UploadReadReduction(task, (int)p[0]);
            }
            return(true);
        }
Example #7
0
 /// <summary>
 /// 清除配置文件
 /// </summary>
 /// <param name="task"></param>
 private void ClearConfig(OssTaskDO task)
 {
     if (task.Status != OssTaskStatus.WAIT && task.Status != OssTaskStatus.RUN && File.Exists(task.UploadPath + @"/" + task.UploadName + ".upload"))
     {
         File.Delete(task.UploadPath + @"/" + task.UploadName + ".upload");
     }
     if (task.Status != OssTaskStatus.WAIT && task.Status != OssTaskStatus.RUN && File.Exists(task.DownloadPath + @"/" + task.DownloadName + ".download"))
     {
         File.Delete(task.DownloadPath + @"/" + task.DownloadName + ".download");
     }
     if (task.Status != OssTaskStatus.WAIT && task.Status != OssTaskStatus.RUN && File.Exists(task.DownloadPath + @"/" + task.DownloadName + ".download.config"))
     {
         File.Delete(task.DownloadPath + @"/" + task.DownloadName + ".download.config");
     }
 }
Example #8
0
        /// <summary>
        /// 刷新任务线程的执行类
        /// </summary>
        /// <param name="task"></param>
        private void RefreshTaskItem(OssTaskDO task)
        {
            //速度
            var speed = task.Progress - task.LastProgress;

            //总体进度百分比
            var progress = task.TotalProgress <= 0 ?
                           0 : task.Progress * 1.0 / task.TotalProgress;

            task.LastProgress = task.Progress;

            //委托调用
            if (task.LastProgress > 0 && taskProgressChangeCallback != null)
            {
                task.ActualProgress = (long)(Math.Round(progress, 2) * 100);
                taskProgressChangeCallback(task, speed, task.ActualProgress);
            }
        }
Example #9
0
        /// <summary>
        /// 读取上传任务进度
        /// </summary>
        /// <param name="task"></param>
        /// <returns>数组1:下标,2:数据</returns>
        private object[] UploadReadAsync(OssTaskDO task)
        {
            lock (task)
            {
                var index = task.DebrisProgress.ToList().FindIndex(r => r == false);
                if (index < 0)
                {
                    return(null);
                }
                task.DebrisProgress[index] = null;

                var data = new byte[task.Stream.Length > (index + 1) * task.DebrisSize
                    ? task.DebrisSize
                    : task.Stream.Length - index * task.DebrisSize];

                task.Stream.Position = index * task.DebrisSize;
                task.Stream.Read(data, 0, data.Length);
                return(new object[] { index, data });
            }
        }
Example #10
0
        /// <summary>
        /// 关闭内存流并回收内存
        /// </summary>
        /// <param name="task"></param>
        private void Close(OssTaskDO task)
        {
            lock (task)
            {
                //回收内存
                task.ThreadTokenList       = null;
                task.ThreadList            = null;
                task.ThreadTokenStatusList = null;
                task.DebrisProgress        = null;

                if (task.Stream != null)
                {
                    task.Stream.Close();
                }
                if (task.ConfigStream != null)
                {
                    task.ConfigStream.Close();
                }
            }
        }
Example #11
0
        /// <summary>
        /// 下载完成详情
        /// </summary>
        /// <param name="task"></param>
        private void DownloadComplete(OssTaskDO task)
        {
            //如果是暂停就跳出
            if (task.Progress < task.TotalProgress - 1)
            {
                //关闭内存流
                this.Close(task);
                //清除配置文件,只有删除状态删除配置
                if (task.Status == OssTaskStatus.DELETE)
                {
                    this.ClearConfig(task);
                }
                return;
            }

            //保存配置
            task.Status         = OssTaskStatus.COMPLETE;
            task.ActualProgress = 100;
            SaveOssTaskConfig(taskList);

            //回收内存
            this.Close(task);

            //重命名, 如果存在文件就加Guid
            if (!File.Exists(task.DownloadPath + @"\" + task.DownloadName))
            {
                File.Move(task.DownloadPath + @"\" + task.DownloadName + ".download", task.DownloadPath + @"\" + task.DownloadName);
            }
            else
            {
                var flag = task.DownloadPath + @"\" + Path.GetFileNameWithoutExtension(task.DownloadName) + Guid.NewGuid().ToString() + Path.GetExtension(task.DownloadName);
                File.Move(task.DownloadPath + @"\" + task.DownloadName + ".download", flag);
            }

            //清除配置文件
            this.ClearConfig(task);


            //执行回执
            downloadCompleteCallBack?.Invoke(task);
        }
Example #12
0
        /// <summary>
        /// 下载任务详情
        /// </summary>
        /// <param name="task"></param>
        /// <returns></returns>
        private object DownloadItem(OssTaskDO task)
        {
            //获取索引
            var p = DownloadReadAsync(task);

            if (p < 0)
            {
                return(null);
            }

            try
            {
                //阿里云下载
                var ossPath = CheckOssPath(task.DownloadOssPath + @"\" + task.DownloadOssName);
                GetObjectRequest request = new GetObjectRequest(aliyunOSSConfig.BucketName, ossPath);

                request.SetRange(p * task.DebrisSize,
                                 p + 1 >= task.DownloadDebrisTotalCount ? task.DownloadFileLength : (p + 1) * task.DebrisSize);

                if (p + 1 >= task.DownloadDebrisTotalCount)
                {
                    request.SetRange(p * task.DebrisSize, task.DownloadFileLength - 1);
                }

                //保存到磁盘
                OssObject result = client.GetObject(request);
                if (result.HttpStatusCode.ToString() != "OK" && result.HttpStatusCode.ToString() != "PartialContent")
                {
                    throw new Exception("Response NOT OK");
                }
                DownloadWriteAsync(task, result.Content, p);
            }
            catch (Exception e)
            {
                Console.WriteLine(e.StackTrace);
                DownloadWriteReduction(task, p);
            }
            return(true);
        }
Example #13
0
        /// <summary>
        /// 读取网络流并写入磁盘文件
        /// </summary>
        /// <param name="task"></param>
        /// <param name="stream"></param>
        /// <param name="i"></param>
        private void DownloadWriteAsync(OssTaskDO task, Stream stream, int i)
        {
            //获取流
            var memoryStream = new MemoryStream();

            CopyStream(stream, memoryStream);
            byte[] bytes = new byte[memoryStream.Length - 1];
            memoryStream.Read(bytes, 0, bytes.Length);



            //保存到文件
            lock (task.Stream)
            {
                task.Stream.Flush();
                task.Stream.Position = i * task.DebrisSize;
                task.Stream.Flush();
                task.Stream.Write(bytes, 0, bytes.Length);
                task.Stream.Flush();
                task.ConfigStream.WriteLine(i + "," + Guid.NewGuid().ToString());
                task.ConfigStream.Flush();
                task.Progress += bytes.Length;
            }
        }
Example #14
0
        /// <summary>
        /// 开始上传
        /// </summary>
        /// <param name="task"></param>
        private void StartUpload(OssTaskDO task)
        {
            //开启扫描
            this.RefreshTask();

            //先上阿里云OSS进行上传初始化
            if (task.UploadOssId == null)
            {
                var ossPath = CheckOssPath(task.UploadOssPath + @"/" + task.UploadOssName);
                var request = new InitiateMultipartUploadRequest(aliyunOSSConfig.BucketName, ossPath);
                var result  = client.InitiateMultipartUpload(request);
                task.UploadOssId = result.UploadId;
            }

            //打开文件流
            task.Stream = new FileStream(task.UploadPath + @"\" + task.UploadName, FileMode.Open);
            var fileLength = task.Stream.Length;

            //存放Etag 读取Tag
            task.UploadETag = new List <string>();
            var fileStream = new FileStream(task.UploadPath + @"\" + task.UploadName + ".upload", FileMode.OpenOrCreate);

            fileStream.Position = 0;
            using (var readStream = new StreamReader(fileStream))
            {
                while (true)
                {
                    var row = readStream.ReadLine();
                    if (row == null)
                    {
                        break;
                    }

                    var values = row.Split(',');
                    var id     = values[0];
                    var index  = task.UploadETag.FindIndex(r => r.Split(',')[0] == id);
                    if (values.Length == 4 && index == -1)
                    {
                        task.UploadETag.Add(row);
                    }
                    else
                    {
                        Console.WriteLine("Read Tag: " + row + ", index:" + index);
                    }
                }
            }
            //隐藏文件
            File.SetAttributes(task.UploadPath + @"\" + task.UploadName + ".upload", FileAttributes.Hidden);
            Console.WriteLine("Tag: " + task.UploadETag.Count);

            //在相同上传文件目录生成一个隐藏配置文件
            fileStream                  = new FileStream(task.UploadPath + @"\" + task.UploadName + ".upload", FileMode.OpenOrCreate);
            fileStream.Position         = fileStream.Length;
            task.ConfigStream           = new StreamWriter(fileStream);
            task.ConfigStream.AutoFlush = false;


            //智能分配碎片
            if (task.DebrisSize <= 0)
            {
                task.DebrisSize = fileLength % 9999 == 0 ? fileLength / 9999 : fileLength / 9999 + 100;
                if (task.DebrisSize < task.MinDebrisSize)
                {
                    task.DebrisSize = task.MinDebrisSize;
                }
            }

            //计算碎片总数和进度
            var debrisTotal    = fileLength % task.DebrisSize == 0 ? fileLength / task.DebrisSize : fileLength / task.DebrisSize + 1;
            var debrisProgress = new bool?[debrisTotal];

            for (int i = 0; i < debrisProgress.Length; i++)
            {
                var s = task.UploadETag.FindIndex(r => r.Split(',')[0] == (i + 1).ToString());
                debrisProgress[i] = s >= 0;
            }
            task.DebrisProgress = debrisProgress;

            //总长度
            task.Progress      = task.DebrisProgress.ToList().Where(r => r == true).ToList().Count *task.DebrisSize;
            task.TotalProgress = task.Stream.Length;

            //保存配置
            SaveOssTaskConfig(taskList);

            //开始任务
            task.ThreadList            = new List <Task>();
            task.ThreadTokenList       = new List <CancellationTokenSource>();
            task.ThreadTokenStatusList = new List <TokenStatus>();
            for (int i = 0; i < threadCount; i++)
            {
                var token = new CancellationTokenSource();
                task.ThreadList.Add(new Task(() =>
                {
                    for (; ;)
                    {
                        try
                        {
                            if (task.ThreadTokenStatusList == null)
                            {
                                return;
                            }
                            var taskStatus = task.ThreadTokenStatusList.Find(it => it.Token == token);
                            if (taskStatus.Status)
                            {
                                return;
                            }
                            if (UploadItem(task) == null)
                            {
                                return;
                            }
                            if (taskStatus.Status)
                            {
                                return;
                            }
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine(ex.StackTrace);
                        }
                    }
                }, token.Token));
                task.ThreadTokenList.Add(token);
                task.ThreadTokenStatusList.Add(new TokenStatus()
                {
                    Token  = token,
                    Status = false
                });
            }

            //等待任务完成结束
            Task.WhenAll(task.ThreadList.ToArray()).ContinueWith(s =>
            {
                UploadComplete(task);
            });

            //开始
            task.ThreadList.ForEach(it => it.Start());
        }
Example #15
0
        /// <summary>
        /// 开始下载任务执行
        /// </summary>
        /// <param name="task"></param>
        private void StartDownload(OssTaskDO task)
        {
            //开启扫描
            this.RefreshTask();

            //是否需要创建文件
            if (!File.Exists(task.DownloadPath + @"\" + task.DownloadName + ".download") || task.DownloadFileLength == 0)
            {
                //如果文件已经存在则删除
                if (File.Exists(task.DownloadPath + @"\" + task.DownloadName + ".download"))
                {
                    File.Delete(task.DownloadPath + @"\" + task.DownloadName + ".download");
                }

                //读取线上文件信息
                var ossPath  = CheckOssPath(task.DownloadOssPath + @"\" + task.DownloadOssName);
                var fileInfo = client.GetObject(aliyunOSSConfig.BucketName, ossPath);
                task.DownloadFileLength = fileInfo.ContentLength;
                fileInfo.Dispose();

                //智能分配碎片
                if (task.DebrisSize <= 0)
                {
                    task.DebrisSize = task.DownloadFileLength % 9999 == 0 ? task.DownloadFileLength / 9999 : task.DownloadFileLength / 9999 + 100;
                    if (task.DebrisSize < task.MinDebrisSize)
                    {
                        task.DebrisSize = task.MinDebrisSize;
                    }
                }

                //磁盘创建一个一样大小空文件
                var fs = new FileStream(task.DownloadPath + @"\" + task.DownloadName + ".download", FileMode.Create);
                task.DownloadDebrisTotalCount = task.DownloadFileLength % task.DebrisSize == 0 ? task.DownloadFileLength / task.DebrisSize : task.DownloadFileLength / task.DebrisSize + 1;
                fs.Position = 0;
                for (int i = 1; i <= task.DownloadDebrisTotalCount; i++)
                {
                    if (i == task.DownloadDebrisTotalCount)
                    {
                        var size = task.DownloadFileLength - task.DebrisSize * (i - 1);
                        fs.Write(new byte[size], 0, (int)size);
                    }
                    else
                    {
                        fs.Write(new byte[task.DebrisSize], 0, (int)task.DebrisSize);
                    }
                }
                task.Stream = fs;
            }
            else
            {
                task.Stream = new FileStream(task.DownloadPath + @"\" + task.DownloadName + ".download", FileMode.Open);
            }

            //存放Etag 读取Tag
            task.UploadETag = new List <string>();
            var fileStream = new FileStream(task.DownloadPath + @"\" + task.DownloadName + ".download.config", FileMode.OpenOrCreate);

            fileStream.Position = 0;
            using (var readStream = new StreamReader(fileStream))
            {
                while (true)
                {
                    var row = readStream.ReadLine();
                    if (row == null)
                    {
                        break;
                    }

                    var values = row.Split(',');
                    var id     = values[0];
                    var index  = task.UploadETag.FindIndex(r => r.Split(',')[0] == id);
                    if (values.Length == 2 && index == -1)
                    {
                        task.UploadETag.Add(row);
                    }
                    else
                    {
                        Console.WriteLine("Read Tag: " + row + ", index:" + index);
                    }
                }
            }
            //隐藏文件
            File.SetAttributes(task.DownloadPath + @"\" + task.DownloadName + ".download.config", FileAttributes.Hidden);
            Console.WriteLine("Tag: " + task.UploadETag.Count);

            //在相同下载文件目录生成一个隐藏配置文件
            fileStream                  = new FileStream(task.DownloadPath + @"\" + task.DownloadName + ".download.config", FileMode.OpenOrCreate);
            fileStream.Position         = fileStream.Length;
            task.ConfigStream           = new StreamWriter(fileStream);
            task.ConfigStream.AutoFlush = false;


            //创建碎片任务
            task.DebrisProgress = new bool?[task.DownloadDebrisTotalCount];
            for (int i = 0; i < task.DownloadDebrisTotalCount; i++)
            {
                var index = task.UploadETag.FindIndex(r => r.Split(',')[0] == i.ToString());
                task.DebrisProgress[i] = index >= 0;
            }
            Console.WriteLine("Tag:" + task.UploadETag.Count);

            //总长度
            task.Progress      = task.UploadETag.Count * task.DebrisSize;
            task.TotalProgress = task.DownloadFileLength;

            //保存配置
            SaveOssTaskConfig(taskList);

            //开启线程下载文件
            task.ThreadList            = new List <Task>();
            task.ThreadTokenList       = new List <CancellationTokenSource>();
            task.ThreadTokenStatusList = new List <TokenStatus>();
            for (int i = 0; i < threadCount; i++)
            {
                var token = new CancellationTokenSource();
                task.ThreadList.Add(new Task(() =>
                {
                    for (; ;)
                    {
                        try
                        {
                            if (task.ThreadTokenStatusList == null)
                            {
                                return;
                            }
                            var taskStatus = task.ThreadTokenStatusList.Find(it => it.Token == token);
                            if (taskStatus.Status)
                            {
                                return;
                            }
                            if (DownloadItem(task) == null)
                            {
                                return;
                            }
                            if (taskStatus.Status)
                            {
                                return;
                            }
                        }
                        catch (Exception ex)
                        {
                            Console.WriteLine(ex.StackTrace);
                        }
                    }
                }, token.Token));
                task.ThreadTokenList.Add(token);
                task.ThreadTokenStatusList.Add(new TokenStatus
                {
                    Token  = token,
                    Status = false
                });
            }


            //等待任务完成结束
            Task.WhenAll(task.ThreadList.ToArray()).ContinueWith(s =>
            {
                DownloadComplete(task);
            });
            //开始
            task.ThreadList.ForEach(it => it.Start());
        }