public void ResumableDownloadWithRetry(DownloadObjectRequest request, ResumableDownloadContext resumableContext) { for (int i = 0; i < _maxRetryTimes; i++) { try { DoResumableDownload(request, resumableContext, request.StreamTransferProgress); break; } catch (NoneRetryableException e) { throw new OssException(e.ToString()); } catch (Exception ex) { if (i != _maxRetryTimes - 1) { Thread.Sleep(1000); continue; } else { throw ex; } } } }
private void DoResumableDownload(DownloadObjectRequest request, ResumableDownloadContext resumableContext, EventHandler <StreamTransferProgressArgs> downloadProgressCallback) { // use single thread if MaxResumableUploadThreads is no bigger than 1 // or the part size is bigger than the conf.MaxPartCachingSize if (resumableContext.PartContextList[0].Length > _conf.MaxPartCachingSize || request.ParallelThreadCount <= 1) { DoResumableDownloadSingleThread(request, resumableContext, downloadProgressCallback); } else { // multi-threaded download the object and write the data DoResumableDownloadMultiThread(request, resumableContext, downloadProgressCallback); } }
private void DoResumableDownloadSingleThread(DownloadObjectRequest request, ResumableDownloadContext resumableContext, EventHandler <StreamTransferProgressArgs> downloadProgressCallback) { _downloadedBytes = resumableContext.GetDownloadedBytes(); if (!request.MatchingETagConstraints.Contains(resumableContext.ETag)) { request.MatchingETagConstraints.Add(resumableContext.ETag); } long totalBytes = resumableContext.GetTotalBytes(); foreach (var part in resumableContext.PartContextList) { if (part.IsCompleted) { continue; } using (Stream fs = File.Open(GetTempDownloadFile(request), FileMode.OpenOrCreate)) { fs.Seek(part.Position, SeekOrigin.Begin); var originalStream = fs; if (downloadProgressCallback != null) { originalStream = _ossClient.SetupDownloadProgressListeners(fs, totalBytes, _downloadedBytes, _conf.ProgressUpdateInterval, downloadProgressCallback); } var getPartRequest = request.ToGetObjectRequest(); getPartRequest.SetRange(part.Position, part.Length + part.Position - 1); var partResult = _ossClient.GetObject(getPartRequest); WriteTo(partResult.Content, originalStream); } part.IsCompleted = true; resumableContext.Dump(); _downloadedBytes += part.Length; } Validate(request, resumableContext); }
private void Validate(DownloadObjectRequest request, ResumableDownloadContext resumableContext) { if (_conf.EnalbeMD5Check && !string.IsNullOrEmpty(resumableContext.ContentMd5)) { using (var fs = File.Open(GetTempDownloadFile(request), FileMode.Open)) { string calcuatedMd5 = OssUtils.ComputeContentMd5(fs, fs.Length); if (calcuatedMd5 != resumableContext.ContentMd5) { throw new OssException(string.Format("The Md5 of the downloaded file {0} does not match the expected. Expected:{1}, actual:{2}", GetTempDownloadFile(request), resumableContext.ContentMd5, calcuatedMd5 )); } } } File.Move(GetTempDownloadFile(request), request.DownloadFile); }
private void Validate(DownloadObjectRequest request, ResumableDownloadContext resumableContext) { if (_conf.EnalbeMD5Check && !string.IsNullOrEmpty(resumableContext.ContentMd5)) { using (var fs = File.Open(GetTempDownloadFile(request), FileMode.Open)) { string calcuatedMd5 = OssUtils.ComputeContentMd5(fs, fs.Length); if (calcuatedMd5 != resumableContext.ContentMd5) { throw new OssException(string.Format("The Md5 of the downloaded file {0} does not match the expected. Expected:{1}, actual:{2}", GetTempDownloadFile(request), resumableContext.ContentMd5, calcuatedMd5 )); } } } else if (_conf.EnableCrcCheck && !string.IsNullOrEmpty(resumableContext.Crc64)) { ulong calculatedCrc = 0; foreach (var part in resumableContext.PartContextList) { calculatedCrc = Crc64.Combine(calculatedCrc, part.Crc64, part.Length); } if (calculatedCrc.ToString() != resumableContext.Crc64) { throw new OssException(string.Format("The Crc64 of the downloaded file {0} does not match the expected. Expected:{1}, actual:{2}", GetTempDownloadFile(request), resumableContext.Crc64, calculatedCrc )); } } File.Move(GetTempDownloadFile(request), request.DownloadFile); }
private void DoResumableDownloadSingleThread(DownloadObjectRequest request, ResumableDownloadContext resumableContext, EventHandler <StreamTransferProgressArgs> downloadProgressCallback) { _downloadedBytes = resumableContext.GetDownloadedBytes(); if (!request.MatchingETagConstraints.Contains(resumableContext.ETag)) { request.MatchingETagConstraints.Add(resumableContext.ETag); } long totalBytes = resumableContext.GetTotalBytes(); foreach (var part in resumableContext.PartContextList) { if (part.IsCompleted) { // is CRC is enabled and part.Crc64 is 0, then redownload the data if (!_conf.EnableCrcCheck || part.Crc64 != 0) { continue; } } using (Stream fs = File.Open(GetTempDownloadFile(request), FileMode.OpenOrCreate)) { fs.Seek(part.Position, SeekOrigin.Begin); var originalStream = fs; if (downloadProgressCallback != null) { originalStream = _ossClient.SetupDownloadProgressListeners(fs, totalBytes, _downloadedBytes, _conf.ProgressUpdateInterval, downloadProgressCallback); } if (_conf.EnableCrcCheck) { originalStream = new Crc64Stream(originalStream, null, part.Length); } var getPartRequest = request.ToGetObjectRequest(); getPartRequest.SetRange(part.Position, part.Length + part.Position - 1); var partResult = _ossClient.GetObject(getPartRequest); WriteTo(partResult.Content, originalStream); if (originalStream is Crc64Stream) { Crc64Stream crcStream = originalStream as Crc64Stream; if (crcStream.CalculatedHash == null) { crcStream.CalculateHash(); } part.Crc64 = BitConverter.ToUInt64(crcStream.CalculatedHash, 0); } } part.IsCompleted = true; resumableContext.Dump(); _downloadedBytes += part.Length; } Validate(request, resumableContext); }
private void DoResumableDownloadMultiThread(DownloadObjectRequest request, ResumableDownloadContext resumableContext, EventHandler <StreamTransferProgressArgs> downloadProgressCallback) { _downloadedBytes = resumableContext.GetDownloadedBytes(); if (!request.MatchingETagConstraints.Contains(resumableContext.ETag)) { request.MatchingETagConstraints.Add(resumableContext.ETag); } Exception e = null; int parallel = Math.Min(Math.Min(request.ParallelThreadCount, resumableContext.PartContextList.Count), Environment.ProcessorCount); ManualResetEvent[] taskFinishedEvents = new ManualResetEvent[parallel]; DownloadTaskParam[] taskParams = new DownloadTaskParam[parallel]; int nextPart = 0; for (nextPart = 0; nextPart < parallel; nextPart++) { var part = resumableContext.PartContextList[nextPart]; taskParams[nextPart] = StartDownloadPartTask(request, part, downloadProgressCallback); taskFinishedEvents[nextPart] = taskParams[nextPart].DownloadFinished; } bool allTasksDone = false; try { long totalBytes = resumableContext.GetTotalBytes(); long lastDownloadedBytes = _downloadedBytes; while (nextPart < resumableContext.PartContextList.Count) { int index = ManualResetEvent.WaitAny(taskFinishedEvents); if (taskParams[index].Error == null) { if (request.StreamTransferProgress != null) { lastDownloadedBytes = _downloadedBytes; StreamTransferProgressArgs args = new StreamTransferProgressArgs(resumableContext.PartContextList[index].Length, lastDownloadedBytes, totalBytes); request.StreamTransferProgress.Invoke(this, args); } resumableContext.Dump(); } else { e = taskParams[index].Error; } taskFinishedEvents[index].Close(); taskParams[index] = StartDownloadPartTask(request, resumableContext.PartContextList[nextPart++], downloadProgressCallback); taskFinishedEvents[index] = taskParams[index].DownloadFinished; } ManualResetEvent.WaitAll(taskFinishedEvents); allTasksDone = true; if (request.StreamTransferProgress != null) { StreamTransferProgressArgs args = new StreamTransferProgressArgs(_downloadedBytes - lastDownloadedBytes, _downloadedBytes, totalBytes); request.StreamTransferProgress.Invoke(this, args); } if (e == null) { Validate(request, resumableContext); } } finally { if (!allTasksDone) { ManualResetEvent.WaitAll(taskFinishedEvents); } for (int i = 0; i < parallel; i++) { taskFinishedEvents[i].Close(); if (taskParams[i].Error != null) { e = taskParams[i].Error; } } if (e != null) { throw e; } } }