コード例 #1
0
        public void StreamChunkVerification()
        {
            var inputStr     = "aaaaaaaaaa";
            var sourceStream = GenerateStreamFromString(inputStr);
            var chunkCreator = new StreamChunkCreator(sourceStream, 2);

            var builder = new StringBuilder();
            var i       = 0;

            foreach (var stream in chunkCreator)
            {
                builder.Append(GetStringFromStream(stream));
                i++;
            }

            Assert.True(inputStr.Equals(builder.ToString()));
            Assert.Equal(inputStr.Length / 2, i);
        }
コード例 #2
0
        private async Task <UploadResponse> MultipartUpload(UploadRequest uploadRequest, bool isNewUpload = true)
        {
            MultipartManifest manifest = null;
            var putobjectRequest       = uploadRequest.PutObjectRequest;
            var chunkCreator           = new StreamChunkCreator(putobjectRequest.PutObjectBody, _configuration.LengthPerUploadPartInMiB * MultipartUtils.MiB);

            // Uses single thread for multipart uploads.
            var executor = new SemaphoreSlim(_configuration.ParallelUploadCount);

            var assembler = new MultipartObjectAssembler(_osClient,
                                                         putobjectRequest.NamespaceName,
                                                         putobjectRequest.BucketName,
                                                         putobjectRequest.ObjectName,
                                                         uploadRequest.AllowOverwrite,
                                                         executor,
                                                         putobjectRequest.OpcClientRequestId,
                                                         _configuration.EnforceMd5MultipartUpload,
                                                         uploadRequest.RetryConfiguration);

            try
            {
                if (isNewUpload && !(uploadRequest is ResumeUploadRequest))
                {
                    var putObjectRequest = uploadRequest.PutObjectRequest;
                    manifest = await assembler.NewRequest(
                        putobjectRequest.ContentEncoding, putobjectRequest.ContentType,
                        putobjectRequest.ContentLanguage, putobjectRequest.ContentDisposition,
                        putobjectRequest.CacheControl, putobjectRequest.OpcMeta);
                }
                else
                {
                    manifest = await assembler.ResumeUpload((uploadRequest as ResumeUploadRequest).UploadId).ConfigureAwait(false);
                }

                ProgressTracker progressTracker = null;

                if (uploadRequest.OnProgress != null)
                {
                    progressTracker = new ProgressTracker(uploadRequest.OnProgress, putobjectRequest.PutObjectBody.Length);
                }

                int partNum = 1;
                foreach (var stream in chunkCreator)
                {
                    if (!manifest.IsPartExists(partNum))
                    {
                        logger.Info($"Creating Chunk: {partNum}");

                        if (progressTracker != null)
                        {
                            var progressStream = new ProgressTrackingInputStream(stream, progressTracker);
                            await assembler.AddPart(progressStream, stream.Length, partNum).ConfigureAwait(false);
                        }
                        else
                        {
                            await assembler.AddPart(stream, stream.Length, partNum).ConfigureAwait(false);
                        }
                    }
                    else
                    {
                        if (progressTracker != null)
                        {
                            progressTracker.OnRead(stream.Length);
                        }
                        logger.Info($"Skipping Part {partNum}");
                    }
                    partNum++;
                }

                var response = await assembler.Commit().ConfigureAwait(false);

                logger.Info("Upload Complete");

                return(new UploadResponse()
                {
                    ETag = response.ETag,
                    OpcClientRequestId = response.OpcClientRequestId,
                    OpcRequestId = response.OpcRequestId,
                    OpcMultipartMd5 = response.OpcMultipartMd5
                });
            }
            catch (Exception e)
            {
                if (manifest != null)
                {
                    logger.Error($"Failed to upload object using multi-part uploads.  Failed part numbers = '{string.Join(" ", manifest.ListFailedParts())}'" +
                                 $".  Successful parts = '{string.Join(" ", manifest.ListCompletedParts())}', Upload Id = '{manifest.UploadId}'");
                }
                if (_configuration.DisableAutoAbort)
                {
                    logger.Info($"Not aborting failed multipart upload {manifest.UploadId} per configuration, client must manually abort it");
                }
                else
                {
                    logger.Info("Aborting in-progress uploads");
                    await assembler.Abort();
                }
                throw e;
            }
            finally
            {
                executor.Dispose();
            }
        }