/// <summary>
        /// Inititate a new Multipart upload request
        /// </summary>
        /// <param name="putObjectRequest"></param>
        /// <returns></returns>
        public async Task <MultipartManifest> NewRequest(
            string contentEncoding, string contentType,
            string contentLanguage, string contentDisposition,
            string cacheControl, Dictionary <string, string> metadata)
        {
            CheckInitialized();
            var ifNoneMatch = _allowOverwrite ? null : "*";

            var request = new CreateMultipartUploadRequest()
            {
                BucketName    = _bucketName,
                NamespaceName = _namespaceName,
                IfNoneMatch   = ifNoneMatch,
                CreateMultipartUploadDetails = new CreateMultipartUploadDetails()
                {
                    Object             = _objectName,
                    ContentEncoding    = contentEncoding,
                    ContentType        = contentType,
                    ContentLanguage    = contentLanguage,
                    Metadata           = metadata,
                    ContentDisposition = contentDisposition,
                    CacheControl       = cacheControl,
                },
                OpcClientRequestId = CreateClientRequestId("New")
            };

            var response = await _service.CreateMultipartUpload(request).ConfigureAwait(false);

            _multipartManifest = new MultipartManifest(response.MultipartUpload.UploadId);
            _transferManager   = new MultipartTransferManager(_multipartManifest, _service, _executorServiceToUse, _tokenSource.Token, _retryConfiguration);
            _initialized       = true;

            return(_multipartManifest);
        }
Exemplo n.º 2
0
        public void Verify()
        {
            var failedParts = new List <int> {
                1, 2, 3, 4, 5
            };
            var completedParts = new List <int> {
                6, 7, 8, 9, 10
            };

            var manifest = new MultipartManifest("DummyId");

            foreach (var i in Enumerable.Range(1, 10))
            {
                manifest.RegisterTransfer(i);
            }

            foreach (var i in completedParts)
            {
                manifest.RegisterSuccess(i, new Responses.UploadPartResponse());
            }

            foreach (var i in failedParts)
            {
                manifest.RegisterFailure(i);
            }

            Assert.True(!manifest.ListCompletedParts()
                        .Select(part => part.PartNum.Value)
                        .Except(completedParts).Any());

            Assert.True(!manifest.ListFailedParts()
                        .Except(failedParts).Any());
        }
Exemplo n.º 3
0
 public MultipartTransferManager(MultipartManifest manifest,
                                 ObjectStorageClient osClient,
                                 SemaphoreSlim executor,
                                 CancellationToken cancellationToken   = default,
                                 RetryConfiguration retryConfiguration = null)
 {
     this.Manifest            = manifest;
     this.OSClient            = osClient;
     this.Executor            = executor;
     this._cancellationToken  = cancellationToken;
     this._retryConfiguration = retryConfiguration;
 }
        /// <summary>
        /// Resumes an existing multi-part upload using the upload ID.
        /// </summary>
        /// <param name="uploadId"></param>
        /// <returns>MultipartManifest</returns>
        public async Task <MultipartManifest> ResumeUpload(string uploadId)
        {
            CheckInitialized();
            var multipartUpload = FindUpload(uploadId);

            if (multipartUpload == null)
            {
                _logger.Error($"No matching upload found for ${uploadId}");
                throw new ArgumentException($"Could not find existing upload with ID: {uploadId} in bucket: {_bucketName} and NamespaceName: {_namespaceName}");
            }

            var    multipartManifest = new MultipartManifest(uploadId);
            string nextPageToken     = null;

            do
            {
                var partsRequest = new ListMultipartUploadPartsRequest()
                {
                    BucketName         = _bucketName,
                    NamespaceName      = _namespaceName,
                    ObjectName         = _objectName,
                    UploadId           = uploadId,
                    Limit              = 100,
                    Page               = nextPageToken,
                    OpcClientRequestId = CreateClientRequestId("List")
                };

                var existingParts = await _service.ListMultipartUploadParts(partsRequest, _retryConfiguration).ConfigureAwait(false);

                nextPageToken = existingParts.OpcNextPage;

                foreach (var part in existingParts.Items)
                {
                    multipartManifest.RegisterExisting(part);
                }
            } while (nextPageToken != null);

            _multipartManifest = multipartManifest;
            _transferManager   = new MultipartTransferManager(_multipartManifest, _service, _executorServiceToUse, _tokenSource.Token, _retryConfiguration);
            _initialized       = true;

            return(multipartManifest);
        }
        private async Task <UploadResponse> MultipartUpload(UploadRequest uploadRequest, bool isNewUpload = true)
        {
            MultipartManifest manifest = null;
            var putobjectRequest       = uploadRequest.PutObjectRequest;
            var chunkCreator           = new StreamChunkCreator(putobjectRequest.PutObjectBody, _configuration.LengthPerUploadPartInMiB * MultipartUtils.MiB);

            // Uses single thread for multipart uploads.
            var executor = new SemaphoreSlim(_configuration.ParallelUploadCount);

            var assembler = new MultipartObjectAssembler(_osClient,
                                                         putobjectRequest.NamespaceName,
                                                         putobjectRequest.BucketName,
                                                         putobjectRequest.ObjectName,
                                                         uploadRequest.AllowOverwrite,
                                                         executor,
                                                         putobjectRequest.OpcClientRequestId,
                                                         _configuration.EnforceMd5MultipartUpload,
                                                         uploadRequest.RetryConfiguration);

            try
            {
                if (isNewUpload && !(uploadRequest is ResumeUploadRequest))
                {
                    var putObjectRequest = uploadRequest.PutObjectRequest;
                    manifest = await assembler.NewRequest(
                        putobjectRequest.ContentEncoding, putobjectRequest.ContentType,
                        putobjectRequest.ContentLanguage, putobjectRequest.ContentDisposition,
                        putobjectRequest.CacheControl, putobjectRequest.OpcMeta);
                }
                else
                {
                    manifest = await assembler.ResumeUpload((uploadRequest as ResumeUploadRequest).UploadId).ConfigureAwait(false);
                }

                ProgressTracker progressTracker = null;

                if (uploadRequest.OnProgress != null)
                {
                    progressTracker = new ProgressTracker(uploadRequest.OnProgress, putobjectRequest.PutObjectBody.Length);
                }

                int partNum = 1;
                foreach (var stream in chunkCreator)
                {
                    if (!manifest.IsPartExists(partNum))
                    {
                        logger.Info($"Creating Chunk: {partNum}");

                        if (progressTracker != null)
                        {
                            var progressStream = new ProgressTrackingInputStream(stream, progressTracker);
                            await assembler.AddPart(progressStream, stream.Length, partNum).ConfigureAwait(false);
                        }
                        else
                        {
                            await assembler.AddPart(stream, stream.Length, partNum).ConfigureAwait(false);
                        }
                    }
                    else
                    {
                        if (progressTracker != null)
                        {
                            progressTracker.OnRead(stream.Length);
                        }
                        logger.Info($"Skipping Part {partNum}");
                    }
                    partNum++;
                }

                var response = await assembler.Commit().ConfigureAwait(false);

                logger.Info("Upload Complete");

                return(new UploadResponse()
                {
                    ETag = response.ETag,
                    OpcClientRequestId = response.OpcClientRequestId,
                    OpcRequestId = response.OpcRequestId,
                    OpcMultipartMd5 = response.OpcMultipartMd5
                });
            }
            catch (Exception e)
            {
                if (manifest != null)
                {
                    logger.Error($"Failed to upload object using multi-part uploads.  Failed part numbers = '{string.Join(" ", manifest.ListFailedParts())}'" +
                                 $".  Successful parts = '{string.Join(" ", manifest.ListCompletedParts())}', Upload Id = '{manifest.UploadId}'");
                }
                if (_configuration.DisableAutoAbort)
                {
                    logger.Info($"Not aborting failed multipart upload {manifest.UploadId} per configuration, client must manually abort it");
                }
                else
                {
                    logger.Info("Aborting in-progress uploads");
                    await assembler.Abort();
                }
                throw e;
            }
            finally
            {
                executor.Dispose();
            }
        }