internal void CompleteMultipartUpload() { CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest { BucketName = this.bucketName, Key = this.fileName, UploadId = this.initResponse.UploadId, }; completeRequest.AddPartETags(this.uploadResponses); this.client.CompleteMultipartUpload(completeRequest); }
public async Task<string> Complete() { var responses = await Task.WhenAll(_blocks).ConfigureAwait(false); var req = new CompleteMultipartUploadRequest { BucketName = _bucket, Key = _key, UploadId = await _uploadId.ConfigureAwait(false), }; req.AddPartETags(responses); var r = await _client.CompleteMultipartUploadAsync(req, _ct).ConfigureAwait(false); return r.VersionId; }
private CompleteMultipartUploadRequest ConstructCompleteMultipartUploadRequest(InitiateMultipartUploadResponse initResponse) { var compRequest = new CompleteMultipartUploadRequest() { BucketName = this._fileTransporterRequest.BucketName, Key = this._fileTransporterRequest.Key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(this._uploadResponses); compRequest.BeforeRequestEvent += this.RequestEventHandler; return compRequest; }
/// <summary> /// Send a CompleteMultipartUploadRequest request and return the response. /// </summary> /// <param name="existingBucketName"></param> /// <param name="keyName"></param> /// <param name="uploadID"></param> /// <returns>Task<CompleteMultipartUploadRequest></returns> public async Task<CompleteMultipartUploadRequest> CompleteMultipartUploadAsync(string existingBucketName, string keyName, string uploadId, CancellationToken token) { token.ThrowIfCancellationRequested(); _log.Debug("Called CompleteMultipartUploadAsync with parameters keyName = \"" + keyName + "\" and uploadID = \"" + uploadId + "\"."); try { CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest { BucketName = existingBucketName, Key = keyName, UploadId = uploadId, }; // in any case request list parts to send etags for each part. var uploadedParts = await this.ListPartsAsync(existingBucketName, keyName, uploadId, token).ConfigureAwait(false); List<PartETag> eTags = new List<PartETag>(); foreach (var part in uploadedParts) { eTags.Add(new PartETag(part.PartNumber, part.ETag)); } completeRequest.AddPartETags(eTags); token.ThrowIfCancellationRequested(); CompleteMultipartUploadResponse completeResponse = await s3Client.CompleteMultipartUploadAsync(completeRequest, token).ConfigureAwait(false); return completeRequest; } catch (Exception e) { if (!(e is TaskCanceledException || e is OperationCanceledException)) { string messagePart = " with parameters keyName = \"" + keyName + "\" and uploadID = \"" + uploadId + "\""; this.LogAmazonException(messagePart, e); } throw; } }
private CompleteMultipartUploadRequest ConstructCompleteMultipartUploadRequest(InitiateMultipartUploadResponse initResponse) { var compRequest = new CompleteMultipartUploadRequest() { BucketName = this._fileTransporterRequest.BucketName, Key = this._fileTransporterRequest.Key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(this._uploadResponses); ((Amazon.Runtime.Internal.IAmazonWebServiceRequest)compRequest).AddBeforeRequestHandler(this.RequestEventHandler); return compRequest; }
/// <summary> /// Runs the multipart upload. /// </summary> public override void Execute() { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = this._fileTransporterRequest.BucketName, Key = this._fileTransporterRequest.Key, CannedACL = this._fileTransporterRequest.CannedACL, ContentType = determineContentType(), StorageClass = this._fileTransporterRequest.StorageClass, ServerSideEncryptionMethod = this._fileTransporterRequest.ServerSideEncryptionMethod }; initRequest.BeforeRequestEvent += this.RequestEventHandler; if (this._fileTransporterRequest.Metadata != null && this._fileTransporterRequest.Metadata.Count > 0) initRequest.Metadata = this._fileTransporterRequest.Metadata; if (this._fileTransporterRequest.Headers != null && this._fileTransporterRequest.Headers.Count > 0) initRequest.Headers = this._fileTransporterRequest.Headers; InitiateMultipartUploadResponse initResponse = this._s3Client.InitiateMultipartUpload(initRequest); _logger.DebugFormat("Initiated upload: {0}", initResponse.UploadId); try { _logger.DebugFormat("Queue up the UploadPartRequests to be executed"); long filePosition = 0; for (int i = 1; filePosition < this._contentLength; i++) { UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = this._fileTransporterRequest.BucketName, Key = this._fileTransporterRequest.Key, UploadId = initResponse.UploadId, PartNumber = i, PartSize = this._partSize, #if (BCL && !BCL45) Timeout = ClientConfig.GetTimeoutValue(this._config.DefaultTimeout,this._fileTransporterRequest.Timeout) #endif }; if ((filePosition + this._partSize >= this._contentLength) && _s3Client is AmazonS3EncryptionClient) { uploadRequest.IsLastPart = true; uploadRequest.PartSize = 0; } uploadRequest.StreamUploadProgressCallback += this.uploadPartProgressEventCallback; uploadRequest.BeforeRequestEvent += this.RequestEventHandler; if (this._fileTransporterRequest.IsSetFilePath()) { uploadRequest.FilePosition = filePosition; uploadRequest.FilePath = this._fileTransporterRequest.FilePath; } else { uploadRequest.InputStream = this._fileTransporterRequest.InputStream; } this._partsToUpload.Enqueue(uploadRequest); filePosition += this._partSize; } this._totalNumberOfParts = this._partsToUpload.Count; _logger.DebugFormat("Starting threads to execute the {0} UploadPartRequests in the queue", this._totalNumberOfParts); startInvokerPool(); _logger.DebugFormat("Waiting for threads to complete. ({0})", initResponse.UploadId); waitTillAllThreadsComplete(); _logger.DebugFormat("Beginning completing multipart. ({0})", initResponse.UploadId); CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = this._fileTransporterRequest.BucketName, Key = this._fileTransporterRequest.Key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(this._uploadResponses); compRequest.BeforeRequestEvent += this.RequestEventHandler; this._s3Client.CompleteMultipartUpload(compRequest); _logger.DebugFormat("Done completing multipart. ({0})", initResponse.UploadId); } catch (Exception e) { _logger.Error(e, "Exception while uploading. ({0})", initResponse.UploadId); shutdown(initResponse.UploadId); throw; } finally { if (this._fileTransporterRequest.InputStream != null && !this._fileTransporterRequest.IsSetFilePath() && this._fileTransporterRequest.AutoCloseStream) { this._fileTransporterRequest.InputStream.Close(); } if (_logger != null) { _logger.Flush(); } } }
public void MultipartEncryptionTestInstructionFile() { string filePath = @"C:\temp\Upload15MegFileIn3PartsViaStream.txt"; string retrievedFilepath = @"C:\temp\Upload15MegFileIn3PartsViaStreamRetreived.txt"; int MEG_SIZE = (int)Math.Pow(2, 20) + 4001; long totalSize = (long)(15 * MEG_SIZE); UtilityMethods.GenerateFile(filePath, totalSize); string key = "MultipartEncryptionTestInstrcutionFile" + random.Next(); s3EncryptionClientFileMode.PutBucket(new PutBucketRequest() { BucketName = bucketName }); Stream inputStream = File.OpenRead(filePath); try { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, StorageClass = S3StorageClass.ReducedRedundancy, ContentType = "text/html", CannedACL = S3CannedACL.PublicRead }; InitiateMultipartUploadResponse initResponse = s3EncryptionClientFileMode.InitiateMultipartUpload(initRequest); // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MEG_SIZE, InputStream = inputStream }; UploadPartResponse up1Response = s3EncryptionClientFileMode.UploadPart(uploadRequest); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MEG_SIZE + 4001, InputStream = inputStream }; UploadPartResponse up2Response = s3EncryptionClientFileMode.UploadPart(uploadRequest); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, IsLastPart = true }; //uploadRequest.setLastPart(); UploadPartResponse up3Response = s3EncryptionClientFileMode.UploadPart(uploadRequest); ListPartsRequest listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest); Assert.AreEqual(3, listPartResponse.Parts.Count); Assert.AreEqual(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber); Assert.AreEqual(up1Response.ETag, listPartResponse.Parts[0].ETag); Assert.AreEqual(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber); Assert.AreEqual(up2Response.ETag, listPartResponse.Parts[1].ETag); Assert.AreEqual(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber); Assert.AreEqual(up3Response.ETag, listPartResponse.Parts[2].ETag); listPartRequest.MaxParts = 1; listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest); Assert.AreEqual(1, listPartResponse.Parts.Count); // Complete the response CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); CompleteMultipartUploadResponse compResponse = s3EncryptionClientFileMode.CompleteMultipartUpload(compRequest); Assert.AreEqual(bucketName, compResponse.BucketName); Assert.IsNotNull(compResponse.ETag); Assert.AreEqual(key, compResponse.Key); Assert.IsNotNull(compResponse.Location); // Get the file back from S3 and make sure it is still the same. GetObjectRequest getRequest = new GetObjectRequest() { BucketName = bucketName, Key = key }; GetObjectResponse getResponse = s3EncryptionClientFileMode.GetObject(getRequest); getResponse.WriteResponseStreamToFile(retrievedFilepath); UtilityMethods.CompareFiles(filePath, retrievedFilepath); GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest() { BucketName = bucketName, Key = key }; GetObjectMetadataResponse metaDataResponse = s3EncryptionClientFileMode.GetObjectMetadata(metaDataRequest); Assert.AreEqual("text/html", metaDataResponse.Headers.ContentType); } finally { inputStream.Close(); if (File.Exists(filePath)) File.Delete(filePath); if (File.Exists(retrievedFilepath)) File.Delete(retrievedFilepath); } }