internal void CompleteMultipartUpload()
{
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = this.bucketName,
Key = this.fileName,
UploadId = this.initResponse.UploadId,
};
completeRequest.AddPartETags(this.uploadResponses);
this.client.CompleteMultipartUpload(completeRequest);
}
public async Task<string> Complete()
{
var responses = await Task.WhenAll(_blocks).ConfigureAwait(false);
var req = new CompleteMultipartUploadRequest
{
BucketName = _bucket,
Key = _key,
UploadId = await _uploadId.ConfigureAwait(false),
};
req.AddPartETags(responses);
var r = await _client.CompleteMultipartUploadAsync(req, _ct).ConfigureAwait(false);
return r.VersionId;
}
private CompleteMultipartUploadRequest ConstructCompleteMultipartUploadRequest(InitiateMultipartUploadResponse initResponse)
{
var compRequest = new CompleteMultipartUploadRequest()
{
BucketName = this._fileTransporterRequest.BucketName,
Key = this._fileTransporterRequest.Key,
UploadId = initResponse.UploadId
};
compRequest.AddPartETags(this._uploadResponses);
compRequest.BeforeRequestEvent += this.RequestEventHandler;
return compRequest;
}
/// <summary>
/// Send a CompleteMultipartUploadRequest request and return the response.
/// </summary>
/// <param name="existingBucketName"></param>
/// <param name="keyName"></param>
/// <param name="uploadID"></param>
/// <returns>Task<CompleteMultipartUploadRequest></returns>
public async Task<CompleteMultipartUploadRequest> CompleteMultipartUploadAsync(string existingBucketName, string keyName, string uploadId,
CancellationToken token)
{
token.ThrowIfCancellationRequested();
_log.Debug("Called CompleteMultipartUploadAsync with parameters keyName = \"" + keyName + "\" and uploadID = \"" + uploadId + "\".");
try
{
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = existingBucketName,
Key = keyName,
UploadId = uploadId,
};
// in any case request list parts to send etags for each part.
var uploadedParts = await this.ListPartsAsync(existingBucketName, keyName, uploadId, token).ConfigureAwait(false);
List<PartETag> eTags = new List<PartETag>();
foreach (var part in uploadedParts)
{
eTags.Add(new PartETag(part.PartNumber, part.ETag));
}
completeRequest.AddPartETags(eTags);
token.ThrowIfCancellationRequested();
CompleteMultipartUploadResponse completeResponse = await s3Client.CompleteMultipartUploadAsync(completeRequest, token).ConfigureAwait(false);
return completeRequest;
}
catch (Exception e)
{
if (!(e is TaskCanceledException || e is OperationCanceledException))
{
string messagePart = " with parameters keyName = \"" + keyName + "\" and uploadID = \"" + uploadId + "\"";
this.LogAmazonException(messagePart, e);
}
throw;
}
}
private CompleteMultipartUploadRequest ConstructCompleteMultipartUploadRequest(InitiateMultipartUploadResponse initResponse)
{
var compRequest = new CompleteMultipartUploadRequest()
{
BucketName = this._fileTransporterRequest.BucketName,
Key = this._fileTransporterRequest.Key,
UploadId = initResponse.UploadId
};
compRequest.AddPartETags(this._uploadResponses);
((Amazon.Runtime.Internal.IAmazonWebServiceRequest)compRequest).AddBeforeRequestHandler(this.RequestEventHandler);
return compRequest;
}
/// <summary>
/// Runs the multipart upload.
/// </summary>
public override void Execute()
{
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
{
BucketName = this._fileTransporterRequest.BucketName,
Key = this._fileTransporterRequest.Key,
CannedACL = this._fileTransporterRequest.CannedACL,
ContentType = determineContentType(),
StorageClass = this._fileTransporterRequest.StorageClass,
ServerSideEncryptionMethod = this._fileTransporterRequest.ServerSideEncryptionMethod
};
initRequest.BeforeRequestEvent += this.RequestEventHandler;
if (this._fileTransporterRequest.Metadata != null && this._fileTransporterRequest.Metadata.Count > 0)
initRequest.Metadata = this._fileTransporterRequest.Metadata;
if (this._fileTransporterRequest.Headers != null && this._fileTransporterRequest.Headers.Count > 0)
initRequest.Headers = this._fileTransporterRequest.Headers;
InitiateMultipartUploadResponse initResponse = this._s3Client.InitiateMultipartUpload(initRequest);
_logger.DebugFormat("Initiated upload: {0}", initResponse.UploadId);
try
{
_logger.DebugFormat("Queue up the UploadPartRequests to be executed");
long filePosition = 0;
for (int i = 1; filePosition < this._contentLength; i++)
{
UploadPartRequest uploadRequest = new UploadPartRequest()
{
BucketName = this._fileTransporterRequest.BucketName,
Key = this._fileTransporterRequest.Key,
UploadId = initResponse.UploadId,
PartNumber = i,
PartSize = this._partSize,
#if (BCL && !BCL45)
Timeout = ClientConfig.GetTimeoutValue(this._config.DefaultTimeout,this._fileTransporterRequest.Timeout)
#endif
};
if ((filePosition + this._partSize >= this._contentLength) && _s3Client is AmazonS3EncryptionClient)
{
uploadRequest.IsLastPart = true;
uploadRequest.PartSize = 0;
}
uploadRequest.StreamUploadProgressCallback += this.uploadPartProgressEventCallback;
uploadRequest.BeforeRequestEvent += this.RequestEventHandler;
if (this._fileTransporterRequest.IsSetFilePath())
{
uploadRequest.FilePosition = filePosition;
uploadRequest.FilePath = this._fileTransporterRequest.FilePath;
}
else
{
uploadRequest.InputStream = this._fileTransporterRequest.InputStream;
}
this._partsToUpload.Enqueue(uploadRequest);
filePosition += this._partSize;
}
this._totalNumberOfParts = this._partsToUpload.Count;
_logger.DebugFormat("Starting threads to execute the {0} UploadPartRequests in the queue", this._totalNumberOfParts);
startInvokerPool();
_logger.DebugFormat("Waiting for threads to complete. ({0})", initResponse.UploadId);
waitTillAllThreadsComplete();
_logger.DebugFormat("Beginning completing multipart. ({0})", initResponse.UploadId);
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
{
BucketName = this._fileTransporterRequest.BucketName,
Key = this._fileTransporterRequest.Key,
UploadId = initResponse.UploadId
};
compRequest.AddPartETags(this._uploadResponses);
compRequest.BeforeRequestEvent += this.RequestEventHandler;
this._s3Client.CompleteMultipartUpload(compRequest);
_logger.DebugFormat("Done completing multipart. ({0})", initResponse.UploadId);
}
catch (Exception e)
{
_logger.Error(e, "Exception while uploading. ({0})", initResponse.UploadId);
shutdown(initResponse.UploadId);
throw;
}
finally
{
if (this._fileTransporterRequest.InputStream != null && !this._fileTransporterRequest.IsSetFilePath() && this._fileTransporterRequest.AutoCloseStream)
{
this._fileTransporterRequest.InputStream.Close();
}
if (_logger != null)
{
_logger.Flush();
}
}
}
public void MultipartEncryptionTestInstructionFile()
{
string filePath = @"C:\temp\Upload15MegFileIn3PartsViaStream.txt";
string retrievedFilepath = @"C:\temp\Upload15MegFileIn3PartsViaStreamRetreived.txt";
int MEG_SIZE = (int)Math.Pow(2, 20) + 4001;
long totalSize = (long)(15 * MEG_SIZE);
UtilityMethods.GenerateFile(filePath, totalSize);
string key = "MultipartEncryptionTestInstrcutionFile" + random.Next();
s3EncryptionClientFileMode.PutBucket(new PutBucketRequest() { BucketName = bucketName });
Stream inputStream = File.OpenRead(filePath);
try
{
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
StorageClass = S3StorageClass.ReducedRedundancy,
ContentType = "text/html",
CannedACL = S3CannedACL.PublicRead
};
InitiateMultipartUploadResponse initResponse = s3EncryptionClientFileMode.InitiateMultipartUpload(initRequest);
// Upload part 1
UploadPartRequest uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 1,
PartSize = 5 * MEG_SIZE,
InputStream = inputStream
};
UploadPartResponse up1Response = s3EncryptionClientFileMode.UploadPart(uploadRequest);
// Upload part 2
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 2,
PartSize = 5 * MEG_SIZE + 4001,
InputStream = inputStream
};
UploadPartResponse up2Response = s3EncryptionClientFileMode.UploadPart(uploadRequest);
// Upload part 3
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 3,
InputStream = inputStream,
IsLastPart = true
};
//uploadRequest.setLastPart();
UploadPartResponse up3Response = s3EncryptionClientFileMode.UploadPart(uploadRequest);
ListPartsRequest listPartRequest = new ListPartsRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
ListPartsResponse listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest);
Assert.AreEqual(3, listPartResponse.Parts.Count);
Assert.AreEqual(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber);
Assert.AreEqual(up1Response.ETag, listPartResponse.Parts[0].ETag);
Assert.AreEqual(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber);
Assert.AreEqual(up2Response.ETag, listPartResponse.Parts[1].ETag);
Assert.AreEqual(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber);
Assert.AreEqual(up3Response.ETag, listPartResponse.Parts[2].ETag);
listPartRequest.MaxParts = 1;
listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest);
Assert.AreEqual(1, listPartResponse.Parts.Count);
// Complete the response
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
compRequest.AddPartETags(up1Response, up2Response, up3Response);
CompleteMultipartUploadResponse compResponse = s3EncryptionClientFileMode.CompleteMultipartUpload(compRequest);
Assert.AreEqual(bucketName, compResponse.BucketName);
Assert.IsNotNull(compResponse.ETag);
Assert.AreEqual(key, compResponse.Key);
Assert.IsNotNull(compResponse.Location);
// Get the file back from S3 and make sure it is still the same.
GetObjectRequest getRequest = new GetObjectRequest()
{
BucketName = bucketName,
Key = key
};
GetObjectResponse getResponse = s3EncryptionClientFileMode.GetObject(getRequest);
getResponse.WriteResponseStreamToFile(retrievedFilepath);
UtilityMethods.CompareFiles(filePath, retrievedFilepath);
GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest()
{
BucketName = bucketName,
Key = key
};
GetObjectMetadataResponse metaDataResponse = s3EncryptionClientFileMode.GetObjectMetadata(metaDataRequest);
Assert.AreEqual("text/html", metaDataResponse.Headers.ContentType);
}
finally
{
inputStream.Close();
if (File.Exists(filePath))
File.Delete(filePath);
if (File.Exists(retrievedFilepath))
File.Delete(retrievedFilepath);
}
}
