public void MultipartUploadProgressTest()
{
// Initiate a multipart upload
var initRequest = new InitiateMultipartUploadRequest(_bucketName, _bigObjectKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
// Set the part size
const int partSize = 1024 * 1024 * 1;
var partFile = new FileInfo(Config.MultiUploadTestFile);
// Calculate the part count
var partCount = OssTestUtils.CalculatePartCount(partFile.Length, partSize);
// Create a list to save the result
var partETags = new List <PartETag>();
//upload the file
using (var fs = new FileStream(partFile.FullName, FileMode.Open))
{
for (var i = 0; i < partCount; i++)
{
// Skip to the start position
long skipBytes = partSize * i;
fs.Position = skipBytes;
// Calculate the part size
var size = partSize < partFile.Length - skipBytes
? partSize
: partFile.Length - skipBytes;
// Create a UploadPartRequest, uploading the part
var uploadPartRequest = new UploadPartRequest(_bucketName, _bigObjectKey, initResult.UploadId)
{
InputStream = fs,
PartSize = size,
PartNumber = (i + 1)
};
uploadPartRequest.StreamTransferProgress += uploadProgressCallback;
var uploadPartResult = _ossClient.UploadPart(uploadPartRequest);
// Save the result
partETags.Add(uploadPartResult.PartETag);
}
}
// Complete the multipart upload
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, _bigObjectKey, initResult.UploadId);
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
_ossClient.CompleteMultipartUpload(completeRequest);
// Download the file to check the hash digest
OssTestUtils.DownloadObject(_ossClient, _bucketName, _bigObjectKey, _tmpLocalFile);
var expectedHashDigest = FileUtils.ComputeContentMd5(Config.MultiUploadTestFile);
var actualHashDigest = FileUtils.ComputeContentMd5(_tmpLocalFile);
Assert.AreEqual(expectedHashDigest, actualHashDigest);
_ossClient.DeleteObject(_bucketName, _bigObjectKey);
File.Delete(_tmpLocalFile);
}
private async Task <bool> UploadObjectBandWidthThrottlingAsync(string bucketName, string localFullFilename, string key, CancellationToken token, int sleepMiliSec, bool publicReadTrueOrFalse = false)
{
bool bReturn = true;
List <UploadPartResponse> uploadResponses = new List <UploadPartResponse>();
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = key
};
if (publicReadTrueOrFalse)
{
initiateRequest.CannedACL = S3CannedACL.PublicRead;
}
IAmazonS3 s3Client = new AmazonS3Client(AccessKey, SecretKey, new AmazonS3Config
{
ServiceURL = ServiceUrl,
BufferSize = 64 * (int)Math.Pow(2, 10),
ProgressUpdateInterval = this.ProgressUpdateInterval,
Timeout = new TimeSpan(1, 0, 0, 0, 0)
});
InitiateMultipartUploadResponse initResponse =
await s3Client.InitiateMultipartUploadAsync(initiateRequest, token);
long contentLength = new FileInfo(localFullFilename).Length;
ContentTotalBytes = contentLength;
long partSize = 5 * (long)Math.Pow(2, 20);
try
{
long filePosition = 0;
for (int i = 1; filePosition < contentLength; i++)
{
if (token.IsCancellationRequested)
{
bReturn = false;
break;
}
var task = Task.Run(() => Thread.Sleep(sleepMiliSec));
await task;
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = i,
PartSize = partSize,
FilePosition = filePosition,
FilePath = localFullFilename
};
uploadRequest.StreamTransferProgress += ProgressEventCallback;
uploadResponses.Add(await s3Client.UploadPartAsync(uploadRequest, token));
filePosition += partSize;
CurrentTransferredBytes = filePosition;
}
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
completeRequest.AddPartETags(uploadResponses);
CompleteMultipartUploadResponse completeUploadResponse =
await s3Client.CompleteMultipartUploadAsync(completeRequest, token);
}
catch (Exception)
{
AbortMultipartUploadRequest abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
await s3Client.AbortMultipartUploadAsync(abortMPURequest);
throw;
}
finally
{
s3Client.Dispose();
}
return(bReturn);
}
private async Task MultipartUploadObject(FileInfo fileInfo, string objectKey, PartSize partSize)
{
if (fileInfo is null)
{
throw new ArgumentNullException(nameof(fileInfo));
}
if (partSize is null)
{
throw new ArgumentNullException(nameof(partSize));
}
var multipartUploadRequest = new InitiateMultipartUploadRequest()
{
BucketName = _bucketName,
Key = objectKey,
};
await Initialize().ConfigureAwait(false);
var multipartUploadResponse = await Client.InitiateMultipartUploadAsync(multipartUploadRequest).ConfigureAwait(false);
var a = (fileInfo.Length > partSize) ? partSize : (int)fileInfo.Length;
try
{
var taskList = new List <Task <UploadPartResponse> >();
var partResponses = new List <UploadPartResponse>();
for (var i = 0; partSize *i < fileInfo.Length; i++)
{
var upload = new UploadPartRequest()
{
BucketName = _bucketName,
Key = objectKey,
UploadId = multipartUploadResponse.UploadId,
PartNumber = i + 1,
PartSize = a,
FilePosition = partSize * i,
FilePath = fileInfo.FullName,
};
if ((i + 1) * partSize > fileInfo.Length)
{
a = (int)fileInfo.Length % partSize;
}
if (taskList.Count >= _parallelParts)
{
await Task.WhenAny(taskList).ConfigureAwait(false);
foreach (var task in taskList)
{
if (task.IsCompleted)
{
partResponses.Add(await task.ConfigureAwait(false));
taskList.Remove(task);
}
}
}
else
{
taskList.Add(Client.UploadPartAsync(upload));
}
}
foreach (var task in taskList)
{
partResponses.Add(await task.ConfigureAwait(false));
}
var completeRequest = new CompleteMultipartUploadRequest
{
BucketName = _bucketName,
Key = objectKey,
UploadId = multipartUploadResponse.UploadId,
};
completeRequest.AddPartETags(partResponses);
var completeUploadResponse = await Client.CompleteMultipartUploadAsync(completeRequest).ConfigureAwait(false);
}
catch
{
var abortMultipartUploadRequest = new AbortMultipartUploadRequest
{
BucketName = _bucketName,
Key = objectKey,
UploadId = multipartUploadResponse.UploadId,
};
await Client.AbortMultipartUploadAsync(abortMultipartUploadRequest).ConfigureAwait(false);
throw;
}
}
internal override void Execute()
{
FileInfo fileInfo = new FileInfo(filePath);
FileStream fileStream = File.OpenRead(filePath);
string uploadId = null;
try
{
this.currentUploadProgressArgs = new StreamTransferProgressArgs(0, 0, fileInfo.Length);
long partSize = CalculatePartSize(fileInfo.Length);
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest()
{
AccountId = this.options.AccountId,
ArchiveDescription = archiveDescription,
VaultName = vaultName,
PartSize = partSize
};
((Amazon.Runtime.Internal.IAmazonWebServiceRequest)initiateRequest).AddBeforeRequestHandler(new UserAgentPostFix("MultiUpload").UserAgentRequestEventHandlerSync);
InitiateMultipartUploadResponse initiateResponse = this.manager.GlacierClient.InitiateMultipartUpload(initiateRequest);
uploadId = initiateResponse.UploadId;
List <string> partTreeHashs = new List <string>();
long currentPosition = 0;
while (currentPosition < fileInfo.Length)
{
long length = partSize;
if (currentPosition + partSize > fileInfo.Length)
{
length = fileInfo.Length - currentPosition;
}
Stream partStream = new PartialWrapperStream(fileStream, length);
string checksum = TreeHashGenerator.CalculateTreeHash(partStream);
partTreeHashs.Add(checksum);
UploadMultipartPartRequest uploadRequest = new UploadMultipartPartRequest()
{
AccountId = this.options.AccountId,
Checksum = checksum,
Range = "bytes " + currentPosition + "-" + (currentPosition + length - 1) + "/*",
UploadId = uploadId,
VaultName = vaultName,
Body = partStream
};
uploadRequest.StreamTransferProgress += this.ProgressCallback;
((Amazon.Runtime.Internal.IAmazonWebServiceRequest)uploadRequest).AddBeforeRequestHandler(new UserAgentPostFix("MultiUpload").UserAgentRequestEventHandlerSync);
this.manager.GlacierClient.UploadMultipartPart(uploadRequest);
currentPosition += partSize;
}
string totalFileChecksum = TreeHashGenerator.CalculateTreeHash(partTreeHashs);
string archiveSize = fileInfo.Length.ToString(CultureInfo.InvariantCulture);
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
{
AccountId = this.options.AccountId,
ArchiveSize = archiveSize,
VaultName = vaultName,
Checksum = totalFileChecksum,
UploadId = uploadId
};
((Amazon.Runtime.Internal.IAmazonWebServiceRequest)compRequest).AddBeforeRequestHandler(new UserAgentPostFix("MultiUpload").UserAgentRequestEventHandlerSync);
CompleteMultipartUploadResponse completeMultipartUploadResponse = this.manager.GlacierClient.CompleteMultipartUpload(compRequest);
string archiveId = completeMultipartUploadResponse.ArchiveId;
this.UploadResult = new UploadResult(archiveId, totalFileChecksum);
}
catch (Exception)
{
// If we got an unrecoverable then abort the upload.
if (!string.IsNullOrEmpty(uploadId))
{
AbortMultipartUploadRequest abortRequest = new AbortMultipartUploadRequest()
{
AccountId = this.options.AccountId,
VaultName = this.vaultName,
UploadId = uploadId
};
((Amazon.Runtime.Internal.IAmazonWebServiceRequest)abortRequest).AddBeforeRequestHandler(new UserAgentPostFix("MultiUpload").UserAgentRequestEventHandlerSync);
this.manager.GlacierClient.AbortMultipartUpload(abortRequest);
}
throw;
}
finally
{
try { fileStream.Close(); }
catch (Exception) { }
}
}
public InitiateMultipartUploadResponse InitiateMultipartUpload(InitiateMultipartUploadRequest request)
{
throw new NotImplementedException();
}
private InitiateMultipartUploadRequest requestFromCtrls()
{
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest();
// Set storage class from control values
if (RadioReducedRedund.Checked)
{
request.StorageClass = S3StorageClass.ReducedRedundancy;
}
else if (RadioStandardIA.Checked)
{
request.StorageClass = S3StorageClass.StandardInfrequentAccess;
}
else
{
request.StorageClass = S3StorageClass.Standard;
}
// Set other fields from control values
request.WebsiteRedirectLocation = TxtWebsite.Text;
request.RequestPayer = (ChkRequestPayer.Checked ? RequestPayer.Requester : null);
// Set metadata from control values
IEnumerable <DataGridViewRow> metaRows = DgvMetadata.Rows.Cast <DataGridViewRow>().Where(r => !r.IsNewRow);
foreach (DataGridViewRow row in metaRows)
{
string key = row.Cells[DgvColMetadataKey.Index].Value.ToString();
string val = row.Cells[DgvColMetadataValue.Index].Value.ToString();
request.Metadata.Add(key, val);
}
// Set headers from control values
request.Headers.ContentType = TxtType.Text;
request.Headers.ContentDisposition = TxtDisposition.Text;
request.Headers.ContentEncoding = TxtEncoding.Text;
request.Headers.Expires = DatePickerExpires.Value.ToUniversalTime();
// Initialize access control
bool useCannedAcl = ChkUseCannedACLs.Checked;
request.CannedACL = (useCannedAcl ? S3CannedACL.FindValue(ComboAcl.Text) : null);
if (useCannedAcl)
{
request.Grants = null;
}
else
{
IEnumerable <S3Grant> grants = DgvGrants.Rows.Cast <DataGridViewRow>()
.Where(r => !r.IsNewRow)
.SelectMany(r => grantsFromRow(r));
request.Grants.AddRange(grants);
}
// Initialize server side encryption method
bool kms = RadioSseKms.Checked;
bool newKey = RadioSseNewKey.Checked;
if (kms)
{
request.ServerSideEncryptionMethod = ServerSideEncryptionMethod.AWSKMS;
}
else if (newKey)
{
request.ServerSideEncryptionMethod = ServerSideEncryptionMethod.AES256;
}
else
{
request.ServerSideEncryptionMethod = ServerSideEncryptionMethod.None;
}
// Set server side encryption from control values
request.ServerSideEncryptionKeyManagementServiceKeyId = (kms ? TxtSseKeyId.Text : null);
request.ServerSideEncryptionCustomerMethod = (newKey ? ServerSideEncryptionCustomerMethod.AES256 : ServerSideEncryptionCustomerMethod.None);
request.ServerSideEncryptionCustomerProvidedKey = (newKey ? TxtSseCustomerKey.Text : null);
request.ServerSideEncryptionCustomerProvidedKeyMD5 = (newKey ? TxtSseCustomerKeyMd5.Text : null);
return(request);
}
private static async Task MPUCopyObjectAsync()
{
// Create a list to store the upload part responses.
List <UploadPartResponse> uploadResponses = new List <UploadPartResponse>();
List <CopyPartResponse> copyResponses = new List <CopyPartResponse>();
// Setup information required to initiate the multipart upload.
InitiateMultipartUploadRequest initiateRequest =
new InitiateMultipartUploadRequest
{
BucketName = targetBucket,
Key = targetObjectKey
};
// Initiate the upload.
InitiateMultipartUploadResponse initResponse =
await s3Client.InitiateMultipartUploadAsync(initiateRequest);
// Save the upload ID.
String uploadId = initResponse.UploadId;
try
{
// Get the size of the object.
GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest
{
BucketName = sourceBucket,
Key = sourceObjectKey
};
GetObjectMetadataResponse metadataResponse =
await s3Client.GetObjectMetadataAsync(metadataRequest);
long objectSize = metadataResponse.ContentLength; // Length in bytes.
// Copy the parts.
long partSize = 5 * (long)Math.Pow(2, 20); // Part size is 5 MB.
long bytePosition = 0;
for (int i = 1; bytePosition < objectSize; i++)
{
CopyPartRequest copyRequest = new CopyPartRequest
{
DestinationBucket = targetBucket,
DestinationKey = targetObjectKey,
SourceBucket = sourceBucket,
SourceKey = sourceObjectKey,
UploadId = uploadId,
FirstByte = bytePosition,
LastByte = bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1,
PartNumber = i
};
copyResponses.Add(await s3Client.CopyPartAsync(copyRequest));
bytePosition += partSize;
}
// Set up to complete the copy.
CompleteMultipartUploadRequest completeRequest =
new CompleteMultipartUploadRequest
{
BucketName = targetBucket,
Key = targetObjectKey,
UploadId = initResponse.UploadId
};
completeRequest.AddPartETags(copyResponses);
// Complete the copy.
CompleteMultipartUploadResponse completeUploadResponse =
await s3Client.CompleteMultipartUploadAsync(completeRequest);
}
catch (AmazonS3Exception e)
{
Console.WriteLine("Error encountered on server. Message:'{0}' when writing an object", e.Message);
}
catch (Exception e)
{
Console.WriteLine("Unknown encountered on server. Message:'{0}' when writing an object", e.Message);
}
}
public void MultipartUploadPartCopyComplexStepTest()
{
//get target object name
var targetObjectKey = OssTestUtils.GetObjectKey(_className);
var initRequest = new InitiateMultipartUploadRequest(_bucketName, targetObjectKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
// Set the part size
const int partSize = 1024 * 512 * 1;
var sourceObjectMeta = _ossClient.GetObjectMetadata(_bucketName, _sourceObjectKey);
// Calculate the part count
var partCount = OssTestUtils.CalculatePartCount(sourceObjectMeta.ContentLength, partSize);
LogUtility.LogMessage("Object {0} is splitted to {1} parts for multipart upload part copy",
_sourceObjectKey, partCount);
// Create a list to save result
var partETags = new List <PartETag>();
for (var i = 0; i < partCount; i++)
{
// Skip to the start position
long skipBytes = partSize * i;
// calculate the part size
var size = partSize < sourceObjectMeta.ContentLength - skipBytes
? partSize
: sourceObjectMeta.ContentLength - skipBytes;
// Create a UploadPartRequest, uploading parts
var uploadPartCopyRequest =
new UploadPartCopyRequest(_bucketName, targetObjectKey, _bucketName, _sourceObjectKey, initResult.UploadId)
{
BeginIndex = skipBytes,
PartSize = size,
PartNumber = (i + 1),
ModifiedSinceConstraint = DateTime.Now.AddDays(-1)
};
uploadPartCopyRequest.MatchingETagConstraints.Add(_objectETag);
var uploadPartCopyResult = _ossClient.UploadPartCopy(uploadPartCopyRequest);
// Save the result
partETags.Add(uploadPartCopyResult.PartETag);
}
var lmuRequest = new ListMultipartUploadsRequest(_bucketName);
var lmuListing = _ossClient.ListMultipartUploads(lmuRequest);
string mpUpload = null;
foreach (var t in lmuListing.MultipartUploads)
{
if (t.UploadId == initResult.UploadId)
{
mpUpload = t.UploadId;
break;
}
}
Assert.IsNotNull(mpUpload, "The multipart uploading should be in progress");
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, targetObjectKey, initResult.UploadId);
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
_ossClient.CompleteMultipartUpload(completeRequest);
Assert.IsTrue(OssTestUtils.ObjectExists(_ossClient, _bucketName, targetObjectKey));
//delete the object
_ossClient.DeleteObject(_bucketName, targetObjectKey);
}
/// <summary>
/// Sets metadata and headers collections for the request.
/// </summary>
/// <param name="request"></param>
/// <param name="metadata"></param>
/// <param name="headers"></param>
public static void SetMetadataAndHeaders(InitiateMultipartUploadRequest request, Hashtable metadata, Hashtable headers)
{
SetMetadata(request.Metadata, metadata);
SetHeaders(request.Headers, headers);
}
public static void MultipartUploadProgress(string bucketName)
{
const string key = "MultipartUploadProgress";
try
{
// 初始化分片上传任务
var initRequest = new InitiateMultipartUploadRequest(bucketName, key);
var initResult = client.InitiateMultipartUpload(initRequest);
// 设置每块为 1M
const int partSize = 1024 * 1024 * 1;
var partFile = new FileInfo(Config.FileToUpload);
// 计算分块数目
var partCount = CalculatePartCount(partFile.Length, partSize);
// 新建一个List保存每个分块上传后的ETag和PartNumber
var partETags = new List <PartETag>();
//upload the file
using (var fs = new FileStream(partFile.FullName, FileMode.Open))
{
for (var i = 0; i < partCount; i++)
{
// 跳到每个分块的开头
long skipBytes = partSize * i;
fs.Position = skipBytes;
// 计算每个分块的大小
var size = partSize < partFile.Length - skipBytes ? partSize : partFile.Length - skipBytes;
// 创建UploadPartRequest，上传分块
var uploadPartRequest = new UploadPartRequest(bucketName, key, initResult.UploadId)
{
InputStream = fs,
PartSize = size,
PartNumber = (i + 1)
};
uploadPartRequest.StreamTransferProgress += streamProgressCallback;
var uploadPartResult = client.UploadPart(uploadPartRequest);
// 将返回的PartETag保存到List中。
partETags.Add(uploadPartResult.PartETag);
}
}
// 提交上传任务
var completeRequest = new CompleteMultipartUploadRequest(bucketName, key, initResult.UploadId);
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
client.CompleteMultipartUpload(completeRequest);
Console.WriteLine("Multipart upload object:{0} succeeded", key);
}
catch (OssException ex)
{
Console.WriteLine("Failed with error code: {0}; Error info: {1}. \nRequestID:{2}\tHostID:{3}",
ex.ErrorCode, ex.Message, ex.RequestId, ex.HostId);
}
catch (Exception ex)
{
Console.WriteLine("Failed with error info: {0}", ex.Message);
}
}
public void MultipartUploadCallbackVarTest()
{
// Initiate a multipart upload
var initRequest = new InitiateMultipartUploadRequest(_bucketName, _bigObjectKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
// Sets the part size as 1M
const int partSize = 1024 * 1024 * 1;
var partFile = new FileInfo(Config.MultiUploadTestFile);
// Calculates the part count
var partCount = OssTestUtils.CalculatePartCount(partFile.Length, partSize);
// creates a list of PartETag
var partETags = new List <PartETag>();
//upload the file
using (var fs = new FileStream(partFile.FullName, FileMode.Open))
{
for (var i = 0; i < partCount; i++)
{
// skip to the start of each part
long skipBytes = partSize * i;
fs.Position = skipBytes;
// calculates the part size
var size = partSize < partFile.Length - skipBytes
? partSize
: partFile.Length - skipBytes;
// uploads the part
var uploadPartRequest = new UploadPartRequest(_bucketName, _bigObjectKey, initResult.UploadId)
{
InputStream = fs,
PartSize = size,
PartNumber = (i + 1)
};
var uploadPartResult = _ossClient.UploadPart(uploadPartRequest);
// adds the result to the list
partETags.Add(uploadPartResult.PartETag);
}
}
string callbackHeaderBuilder = new CallbackHeaderBuilder(_callbackUrl, _callbackBody).Build();
string CallbackVariableHeaderBuilder = new CallbackVariableHeaderBuilder().
AddCallbackVariable("x:var1", "x:value1").AddCallbackVariable("x:var2", "x:value2").Build();
var metadata = new ObjectMetadata();
metadata.AddHeader(HttpHeaders.Callback, callbackHeaderBuilder);
metadata.AddHeader(HttpHeaders.CallbackVar, CallbackVariableHeaderBuilder);
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, _bigObjectKey, initResult.UploadId);
completeRequest.Metadata = metadata;
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
var completeMultipartUploadResult = _ossClient.CompleteMultipartUpload(completeRequest);
Assert.IsTrue(completeMultipartUploadResult.IsSetResponseStream());
Assert.AreEqual(_callbackOkResponse, GetCallbackResponse(completeMultipartUploadResult));
Assert.AreEqual(completeMultipartUploadResult.HttpStatusCode, HttpStatusCode.OK);
Assert.AreEqual(completeMultipartUploadResult.ContentLength, _callbackOkResponse.Length);
Assert.AreEqual(completeMultipartUploadResult.RequestId.Length, "58DB0ACB686D42D5B4163D75".Length);
Assert.AreEqual(completeMultipartUploadResult.ResponseMetadata[HttpHeaders.ContentType], "application/json");
Assert.AreEqual(completeMultipartUploadResult.ResponseMetadata[HttpHeaders.ETag], completeMultipartUploadResult.ETag);
Assert.IsFalse(completeMultipartUploadResult.ResponseMetadata.ContainsKey(HttpHeaders.ContentMd5));
Assert.IsTrue(completeMultipartUploadResult.ResponseMetadata[HttpHeaders.HashCrc64Ecma].Length > 0);
Assert.IsTrue(completeMultipartUploadResult.ResponseMetadata[HttpHeaders.ServerElapsedTime].Length > 0);
Assert.AreEqual(completeMultipartUploadResult.ResponseMetadata[HttpHeaders.Date].Length, "Wed, 29 Mar 2017 01:15:58 GMT".Length);
//delete the object
_ossClient.DeleteObject(_bucketName, _bigObjectKey);
}
//[Test]
public void ListMultipartUploadsWithHiddenCharacters()
{
char[] buffer = new char[2];
buffer[0] = Convert.ToChar(0x1c);
buffer[1] = Convert.ToChar(0x1a);
var newKey = _keyName + (new string(buffer)) + ".1.cd";
try
{
var initRequest = new InitiateMultipartUploadRequest(_bucketName, newKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
Assert.AreEqual(newKey, initRequest.Key);
// 设置每块为 5M
const int partSize = 5 * 1024 * 1024;
var fileInfo = new FileInfo(Config.MultiUploadSampleFile);
// 计算分块数目
var partCount = OssTestUtils.CalculatePartCount(fileInfo.Length, partSize);
// 新建一个List保存每个分块上传后的ETag和PartNumber
var partETags = new List <PartETag>();
//upload the file
using (var fs = new FileStream(fileInfo.FullName, FileMode.Open))
{
for (var i = 0; i < partCount; i++)
{
// 跳到每个分块的开头
long skipBytes = partSize * i;
fs.Position = skipBytes;
// 计算每个分块的大小
long size = partSize < fileInfo.Length - skipBytes ? partSize : fileInfo.Length - skipBytes;
// 创建UploadPartRequest，上传分块
var uploadPartRequest = new UploadPartRequest(_bucketName, newKey, initResult.UploadId);
uploadPartRequest.InputStream = fs;
uploadPartRequest.PartSize = size;
uploadPartRequest.PartNumber = (i + 1);
var uploadPartResult = _ossClient.UploadPart(uploadPartRequest);
// 将返回的PartETag保存到List中。
partETags.Add(uploadPartResult.PartETag);
var multipartListing = _ossClient.ListMultipartUploads(new ListMultipartUploadsRequest(_bucketName));
foreach (var upload in multipartListing.MultipartUploads)
{
Console.WriteLine(upload.ToString());
Assert.AreEqual(newKey, upload.Key);
}
}
}
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, newKey, initResult.UploadId);
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
_ossClient.CompleteMultipartUpload(completeRequest);
}
finally
{
var multipartListing = _ossClient.ListMultipartUploads(new ListMultipartUploadsRequest(_bucketName));
foreach (var upload in multipartListing.MultipartUploads)
{
var abortRequest = new AbortMultipartUploadRequest(_bucketName, newKey, upload.UploadId);
_ossClient.AbortMultipartUpload(abortRequest);
}
_ossClient.DeleteObject(_bucketName, newKey);
}
}
public static async Task MultipartEncryptionTestAsync(AmazonS3Client s3EncryptionClient,
AmazonS3Client s3DecryptionClient, string bucketName)
{
var filePath = Path.GetTempFileName();
var retrievedFilepath = Path.GetTempFileName();
var totalSize = MegaBytesSize * 15;
UtilityMethods.GenerateFile(filePath, totalSize);
var key = Guid.NewGuid().ToString();
Stream inputStream = File.OpenRead(filePath);
try
{
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
StorageClass = S3StorageClass.OneZoneInfrequentAccess,
ContentType = "text/html",
};
InitiateMultipartUploadResponse initResponse =
await s3EncryptionClient.InitiateMultipartUploadAsync(initRequest).ConfigureAwait(false);
// Upload part 1
UploadPartRequest uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 1,
PartSize = 5 * MegaBytesSize,
InputStream = inputStream,
};
UploadPartResponse up1Response =
await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false);
// Upload part 2
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 2,
PartSize = 5 * MegaBytesSize,
InputStream = inputStream
};
UploadPartResponse up2Response =
await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false);
// Upload part 3
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 3,
InputStream = inputStream,
IsLastPart = true
};
UploadPartResponse up3Response =
await s3EncryptionClient.UploadPartAsync(uploadRequest).ConfigureAwait(false);
ListPartsRequest listPartRequest = new ListPartsRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
ListPartsResponse listPartResponse =
await s3EncryptionClient.ListPartsAsync(listPartRequest).ConfigureAwait(false);
Assert.Equal(3, listPartResponse.Parts.Count);
Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber);
Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag);
Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber);
Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag);
Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber);
Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag);
listPartRequest.MaxParts = 1;
listPartResponse = await s3EncryptionClient.ListPartsAsync(listPartRequest).ConfigureAwait(false);
Assert.Single(listPartResponse.Parts);
// Complete the response
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
compRequest.AddPartETags(up1Response, up2Response, up3Response);
CompleteMultipartUploadResponse compResponse =
await s3EncryptionClient.CompleteMultipartUploadAsync(compRequest).ConfigureAwait(false);
Assert.Equal(bucketName, compResponse.BucketName);
Assert.NotNull(compResponse.ETag);
Assert.Equal(key, compResponse.Key);
Assert.NotNull(compResponse.Location);
// Get the file back from S3 and make sure it is still the same.
GetObjectRequest getRequest = new GetObjectRequest()
{
BucketName = bucketName,
Key = key
};
GetObjectResponse getResponse =
await s3DecryptionClient.GetObjectAsync(getRequest).ConfigureAwait(false);
await getResponse.WriteResponseStreamToFileAsync(retrievedFilepath, false, CancellationToken.None);
UtilityMethods.CompareFiles(filePath, retrievedFilepath);
GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest()
{
BucketName = bucketName,
Key = key
};
GetObjectMetadataResponse metaDataResponse =
await s3DecryptionClient.GetObjectMetadataAsync(metaDataRequest).ConfigureAwait(false);
Assert.Equal("text/html", metaDataResponse.Headers.ContentType);
}
finally
{
inputStream.Dispose();
if (File.Exists(filePath))
{
File.Delete(filePath);
}
if (File.Exists(retrievedFilepath))
{
File.Delete(retrievedFilepath);
}
}
}
/// <summary>
/// Copies large file as chunks from Azure BLOB to Amazon S3.
/// </summary>
/// <returns></returns>
public async Task CopyLargeFileFromAzureBlobToAwsS3()
{
AmazonS3Client s3Client = new AmazonS3Client(AwsAccessKeyId, AwsSecretKey, Amazon.RegionEndpoint.APSouth1);
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(StorageAccount); //Create Storage account reference.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Create the blob client.
CloudBlobContainer container = blobClient.GetContainerReference(ContainerName); // Retrieve reference to a container.
container.CreateIfNotExists();
CloudBlockBlob blob = container.GetBlockBlobReference(BlobFileName); // Create Blob reference.
blob.FetchAttributes(); // Prepare blob instance To get the file length.
var remainingBytes = blob.Properties.Length;
long readPosition = 0; // To be used offset / position from where to start reading from BLOB.
InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest
{
BucketName = AwsS3BucketName,
Key = TargetFileName
};
// Will use UploadId from this response.
InitiateMultipartUploadResponse initiateMultipartUploadResponse = s3Client.InitiateMultipartUpload(initiateMultipartUploadRequest);
List <UploadPartResponse> uploadPartResponses = new List <UploadPartResponse>();
Stopwatch stopwatch = Stopwatch.StartNew();
try
{
int partCounter = 0; // To increment on each read of parts and use it as part number.
while (remainingBytes > 0)
{
// Determine the size when final block reached as it might be less than Part size.
// Will be PartSize except final block.
long bytesToCopy = Math.Min(PartSize, remainingBytes);
using (MemoryStream memoryStream = new MemoryStream())
{
// To download part from BLOB.
await blob.DownloadRangeToStreamAsync(memoryStream, readPosition, bytesToCopy).ConfigureAwait(false);
memoryStream.Position = 0;
partCounter++;
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = AwsS3BucketName,
Key = TargetFileName,
UploadId = initiateMultipartUploadResponse.UploadId,
PartNumber = partCounter,
PartSize = bytesToCopy,
InputStream = memoryStream
};
UploadPartResponse uploadPartResponse = s3Client.UploadPart(uploadRequest);
uploadPartResponses.Add(uploadPartResponse);
remainingBytes -= bytesToCopy;
readPosition += bytesToCopy;
this.logger.WriteLine($"Uploaded part with part number {partCounter}, size {bytesToCopy}bytes and remaining {remainingBytes}bytes to read.");
}
}
CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest
{
BucketName = AwsS3BucketName,
Key = TargetFileName,
UploadId = initiateMultipartUploadResponse.UploadId
};
completeMultipartUploadRequest.AddPartETags(uploadPartResponses);
CompleteMultipartUploadResponse completeMultipartUploadResponse = await s3Client.CompleteMultipartUploadAsync(completeMultipartUploadRequest).ConfigureAwait(false);
}
catch (Exception exception)
{
this.logger.WriteLine($"Exception : {exception.Message}");
AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest
{
BucketName = AwsS3BucketName,
Key = TargetFileName,
UploadId = initiateMultipartUploadResponse.UploadId
};
await s3Client.AbortMultipartUploadAsync(abortMultipartUploadRequest).ConfigureAwait(false);
}
finally
{
stopwatch.Stop();
this.logger.WriteLine($"Execution time in mins: {stopwatch.Elapsed.TotalMinutes}");
}
}
public void MutiPartUpload(string fileName, string key)
{
try
{
setTexItems("开始上传:" + key);
InitiateMultipartUploadRequest initRequest =
new InitiateMultipartUploadRequest(ossConfig.BucketName, key);
InitiateMultipartUploadResult initResult = ossClient.InitiateMultipartUpload(initRequest);
// 设置每块为 1M
int partSize = 1024 * 1024 * 1;
FileInfo partFile = new FileInfo(fileName);
// 计算分块数目
int partCount = (int)(partFile.Length / partSize);
if (partFile.Length % partSize != 0)
{
partCount++;
}
setTexItems("数据分块上传，一共:" + partCount + "块");
// 新建一个List保存每个分块上传后的ETag和PartNumber
List <PartETag> partETags = new List <PartETag>();
for (int i = 0; i < partCount; i++)
{
int number = i + 1;
setTexItems("第" + number + "块,上传开始");
// 获取文件流
FileStream fis = new FileStream(partFile.FullName, FileMode.Open);
// 跳到每个分块的开头
long skipBytes = partSize * i;
fis.Position = skipBytes;
//fis.skip(skipBytes);
// 计算每个分块的大小
long size = partSize < partFile.Length - skipBytes ?
partSize : partFile.Length - skipBytes;
// 创建UploadPartRequest，上传分块
UploadPartRequest uploadPartRequest = new UploadPartRequest(ossConfig.BucketName, key, initResult.UploadId);
uploadPartRequest.InputStream = fis;
uploadPartRequest.PartSize = size;
uploadPartRequest.PartNumber = (i + 1);
UploadPartResult uploadPartResult = ossClient.UploadPart(uploadPartRequest);
// 将返回的PartETag保存到List中。
partETags.Add(uploadPartResult.PartETag);
// 关闭文件
fis.Close();
setTexItems("第" + number + "块,上传完毕");
}
CompleteMultipartUploadRequest completeReq = new CompleteMultipartUploadRequest(ossConfig.BucketName, key, initResult.UploadId);
foreach (PartETag partETag in partETags)
{
completeReq.PartETags.Add(partETag);
}
// 红色标注的是与JAVA的SDK有区别的地方
//完成分块上传
setTexItems("合并数据块开始");
CompleteMultipartUploadResult completeResult = ossClient.CompleteMultipartUpload(completeReq);
setTexItems("合并数据块结束");
// 返回最终文件的MD5，用于用户进行校验
setTexItems(key + " 上传成功.");
}
catch (Exception ex)
{
logHelper.Error(ex.Message);
}
}
private static async Task CopyObjectAsync(IAmazonS3 s3Client, string base64Key)
{
List <CopyPartResponse> uploadResponses = new List <CopyPartResponse>();
// 1. Initialize.
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = existingBucketName,
Key = targetKeyName,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = base64Key,
};
InitiateMultipartUploadResponse initResponse =
await s3Client.InitiateMultipartUploadAsync(initiateRequest);
// 2. Upload Parts.
long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB
long firstByte = 0;
long lastByte = partSize;
try
{
// First find source object size. Because object is stored encrypted with
// customer provided key you need to provide encryption information in your request.
GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest()
{
BucketName = existingBucketName,
Key = sourceKeyName,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = base64Key // " * **source object encryption key ***"
};
GetObjectMetadataResponse getObjectMetadataResponse = await s3Client.GetObjectMetadataAsync(getObjectMetadataRequest);
long filePosition = 0;
for (int i = 1; filePosition < getObjectMetadataResponse.ContentLength; i++)
{
CopyPartRequest copyPartRequest = new CopyPartRequest
{
UploadId = initResponse.UploadId,
// Source.
SourceBucket = existingBucketName,
SourceKey = sourceKeyName,
// Source object is stored using SSE-C. Provide encryption information.
CopySourceServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
CopySourceServerSideEncryptionCustomerProvidedKey = base64Key, //"***source object encryption key ***",
FirstByte = firstByte,
// If the last part is smaller then our normal part size then use the remaining size.
LastByte = lastByte > getObjectMetadataResponse.ContentLength ?
getObjectMetadataResponse.ContentLength - 1 : lastByte,
// Target.
DestinationBucket = existingBucketName,
DestinationKey = targetKeyName,
PartNumber = i,
// Encryption information for the target object.
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = base64Key
};
uploadResponses.Add(await s3Client.CopyPartAsync(copyPartRequest));
filePosition += partSize;
firstByte += partSize;
lastByte += partSize;
}
// Step 3: complete.
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = existingBucketName,
Key = targetKeyName,
UploadId = initResponse.UploadId,
};
completeRequest.AddPartETags(uploadResponses);
CompleteMultipartUploadResponse completeUploadResponse =
await s3Client.CompleteMultipartUploadAsync(completeRequest);
}
catch (Exception exception)
{
Console.WriteLine("Exception occurred: {0}", exception.Message);
AbortMultipartUploadRequest abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = existingBucketName,
Key = targetKeyName,
UploadId = initResponse.UploadId
};
s3Client.AbortMultipartUpload(abortMPURequest);
}
}
public void MultipartUploadWithoutCallbackTest()
{
// 初始化分片上传任务
var initRequest = new InitiateMultipartUploadRequest(_bucketName, _bigObjectKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
Assert.AreEqual(initResult.HttpStatusCode, HttpStatusCode.OK);
Assert.IsTrue(initResult.ContentLength > 0);
Assert.AreEqual(initResult.RequestId.Length, "58DB0ACB686D42D5B4163D75".Length);
Assert.AreEqual(initResult.ResponseMetadata[HttpHeaders.ContentType], "application/xml");
Assert.IsFalse(initResult.ResponseMetadata.ContainsKey(HttpHeaders.ETag));
Assert.IsFalse(initResult.ResponseMetadata.ContainsKey(HttpHeaders.ContentMd5));
Assert.IsFalse(initResult.ResponseMetadata.ContainsKey(HttpHeaders.HashCrc64Ecma));
Assert.IsTrue(initResult.ResponseMetadata[HttpHeaders.ServerElapsedTime].Length > 0);
Assert.AreEqual(initResult.ResponseMetadata[HttpHeaders.Date].Length, "Wed, 29 Mar 2017 01:15:58 GMT".Length);
// 设置每块为 1M
const int partSize = 1024 * 1024 * 1;
var partFile = new FileInfo(Config.MultiUploadTestFile);
// 计算分块数目
var partCount = OssTestUtils.CalculatePartCount(partFile.Length, partSize);
// 新建一个List保存每个分块上传后的ETag和PartNumber
var partETags = new List <PartETag>();
//upload the file
using (var fs = new FileStream(partFile.FullName, FileMode.Open))
{
for (var i = 0; i < partCount; i++)
{
// 跳到每个分块的开头
long skipBytes = partSize * i;
fs.Position = skipBytes;
// 计算每个分块的大小
var size = partSize < partFile.Length - skipBytes
? partSize
: partFile.Length - skipBytes;
// 创建UploadPartRequest，上传分块
var uploadPartRequest = new UploadPartRequest(_bucketName, _bigObjectKey, initResult.UploadId)
{
InputStream = fs,
PartSize = size,
PartNumber = (i + 1)
};
var uploadPartResult = _ossClient.UploadPart(uploadPartRequest);
Assert.AreEqual(uploadPartResult.HttpStatusCode, HttpStatusCode.OK);
Assert.AreEqual(uploadPartResult.ContentLength, 0);
Assert.AreEqual(uploadPartResult.RequestId.Length, "58DB0ACB686D42D5B4163D75".Length);
Assert.IsFalse(uploadPartResult.ResponseMetadata.ContainsKey(HttpHeaders.ContentType));
Assert.AreEqual(uploadPartResult.ResponseMetadata[HttpHeaders.ETag], uploadPartResult.ETag);
Assert.AreEqual(uploadPartResult.ResponseMetadata[HttpHeaders.ContentMd5].Length, "7GoU4OYaYroKXsbsG1f/lw==".Length);
Assert.IsTrue(uploadPartResult.ResponseMetadata[HttpHeaders.ServerElapsedTime].Length > 0);
Assert.AreEqual(uploadPartResult.ResponseMetadata[HttpHeaders.Date].Length, "Wed, 29 Mar 2017 01:15:58 GMT".Length);
// 将返回的PartETag保存到List中。
partETags.Add(uploadPartResult.PartETag);
}
}
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, _bigObjectKey, initResult.UploadId);
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
var completeMultipartUploadResult = _ossClient.CompleteMultipartUpload(completeRequest);
Assert.IsFalse(completeMultipartUploadResult.IsSetResponseStream());
Assert.AreEqual(completeMultipartUploadResult.HttpStatusCode, HttpStatusCode.OK);
Assert.IsTrue(completeMultipartUploadResult.ContentLength > 0);
Assert.AreEqual(completeMultipartUploadResult.RequestId.Length, "58DB0ACB686D42D5B4163D75".Length);
Assert.AreEqual(completeMultipartUploadResult.ResponseMetadata[HttpHeaders.ContentType], "application/xml");
Assert.AreEqual(completeMultipartUploadResult.ResponseMetadata[HttpHeaders.ETag], completeMultipartUploadResult.ETag);
Assert.IsFalse(completeMultipartUploadResult.ResponseMetadata.ContainsKey(HttpHeaders.ContentMd5));
Assert.IsTrue(completeMultipartUploadResult.ResponseMetadata[HttpHeaders.HashCrc64Ecma].Length > 0);
Assert.IsTrue(completeMultipartUploadResult.ResponseMetadata[HttpHeaders.ServerElapsedTime].Length > 0);
Assert.AreEqual(completeMultipartUploadResult.ResponseMetadata[HttpHeaders.Date].Length, "Wed, 29 Mar 2017 01:15:58 GMT".Length);
//delete the object
_ossClient.DeleteObject(_bucketName, _bigObjectKey);
}
public void CompleteMultipartUploadWithListParts()
{
var sourceFile = Config.MultiUploadTestFile;
//get target object name
var targetObjectKey = OssTestUtils.GetObjectKey(_className);
var initRequest = new InitiateMultipartUploadRequest(_bucketName, targetObjectKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
// Set the part size
const int partSize = 1024 * 1024 * 1;
var partFile = new FileInfo(sourceFile);
// Calculate the part count
var partCount = OssTestUtils.CalculatePartCount(partFile.Length, partSize);
LogUtility.LogMessage("File {0} is splitted to {1} parts for multipart upload",
sourceFile, partCount);
//upload the file
using (var fs = new FileStream(partFile.FullName, FileMode.Open))
{
for (var i = 0; i < partCount; i++)
{
// Skip to the start position
long skipBytes = partSize * i;
fs.Position = skipBytes;
// calculate the part size
var size = partSize < partFile.Length - skipBytes
? partSize
: partFile.Length - skipBytes;
// Create a UploadPartRequest, uploading parts
var uploadPartRequest = new UploadPartRequest(_bucketName, targetObjectKey, initResult.UploadId)
{
InputStream = fs,
PartSize = size,
PartNumber = (i + 1)
};
_ossClient.UploadPart(uploadPartRequest);
}
}
var lmuRequest = new ListMultipartUploadsRequest(_bucketName);
var lmuListing = _ossClient.ListMultipartUploads(lmuRequest);
string mpUpload = null;
foreach (var t in lmuListing.MultipartUploads)
{
if (t.UploadId == initResult.UploadId)
{
mpUpload = t.UploadId;
}
}
Assert.IsNotNull(mpUpload, "The multipart uploading should be in progress");
PartListing partList = _ossClient.ListParts(new ListPartsRequest(_bucketName, targetObjectKey, mpUpload));
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, targetObjectKey, initResult.UploadId);
foreach (var part in partList.Parts)
{
completeRequest.PartETags.Add(part.PartETag);
}
_ossClient.CompleteMultipartUpload(completeRequest);
Assert.IsTrue(OssTestUtils.ObjectExists(_ossClient, _bucketName, targetObjectKey));
//delete the object
_ossClient.DeleteObject(_bucketName, targetObjectKey);
}
public void MultipartUploadCallbackUriInvalidNegativeTest()
{
// 初始化分片上传任务
var initRequest = new InitiateMultipartUploadRequest(_bucketName, _bigObjectKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
// 设置每块为 1M
const int partSize = 1024 * 1024 * 1;
var partFile = new FileInfo(Config.MultiUploadTestFile);
// 计算分块数目
var partCount = OssTestUtils.CalculatePartCount(partFile.Length, partSize);
// 新建一个List保存每个分块上传后的ETag和PartNumber
var partETags = new List <PartETag>();
//upload the file
using (var fs = new FileStream(partFile.FullName, FileMode.Open))
{
for (var i = 0; i < partCount; i++)
{
// 跳到每个分块的开头
long skipBytes = partSize * i;
fs.Position = skipBytes;
// 计算每个分块的大小
var size = partSize < partFile.Length - skipBytes
? partSize
: partFile.Length - skipBytes;
// 创建UploadPartRequest，上传分块
var uploadPartRequest = new UploadPartRequest(_bucketName, _bigObjectKey, initResult.UploadId)
{
InputStream = fs,
PartSize = size,
PartNumber = (i + 1)
};
var uploadPartResult = _ossClient.UploadPart(uploadPartRequest);
// 将返回的PartETag保存到List中。
partETags.Add(uploadPartResult.PartETag);
}
}
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, _bigObjectKey, initResult.UploadId);
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
string callbackHeaderBuilder = new CallbackHeaderBuilder("https://wwww.aliyun.com/", _callbackBody).Build();
var metadata = new ObjectMetadata();
metadata.AddHeader(HttpHeaders.Callback, callbackHeaderBuilder);
completeRequest.Metadata = metadata;
try
{
_ossClient.CompleteMultipartUpload(completeRequest);
Assert.Fail("Multipart upload callback should be not successfully with invald callback uri");
}
catch (OssException e)
{
Assert.AreEqual(OssErrorCode.CallbackFailed, e.ErrorCode);
Assert.AreEqual("Error status : 502.", e.Message);
}
finally
{
_ossClient.DeleteObject(_bucketName, _bigObjectKey);
}
}
public static void Main(string[] args)
{
// create the ECS S3 client
ECSS3Client s3 = ECSS3Factory.getS3Client();
// retrieve the object key/value from user
Console.Write("Enter the object key: ");
string key = Console.ReadLine();
Console.Write("Enter the file location: ");
string filePath = Console.ReadLine();
// grab the start time of upload
DateTime startDate = DateTime.Now;
// part size for chunking in multi-part
long partSize = 1024 * 1024 * 8; // 2 MB
// list of upload part response objects for each part that is uploaded
List <PartETag> partETags = new List <PartETag>();
// Step 1: Initialize
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
{
BucketName = ECSS3Factory.S3_BUCKET,
Key = key,
};
InitiateMultipartUploadResponse initResponse = s3.InitiateMultipartUpload(initRequest);
// get the file and file length
long contentLength = new FileInfo(filePath).Length;
Console.WriteLine(string.Format("Starting multi-part upload for object {0}/{1} with file path {2} and size {3} in {4} MB size chunks",
ECSS3Factory.S3_BUCKET, key, filePath, Convert.ToString(contentLength), partSize / 1024 / 1024));
try
{
// Step 2: Upload parts
long filePosition = 0;
for (int i = 1; filePosition < contentLength; i++)
{
// get the size of the chunk. Note - the last part can be less than the chunk size
partSize = Math.Min(partSize, (contentLength - filePosition));
Console.WriteLine(string.Format("Sending chunk {0} starting at position {1}", i, filePosition));
// create request to upload a part
UploadPartRequest uploadRequest = new UploadPartRequest()
{
BucketName = ECSS3Factory.S3_BUCKET,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = i,
FilePosition = filePosition,
FilePath = filePath,
PartSize = partSize
};
UploadPartResponse partResponse = s3.UploadPart(uploadRequest);
PartETag eTagPart = new PartETag(partResponse.PartNumber, partResponse.ETag);
partETags.Add(eTagPart);
filePosition = filePosition += partSize;
}
// Step 3: complete
Console.WriteLine("Waiting for completion of multi-part upload");
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
{
BucketName = ECSS3Factory.S3_BUCKET,
Key = key,
UploadId = initResponse.UploadId,
PartETags = partETags
};
s3.CompleteMultipartUpload(compRequest);
}
catch (Exception e)
{
s3.AbortMultipartUpload(new AbortMultipartUploadRequest()
{
BucketName = ECSS3Factory.S3_BUCKET,
Key = key,
UploadId = initResponse.UploadId
});
Console.WriteLine(e);
}
// grab the end time of upload
DateTime endDate = DateTime.Now;
Console.WriteLine(string.Format("Completed multi-part upload for object {0}/{1} with file path: {2}", ECSS3Factory.S3_BUCKET, key, filePath));
Console.WriteLine(string.Format("Process took: {0} seconds.", (endDate - startDate).TotalSeconds.ToString()));
Console.ReadLine();
}
private static void Main(string[] args)
{
BosClient client = SampleUtil.GetSampleBosClient();
const string bucketName = "sample-bucket-multi-upload1"; //示例Bucket名称
// 初始化：创建示例Bucket
client.CreateBucket(bucketName); //指定Bucket名称
string objectKey = "sample";
// 1.开始Multipart Upload
InitiateMultipartUploadRequest initiateMultipartUploadRequest =
new InitiateMultipartUploadRequest()
{
BucketName = bucketName, Key = objectKey
};
InitiateMultipartUploadResponse initiateMultipartUploadResponse =
client.InitiateMultipartUpload(initiateMultipartUploadRequest);
// 2.获取Bucket内的Multipart Upload
ListMultipartUploadsRequest listMultipartUploadsRequest =
new ListMultipartUploadsRequest()
{
BucketName = bucketName
};
ListMultipartUploadsResponse listMultipartUploadsResponse =
client.ListMultipartUploads(listMultipartUploadsRequest);
foreach (MultipartUploadSummary multipartUpload in listMultipartUploadsResponse.Uploads)
{
Console.WriteLine("Key: " + multipartUpload.Key + " UploadId: " + multipartUpload.UploadId);
}
// 3.分块上传，首先设置每块为 5Mb
long partSize = 1024 * 1024 * 5L;
FileInfo partFile = new FileInfo("d:\\lzb\\sample");
// 计算分块数目
int partCount = (int)(partFile.Length / partSize);
if (partFile.Length % partSize != 0)
{
partCount++;
}
// 新建一个List保存每个分块上传后的ETag和PartNumber
List <PartETag> partETags = new List <PartETag>();
for (int i = 0; i < partCount; i++)
{
// 获取文件流
Stream stream = partFile.OpenRead();
// 跳到每个分块的开头
long skipBytes = partSize * i;
stream.Seek(skipBytes, SeekOrigin.Begin);
// 计算每个分块的大小
long size = Math.Min(partSize, partFile.Length - skipBytes);
// 创建UploadPartRequest，上传分块
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.BucketName = bucketName;
uploadPartRequest.Key = objectKey;
uploadPartRequest.UploadId = initiateMultipartUploadResponse.UploadId;
uploadPartRequest.InputStream = stream;
uploadPartRequest.PartSize = size;
uploadPartRequest.PartNumber = i + 1;
UploadPartResponse uploadPartResponse = client.UploadPart(uploadPartRequest);
// 将返回的PartETag保存到List中。
partETags.Add(new PartETag()
{
ETag = uploadPartResponse.ETag,
PartNumber = uploadPartResponse.PartNumber
});
// 关闭文件
stream.Close();
}
// 4. 获取UploadId的所有Upload Part
ListPartsRequest listPartsRequest = new ListPartsRequest()
{
BucketName = bucketName,
Key = objectKey,
UploadId = initiateMultipartUploadResponse.UploadId,
};
// 获取上传的所有Part信息
ListPartsResponse listPartsResponse = client.ListParts(listPartsRequest);
// 遍历所有Part
foreach (PartSummary part in listPartsResponse.Parts)
{
Console.WriteLine("PartNumber: " + part.PartNumber + " ETag: " + part.ETag);
}
// 5. 完成分块上传
CompleteMultipartUploadRequest completeMultipartUploadRequest =
new CompleteMultipartUploadRequest()
{
BucketName = bucketName,
Key = objectKey,
UploadId = initiateMultipartUploadResponse.UploadId,
PartETags = partETags
};
CompleteMultipartUploadResponse completeMultipartUploadResponse =
client.CompleteMultipartUpload(completeMultipartUploadRequest);
Console.WriteLine(completeMultipartUploadResponse.ETag);
Console.ReadKey();
}
private void initRequestCtrls(InitiateMultipartUploadRequest request)
{
// Initialize storage class
if (request.StorageClass == S3StorageClass.ReducedRedundancy)
{
RadioReducedRedund.Checked = true;
}
else if (request.StorageClass == S3StorageClass.StandardInfrequentAccess)
{
RadioStandardIA.Checked = true;
}
else
{
RadioStandard.Checked = true;
}
// Initialize other fields
ComboAcl.SelectedItem = request.CannedACL?.Value;
TxtWebsite.Text = request.WebsiteRedirectLocation;
ChkRequestPayer.Checked = (request.RequestPayer == null ? false : true);
// Initialize metadata
MetadataCollection metadata = request.Metadata;
foreach (string key in metadata.Keys)
{
DgvMetadata.Rows.Add(key, metadata[key]);
}
// Initialize headers
TxtType.Text = request.Headers.ContentType;
TxtDisposition.Text = request.Headers.ContentDisposition;
TxtEncoding.Text = request.Headers.ContentEncoding;
// Initialize access control
DataGridViewRow[] rows = request.Grants
.GroupBy(g => g.Grantee, g => g.Permission)
.Select(g => rowFromGrants(g.Key, g.AsEnumerable()))
.ToArray();
DgvGrants.Rows.AddRange(rows);
// Initialize SSE method
bool kms = (request.ServerSideEncryptionMethod == ServerSideEncryptionMethod.AWSKMS);
bool newKey = (request.ServerSideEncryptionMethod == ServerSideEncryptionMethod.AES256);
if (kms)
{
RadioSseKms.Checked = true;
}
else if (newKey)
{
RadioSseNewKey.Checked = true;
}
else
{
RadioSseNone.Checked = true;
}
// Initialize other SSE controls
TxtSseKeyId.Text = (kms ? request.ServerSideEncryptionKeyManagementServiceKeyId : null);
TxtSseCustomerKey.Text = (newKey ? request.ServerSideEncryptionCustomerProvidedKey : null);
TxtSseCustomerKey.Text = request.ServerSideEncryptionCustomerProvidedKey;
TxtSseCustomerKeyMd5.Text = request.ServerSideEncryptionCustomerProvidedKeyMD5;
}
public void mutiPartUpload(DoWorkEventArgs e)
{
for (; ;)
{
getListItemName();
if (string.IsNullOrEmpty(path))
{
return;
}
//判断是否取消操作
if (bw.CancellationPending)
{
e.Cancel = true; //这里才真正取消
SetText("已取消，取消速度慢以防止产生碎片");
return;
}
key = Guid.NewGuid().ToString() + path.Substring(path.LastIndexOf('.'));
OssClient ossClient = new OssClient(endPoint, accessKeyID, accessKeySecret);
InitiateMultipartUploadRequest initRequest =
new InitiateMultipartUploadRequest(bucketName, "bbb1\\" + key);
InitiateMultipartUploadResult initResult = ossClient.InitiateMultipartUpload(initRequest);
// 设置每块为 5M ,不允许小于5M
int partSize = 1024 * 100;
FileInfo partFile = new FileInfo(path);
// 计算分块数目
int partCount = (int)(partFile.Length / partSize);
if (partFile.Length % partSize != 0)
{
partCount++;
}
// 新建一个List保存每个分块上传后的ETag和PartNumber
List <PartETag> partETags = new List <PartETag>();
for (int i = 0; i < partCount; i++)
{
//Bar.Value = (i * 100) / partCount;
//UploadInfo(Bar.Value.ToString());
// 获取文件流
FileStream fis = new FileStream(partFile.FullName, FileMode.Open);
// 跳到每个分块的开头
long skipBytes = partSize * i;
fis.Position = skipBytes;
//fis.skip(skipBytes);
// 计算每个分块的大小
long size = partSize < partFile.Length - skipBytes ?
partSize : partFile.Length - skipBytes;
// 创建UploadPartRequest，上传分块
UploadPartRequest uploadPartRequest = new UploadPartRequest(bucketName, "bbb1\\" + key, initResult.UploadId);
uploadPartRequest.InputStream = fis;
uploadPartRequest.PartSize = size;
uploadPartRequest.PartNumber = (i + 1);
UploadPartResult uploadPartResult = ossClient.UploadPart(uploadPartRequest);
// 将返回的PartETag保存到List中。
partETags.Add(uploadPartResult.PartETag);
// 关闭文件
fis.Close();
manualReset.WaitOne();//如果ManualResetEvent的初始化为终止状态（true），那么该方法将一直工作，直到收到Reset信号。然后，直到收到Set信号，就继续工作。
}
CompleteMultipartUploadRequest completeReq = new CompleteMultipartUploadRequest(bucketName, "bbb1\\" + key, initResult.UploadId);
foreach (PartETag partETag in partETags)
{
completeReq.PartETags.Add(partETag);
}
// 红色标注的是与JAVA的SDK有区别的地方
//完成分块上传
CompleteMultipartUploadResult completeResult = ossClient.CompleteMultipartUpload(completeReq);
//Bar.Value = 100;
//UploadInfo(Bar.Value.ToString());
// 返回最终文件的MD5，用于用户进行校验
//Console.WriteLine(completeResult.ETag);
setListItemValue(key);
}
}
public void InitiateMultipartUploadAsync(InitiateMultipartUploadRequest request, AmazonServiceCallback <InitiateMultipartUploadRequest, InitiateMultipartUploadResponse> callback, AsyncOptions options = null)
{
throw new System.NotImplementedException();
}
/// <summary> /// Update InitiateMultipartUploadRequest request with given encryption instructions /// </summary> /// <param name="instructions">EncryptionInstructions which used for encrypting the UploadPartRequest request</param> /// <param name="initiateMultiPartUploadRequest">InitiateMultipartUploadRequest whose encryption context needs to updated</param> /// <param name="useKmsKeyWrapping">If true, KMS mode of encryption is used</param> protected abstract void GenerateInitiateMultiPartUploadRequest(EncryptionInstructions instructions, InitiateMultipartUploadRequest initiateMultiPartUploadRequest, bool useKmsKeyWrapping);
public void MultipartEncryptionTestInstructionFile()
{
string filePath = @"C:\temp\Upload15MegFileIn3PartsViaStream.txt";
string retrievedFilepath = @"C:\temp\Upload15MegFileIn3PartsViaStreamRetreived.txt";
int MEG_SIZE = (int)Math.Pow(2, 20) + 4001;
long totalSize = (long)(15 * MEG_SIZE);
UtilityMethods.GenerateFile(filePath, totalSize);
string key = "MultipartEncryptionTestInstrcutionFile" + random.Next();
s3EncryptionClientFileMode.PutBucket(new PutBucketRequest()
{
BucketName = bucketName
});
Stream inputStream = File.OpenRead(filePath);
try
{
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
StorageClass = S3StorageClass.ReducedRedundancy,
ContentType = "text/html",
CannedACL = S3CannedACL.PublicRead
};
InitiateMultipartUploadResponse initResponse = s3EncryptionClientFileMode.InitiateMultipartUpload(initRequest);
// Upload part 1
UploadPartRequest uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 1,
PartSize = 5 * MEG_SIZE,
InputStream = inputStream
};
UploadPartResponse up1Response = s3EncryptionClientFileMode.UploadPart(uploadRequest);
// Upload part 2
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 2,
PartSize = 5 * MEG_SIZE + 4001,
InputStream = inputStream
};
UploadPartResponse up2Response = s3EncryptionClientFileMode.UploadPart(uploadRequest);
// Upload part 3
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 3,
InputStream = inputStream,
IsLastPart = true
};
//uploadRequest.setLastPart();
UploadPartResponse up3Response = s3EncryptionClientFileMode.UploadPart(uploadRequest);
ListPartsRequest listPartRequest = new ListPartsRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
ListPartsResponse listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest);
Assert.AreEqual(3, listPartResponse.Parts.Count);
Assert.AreEqual(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber);
Assert.AreEqual(up1Response.ETag, listPartResponse.Parts[0].ETag);
Assert.AreEqual(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber);
Assert.AreEqual(up2Response.ETag, listPartResponse.Parts[1].ETag);
Assert.AreEqual(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber);
Assert.AreEqual(up3Response.ETag, listPartResponse.Parts[2].ETag);
listPartRequest.MaxParts = 1;
listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest);
Assert.AreEqual(1, listPartResponse.Parts.Count);
// Complete the response
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
compRequest.AddPartETags(up1Response, up2Response, up3Response);
CompleteMultipartUploadResponse compResponse = s3EncryptionClientFileMode.CompleteMultipartUpload(compRequest);
Assert.AreEqual(bucketName, compResponse.BucketName);
Assert.IsNotNull(compResponse.ETag);
Assert.AreEqual(key, compResponse.Key);
Assert.IsNotNull(compResponse.Location);
// Get the file back from S3 and make sure it is still the same.
GetObjectRequest getRequest = new GetObjectRequest()
{
BucketName = bucketName,
Key = key
};
GetObjectResponse getResponse = s3EncryptionClientFileMode.GetObject(getRequest);
getResponse.WriteResponseStreamToFile(retrievedFilepath);
UtilityMethods.CompareFiles(filePath, retrievedFilepath);
GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest()
{
BucketName = bucketName,
Key = key
};
GetObjectMetadataResponse metaDataResponse = s3EncryptionClientFileMode.GetObjectMetadata(metaDataRequest);
Assert.AreEqual("text/html", metaDataResponse.Headers.ContentType);
}
finally
{
inputStream.Close();
if (File.Exists(filePath))
{
File.Delete(filePath);
}
if (File.Exists(retrievedFilepath))
{
File.Delete(retrievedFilepath);
}
}
}
// Commented out because the would leave data in glacier that would cost money
//[Test]
//[Category("Glacier")]
public void TestMultiPartUpload()
{
var testingVaultName = "dotnet-sdk-test" + DateTime.Now.Ticks.ToString();
Client.CreateVaultAsync(new CreateVaultRequest()
{
VaultName = testingVaultName
}).Wait();
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
{
VaultName = testingVaultName,
ArchiveDescription = "dotnet mp upload",
PartSize = 1048576
};
InitiateMultipartUploadResponse initResponse = Client.InitiateMultipartUploadAsync(initRequest).Result;
string uploadId = initResponse.UploadId;
MemoryStream totalStream = new MemoryStream();
for (int i = 0; i < 1048576 + 1048576 / 2; i++)
{
totalStream.WriteByte((byte)(i % byte.MaxValue));
}
totalStream.Position = 0;
List <string> md5s = new List <string>();
long currentPosition = 0;
long partSize = 1048576;
while (totalStream.Position < totalStream.Length)
{
Stream partStream = GlacierUtils.CreatePartStream(totalStream, partSize);
string checkSum = TreeHashGenerator.CalculateTreeHash(partStream);
md5s.Add(checkSum);
UploadMultipartPartRequest partRequest = new UploadMultipartPartRequest()
{
VaultName = testingVaultName,
UploadId = uploadId,
Body = partStream,
Checksum = checkSum
};
partRequest.SetRange(currentPosition, currentPosition + partStream.Length - 1);
Client.UploadMultipartPartAsync(partRequest).Wait();
currentPosition += partStream.Length;
}
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
{
VaultName = testingVaultName,
UploadId = uploadId,
ArchiveSize = totalStream.Length.ToString(),
Checksum = TreeHashGenerator.CalculateTreeHash(md5s)
};
CompleteMultipartUploadResponse compResponse = Client.CompleteMultipartUploadAsync(compRequest).Result;
Assert.IsNotNull(compResponse.Location);
Assert.IsNotNull(compResponse.Checksum);
string archiveId = compResponse.ArchiveId;
DeleteArchiveRequest delArchiveRequest = new DeleteArchiveRequest()
{
VaultName = testingVaultName,
ArchiveId = archiveId
};
DeleteArchiveResponse delArchiveResponse = Client.DeleteArchiveAsync(delArchiveRequest).Result;
}
private static async Task UploadObjectAsync()
{
// Create list to store upload part responses.
List <UploadPartResponse> uploadResponses = new List <UploadPartResponse>();
// Setup information required to initiate the multipart upload.
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName
};
// Initiate the upload.
InitiateMultipartUploadResponse initResponse =
await s3Client.InitiateMultipartUploadAsync(initiateRequest);
// Upload parts.
long contentLength = new FileInfo(filePath).Length;
long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB
try
{
Console.WriteLine("Uploading parts");
long filePosition = 0;
for (int i = 1; filePosition < contentLength; i++)
{
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId,
PartNumber = i,
PartSize = partSize,
FilePosition = filePosition,
FilePath = filePath
};
// Track upload progress.
uploadRequest.StreamTransferProgress +=
new EventHandler <StreamTransferProgressArgs>(UploadPartProgressEventCallback);
// Upload a part and add the response to our list.
uploadResponses.Add(await s3Client.UploadPartAsync(uploadRequest));
filePosition += partSize;
}
// Setup to complete the upload.
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
completeRequest.AddPartETags(uploadResponses);
// Complete the upload.
CompleteMultipartUploadResponse completeUploadResponse =
await s3Client.CompleteMultipartUploadAsync(completeRequest);
}
catch (Exception exception)
{
Console.WriteLine("An AmazonS3Exception was thrown: { 0}", exception.Message);
// Abort the upload.
AbortMultipartUploadRequest abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = bucketName,
Key = keyName,
UploadId = initResponse.UploadId
};
await s3Client.AbortMultipartUploadAsync(abortMPURequest);
}
}
private static async Task CreateSampleObjUsingClientEncryptionKeyAsync(string base64Key, IAmazonS3 s3Client)
{
// List to store upload part responses.
List <UploadPartResponse> uploadResponses = new List <UploadPartResponse>();
// 1. Initialize.
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = existingBucketName,
Key = sourceKeyName,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = base64Key
};
InitiateMultipartUploadResponse initResponse =
await s3Client.InitiateMultipartUploadAsync(initiateRequest);
// 2. Upload Parts.
long contentLength = new FileInfo(filePath).Length;
long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB
try
{
long filePosition = 0;
for (int i = 1; filePosition < contentLength; i++)
{
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = existingBucketName,
Key = sourceKeyName,
UploadId = initResponse.UploadId,
PartNumber = i,
PartSize = partSize,
FilePosition = filePosition,
FilePath = filePath,
ServerSideEncryptionCustomerMethod = ServerSideEncryptionCustomerMethod.AES256,
ServerSideEncryptionCustomerProvidedKey = base64Key
};
// Upload part and add response to our list.
uploadResponses.Add(await s3Client.UploadPartAsync(uploadRequest));
filePosition += partSize;
}
// Step 3: complete.
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = existingBucketName,
Key = sourceKeyName,
UploadId = initResponse.UploadId,
//PartETags = new List<PartETag>(uploadResponses)
};
completeRequest.AddPartETags(uploadResponses);
CompleteMultipartUploadResponse completeUploadResponse =
await s3Client.CompleteMultipartUploadAsync(completeRequest);
}
catch (Exception exception)
{
Console.WriteLine("Exception occurred: {0}", exception.Message);
AbortMultipartUploadRequest abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = existingBucketName,
Key = sourceKeyName,
UploadId = initResponse.UploadId
};
await s3Client.AbortMultipartUploadAsync(abortMPURequest);
}
}
private static string InitiateMultipartUpload(String bucketName, String objectName)
{
var request = new InitiateMultipartUploadRequest(bucketName, objectName);
var result = _ossClient.InitiateMultipartUpload(request);
return result.UploadId;
}
public static void UploadToOSS(string fileMD5, string fileExt, Stream fs)
{
fs.Position = 0;
var key = fileMD5 + "." + fileExt;
//var uploadFile = fileLocalPath + @"\" + date + @"\" + key;
try
{
bool UploadStatus = false;
#region 从本地读取视频文件并上传
var content = fs;
//using (var content = File.Open(uploadFile, FileMode.Open))
//{
if (content.Length < 50 * 1024 * 1024) //50M
{
//Common.WriteLog(string.Format("文件{0}上传开始", key));
var resultS = client.PutObject(bucketName, key, content);
UploadStatus = true;
//Common.WriteLog(string.Format("文件{0}上传成功，返回信息为{1}", key, resultS.ETag));
}
else
{
//初始化分片上传
//Common.WriteLog(string.Format("文件{0}开始分片上传", key));
var request1 = new InitiateMultipartUploadRequest(bucketName, key);
var UploadId = client.InitiateMultipartUpload(request1).UploadId;
int partCount = 0;
var fileSize = content.Length;
int partSize = 10 * 1024 * 1024;
partCount = (int)(fileSize / partSize + (fileSize % partSize == 0 ? 0 : 1));
// 开始分片上传
var partETags = new List <PartETag>();
for (var i = 0; i < partCount; i++)
{
var skipBytes = (long)partSize * i;
//定位到本次上传片应该开始的位置
content.Seek(skipBytes, 0);
//计算本次上传的片大小，最后一片为剩余的数据大小，其余片都是part size大小。
var size = (partSize < fileSize - skipBytes) ? partSize : (fileSize - skipBytes);
var request = new UploadPartRequest(bucketName, key, UploadId)
{
InputStream = content,
PartSize = size,
PartNumber = i + 1
};
//调用UploadPart接口执行上传功能，返回结果中包含了这个数据片的ETag值
var result2 = client.UploadPart(request);
partETags.Add(result2.PartETag);
}
//完成分片上传
var completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, key, UploadId);
foreach (var partETag in partETags)
{
completeMultipartUploadRequest.PartETags.Add(partETag);
}
var resultEnd = client.CompleteMultipartUpload(completeMultipartUploadRequest);
UploadStatus = true;
//Common.WriteLog(string.Format("文件{0}分片上传结束", key));
}
//}
#endregion
}
catch (Exception ex)
{
Logger.LogError("阿里云上传问题：" + ex.Message);
}
}
