private static List<PartETag> UploadPartCopys(String targetBucket, String targetObject, String sourceBucket, String sourceObject,
String uploadId, int partSize)
{
var metadata = _ossClient.GetObjectMetadata(sourceBucket, sourceObject);
var fileSize = metadata.ContentLength;
var partCount = (int)fileSize / partSize;
if (fileSize % partSize != 0)
{
partCount++;
}
var partETags = new List<PartETag>();
for (var i = 0; i < partCount; i++)
{
var skipBytes = (long)partSize * i;
var size = (partSize < fileSize - skipBytes) ? partSize : (fileSize - skipBytes);
var request =
new UploadPartCopyRequest(targetBucket, targetObject, sourceBucket, sourceObject, uploadId)
{
PartSize = size,
PartNumber = i + 1,
BeginIndex = skipBytes
};
var result = _ossClient.UploadPartCopy(request);
partETags.Add(result.PartETag);
}
return partETags;
}
private static void AsyncUploadPartCopys(String targetBucket, String targetObject, String sourceBucket, String sourceObject,
String uploadId, int partSize)
{
var metadata = _ossClient.GetObjectMetadata(sourceBucket, sourceObject);
var fileSize = metadata.ContentLength;
var partCount = (int)fileSize / partSize;
if (fileSize % partSize != 0)
{
partCount++;
}
var ctx = new UploadPartCopyContext()
{
TargetBucket = targetBucket,
TargetObject = targetObject,
UploadId = uploadId,
TotalParts = partCount,
CompletedParts = 0,
SyncLock = new object(),
PartETags = new List<PartETag>(),
WaitEvent = new ManualResetEvent(false)
};
for (var i = 0; i < partCount; i++)
{
var skipBytes = (long)partSize * i;
var size = (partSize < fileSize - skipBytes) ? partSize : (fileSize - skipBytes);
var request =
new UploadPartCopyRequest(targetBucket, targetObject, sourceBucket, sourceObject, uploadId)
{
PartSize = size,
PartNumber = i + 1,
BeginIndex = skipBytes
};
_ossClient.BeginUploadPartCopy(request, UploadPartCopyCallback, new UploadPartCopyContextWrapper(ctx, i + 1));
}
ctx.WaitEvent.WaitOne();
}
public void MultipartUploadCopyTest()
{
//get target object name
var sourceObjectKey = OssTestUtils.GetObjectKey(_className);
var targetObjectKey = OssTestUtils.GetObjectKey(_className);
OssTestUtils.UploadObject(_ossClient, _bucketName, sourceObjectKey, Config.UploadTestFile);
var initRequest = new InitiateMultipartUploadRequest(_bucketName, targetObjectKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
// Set the part size
const int partSize = 100 * 1024;
var sourceObjectMeta = _ossClient.GetObjectMetadata(_bucketName, sourceObjectKey);
// Calculate the part count
var partCount = OssTestUtils.CalculatePartCount(sourceObjectMeta.ContentLength, partSize);
// Create a list to save result
var partETags = new List <PartETag>();
for (var i = 0; i < partCount; i++)
{
// Skip to the start position
long skipBytes = partSize * i;
// calculate the part size
var size = partSize < sourceObjectMeta.ContentLength - skipBytes
? partSize
: sourceObjectMeta.ContentLength - skipBytes;
// Create a UploadPartRequest, uploading parts
var uploadPartCopyRequest =
new UploadPartCopyRequest(_bucketName, targetObjectKey, _bucketName, sourceObjectKey, initResult.UploadId)
{
BeginIndex = skipBytes,
PartSize = size,
PartNumber = (i + 1),
ModifiedSinceConstraint = DateTime.Now.AddDays(-1),
RequestPayer = RequestPayer.Requester
};
uploadPartCopyRequest.MatchingETagConstraints.Add(sourceObjectMeta.ETag);
var uploadPartCopyResult = _ossPayerClient.UploadPartCopy(uploadPartCopyRequest);
// Save the result
partETags.Add(uploadPartCopyResult.PartETag);
}
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, targetObjectKey, initResult.UploadId);
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
_ossClient.CompleteMultipartUpload(completeRequest);
var targetObjectMeta = _ossClient.GetObjectMetadata(_bucketName, targetObjectKey);
Assert.AreEqual(targetObjectMeta.Crc64, sourceObjectMeta.Crc64);
//delete the object
_ossClient.DeleteObject(_bucketName, targetObjectKey);
}
/// <summary>
/// 文件拷贝，默认采用分片拷贝的方式
/// </summary>
/// <param name="bucketName"></param>
/// <param name="objectName"></param>
/// <param name="destBucketName"></param>
/// <param name="destObjectName"></param>
/// <returns></returns>
public async Task <bool> CopyObjectAsync(string bucketName, string objectName, string destBucketName, string destObjectName = null)
{
if (string.IsNullOrEmpty(bucketName))
{
throw new ArgumentNullException(nameof(bucketName));
}
if (string.IsNullOrEmpty(objectName))
{
throw new ArgumentNullException(nameof(objectName));
}
if (string.IsNullOrEmpty(destBucketName))
{
destBucketName = bucketName;
}
if (string.IsNullOrEmpty(destObjectName))
{
destObjectName = objectName;
}
var partSize = 50 * 1024 * 1024;
// 创建OssClient实例。
// 初始化拷贝任务。可以通过InitiateMultipartUploadRequest指定目标文件元信息。
var request = new InitiateMultipartUploadRequest(destBucketName, destObjectName);
var result = _client.InitiateMultipartUpload(request);
// 计算分片数。
var metadata = _client.GetObjectMetadata(bucketName, objectName);
var fileSize = metadata.ContentLength;
var partCount = (int)fileSize / partSize;
if (fileSize % partSize != 0)
{
partCount++;
}
// 开始分片拷贝。
var partETags = new List <PartETag>();
for (var i = 0; i < partCount; i++)
{
var skipBytes = (long)partSize * i;
var size = (partSize < fileSize - skipBytes) ? partSize : (fileSize - skipBytes);
// 创建UploadPartCopyRequest。可以通过UploadPartCopyRequest指定限定条件。
var uploadPartCopyRequest = new UploadPartCopyRequest(destBucketName, destObjectName, bucketName, objectName, result.UploadId)
{
PartSize = size,
PartNumber = i + 1,
// BeginIndex用来定位此次上传分片开始所对应的位置。
BeginIndex = skipBytes
};
// 调用uploadPartCopy方法来拷贝每一个分片。
var uploadPartCopyResult = _client.UploadPartCopy(uploadPartCopyRequest);
partETags.Add(uploadPartCopyResult.PartETag);
}
// 完成分片拷贝。
var completeMultipartUploadRequest =
new CompleteMultipartUploadRequest(destBucketName, destObjectName, result.UploadId);
// partETags为分片上传中保存的partETag的列表，OSS收到用户提交的此列表后，会逐一验证每个数据分片的有效性。全部验证通过后，OSS会将这些分片合成一个完整的文件。
foreach (var partETag in partETags)
{
completeMultipartUploadRequest.PartETags.Add(partETag);
}
_client.CompleteMultipartUpload(completeMultipartUploadRequest);
return(await Task.FromResult(true));
}
private void PartCopy()
{
int size = sliceList.Count;
sliceCount = size;
uploadCopyCopyRequestList = new List <UploadPartCopyRequest>(size);
for (int i = 0; i < size; i++)
{
if (isExit)
{
return;
}
SliceStruct sliceStruct = sliceList[i];
if (!sliceStruct.isAlreadyUpload)
{
UploadPartCopyRequest uploadPartCopyRequest = new UploadPartCopyRequest(bucket, key, sliceStruct.partNumber, uploadId);
uploadPartCopyRequest.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), 600);
uploadPartCopyRequest.SetCopySource(copySource);
uploadPartCopyRequest.SetCopyRange(sliceStruct.sliceStart, sliceStruct.sliceEnd);
uploadCopyCopyRequestList.Add(uploadPartCopyRequest);
cosXmlServer.PartCopy(uploadPartCopyRequest, delegate(CosResult result)
{
lock (syncExit)
{
if (isExit)
{
return;
}
}
UploadPartCopyResult uploadPartCopyResult = result as UploadPartCopyResult;
sliceStruct.eTag = uploadPartCopyResult.copyObject.eTag;
lock (syncPartCopyCount)
{
sliceCount--;
if (sliceCount == 0)
{
OnPart();
return;
}
}
}, delegate(CosClientException clientEx, CosServerException serverEx)
{
lock (syncExit)
{
if (isExit)
{
return;
}
}
if (UpdateTaskState(TaskState.FAILED))
{
OnFailed(clientEx, serverEx);
}
return;
});
}
else
{
lock (syncPartCopyCount)
{
sliceCount--;
if (sliceCount == 0)
{
OnPart();
return;
}
}
}
}
}
public void MultipartUploadNegativeTest()
{
var key = OssTestUtils.GetObjectKey(_className);
// Initiate a multipart upload
var initRequest = new InitiateMultipartUploadRequest(_bucketName, key);
Assert.AreEqual(initRequest.RequestPayer, RequestPayer.BucketOwner);
try
{
var initResult = _ossPayerClient.InitiateMultipartUpload(initRequest);
Assert.Fail("should not here.");
}
catch (OssException e)
{
Assert.AreEqual(e.ErrorCode, "AccessDenied");
}
// uploads the part
var uploadPartRequest = new UploadPartRequest(_bucketName, key, "initId")
{
InputStream = new MemoryStream(Encoding.ASCII.GetBytes("hello world")),
PartSize = 11,
PartNumber = 1
};
Assert.AreEqual(uploadPartRequest.RequestPayer, RequestPayer.BucketOwner);
try
{
var uploadPartResult = _ossPayerClient.UploadPart(uploadPartRequest);
Assert.Fail("should not here.");
}
catch (OssException e)
{
Assert.AreEqual(e.ErrorCode, "AccessDenied");
}
//list part
var listpartRequest = new ListPartsRequest(_bucketName, key, "initId");
Assert.AreEqual(listpartRequest.RequestPayer, RequestPayer.BucketOwner);
try
{
var listPartResult = _ossPayerClient.ListParts(listpartRequest);
Assert.Fail("should not here.");
}
catch (OssException e)
{
Assert.AreEqual(e.ErrorCode, "AccessDenied");
}
//abort
var abortRequest = new AbortMultipartUploadRequest(_bucketName, key, "initId");
Assert.AreEqual(abortRequest.RequestPayer, RequestPayer.BucketOwner);
try
{
_ossPayerClient.AbortMultipartUpload(abortRequest);
Assert.Fail("should not here.");
}
catch (OssException e)
{
Assert.AreEqual(e.ErrorCode, "AccessDenied");
}
//complete
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, key, "initId");
Assert.AreEqual(completeRequest.RequestPayer, RequestPayer.BucketOwner);
try
{
var listPartResult = _ossPayerClient.CompleteMultipartUpload(completeRequest);
Assert.Fail("should not here.");
}
catch (OssException e)
{
Assert.AreEqual(e.ErrorCode, "AccessDenied");
}
//list upload
var listUploadsRequest = new ListMultipartUploadsRequest(_bucketName);
Assert.AreEqual(listUploadsRequest.RequestPayer, RequestPayer.BucketOwner);
try
{
var listUploadsResult = _ossPayerClient.ListMultipartUploads(listUploadsRequest);
Assert.Fail("should not here.");
}
catch (OssException e)
{
Assert.AreEqual(e.ErrorCode, "AccessDenied");
}
//uploadpart copy
var uploadpartcopyRequet = new UploadPartCopyRequest(_bucketName, key, _bucketName, key, "uploadId")
{
BeginIndex = 0,
PartNumber = 1,
PartSize = 100 * 1024
};
Assert.AreEqual(uploadpartcopyRequet.RequestPayer, RequestPayer.BucketOwner);
try
{
var uploadpartcopyResult = _ossPayerClient.UploadPartCopy(uploadpartcopyRequet);
Assert.Fail("should not here.");
}
catch (OssException e)
{
Assert.AreEqual(e.ErrorCode, "AccessDenied");
}
}
public void MultipartUploadPartCopyComplexStepTest()
{
//get target object name
var targetObjectKey = OssTestUtils.GetObjectKey(_className);
var initRequest = new InitiateMultipartUploadRequest(_bucketName, targetObjectKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
// Set the part size
const int partSize = 1024 * 512 * 1;
var sourceObjectMeta = _ossClient.GetObjectMetadata(_bucketName, _sourceObjectKey);
// Calculate the part count
var partCount = OssTestUtils.CalculatePartCount(sourceObjectMeta.ContentLength, partSize);
LogUtility.LogMessage("Object {0} is splitted to {1} parts for multipart upload part copy",
_sourceObjectKey, partCount);
// Create a list to save result
var partETags = new List <PartETag>();
for (var i = 0; i < partCount; i++)
{
// Skip to the start position
long skipBytes = partSize * i;
// calculate the part size
var size = partSize < sourceObjectMeta.ContentLength - skipBytes
? partSize
: sourceObjectMeta.ContentLength - skipBytes;
// Create a UploadPartRequest, uploading parts
var uploadPartCopyRequest =
new UploadPartCopyRequest(_bucketName, targetObjectKey, _bucketName, _sourceObjectKey, initResult.UploadId)
{
BeginIndex = skipBytes,
PartSize = size,
PartNumber = (i + 1),
ModifiedSinceConstraint = DateTime.Now.AddDays(-1)
};
uploadPartCopyRequest.MatchingETagConstraints.Add(_objectETag);
var uploadPartCopyResult = _ossClient.UploadPartCopy(uploadPartCopyRequest);
// Save the result
partETags.Add(uploadPartCopyResult.PartETag);
}
var lmuRequest = new ListMultipartUploadsRequest(_bucketName);
var lmuListing = _ossClient.ListMultipartUploads(lmuRequest);
string mpUpload = null;
foreach (var t in lmuListing.MultipartUploads)
{
if (t.UploadId == initResult.UploadId)
{
mpUpload = t.UploadId;
break;
}
}
Assert.IsNotNull(mpUpload, "The multipart uploading should be in progress");
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, targetObjectKey, initResult.UploadId);
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
_ossClient.CompleteMultipartUpload(completeRequest);
Assert.IsTrue(OssTestUtils.ObjectExists(_ossClient, _bucketName, targetObjectKey));
//delete the object
_ossClient.DeleteObject(_bucketName, targetObjectKey);
}
public void PartCopy(UploadPartCopyRequest request, Callback.OnSuccessCallback <CosResult> successCallback, Callback.OnFailedCallback failCallback)
{
schedue(request, new UploadPartCopyResult(), successCallback, failCallback);
}
public UploadPartCopyResult PartCopy(UploadPartCopyRequest request)
{
return((Model.Object.UploadPartCopyResult)excute(request, new Model.Object.UploadPartCopyResult()));
}
public void MultipartUploadPartCopyComplexStepTest()
{
//get target object name
var targetObjectKey = OssTestUtils.GetObjectKey(_className);
var initRequest = new InitiateMultipartUploadRequest(_bucketName, targetObjectKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
// 设置每块为 512K
const int partSize = 1024 * 512 * 1;
var sourceObjectMeta = _ossClient.GetObjectMetadata(_bucketName, _sourceObjectKey);
// 计算分块数目
var partCount = OssTestUtils.CalculatePartCount(sourceObjectMeta.ContentLength, partSize);
LogUtility.LogMessage("Object {0} is splitted to {1} parts for multipart upload part copy",
_sourceObjectKey, partCount);
// 新建一个List保存每个分块上传后的ETag和PartNumber
var partETags = new List <PartETag>();
for (var i = 0; i < partCount; i++)
{
// 跳到每个分块的开头
long skipBytes = partSize * i;
// 计算每个分块的大小
var size = partSize < sourceObjectMeta.ContentLength - skipBytes
? partSize
: sourceObjectMeta.ContentLength - skipBytes;
// 创建UploadPartRequest，上传分块
var uploadPartCopyRequest =
new UploadPartCopyRequest(_bucketName, targetObjectKey, _bucketName, _sourceObjectKey, initResult.UploadId)
{
BeginIndex = skipBytes,
PartSize = size,
PartNumber = (i + 1),
ModifiedSinceConstraint = DateTime.Now.AddDays(-1)
};
uploadPartCopyRequest.MatchingETagConstraints.Add(_objectETag);
var uploadPartCopyResult = _ossClient.UploadPartCopy(uploadPartCopyRequest);
// 将返回的PartETag保存到List中。
partETags.Add(uploadPartCopyResult.PartETag);
}
var lmuRequest = new ListMultipartUploadsRequest(_bucketName);
var lmuListing = _ossClient.ListMultipartUploads(lmuRequest);
var mpUpload = lmuListing.MultipartUploads.Single(t => t.UploadId == initResult.UploadId);
Assert.IsNotNull(mpUpload, "The multipart uploading should be in progress");
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, targetObjectKey, initResult.UploadId);
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
_ossClient.CompleteMultipartUpload(completeRequest);
Assert.IsTrue(OssTestUtils.ObjectExists(_ossClient, _bucketName, targetObjectKey));
//delete the object
_ossClient.DeleteObject(_bucketName, targetObjectKey);
}
public void MultipartUploadUploadPartCopyRequestArgumentCheck()
{
try
{
var uploadPartCopyRequest = new UploadPartCopyRequest("targetbucket",
"targetKey", "sourcebucket", "sourceKey", "upload-id")
{
PartNumber = null,
PartSize = null,
BeginIndex = null
};
var uploadPartResult = _ossClient.UploadPartCopy(uploadPartCopyRequest);
Assert.Fail("the arg is null, should throw exception.");
}
catch (Exception e)
{
Assert.IsTrue(e.Message.Contains("partNumber should be specfied"));
}
try
{
var uploadPartCopyRequest = new UploadPartCopyRequest("targetbucket",
"targetKey", "sourcebucket", "sourceKey", "upload-id")
{
PartNumber = 1,
PartSize = null,
BeginIndex = null
};
var uploadPartResult = _ossClient.UploadPartCopy(uploadPartCopyRequest);
Assert.Fail("the arg is null, should throw exception.");
}
catch (Exception e)
{
Assert.IsTrue(e.Message.Contains("partSize should be specfied"));
}
try
{
var uploadPartCopyRequest = new UploadPartCopyRequest("targetbucket",
"targetKey", "sourcebucket", "sourceKey", "upload-id")
{
PartNumber = 1,
PartSize = 1024 * 1024 * 5,
BeginIndex = null
};
var uploadPartResult = _ossClient.UploadPartCopy(uploadPartCopyRequest);
Assert.Fail("the arg is null, should throw exception.");
}
catch (Exception e)
{
Assert.IsTrue(e.Message.Contains("beginIndex should be specfied"));
}
try
{
var uploadPartCopyRequest = new UploadPartCopyRequest("targetbucket",
"targetKey", "sourcebucket", "sourceKey", "upload-id")
{
PartNumber = 1,
PartSize = -1,
BeginIndex = 1
};
var uploadPartResult = _ossClient.UploadPartCopy(uploadPartCopyRequest);
Assert.Fail("the arg is null, should throw exception.");
}
catch (Exception e)
{
Assert.IsTrue(e.Message.Contains("partSize not live in valid range"));
}
try
{
var uploadPartCopyRequest = new UploadPartCopyRequest("targetbucket",
"targetKey", "sourcebucket", "sourceKey", "upload-id")
{
PartNumber = 1,
PartSize = 5 * 1024 * 1024 * 1024L + 1L,
BeginIndex = 1
};
var uploadPartResult = _ossClient.UploadPartCopy(uploadPartCopyRequest);
Assert.Fail("the arg is null, should throw exception.");
}
catch (Exception e)
{
Assert.IsTrue(e.Message.Contains("partSize not live in valid range"));
}
try
{
var uploadPartCopyRequest = new UploadPartCopyRequest("targetbucket",
"targetKey", "sourcebucket", "sourceKey", "upload-id")
{
PartNumber = 10001,
PartSize = 1024 * 1024 * 5,
BeginIndex = 1
};
var uploadPartResult = _ossClient.UploadPartCopy(uploadPartCopyRequest);
Assert.Fail("the arg is null, should throw exception.");
}
catch (Exception e)
{
Assert.IsTrue(e.Message.Contains("partNumber not live in valid range"));
}
}
