private void DoResumableUploadSingleThread(string bucketName, string key, ResumableContext resumableContext, Stream fs,
EventHandler <StreamTransferProgressArgs> uploadProgressCallback)
{
var uploadedBytes = resumableContext.GetUploadedBytes();
foreach (var part in resumableContext.PartContextList)
{
if (part.IsCompleted)
{
continue;
}
fs.Seek(part.Position, SeekOrigin.Begin);
var originalStream = fs;
if (uploadProgressCallback != null)
{
originalStream = _ossClient.SetupProgressListeners(originalStream,
fs.Length,
uploadedBytes,
_conf.ProgressUpdateInterval,
uploadProgressCallback);
}
var request = new UploadPartRequest(bucketName, key, resumableContext.UploadId)
{
InputStream = originalStream,
PartSize = part.Length,
PartNumber = part.PartId,
RequestPayer = _request.RequestPayer,
TrafficLimit = _request.TrafficLimit
};
var partResult = _ossClient.UploadPart(request);
part.PartETag = partResult.PartETag;
part.IsCompleted = true;
if (partResult.ResponseMetadata.ContainsKey(HttpHeaders.HashCrc64Ecma))
{
part.Crc64 = ulong.Parse(partResult.ResponseMetadata[HttpHeaders.HashCrc64Ecma]);
}
resumableContext.Dump();
uploadedBytes += part.Length;
}
}
public void MultipartUploadAbortInMiddleTest()
{
var sourceFile = Config.MultiUploadSampleFile;
//get target object name
var targetObjectKey = OssTestUtils.GetObjectKey(_className);
var initRequest = new InitiateMultipartUploadRequest(_bucketName, targetObjectKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
// 设置每块为 1M
const int partSize = 1024 * 1024 * 1;
var partFile = new FileInfo(sourceFile);
// 计算分块数目
var partCount = OssTestUtils.CalculatePartCount(partFile.Length, partSize);
Assert.IsTrue(partCount > 1, "Source file is too small to perform multipart upload");
LogUtility.LogMessage("File {0} is splitted to {1} parts for multipart upload",
sourceFile, partCount);
// 新建一个List保存每个分块上传后的ETag和PartNumber
var partETags = new List<PartETag>();
//upload the file
using (var fs = new FileStream(partFile.FullName, FileMode.Open))
{
//use partCount - 1, so that the last part is left
for (var i = 0; i < partCount - 1; i++)
{
// 跳到每个分块的开头
long skipBytes = partSize * i;
fs.Position = skipBytes;
// 计算每个分块的大小
var size = partSize < partFile.Length - skipBytes
? partSize
: partFile.Length - skipBytes;
// 创建UploadPartRequest，上传分块
var uploadPartRequest = new UploadPartRequest(_bucketName, targetObjectKey, initResult.UploadId);
uploadPartRequest.InputStream = fs;
uploadPartRequest.PartSize = size;
uploadPartRequest.PartNumber = (i + 1);
var uploadPartResult = _ossClient.UploadPart(uploadPartRequest);
// 将返回的PartETag保存到List中。
partETags.Add(uploadPartResult.PartETag);
//list parts which are uploaded
var listPartsRequest = new ListPartsRequest(_bucketName, targetObjectKey, initResult.UploadId);
var listPartsResult = _ossClient.ListParts(listPartsRequest);
//there should be only 1 part was not uploaded
Assert.AreEqual(i + 1, listPartsResult.Parts.Count(), "uploaded parts is not expected");
}
}
//abort the upload
var abortRequest = new AbortMultipartUploadRequest(_bucketName, targetObjectKey, initResult.UploadId);
_ossClient.AbortMultipartUpload(abortRequest);
}
private void UploadPart(object state)
{
UploadTask taskParam = state as UploadTask;
if (taskParam == null)
{
throw new ClientException("Internal error. The state object should be an instance of class UploadTaskParam");
}
try
{
ResumablePartContext part = taskParam.ResumableUploadPartContext;
if (part.IsCompleted)
{
return;
}
const int retryCount = 3;
Stream stream = taskParam.InputStream;
for (int i = 0; i < retryCount; i++)
{
if (stream is FileStream)
{
stream.Seek(part.Position, SeekOrigin.Begin);
}
else
{
stream.Seek(0, SeekOrigin.Begin);
}
Stream progressCallbackStream = null;
try
{
if (taskParam.UploadProgressCallback != null)
{
progressCallbackStream = _ossClient.SetupProgressListeners(stream,
taskParam.UploadFileStream.Length, // does not matter
_uploadedBytes, // does not matter
Math.Min(taskParam.ProgressUpdateInterval, 1024 * 4),
this.ProcessCallbackInternal);
}
var request = new UploadPartRequest(taskParam.ResumableUploadContext.BucketName, taskParam.ResumableUploadContext.Key, taskParam.ResumableUploadContext.UploadId)
{
InputStream = progressCallbackStream ?? stream,
PartSize = part.Length,
PartNumber = part.PartId,
RequestPayer = _request.RequestPayer,
TrafficLimit = _request.TrafficLimit
};
var partResult = _ossClient.UploadPart(request);
part.PartETag = partResult.PartETag;
if (partResult.ResponseMetadata.ContainsKey(HttpHeaders.HashCrc64Ecma))
{
part.Crc64 = ulong.Parse(partResult.ResponseMetadata[HttpHeaders.HashCrc64Ecma]);
}
part.IsCompleted = true;
break;
}
catch (Exception ex) // when the connection is closed while sending the data, it will run into ObjectDisposedException.
{
if (!(ex is ObjectDisposedException || ex is WebException) || i == retryCount - 1)
{
throw;
}
}
finally
{
if (progressCallbackStream != null)
{
progressCallbackStream.Dispose();
}
}
}
}
catch (Exception e)
{
taskParam.Error = e;
}
finally
{
taskParam.Finished.Set();
}
}
public void MultipartUploadComplexStepTest()
{
var sourceFile = Config.MultiUploadSampleFile;
//get target object name
var targetObjectKey = OssTestUtils.GetObjectKey(_className);
var initRequest = new InitiateMultipartUploadRequest(_bucketName, targetObjectKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
// 设置每块为 1M
const int partSize = 1024 * 1024 * 1;
var partFile = new FileInfo(sourceFile);
// 计算分块数目
var partCount = OssTestUtils.CalculatePartCount(partFile.Length, partSize);
LogUtility.LogMessage("File {0} is splitted to {1} parts for multipart upload",
sourceFile, partCount);
// 新建一个List保存每个分块上传后的ETag和PartNumber
var partETags = new List<PartETag>();
//upload the file
using (var fs = new FileStream(partFile.FullName, FileMode.Open))
{
for (var i = 0; i < partCount; i++)
{
// 跳到每个分块的开头
long skipBytes = partSize*i;
fs.Position = skipBytes;
// 计算每个分块的大小
var size = partSize < partFile.Length - skipBytes
? partSize
: partFile.Length - skipBytes;
// 创建UploadPartRequest，上传分块
var uploadPartRequest = new UploadPartRequest(_bucketName, targetObjectKey, initResult.UploadId)
{
InputStream = fs,
PartSize = size,
PartNumber = (i + 1)
};
var uploadPartResult = _ossClient.UploadPart(uploadPartRequest);
// 将返回的PartETag保存到List中。
partETags.Add(uploadPartResult.PartETag);
}
}
var lmuRequest = new ListMultipartUploadsRequest(_bucketName);
var lmuListing = _ossClient.ListMultipartUploads(lmuRequest);
var mpUpload = lmuListing.MultipartUploads.Single(t => t.UploadId == initResult.UploadId);
Assert.IsNotNull(mpUpload, "The multipart uploading should be in progress");
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, targetObjectKey, initResult.UploadId);
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
_ossClient.CompleteMultipartUpload(completeRequest);
Assert.IsTrue(OssTestUtils.ObjectExists(_ossClient, _bucketName, targetObjectKey));
//delete the object
_ossClient.DeleteObject(_bucketName, targetObjectKey);
}
public static void MultiPartUpload(IOss ossClient, string bucketName,
string objectKeyName, string originalFile, int numberOfParts, int totalSize)
{
var initRequest = new InitiateMultipartUploadRequest(bucketName, objectKeyName);
var initResult = ossClient.InitiateMultipartUpload(initRequest);
var partSize = totalSize%numberOfParts == 0 ? totalSize/numberOfParts : totalSize/numberOfParts + 1;
LogUtility.LogMessage("Each part size is {0} KB", partSize);
//change to Byte
partSize *= 1024;
var partFile = new FileInfo(originalFile);
// 新建一个List保存每个分块上传后的ETag和PartNumber
var partETags = new List<PartETag>();
//upload the file
using (var fs = new FileStream(partFile.FullName, FileMode.Open))
{
for (var i = 0; i < numberOfParts; i++)
{
// 跳到每个分块的开头
long skipBytes = partSize * i;
fs.Position = skipBytes;
// 计算每个分块的大小
long size = partSize < partFile.Length - skipBytes
? partSize
: partFile.Length - skipBytes;
// 创建UploadPartRequest，上传分块
var uploadPartRequest = new UploadPartRequest(bucketName, objectKeyName, initResult.UploadId);
uploadPartRequest.InputStream = fs;
uploadPartRequest.PartSize = size;
uploadPartRequest.PartNumber = (i + 1);
var uploadPartResult = ossClient.UploadPart(uploadPartRequest);
// 将返回的PartETag保存到List中。
partETags.Add(uploadPartResult.PartETag);
}
}
var completeRequest = new CompleteMultipartUploadRequest(bucketName, objectKeyName, initResult.UploadId);
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
ossClient.CompleteMultipartUpload(completeRequest);
}
//[Test]
public void ListMultipartUploadsWithHiddenCharacters()
{
char[] buffer = new char[2];
buffer[0] = Convert.ToChar(0x1c);
buffer[1] = Convert.ToChar(0x1a);
var newKey = _keyName + (new string(buffer)) + ".1.cd";
try
{
var initRequest = new InitiateMultipartUploadRequest(_bucketName, newKey);
var initResult = _ossClient.InitiateMultipartUpload(initRequest);
Assert.AreEqual(newKey, initRequest.Key);
// 设置每块为 5M
const int partSize = 5 * 1024 * 1024;
var fileInfo = new FileInfo(Config.MultiUploadSampleFile);
// 计算分块数目
var partCount = OssTestUtils.CalculatePartCount(fileInfo.Length, partSize);
// 新建一个List保存每个分块上传后的ETag和PartNumber
var partETags = new List<PartETag>();
//upload the file
using (var fs = new FileStream(fileInfo.FullName, FileMode.Open))
{
for (var i = 0; i < partCount; i++)
{
// 跳到每个分块的开头
long skipBytes = partSize * i;
fs.Position = skipBytes;
// 计算每个分块的大小
long size = partSize < fileInfo.Length - skipBytes ? partSize : fileInfo.Length - skipBytes;
// 创建UploadPartRequest，上传分块
var uploadPartRequest = new UploadPartRequest(_bucketName, newKey, initResult.UploadId);
uploadPartRequest.InputStream = fs;
uploadPartRequest.PartSize = size;
uploadPartRequest.PartNumber = (i + 1);
var uploadPartResult = _ossClient.UploadPart(uploadPartRequest);
// 将返回的PartETag保存到List中。
partETags.Add(uploadPartResult.PartETag);
var multipartListing = _ossClient.ListMultipartUploads(new ListMultipartUploadsRequest(_bucketName));
foreach (var upload in multipartListing.MultipartUploads)
{
Assert.AreEqual(newKey, upload.Key);
}
}
}
var completeRequest = new CompleteMultipartUploadRequest(_bucketName, newKey, initResult.UploadId);
foreach (var partETag in partETags)
{
completeRequest.PartETags.Add(partETag);
}
_ossClient.CompleteMultipartUpload(completeRequest);
}
finally
{
var multipartListing = _ossClient.ListMultipartUploads(new ListMultipartUploadsRequest(_bucketName));
foreach (var upload in multipartListing.MultipartUploads)
{
var abortRequest = new AbortMultipartUploadRequest(_bucketName, newKey, upload.UploadId);
_ossClient.AbortMultipartUpload(abortRequest);
}
_ossClient.DeleteObject(_bucketName, newKey);
}
}
private static List<PartETag> UploadParts(String bucketName, String objectName, String fileToUpload,
String uploadId, int partSize)
{
var fi = new FileInfo(fileToUpload);
var fileSize = fi.Length;
var partCount = fileSize / partSize;
if (fileSize % partSize != 0)
{
partCount++;
}
var partETags = new List<PartETag>();
using (var fs = File.Open(fileToUpload, FileMode.Open))
{
for (var i = 0; i < partCount; i++)
{
var skipBytes = (long)partSize * i;
fs.Seek(skipBytes, 0);
var size = (partSize < fileSize - skipBytes) ? partSize : (fileSize - skipBytes);
var request = new UploadPartRequest(bucketName, objectName, uploadId)
{
InputStream = fs,
PartSize = size,
PartNumber = i + 1
};
var result = client.UploadPart(request);
Console.WriteLine("oss:" + result.PartETag);
partETags.Add(result.PartETag);
}
}
return partETags;
}
public static Task <UploadPartResult> UploadPartAsync(this IOss client, UploadPartRequest request)
{
return(Task.Factory.FromAsync(client.BeginUploadPart, client.EndUploadPart, request, null));
}
