public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems)
{
AmazonGlacierConfig config = new AmazonGlacierConfig();
config.RegionEndpoint = region;
ConfigureClient(config);
AmazonGlacierClient client = new AmazonGlacierClient(creds, config);
ListMultipartUploadsResponse resp = new ListMultipartUploadsResponse();
do
{
ListMultipartUploadsRequest req = new ListMultipartUploadsRequest
{
Limit = maxItems
};
resp = client.ListMultipartUploads(req);
CheckError(resp.HttpStatusCode, "200");
foreach (var obj in resp.UploadsList)
{
AddObject(obj);
}
}while (!string.IsNullOrEmpty(resp.Marker));
}
public async Task ListMultipartUploads()
{
await CreateTempBucketAsync(async bucket =>
{
string objName = nameof(ListMultipartUploads) + "%";
CreateMultipartUploadResponse createResp = await MultipartClient.CreateMultipartUploadAsync(bucket, objName).ConfigureAwait(false);
byte[] file = new byte[5 * 1024];
using (MemoryStream ms = new MemoryStream(file))
await MultipartClient.UploadPartAsync(bucket, objName, 1, createResp.UploadId, ms).ConfigureAwait(false);
ListMultipartUploadsResponse listResp = await MultipartClient.ListMultipartUploadsAsync(bucket, req => req.EncodingType = EncodingType.Url).ConfigureAwait(false);
Assert.Equal(bucket, listResp.Bucket);
Assert.Equal(WebUtility.UrlEncode(objName), listResp.NextKeyMarker);
Assert.NotEmpty(listResp.NextUploadIdMarker);
Assert.Equal(1000, listResp.MaxUploads);
Assert.False(listResp.IsTruncated);
S3Upload upload = Assert.Single(listResp.Uploads);
Assert.Equal(listResp.NextKeyMarker, upload.ObjectKey);
Assert.Equal(listResp.NextUploadIdMarker, upload.UploadId);
Assert.Equal(StorageClass.Standard, upload.StorageClass);
Assert.Equal(DateTime.UtcNow, upload.Initiated.DateTime, TimeSpan.FromSeconds(5));
}).ConfigureAwait(false);
}
public async Task ListIncompleteParts()
{
await CreateTempBucketAsync(async bucket =>
{
InitiateMultipartUploadResponse initResp = await ObjectClient.InitiateMultipartUploadAsync(bucket, nameof(ListIncompleteParts)).ConfigureAwait(false);
byte[] file = new byte[5 * 1024];
using (MemoryStream ms = new MemoryStream(file))
await ObjectClient.UploadPartAsync(bucket, nameof(ListIncompleteParts), 1, initResp.UploadId, ms).ConfigureAwait(false);
ListMultipartUploadsResponse listResp = await BucketClient.ListMultipartUploadsAsync(bucket).ConfigureAwait(false);
Assert.Equal(bucket, listResp.Bucket);
Assert.Equal("ListIncompleteParts", listResp.NextKeyMarker);
Assert.NotEmpty(listResp.NextUploadIdMarker);
Assert.Equal(1000, listResp.MaxUploads);
Assert.False(listResp.IsTruncated);
S3Upload upload = Assert.Single(listResp.Uploads);
Assert.Equal(listResp.NextKeyMarker, upload.Name);
Assert.Equal(listResp.NextUploadIdMarker, upload.UploadId);
Assert.Equal(TestConstants.TestUsername, upload.Initiator.Name);
Assert.Equal(StorageClass.Standard, upload.StorageClass);
Assert.Equal(DateTime.UtcNow, upload.Initiated.DateTime, TimeSpan.FromSeconds(5));
}).ConfigureAwait(false);
}
public override void Execute()
{
if (string.IsNullOrEmpty(this._bucketName))
{
throw new ArgumentNullException("bucketName", "The bucketName Specified is null or empty!");
}
ListMultipartUploadsResponse listResponse = new ListMultipartUploadsResponse();
do
{
ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest()
.WithBucketName(this._bucketName)
.WithKeyMarker(listResponse.KeyMarker)
.WithUploadIdMarker(listResponse.NextUploadIdMarker)
.WithBeforeRequestHandler(RequestEventHandler) as ListMultipartUploadsRequest;
listResponse = this._s3Client.ListMultipartUploads(listRequest);
foreach (MultipartUpload upload in listResponse.MultipartUploads)
{
if (upload.Initiated < this._initiatedDate)
{
this._s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest()
.WithBucketName(this._bucketName)
.WithKey(upload.Key)
.WithUploadId(upload.UploadId)
.WithBeforeRequestHandler(RequestEventHandler) as AbortMultipartUploadRequest);
}
}
}while (listResponse.IsTruncated);
}
static void ListMultipartUploads()
{
try
{
ListMultipartUploadsRequest request = new ListMultipartUploadsRequest()
{
BucketName = bucketName,
Delimiter = "delimiter",
Prefix = "prefix",
KeyMarker = "keymarker",
MaxUploads = 10,
UploadIdMarker = "uploadidmarker"
};
ListMultipartUploadsResponse response = client.ListMultipartUploads(request);
Console.WriteLine("List multipart uploads response: {0}", response.StatusCode);
foreach (MultipartUpload multipart in response.MultipartUploads)
{
Console.WriteLine("MultipartUpload object key: {0}", multipart.ObjectKey);
Console.WriteLine("MultipartUpload upload id: {0}", multipart.UploadId);
}
}
catch (ObsException ex)
{
Console.WriteLine("Exception errorcode: {0}, when list multipart uploads.", ex.ErrorCode);
Console.WriteLine("Exception errormessage: {0}", ex.ErrorMessage);
}
}
/// <summary>
/// Unmarshaller the response from the service to the response class.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
ListMultipartUploadsResponse response = new ListMultipartUploadsResponse();
context.Read();
int targetDepth = context.CurrentDepth;
while (context.ReadAtDepth(targetDepth))
{
if (context.TestExpression("Marker", targetDepth))
{
var unmarshaller = StringUnmarshaller.Instance;
response.Marker = unmarshaller.Unmarshall(context);
continue;
}
if (context.TestExpression("UploadsList", targetDepth))
{
var unmarshaller = new ListUnmarshaller <UploadListElement, UploadListElementUnmarshaller>(UploadListElementUnmarshaller.Instance);
response.UploadsList = unmarshaller.Unmarshall(context);
continue;
}
}
return(response);
}
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
ListMultipartUploadsResponse response = new ListMultipartUploadsResponse();
context.Read();
UnmarshallResult(context, response);
return(response);
}
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
ListMultipartUploadsResponse response = new ListMultipartUploadsResponse();
context.Read();
response.ListMultipartUploadsResult = ListMultipartUploadsResultUnmarshaller.GetInstance().Unmarshall(context);
return(response);
}
private ListMultipartUploadsRequest ConstructListMultipartUploadsRequest(ListMultipartUploadsResponse listResponse)
{
ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest()
{
BucketName = this._bucketName,
KeyMarker = listResponse.KeyMarker,
UploadIdMarker = listResponse.NextUploadIdMarker,
};
listRequest.BeforeRequestEvent += this.RequestEventHandler;
return(listRequest);
}
private ListMultipartUploadsRequest ConstructListMultipartUploadsRequest(ListMultipartUploadsResponse listResponse)
{
ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest()
{
BucketName = this._bucketName,
KeyMarker = listResponse.KeyMarker,
UploadIdMarker = listResponse.NextUploadIdMarker,
};
((Amazon.Runtime.Internal.IAmazonWebServiceRequest)listRequest).AddBeforeRequestHandler(this.RequestEventHandler);
return(listRequest);
}
public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext context)
{
ListMultipartUploadsResponse listMultipartUploadsResponse = new ListMultipartUploadsResponse();
while (context.Read())
{
if (context.get_IsStartElement())
{
UnmarshallResult(context, listMultipartUploadsResponse);
}
}
return(listMultipartUploadsResponse);
}
private static void UnmarshallResult(JsonUnmarshallerContext context, ListMultipartUploadsResponse response)
{
int originalDepth = context.CurrentDepth;
int targetDepth = originalDepth + 1;
while (context.Read())
{
if (context.TestExpression("UploadsList", targetDepth))
{
context.Read();
if (context.CurrentTokenType == JsonToken.Null)
{
response.UploadsList = null;
continue;
}
response.UploadsList = new List <UploadListElement>();
UploadListElementUnmarshaller unmarshaller = UploadListElementUnmarshaller.GetInstance();
while (context.Read())
{
JsonToken token = context.CurrentTokenType;
if (token == JsonToken.ArrayStart)
{
continue;
}
if (token == JsonToken.ArrayEnd)
{
break;
}
response.UploadsList.Add(unmarshaller.Unmarshall(context));
}
continue;
}
if (context.TestExpression("Marker", targetDepth))
{
context.Read();
response.Marker = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.CurrentDepth <= originalDepth)
{
return;
}
}
return;
}
public void RemoveAllMultiUploads(String bucketName)
{
BosClient client = this.GenerateBosClient();
ListMultipartUploadsResponse list = client.ListMultipartUploads(bucketName);
foreach (MultipartUploadSummary summary in list.Uploads)
{
String uploadID = summary.UploadId;
String key = summary.Key;
String name = summary.Owner.DisplayName;
client.AbortMultipartUpload(bucketName, key, uploadID);
String sss = "";
}
}
public async Task ListMultipartUploads(S3Provider provider, string _, ISimpleClient client)
{
await CreateTempBucketAsync(provider, client, async tempBucket =>
{
//The percentage sign at the end is to test if encoding works correctly
string objName = nameof(ListMultipartUploads) + "%";
CreateMultipartUploadResponse createResp = await client.CreateMultipartUploadAsync(tempBucket, objName).ConfigureAwait(false);
Assert.Equal(200, createResp.StatusCode);
byte[] file = new byte[5 * 1024];
await using (MemoryStream ms = new MemoryStream(file))
await client.UploadPartAsync(tempBucket, objName, 1, createResp.UploadId, ms).ConfigureAwait(false);
ListMultipartUploadsResponse listResp = await client.ListMultipartUploadsAsync(tempBucket, r => r.EncodingType = EncodingType.Url).ConfigureAwait(false);
Assert.Equal(200, listResp.StatusCode);
Assert.Equal(tempBucket, listResp.Bucket);
if (provider == S3Provider.AmazonS3)
{
Assert.Equal(WebUtility.UrlEncode(objName), listResp.NextKeyMarker);
Assert.NotEmpty(listResp.NextUploadIdMarker);
}
Assert.Equal(1000, listResp.MaxUploads);
Assert.False(listResp.IsTruncated);
S3Upload?upload = Assert.Single(listResp.Uploads);
Assert.Equal(WebUtility.UrlEncode(objName), upload.ObjectKey);
if (provider == S3Provider.AmazonS3)
{
Assert.Equal(listResp.NextUploadIdMarker, upload.UploadId);
}
Assert.Equal(StorageClass.Standard, upload.StorageClass);
Assert.Equal(DateTime.UtcNow, upload.Initiated.DateTime, TimeSpan.FromSeconds(5));
}).ConfigureAwait(false);
}
public override void Execute()
{
if (string.IsNullOrEmpty(this._bucketName))
{
throw new InvalidOperationException("The bucketName Specified is null or empty!");
}
ListMultipartUploadsResponse listResponse = new ListMultipartUploadsResponse();
do
{
ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest()
{
BucketName = this._bucketName,
KeyMarker = listResponse.KeyMarker,
UploadIdMarker = listResponse.NextUploadIdMarker,
};
listRequest.BeforeRequestEvent += this.RequestEventHandler;
listResponse = this._s3Client.ListMultipartUploads(listRequest);
foreach (MultipartUpload upload in listResponse.MultipartUploads)
{
if (upload.Initiated < this._initiatedDate)
{
var abortRequest = new AbortMultipartUploadRequest()
{
BucketName = this._bucketName,
Key = upload.Key,
UploadId = upload.UploadId,
};
abortRequest.BeforeRequestEvent += this.RequestEventHandler;
this._s3Client.AbortMultipartUpload(abortRequest);
}
}
}while (listResponse.IsTruncated);
}
public override void Execute()
{
if (string.IsNullOrEmpty(this._bucketName))
{
throw new InvalidOperationException("The bucketName Specified is null or empty!");
}
ListMultipartUploadsResponse listResponse = new ListMultipartUploadsResponse();
do
{
ListMultipartUploadsRequest listRequest = ConstructListMultipartUploadsRequest(listResponse);
listResponse = this._s3Client.ListMultipartUploads(listRequest);
foreach (MultipartUpload upload in listResponse.MultipartUploads)
{
if (upload.Initiated < this._initiatedDate)
{
var abortRequest = ConstructAbortMultipartUploadRequest(upload);
this._s3Client.AbortMultipartUpload(abortRequest);
}
}
}while (listResponse.IsTruncated);
}
private static void UnmarshallResult(XmlUnmarshallerContext context, ListMultipartUploadsResponse response)
{
int currentDepth = context.get_CurrentDepth();
int num = currentDepth + 1;
if (context.get_IsStartOfDocument())
{
num += 2;
}
while (context.Read())
{
if (context.get_IsStartElement() || context.get_IsAttribute())
{
if (context.TestExpression("Bucket", num))
{
response.BucketName = StringUnmarshaller.GetInstance().Unmarshall(context);
}
else if (context.TestExpression("KeyMarker", num))
{
response.KeyMarker = StringUnmarshaller.GetInstance().Unmarshall(context);
}
else if (context.TestExpression("UploadIdMarker", num))
{
response.UploadIdMarker = StringUnmarshaller.GetInstance().Unmarshall(context);
}
else if (context.TestExpression("NextKeyMarker", num))
{
response.NextKeyMarker = StringUnmarshaller.GetInstance().Unmarshall(context);
}
else if (context.TestExpression("NextUploadIdMarker", num))
{
response.NextUploadIdMarker = StringUnmarshaller.GetInstance().Unmarshall(context);
}
else if (context.TestExpression("MaxUploads", num))
{
response.MaxUploads = IntUnmarshaller.GetInstance().Unmarshall(context);
}
else if (context.TestExpression("IsTruncated", num))
{
response.IsTruncated = BoolUnmarshaller.GetInstance().Unmarshall(context);
}
else if (context.TestExpression("Upload", num))
{
response.MultipartUploads.Add(MultipartUploadUnmarshaller.Instance.Unmarshall(context));
}
else if (context.TestExpression("Delimiter", num))
{
response.Delimiter = StringUnmarshaller.GetInstance().Unmarshall(context);
}
else if (context.TestExpression("Prefix", num))
{
response.Prefix = StringUnmarshaller.GetInstance().Unmarshall(context);
}
else if (context.TestExpression("CommonPrefixes", num))
{
string text = CommonPrefixesItemUnmarshaller.Instance.Unmarshall(context);
if (text != null)
{
response.CommonPrefixes.Add(text);
}
}
}
else if (context.get_IsEndElement() && context.get_CurrentDepth() < currentDepth)
{
break;
}
}
}
private static void Main(string[] args)
{
BosClient client = SampleUtil.GetSampleBosClient();
const string bucketName = "sample-bucket-multi-upload1"; //示例Bucket名称
// 初始化：创建示例Bucket
client.CreateBucket(bucketName); //指定Bucket名称
string objectKey = "sample";
// 1.开始Multipart Upload
InitiateMultipartUploadRequest initiateMultipartUploadRequest =
new InitiateMultipartUploadRequest()
{
BucketName = bucketName, Key = objectKey
};
InitiateMultipartUploadResponse initiateMultipartUploadResponse =
client.InitiateMultipartUpload(initiateMultipartUploadRequest);
// 2.获取Bucket内的Multipart Upload
ListMultipartUploadsRequest listMultipartUploadsRequest =
new ListMultipartUploadsRequest()
{
BucketName = bucketName
};
ListMultipartUploadsResponse listMultipartUploadsResponse =
client.ListMultipartUploads(listMultipartUploadsRequest);
foreach (MultipartUploadSummary multipartUpload in listMultipartUploadsResponse.Uploads)
{
Console.WriteLine("Key: " + multipartUpload.Key + " UploadId: " + multipartUpload.UploadId);
}
// 3.分块上传，首先设置每块为 5Mb
long partSize = 1024 * 1024 * 5L;
FileInfo partFile = new FileInfo("d:\\lzb\\sample");
// 计算分块数目
int partCount = (int)(partFile.Length / partSize);
if (partFile.Length % partSize != 0)
{
partCount++;
}
// 新建一个List保存每个分块上传后的ETag和PartNumber
List <PartETag> partETags = new List <PartETag>();
for (int i = 0; i < partCount; i++)
{
// 获取文件流
Stream stream = partFile.OpenRead();
// 跳到每个分块的开头
long skipBytes = partSize * i;
stream.Seek(skipBytes, SeekOrigin.Begin);
// 计算每个分块的大小
long size = Math.Min(partSize, partFile.Length - skipBytes);
// 创建UploadPartRequest，上传分块
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.BucketName = bucketName;
uploadPartRequest.Key = objectKey;
uploadPartRequest.UploadId = initiateMultipartUploadResponse.UploadId;
uploadPartRequest.InputStream = stream;
uploadPartRequest.PartSize = size;
uploadPartRequest.PartNumber = i + 1;
UploadPartResponse uploadPartResponse = client.UploadPart(uploadPartRequest);
// 将返回的PartETag保存到List中。
partETags.Add(new PartETag()
{
ETag = uploadPartResponse.ETag,
PartNumber = uploadPartResponse.PartNumber
});
// 关闭文件
stream.Close();
}
// 4. 获取UploadId的所有Upload Part
ListPartsRequest listPartsRequest = new ListPartsRequest()
{
BucketName = bucketName,
Key = objectKey,
UploadId = initiateMultipartUploadResponse.UploadId,
};
// 获取上传的所有Part信息
ListPartsResponse listPartsResponse = client.ListParts(listPartsRequest);
// 遍历所有Part
foreach (PartSummary part in listPartsResponse.Parts)
{
Console.WriteLine("PartNumber: " + part.PartNumber + " ETag: " + part.ETag);
}
// 5. 完成分块上传
CompleteMultipartUploadRequest completeMultipartUploadRequest =
new CompleteMultipartUploadRequest()
{
BucketName = bucketName,
Key = objectKey,
UploadId = initiateMultipartUploadResponse.UploadId,
PartETags = partETags
};
CompleteMultipartUploadResponse completeMultipartUploadResponse =
client.CompleteMultipartUpload(completeMultipartUploadRequest);
Console.WriteLine(completeMultipartUploadResponse.ETag);
Console.ReadKey();
}
private static void UnmarshallResult(XmlUnmarshallerContext context, ListMultipartUploadsResponse response)
{
int originalDepth = context.CurrentDepth;
int targetDepth = originalDepth + 1;
if (context.IsStartOfDocument)
{
targetDepth += 2;
}
while (context.Read())
{
if (context.IsStartElement || context.IsAttribute)
{
if (context.TestExpression("Bucket", targetDepth))
{
response.BucketName = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("KeyMarker", targetDepth))
{
response.KeyMarker = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("UploadIdMarker", targetDepth))
{
response.UploadIdMarker = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("NextKeyMarker", targetDepth))
{
response.NextKeyMarker = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("NextUploadIdMarker", targetDepth))
{
response.NextUploadIdMarker = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("MaxUploads", targetDepth))
{
response.MaxUploads = IntUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("IsTruncated", targetDepth))
{
response.IsTruncated = BoolUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("Upload", targetDepth))
{
response.MultipartUploads.Add(MultipartUploadUnmarshaller.Instance.Unmarshall(context));
continue;
}
if (context.TestExpression("Delimiter", targetDepth))
{
response.Delimiter = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("Prefix", targetDepth))
{
response.Prefix = StringUnmarshaller.GetInstance().Unmarshall(context);
continue;
}
if (context.TestExpression("CommonPrefixes", targetDepth))
{
var prefix = CommonPrefixesItemUnmarshaller.Instance.Unmarshall(context);
if (prefix != null)
{
response.CommonPrefixes.Add(prefix);
}
continue;
}
}
else if (context.IsEndElement && context.CurrentDepth < originalDepth)
{
return;
}
}
return;
}
public override async Task ExecuteAsync(CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(this._bucketName))
{
throw new InvalidOperationException("The bucketName specified is null or empty!");
}
SemaphoreSlim asyncThrottler = null;
CancellationTokenSource internalCts = null;
try
{
asyncThrottler = new SemaphoreSlim(_config.ConcurrentServiceRequests);
internalCts = new CancellationTokenSource();
var internalCancellationToken = internalCts.Token;
ListMultipartUploadsResponse listResponse = new ListMultipartUploadsResponse();
var pendingTasks = new List <Task <AbortMultipartUploadResponse> >();
do
{
ListMultipartUploadsRequest listRequest = ConstructListMultipartUploadsRequest(listResponse);
listResponse = await this._s3Client.ListMultipartUploadsAsync(listRequest, cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
foreach (MultipartUpload upload in listResponse.MultipartUploads)
{
cancellationToken.ThrowIfCancellationRequested();
if (internalCancellationToken.IsCancellationRequested)
{
// Operation cancelled as one of the AbortMultipartUpload requests failed with an exception,
// don't schedule any more AbortMultipartUpload tasks.
// Don't throw an OperationCanceledException here as we want to process the
// responses and throw the original exception.
break;
}
if (upload.Initiated < this._initiatedDate)
{
await asyncThrottler.WaitAsync(cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
var abortRequest = ConstructAbortMultipartUploadRequest(upload);
var task = AbortAsync(abortRequest, internalCts, cancellationToken, asyncThrottler);
pendingTasks.Add(task);
}
}
}while (listResponse.IsTruncated);
await WhenAllOrFirstExceptionAsync(pendingTasks, cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
}
finally
{
if (internalCts != null)
{
internalCts.Dispose();
}
if (asyncThrottler != null)
{
asyncThrottler.Dispose();
}
}
}
