public async Task MultipartTooSmall()
{
string objectKey = nameof(MultipartTooSmall);
CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey).ConfigureAwait(false);
Assert.Equal(BucketName, initResp.Bucket);
Assert.Equal(objectKey, initResp.ObjectKey);
Assert.NotNull(initResp.UploadId);
//4 MB is below the 5 MB limit. See https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html
//Note that if there only is 1 part, then it is technically the last part, and can be of any size. That's why this test has 2 parts.
byte[] file = new byte[1024 * 1024 * 4];
byte[][] chunks = file.Chunk(file.Length / 2).Select(x => x.ToArray()).ToArray();
UploadPartResponse uploadResp1 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(chunks[0])).ConfigureAwait(false);
Assert.True(uploadResp1.IsSuccess);
Assert.NotNull(uploadResp1.ETag);
UploadPartResponse uploadResp2 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 2, initResp.UploadId, new MemoryStream(chunks[1])).ConfigureAwait(false);
Assert.True(uploadResp2.IsSuccess);
Assert.NotNull(uploadResp2.ETag);
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp1, uploadResp2 }).ConfigureAwait(false);
Assert.False(completeResp.IsSuccess);
Assert.Equal(400, completeResp.StatusCode);
}
public async Task MultipartCustomerEncryption(S3Provider _, string bucket, ISimpleClient client)
{
string objectKey = nameof(MultipartCustomerEncryption);
byte[] key = { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 };
byte[] keyMd5 = CryptoHelper.Md5Hash(key);
CreateMultipartUploadResponse initResp = await client.CreateMultipartUploadAsync(bucket, objectKey, r =>
{
r.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
r.SseCustomerKey = key;
r.SseCustomerKeyMd5 = keyMd5;
}).ConfigureAwait(false);
Assert.Equal(200, initResp.StatusCode);
await using MemoryStream ms = new MemoryStream(new byte[1024 * 1024 * 5]);
UploadPartResponse uploadResp1 = await client.UploadPartAsync(bucket, objectKey, 1, initResp.UploadId, ms, r =>
{
r.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
r.SseCustomerKey = key;
r.SseCustomerKeyMd5 = keyMd5;
}).ConfigureAwait(false);
Assert.Equal(200, uploadResp1.StatusCode);
Assert.Equal(SseCustomerAlgorithm.Aes256, uploadResp1.SseCustomerAlgorithm);
Assert.Equal(keyMd5, uploadResp1.SseCustomerKeyMd5);
CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, new[] { uploadResp1 }).ConfigureAwait(false);
Assert.Equal(200, completeResp.StatusCode);
Assert.Equal(SseCustomerAlgorithm.Aes256, completeResp.SseCustomerAlgorithm);
}
public async Task MultipartSinglePart()
{
string objectKey = nameof(MultipartSinglePart);
CreateMultipartUploadResponse createResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey).ConfigureAwait(false);
Assert.True(createResp.IsSuccess);
Assert.Equal(BucketName, createResp.Bucket);
Assert.Equal(objectKey, createResp.ObjectKey);
Assert.NotNull(createResp.UploadId);
//Test lifecycle expiration
Assert.Equal(DateTime.UtcNow.AddDays(2).Date, createResp.AbortsOn.Value.UtcDateTime.Date);
Assert.Equal("ExpireAll", createResp.AbortRuleId);
byte[] file = new byte[1024 * 1024 * 5];
file[0] = (byte)'a';
UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, createResp.UploadId, new MemoryStream(file)).ConfigureAwait(false);
Assert.True(uploadResp.IsSuccess);
Assert.Equal("\"10f74ef02085310ccd1f87150b83e537\"", uploadResp.ETag);
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.True(completeResp.IsSuccess);
Assert.Equal("\"bd74e21dfa8678d127240f76e518e9c2-1\"", completeResp.ETag);
Assert.NotNull(completeResp.VersionId);
//Test lifecycle expiration
Assert.Equal(DateTime.UtcNow.AddDays(2).Date, completeResp.LifeCycleExpiresOn.Value.UtcDateTime.Date);
Assert.Equal("ExpireAll", completeResp.LifeCycleRuleId);
}
public async Task MultipartWithEncryption(SseAlgorithm algorithm)
{
string objectKey = nameof(MultipartWithEncryption);
CreateMultipartUploadResponse createResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req => req.SseAlgorithm = algorithm).ConfigureAwait(false);
Assert.Equal(algorithm, createResp.SseAlgorithm);
if (algorithm == SseAlgorithm.AwsKms)
{
Assert.NotNull(createResp.SseKmsKeyId);
}
byte[] file = new byte[1024 * 1024 * 5];
UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, createResp.UploadId, new MemoryStream(file)).ConfigureAwait(false);
Assert.Equal(algorithm, uploadResp.SseAlgorithm);
if (algorithm == SseAlgorithm.AwsKms)
{
Assert.NotNull(uploadResp.SseKmsKeyId);
}
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.Equal(algorithm, completeResp.SseAlgorithm);
if (algorithm == SseAlgorithm.AwsKms)
{
Assert.NotNull(completeResp.SseKmsKeyId);
}
}
public async Task ListMultipartUploads()
{
await CreateTempBucketAsync(async bucket =>
{
string objName = nameof(ListMultipartUploads) + "%";
CreateMultipartUploadResponse createResp = await MultipartClient.CreateMultipartUploadAsync(bucket, objName).ConfigureAwait(false);
byte[] file = new byte[5 * 1024];
using (MemoryStream ms = new MemoryStream(file))
await MultipartClient.UploadPartAsync(bucket, objName, 1, createResp.UploadId, ms).ConfigureAwait(false);
ListMultipartUploadsResponse listResp = await MultipartClient.ListMultipartUploadsAsync(bucket, req => req.EncodingType = EncodingType.Url).ConfigureAwait(false);
Assert.Equal(bucket, listResp.Bucket);
Assert.Equal(WebUtility.UrlEncode(objName), listResp.NextKeyMarker);
Assert.NotEmpty(listResp.NextUploadIdMarker);
Assert.Equal(1000, listResp.MaxUploads);
Assert.False(listResp.IsTruncated);
S3Upload upload = Assert.Single(listResp.Uploads);
Assert.Equal(listResp.NextKeyMarker, upload.ObjectKey);
Assert.Equal(listResp.NextUploadIdMarker, upload.UploadId);
Assert.Equal(StorageClass.Standard, upload.StorageClass);
Assert.Equal(DateTime.UtcNow, upload.Initiated.DateTime, TimeSpan.FromSeconds(5));
}).ConfigureAwait(false);
}
public async Task MultipartTooSmall(S3Provider provider, string bucket, ISimpleClient client)
{
string objectKey = nameof(MultipartTooSmall);
CreateMultipartUploadResponse initResp = await client.CreateMultipartUploadAsync(bucket, objectKey).ConfigureAwait(false);
Assert.Equal(200, initResp.StatusCode);
Assert.Equal(bucket, initResp.BucketName);
Assert.Equal(objectKey, initResp.ObjectKey);
Assert.NotNull(initResp.UploadId);
//4 MB is below the 5 MB limit. See https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html
//Note that if there only is 1 part, then it is technically the last part, and can be of any size. That's why this test has 2 parts.
byte[] file = new byte[1024 * 1024 * 4];
byte[][] parts = file.Chunk(file.Length / 2).Select(x => x.ToArray()).ToArray();
await using MemoryStream ms1 = new MemoryStream(parts[0]);
UploadPartResponse uploadResp1 = await client.UploadPartAsync(bucket, objectKey, 1, initResp.UploadId, ms1).ConfigureAwait(false);
Assert.Equal(provider == S3Provider.BackBlazeB2 ? 400 : 200, uploadResp1.StatusCode);
await using MemoryStream ms2 = new MemoryStream(parts[0]);
UploadPartResponse uploadResp2 = await client.UploadPartAsync(bucket, objectKey, 2, initResp.UploadId, ms2).ConfigureAwait(false);
Assert.Equal(provider == S3Provider.BackBlazeB2 ? 400 : 200, uploadResp2.StatusCode);
CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, new[] { uploadResp1, uploadResp2 }).ConfigureAwait(false);
Assert.Equal(provider == S3Provider.BackBlazeB2 ? 500 : 400, completeResp.StatusCode);
}
public async Task MultipartCustomerEncryption()
{
string objectKey = nameof(MultipartCustomerEncryption);
byte[] key = { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 };
byte[] keyMd5 = CryptoHelper.Md5Hash(key);
CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req =>
{
req.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
req.SseCustomerKey = key;
req.SseCustomerKeyMd5 = keyMd5;
}).ConfigureAwait(false);
Assert.True(initResp.IsSuccess);
byte[] file = new byte[1024 * 1024 * 5];
UploadPartResponse uploadResp1 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(file), req =>
{
req.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
req.SseCustomerKey = key;
req.SseCustomerKeyMd5 = keyMd5;
}).ConfigureAwait(false);
Assert.Equal(SseCustomerAlgorithm.Aes256, uploadResp1.SseCustomerAlgorithm);
Assert.Equal(keyMd5, uploadResp1.SseCustomerKeyMd5);
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp1 }).ConfigureAwait(false);
Assert.True(completeResp.IsSuccess);
Assert.Equal(SseCustomerAlgorithm.Aes256, completeResp.SseCustomerAlgorithm);
}
public async Task ListParts()
{
await CreateTempBucketAsync(async bucket =>
{
//We add the special characters at the end to test EncodingType support.
string objName = nameof(ListParts) + "%";
CreateMultipartUploadResponse createResp = await MultipartClient.CreateMultipartUploadAsync(bucket, objName).ConfigureAwait(false);
ListPartsResponse listResp1 = await MultipartClient.ListPartsAsync(bucket, objName, createResp.UploadId).ConfigureAwait(false);
Assert.Equal(bucket, listResp1.BucketName);
Assert.Equal(objName, listResp1.ObjectKey);
Assert.Equal(createResp.UploadId, listResp1.UploadId);
Assert.Equal(StorageClass.Standard, listResp1.StorageClass);
Assert.Equal(0, listResp1.PartNumberMarker);
Assert.Equal(0, listResp1.NextPartNumberMarker);
Assert.Equal(1000, listResp1.MaxParts);
Assert.False(listResp1.IsTruncated);
Assert.Equal(TestConstants.TestUsername, listResp1.Owner.Name);
Assert.Empty(listResp1.Parts);
UploadPartResponse uploadResp;
byte[] file = new byte[5 * 1024];
using (MemoryStream ms = new MemoryStream(file))
uploadResp = await MultipartClient.UploadPartAsync(bucket, objName, 1, createResp.UploadId, ms).ConfigureAwait(false);
ListPartsResponse listResp2 = await MultipartClient.ListPartsAsync(bucket, objName, createResp.UploadId, req => req.EncodingType = EncodingType.Url).ConfigureAwait(false);
Assert.Equal(bucket, listResp2.BucketName);
Assert.Equal(WebUtility.UrlEncode(objName), listResp2.ObjectKey); //It should be encoded at this point
Assert.Equal(createResp.UploadId, listResp2.UploadId);
Assert.Equal(StorageClass.Standard, listResp2.StorageClass);
Assert.Equal(0, listResp2.PartNumberMarker);
Assert.Equal(1, listResp2.NextPartNumberMarker);
Assert.Equal(1000, listResp2.MaxParts);
Assert.False(listResp2.IsTruncated);
Assert.Equal(TestConstants.TestUsername, listResp2.Owner.Name);
S3Part part = Assert.Single(listResp2.Parts);
Assert.Equal(1, part.PartNumber);
Assert.Equal(DateTime.UtcNow, part.LastModified.DateTime, TimeSpan.FromSeconds(5));
Assert.Equal("\"32ca18808933aa12e979375d07048a11\"", part.ETag);
Assert.Equal(file.Length, part.Size);
await MultipartClient.CompleteMultipartUploadAsync(bucket, objName, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
ListPartsResponse listResp3 = await MultipartClient.ListPartsAsync(bucket, objName, createResp.UploadId).ConfigureAwait(false);
Assert.False(listResp3.IsSuccess);
Assert.Equal(404, listResp3.StatusCode);
}).ConfigureAwait(false);
}
public async Task MultipartUpload(S3Provider provider, string bucket, ISimpleClient client)
{
string objectKey = nameof(MultipartUpload);
CreateMultipartUploadResponse initResp = await client.CreateMultipartUploadAsync(bucket, objectKey).ConfigureAwait(false);
Assert.Equal(200, initResp.StatusCode);
Assert.Equal(bucket, initResp.BucketName);
Assert.Equal(objectKey, initResp.ObjectKey);
Assert.NotNull(initResp.UploadId);
byte[] file = new byte[1024 * 1024 * 10];
file[0] = (byte)'a';
file[^ 1] = (byte)'b';
public async Task AbortIncompleteUpload(S3Provider _, string bucket, ISimpleClient client)
{
string objectKey = nameof(AbortIncompleteUpload);
CreateMultipartUploadResponse createResp = await client.CreateMultipartUploadAsync(bucket, objectKey).ConfigureAwait(false);
Assert.Equal(200, createResp.StatusCode);
Assert.Equal(bucket, createResp.BucketName);
Assert.Equal(objectKey, createResp.ObjectKey);
Assert.NotNull(createResp.UploadId);
AbortMultipartUploadResponse abortResp = await client.AbortMultipartUploadAsync(bucket, objectKey, createResp.UploadId).ConfigureAwait(false);
Assert.Equal(204, abortResp.StatusCode);
}
public async Task AbortIncompleteUpload()
{
string objectKey = nameof(AbortIncompleteUpload);
CreateMultipartUploadResponse createResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey).ConfigureAwait(false);
Assert.Equal(BucketName, createResp.Bucket);
Assert.Equal(objectKey, createResp.ObjectKey);
Assert.NotNull(createResp.UploadId);
AbortMultipartUploadResponse abortResp = await MultipartClient.AbortMultipartUploadAsync(BucketName, objectKey, createResp.UploadId).ConfigureAwait(false);
Assert.True(abortResp.IsSuccess);
Assert.Equal(204, abortResp.StatusCode);
}
public async Task MultipartWithEncryption(S3Provider provider, string bucket, ISimpleClient client, SseAlgorithm algorithm)
{
string objectKey = nameof(MultipartWithEncryption);
CreateMultipartUploadResponse createResp = await client.CreateMultipartUploadAsync(bucket, objectKey, req => req.SseAlgorithm = algorithm).ConfigureAwait(false);
Assert.Equal(200, createResp.StatusCode);
if (provider == S3Provider.AmazonS3)
{
Assert.Equal(algorithm, createResp.SseAlgorithm);
}
if (algorithm == SseAlgorithm.AwsKms)
{
Assert.NotNull(createResp.SseKmsKeyId);
}
await using MemoryStream ms = new MemoryStream(new byte[1024 * 1024 * 5]);
UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, createResp.UploadId, ms).ConfigureAwait(false);
Assert.Equal(200, uploadResp.StatusCode);
if (provider == S3Provider.AmazonS3)
{
Assert.Equal(algorithm, uploadResp.SseAlgorithm);
}
if (algorithm == SseAlgorithm.AwsKms)
{
Assert.NotNull(uploadResp.SseKmsKeyId);
}
CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.Equal(200, completeResp.StatusCode);
if (provider == S3Provider.AmazonS3)
{
Assert.Equal(algorithm, completeResp.SseAlgorithm);
}
if (algorithm == SseAlgorithm.AwsKms)
{
Assert.NotNull(completeResp.SseKmsKeyId);
}
}
public async Task MultipartUpload()
{
string objectKey = nameof(MultipartUpload);
CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey).ConfigureAwait(false);
Assert.True(initResp.IsSuccess);
Assert.Equal(BucketName, initResp.Bucket);
Assert.Equal(objectKey, initResp.ObjectKey);
Assert.NotNull(initResp.UploadId);
byte[] file = new byte[1024 * 1024 * 10];
file[0] = (byte)'a';
file[file.Length - 1] = (byte)'b';
byte[][] parts = file.Chunk(file.Length / 2).Select(x => x.ToArray()).ToArray();
UploadPartResponse uploadResp1 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(parts[0])).ConfigureAwait(false);
Assert.True(uploadResp1.IsSuccess);
Assert.NotNull(uploadResp1.ETag);
UploadPartResponse uploadResp2 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 2, initResp.UploadId, new MemoryStream(parts[1])).ConfigureAwait(false);
Assert.True(uploadResp2.IsSuccess);
Assert.NotNull(uploadResp2.ETag);
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp1, uploadResp2 }).ConfigureAwait(false);
Assert.True(completeResp.IsSuccess);
Assert.NotNull(uploadResp2.ETag);
GetObjectResponse getResp = await ObjectClient.GetObjectAsync(BucketName, objectKey).ConfigureAwait(false);
//Provoke an 'InvalidArgument' error
GetObjectResponse gResp1 = await ObjectClient.GetObjectAsync(BucketName, nameof(MultipartUpload), req => req.PartNumber = 0).ConfigureAwait(false);
Assert.False(gResp1.IsSuccess);
Assert.IsType <InvalidArgumentError>(gResp1.Error);
GetObjectResponse gResp2 = await ObjectClient.GetObjectAsync(BucketName, nameof(MultipartUpload), req => req.PartNumber = 1).ConfigureAwait(false);
Assert.True(gResp2.IsSuccess);
Assert.Equal(file.Length / 2, (await gResp2.Content.AsDataAsync().ConfigureAwait(false)).Length);
Assert.Equal(file, await getResp.Content.AsDataAsync().ConfigureAwait(false));
}
public async Task ListMultipartUploads(S3Provider provider, string _, ISimpleClient client)
{
await CreateTempBucketAsync(provider, client, async tempBucket =>
{
//The percentage sign at the end is to test if encoding works correctly
string objName = nameof(ListMultipartUploads) + "%";
CreateMultipartUploadResponse createResp = await client.CreateMultipartUploadAsync(tempBucket, objName).ConfigureAwait(false);
Assert.Equal(200, createResp.StatusCode);
byte[] file = new byte[5 * 1024];
await using (MemoryStream ms = new MemoryStream(file))
await client.UploadPartAsync(tempBucket, objName, 1, createResp.UploadId, ms).ConfigureAwait(false);
ListMultipartUploadsResponse listResp = await client.ListMultipartUploadsAsync(tempBucket, r => r.EncodingType = EncodingType.Url).ConfigureAwait(false);
Assert.Equal(200, listResp.StatusCode);
Assert.Equal(tempBucket, listResp.Bucket);
if (provider == S3Provider.AmazonS3)
{
Assert.Equal(WebUtility.UrlEncode(objName), listResp.NextKeyMarker);
Assert.NotEmpty(listResp.NextUploadIdMarker);
}
Assert.Equal(1000, listResp.MaxUploads);
Assert.False(listResp.IsTruncated);
S3Upload?upload = Assert.Single(listResp.Uploads);
Assert.Equal(WebUtility.UrlEncode(objName), upload.ObjectKey);
if (provider == S3Provider.AmazonS3)
{
Assert.Equal(listResp.NextUploadIdMarker, upload.UploadId);
}
Assert.Equal(StorageClass.Standard, upload.StorageClass);
Assert.Equal(DateTime.UtcNow, upload.Initiated.DateTime, TimeSpan.FromSeconds(5));
}).ConfigureAwait(false);
}
public async Task MultipartSinglePart(S3Provider provider, string bucket, ISimpleClient client)
{
string objectKey = nameof(MultipartSinglePart);
CreateMultipartUploadResponse createResp = await client.CreateMultipartUploadAsync(bucket, objectKey).ConfigureAwait(false);
Assert.Equal(200, createResp.StatusCode);
Assert.Equal(bucket, createResp.BucketName);
Assert.Equal(objectKey, createResp.ObjectKey);
Assert.NotNull(createResp.UploadId);
if (provider == S3Provider.AmazonS3)
{
//Test lifecycle expiration
Assert.Equal(DateTime.UtcNow.AddDays(2).Date, createResp.AbortsOn !.Value.UtcDateTime.Date);
Assert.Equal("ExpireAll", createResp.AbortRuleId);
}
byte[] file = new byte[1024 * 1024 * 5];
file[0] = (byte)'a';
await using MemoryStream ms = new MemoryStream(file);
UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, createResp.UploadId, ms).ConfigureAwait(false);
Assert.Equal(200, uploadResp.StatusCode);
Assert.Equal("\"10f74ef02085310ccd1f87150b83e537\"", uploadResp.ETag);
CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.Equal(200, completeResp.StatusCode);
Assert.Equal("\"bd74e21dfa8678d127240f76e518e9c2-1\"", completeResp.ETag);
if (provider == S3Provider.AmazonS3)
{
//Test lifecycle expiration
Assert.Equal(DateTime.UtcNow.AddDays(2).Date, completeResp.LifeCycleExpiresOn !.Value.UtcDateTime.Date);
Assert.Equal("ExpireAll", completeResp.LifeCycleRuleId);
}
}
public async Task MultipartRequestPayer()
{
string objectKey = nameof(MultipartRequestPayer);
CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);
Assert.True(initResp.RequestCharged);
byte[] file = new byte[1024 * 1024 * 5];
UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(file), req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);
Assert.True(uploadResp.RequestCharged);
ListPartsResponse listResp = await MultipartClient.ListPartsAsync(BucketName, objectKey, initResp.UploadId, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);
Assert.True(listResp.RequestCharged);
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp }, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);
Assert.True(completeResp.RequestCharged);
}
public async Task MultipartLockMode()
{
string objectKey = nameof(MultipartLockMode);
CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req =>
{
req.LockMode = LockMode.Governance;
req.LockRetainUntil = DateTimeOffset.UtcNow.AddMinutes(5);
}).ConfigureAwait(false);
Assert.True(initResp.IsSuccess);
byte[] file = new byte[1024 * 1024 * 5];
UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(file), req => req.ContentMd5 = CryptoHelper.Md5Hash(file)).ConfigureAwait(false);
Assert.True(uploadResp.IsSuccess);
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.True(completeResp.IsSuccess);
}
public async Task MultipartLockMode(S3Provider _, string bucket, ISimpleClient client)
{
string objectKey = nameof(MultipartLockMode);
CreateMultipartUploadResponse initResp = await client.CreateMultipartUploadAsync(bucket, objectKey, r =>
{
r.LockMode = LockMode.Governance;
r.LockRetainUntil = DateTimeOffset.UtcNow.AddMinutes(5);
}).ConfigureAwait(false);
Assert.Equal(200, initResp.StatusCode);
byte[] file = new byte[1024 * 1024 * 5];
await using MemoryStream ms = new MemoryStream(file);
UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, initResp.UploadId, ms, r => r.ContentMd5 = CryptoHelper.Md5Hash(file)).ConfigureAwait(false);
Assert.Equal(200, uploadResp.StatusCode);
CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.Equal(200, completeResp.StatusCode);
}
public static async IAsyncEnumerable <UploadPartResponse> MultipartUploadAsync(this IMultipartOperations operations, CreateMultipartUploadRequest req, Stream data, int partSize = 16777216, int numParallelParts = 4, [EnumeratorCancellation] CancellationToken token = default)
{
Validator.RequireNotNull(req, nameof(req));
Validator.RequireNotNull(data, nameof(data));
foreach (IRequestWrapper wrapper in operations.RequestWrappers)
{
if (wrapper.IsSupported(req))
{
data = wrapper.Wrap(data, req);
}
}
string bucket = req.BucketName;
string objectKey = req.ObjectKey;
CreateMultipartUploadResponse initResp = await operations.CreateMultipartUploadAsync(req, token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
yield break;
}
if (!initResp.IsSuccess)
{
throw new S3RequestException(initResp.StatusCode, "CreateMultipartUploadRequest was unsuccessful");
}
Queue <Task <UploadPartResponse> > uploads = new Queue <Task <UploadPartResponse> >();
using (SemaphoreSlim semaphore = new SemaphoreSlim(numParallelParts))
{
long offset = 0;
for (int i = 1; offset < data.Length; i++)
{
await semaphore.WaitAsync(token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
break;
}
byte[] partData = new byte[partSize];
int read = await data.ReadUpToAsync(partData, 0, partData.Length, token).ConfigureAwait(false);
TaskCompletionSource <UploadPartResponse> completionSource = new TaskCompletionSource <UploadPartResponse>();
uploads.Enqueue(completionSource.Task);
UploadPartAsync(completionSource, operations, bucket, objectKey, partData, read, i, initResp.UploadId, semaphore, token);
offset += partSize;
}
Queue <UploadPartResponse> responses = new Queue <UploadPartResponse>(uploads.Count);
while (uploads.TryDequeue(out Task <UploadPartResponse>?task))
{
if (token.IsCancellationRequested)
{
yield break;
}
UploadPartResponse response = await task !.ConfigureAwait(false);
responses.Enqueue(response);
yield return(response);
}
CompleteMultipartUploadRequest completeReq = new CompleteMultipartUploadRequest(bucket, objectKey, initResp.UploadId, responses);
CompleteMultipartUploadResponse completeResp = await operations.CompleteMultipartUploadAsync(completeReq, token).ConfigureAwait(false);
if (!completeResp.IsSuccess)
{
throw new S3RequestException(completeResp.StatusCode, "CompleteMultipartUploadRequest was unsuccessful");
}
}
}
public async Task <CompleteMultipartUploadResponse> MultipartUploadAsync(CreateMultipartUploadRequest req, Stream data, int partSize = 16777216, int numParallelParts = 4, Action <UploadPartResponse>?onPartResponse = null, CancellationToken token = default)
{
Validator.RequireNotNull(req, nameof(req));
Validator.RequireNotNull(data, nameof(data));
foreach (IRequestWrapper wrapper in _requestWrappers)
{
if (wrapper.IsSupported(req))
{
data = wrapper.Wrap(data, req);
}
}
string bucket = req.BucketName;
string objectKey = req.ObjectKey;
byte[]? encryptionKey = null;
try
{
if (req.SseCustomerKey != null)
{
encryptionKey = new byte[req.SseCustomerKey.Length];
Array.Copy(req.SseCustomerKey, 0, encryptionKey, 0, encryptionKey.Length);
}
CreateMultipartUploadResponse initResp = await _multipartOperations.CreateMultipartUploadAsync(req, token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
return new CompleteMultipartUploadResponse {
BucketName = bucket, ObjectKey = objectKey
}
}
;
if (!initResp.IsSuccess)
{
throw new S3RequestException(initResp, "CreateMultipartUploadRequest was unsuccessful");
}
IEnumerable <ArraySegment <byte> > chunks = ReadChunks(data, partSize);
int partNumber = 0;
IEnumerable <UploadPartResponse> responses = await ParallelHelper.ExecuteAsync(chunks, async (bytes, innerToken) =>
{
Interlocked.Increment(ref partNumber);
using (MemoryStream ms = new MemoryStream(bytes.Array !, 0, bytes.Count))
{
UploadPartResponse resp = await _multipartClient.UploadPartAsync(bucket, objectKey, partNumber, initResp.UploadId, ms, uploadPart =>
{
uploadPart.SseCustomerAlgorithm = req.SseCustomerAlgorithm;
uploadPart.SseCustomerKey = encryptionKey;
uploadPart.SseCustomerKeyMd5 = req.SseCustomerKeyMd5;
}, innerToken).ConfigureAwait(false);
onPartResponse?.Invoke(resp);
return(resp);
}
}, numParallelParts, token);
CompleteMultipartUploadResponse completeResp = await _multipartClient.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, responses.OrderBy(x => x.PartNumber), null, token).ConfigureAwait(false);
return(completeResp);
}
finally
{
if (encryptionKey != null)
{
Array.Clear(encryptionKey, 0, encryptionKey.Length);
}
}
}
