/// <summary>
/// Use the S3 protocol to upload the local file at filePath to the server at uploadTarget.
/// </summary>
/// <param name="filePath"></param>
/// <param name="uploadTarget"></param>
public static void TranslocateFile(string uploadTarget, string filePath)
{
// Create an S3 client.
AmazonS3Client s3Client = Common.CreateS3Client(
uploadTarget);
// Initiate an upload to the S3 server.
InitiateMultipartUploadResponse initMultiPartUploadResponse = Common.OpenUpload(
s3Client,
uploadTarget,
filePath);
string uploadId = initMultiPartUploadResponse.UploadId;
// Upload the parts to the S3 server.
long partSize = 5 * Common.MiB;
List <UploadPartResponse> uploadPartResponses = Common.UploadParts(
s3Client,
uploadTarget,
filePath,
uploadId,
partSize);
// Complete the upload to the S3 server.
CompleteMultipartUploadResponse completeMultipartUploadResponse = Common.CloseUpload(
s3Client,
uploadTarget,
filePath,
uploadId,
uploadPartResponses);
}
/// <summary>Puts data from stream to Amazon S3 storage.</summary>
/// <param name="obj">Object info.</param>
/// <param name="stream">Stream to upload.</param>
public void PutDataFromStreamToObject(IS3ObjectInfo obj, System.IO.Stream stream)
{
if (obj.IsLocked)
{
throw new Exception($"[IS3ObjectInfoProvider.PutDataFromStreamToObject]: Couldn't upload object {obj.Key} because it is used by another process.");
}
obj.Lock();
string bucketName = obj.BucketName;
long length = stream.Length;
if (length > RECOMMENDED_SIZE_FOR_MULTIPART_UPLOAD)
{
CompleteMultipartUploadResponse response = this.MultiPartUploader.UploadFromStream(obj.Key, bucketName, stream);
this.SetS3ObjectMetadaFromResponse(obj, response, length);
}
else
{
PutObjectRequest putRequest = CreatePutRequest(obj.Key, bucketName);
putRequest.InputStream = stream;
PutObjectResponse response = this.S3Client.PutObject(putRequest);
this.SetS3ObjectMetadaFromResponse(obj, response, length);
}
FileDebug.LogFileOperation(PathHelper.GetPathFromObjectKey(obj.Key, true), "PutStreamToObject", "Custom Amazon");
obj.UnLock();
RemoveRequestCache(obj.Key);
}
static void Main(string[] args)
{
IAmazonS3 s3Client = new AmazonS3Client(Amazon.RegionEndpoint.USEast1);
// List to store upload part responses.
List<UploadPartResponse> uploadResponses = new List<UploadPartResponse>();
List<CopyPartResponse> copyResponses = new List<CopyPartResponse>();
InitiateMultipartUploadRequest initiateRequest =
new InitiateMultipartUploadRequest
{
BucketName = targetBucket,
Key = targetObjectKey
};
InitiateMultipartUploadResponse initResponse =
s3Client.InitiateMultipartUpload(initiateRequest);
String uploadId = initResponse.UploadId;
try
{
// Get object size.
GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest
{
BucketName = sourceBucket,
Key = sourceObjectKey
};
GetObjectMetadataResponse metadataResponse =
s3Client.GetObjectMetadata(metadataRequest);
long objectSize = metadataResponse.ContentLength; // in bytes
// Copy parts.
long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB
long bytePosition = 0;
for (int i = 1; bytePosition < objectSize; i++)
{
CopyPartRequest copyRequest = new CopyPartRequest
{
DestinationBucket = targetBucket,
DestinationKey = targetObjectKey,
SourceBucket = sourceBucket,
SourceKey = sourceObjectKey,
UploadId = uploadId,
FirstByte = bytePosition,
LastByte = bytePosition + partSize - 1 >= objectSize ? objectSize - 1 : bytePosition + partSize - 1,
PartNumber = i
};
copyResponses.Add(s3Client.CopyPart(copyRequest));
bytePosition += partSize;
}
CompleteMultipartUploadRequest completeRequest =
new CompleteMultipartUploadRequest
{
BucketName = targetBucket,
Key = targetObjectKey,
UploadId = initResponse.UploadId
};
completeRequest.AddPartETags(copyResponses);
CompleteMultipartUploadResponse completeUploadResponse = s3Client.CompleteMultipartUpload(completeRequest);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
public async Task ManualSinglePartUpload()
{
string resourceName = nameof(ManualSinglePartUpload);
InitiateMultipartUploadResponse initResp = await ObjectClient.InitiateMultipartUploadAsync(BucketName, resourceName, request =>
{
request.SseAlgorithm = SseAlgorithm.Aes256;
request.StorageClass = StorageClass.StandardIa;
}).ConfigureAwait(false);
Assert.True(initResp.IsSuccess);
Assert.Equal(BucketName, initResp.Bucket);
Assert.Equal(resourceName, initResp.Key);
Assert.NotNull(initResp.UploadId);
byte[] file = new byte[1024 * 1024 * 5];
file[0] = (byte)'a';
UploadPartResponse uploadResp = await ObjectClient.UploadPartAsync(BucketName, resourceName, 1, initResp.UploadId, new MemoryStream(file)).ConfigureAwait(false);
Assert.True(uploadResp.IsSuccess);
Assert.NotNull(uploadResp.ETag);
Assert.Equal(SseAlgorithm.Aes256, uploadResp.SseAlgorithm);
Assert.Equal(StorageClass.StandardIa, uploadResp.StorageClass);
CompleteMultipartUploadResponse completeResp = await ObjectClient.CompleteMultipartUploadAsync(BucketName, resourceName, initResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.True(completeResp.IsSuccess);
Assert.NotNull(completeResp.ETag);
GetObjectResponse resp = await ObjectClient.GetObjectAsync(BucketName, resourceName).ConfigureAwait(false);
Assert.True(resp.IsSuccess);
Assert.Equal(file, await resp.Content.AsDataAsync().ConfigureAwait(false));
}
public async Task MultipartCustomerEncryption(S3Provider _, string bucket, ISimpleClient client)
{
string objectKey = nameof(MultipartCustomerEncryption);
byte[] key = { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 };
byte[] keyMd5 = CryptoHelper.Md5Hash(key);
CreateMultipartUploadResponse initResp = await client.CreateMultipartUploadAsync(bucket, objectKey, r =>
{
r.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
r.SseCustomerKey = key;
r.SseCustomerKeyMd5 = keyMd5;
}).ConfigureAwait(false);
Assert.Equal(200, initResp.StatusCode);
await using MemoryStream ms = new MemoryStream(new byte[1024 * 1024 * 5]);
UploadPartResponse uploadResp1 = await client.UploadPartAsync(bucket, objectKey, 1, initResp.UploadId, ms, r =>
{
r.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
r.SseCustomerKey = key;
r.SseCustomerKeyMd5 = keyMd5;
}).ConfigureAwait(false);
Assert.Equal(200, uploadResp1.StatusCode);
Assert.Equal(SseCustomerAlgorithm.Aes256, uploadResp1.SseCustomerAlgorithm);
Assert.Equal(keyMd5, uploadResp1.SseCustomerKeyMd5);
CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, new[] { uploadResp1 }).ConfigureAwait(false);
Assert.Equal(200, completeResp.StatusCode);
Assert.Equal(SseCustomerAlgorithm.Aes256, completeResp.SseCustomerAlgorithm);
}
public async Task MultipartWithEncryption(SseAlgorithm algorithm)
{
string objectKey = nameof(MultipartWithEncryption);
CreateMultipartUploadResponse createResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req => req.SseAlgorithm = algorithm).ConfigureAwait(false);
Assert.Equal(algorithm, createResp.SseAlgorithm);
if (algorithm == SseAlgorithm.AwsKms)
{
Assert.NotNull(createResp.SseKmsKeyId);
}
byte[] file = new byte[1024 * 1024 * 5];
UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, createResp.UploadId, new MemoryStream(file)).ConfigureAwait(false);
Assert.Equal(algorithm, uploadResp.SseAlgorithm);
if (algorithm == SseAlgorithm.AwsKms)
{
Assert.NotNull(uploadResp.SseKmsKeyId);
}
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.Equal(algorithm, completeResp.SseAlgorithm);
if (algorithm == SseAlgorithm.AwsKms)
{
Assert.NotNull(completeResp.SseKmsKeyId);
}
}
public async Task MultipartTooSmall(S3Provider provider, string bucket, ISimpleClient client)
{
string objectKey = nameof(MultipartTooSmall);
CreateMultipartUploadResponse initResp = await client.CreateMultipartUploadAsync(bucket, objectKey).ConfigureAwait(false);
Assert.Equal(200, initResp.StatusCode);
Assert.Equal(bucket, initResp.BucketName);
Assert.Equal(objectKey, initResp.ObjectKey);
Assert.NotNull(initResp.UploadId);
//4 MB is below the 5 MB limit. See https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html
//Note that if there only is 1 part, then it is technically the last part, and can be of any size. That's why this test has 2 parts.
byte[] file = new byte[1024 * 1024 * 4];
byte[][] parts = file.Chunk(file.Length / 2).Select(x => x.ToArray()).ToArray();
await using MemoryStream ms1 = new MemoryStream(parts[0]);
UploadPartResponse uploadResp1 = await client.UploadPartAsync(bucket, objectKey, 1, initResp.UploadId, ms1).ConfigureAwait(false);
Assert.Equal(provider == S3Provider.BackBlazeB2 ? 400 : 200, uploadResp1.StatusCode);
await using MemoryStream ms2 = new MemoryStream(parts[0]);
UploadPartResponse uploadResp2 = await client.UploadPartAsync(bucket, objectKey, 2, initResp.UploadId, ms2).ConfigureAwait(false);
Assert.Equal(provider == S3Provider.BackBlazeB2 ? 400 : 200, uploadResp2.StatusCode);
CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, new[] { uploadResp1, uploadResp2 }).ConfigureAwait(false);
Assert.Equal(provider == S3Provider.BackBlazeB2 ? 500 : 400, completeResp.StatusCode);
}
public async Task MultipartCustomerEncryption()
{
string objectKey = nameof(MultipartCustomerEncryption);
byte[] key = { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 };
byte[] keyMd5 = CryptoHelper.Md5Hash(key);
CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req =>
{
req.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
req.SseCustomerKey = key;
req.SseCustomerKeyMd5 = keyMd5;
}).ConfigureAwait(false);
Assert.True(initResp.IsSuccess);
byte[] file = new byte[1024 * 1024 * 5];
UploadPartResponse uploadResp1 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(file), req =>
{
req.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
req.SseCustomerKey = key;
req.SseCustomerKeyMd5 = keyMd5;
}).ConfigureAwait(false);
Assert.Equal(SseCustomerAlgorithm.Aes256, uploadResp1.SseCustomerAlgorithm);
Assert.Equal(keyMd5, uploadResp1.SseCustomerKeyMd5);
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp1 }).ConfigureAwait(false);
Assert.True(completeResp.IsSuccess);
Assert.Equal(SseCustomerAlgorithm.Aes256, completeResp.SseCustomerAlgorithm);
}
static void CompleteMultipartUpload()
{
try
{
List <PartETag> partEtags = new List <PartETag>();
PartETag partEtag1 = new PartETag();
partEtag1.PartNumber = 1;
partEtag1.ETag = etag;
partEtags.Add(partEtag1);
CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest()
{
BucketName = bucketName,
ObjectKey = objectName,
UploadId = uploadId,
PartETags = partEtags
};
CompleteMultipartUploadResponse response = client.CompleteMultipartUpload(request);
Console.WriteLine("Complete multipart upload response: {0}", response.StatusCode);
}
catch (ObsException ex)
{
Console.WriteLine("Exception errorcode: {0}, when complete multipart upload.", ex.ErrorCode);
Console.WriteLine("Exception errormessage: {0}", ex.ErrorMessage);
}
}
AWSArchiveResult CompleteMPU(string uploadID, AmazonGlacier client, List <string> partChecksumList, FileInfo fio)
{
try
{
long fileLength = fio.Length;
fileLength = new FileInfo(archiveToUpload).Length;
FileStream inputFile = File.Open(archiveToUpload, FileMode.Open, FileAccess.Read);
byte[] treeHash = Form1.ComputeSHA256TreeHash(inputFile);
String localChecksum = BitConverter.ToString(treeHash).Replace("-", "").ToLower();
CompleteMultipartUploadRequest completeMPUrequest = new CompleteMultipartUploadRequest()
{
UploadId = uploadID,
ArchiveSize = fileLength.ToString(),
Checksum = localChecksum,
VaultName = vaultName
};
CompleteMultipartUploadResponse completeMPUresponse = client.CompleteMultipartUpload(completeMPUrequest);
AWSArchiveResult ar = new AWSArchiveResult();
ar.ArchiveID = completeMPUresponse.CompleteMultipartUploadResult.ArchiveId;
ar.Checksum = localChecksum;
return(ar);
}
catch (Exception e)
{
Form1.log.Error(e.ToString());
return(new AWSArchiveResult());
}
}
public async Task MultipartTooSmall()
{
string objectKey = nameof(MultipartTooSmall);
CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey).ConfigureAwait(false);
Assert.Equal(BucketName, initResp.Bucket);
Assert.Equal(objectKey, initResp.ObjectKey);
Assert.NotNull(initResp.UploadId);
//4 MB is below the 5 MB limit. See https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html
//Note that if there only is 1 part, then it is technically the last part, and can be of any size. That's why this test has 2 parts.
byte[] file = new byte[1024 * 1024 * 4];
byte[][] chunks = file.Chunk(file.Length / 2).Select(x => x.ToArray()).ToArray();
UploadPartResponse uploadResp1 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(chunks[0])).ConfigureAwait(false);
Assert.True(uploadResp1.IsSuccess);
Assert.NotNull(uploadResp1.ETag);
UploadPartResponse uploadResp2 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 2, initResp.UploadId, new MemoryStream(chunks[1])).ConfigureAwait(false);
Assert.True(uploadResp2.IsSuccess);
Assert.NotNull(uploadResp2.ETag);
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp1, uploadResp2 }).ConfigureAwait(false);
Assert.False(completeResp.IsSuccess);
Assert.Equal(400, completeResp.StatusCode);
}
public async Task MultipartSinglePart()
{
string objectKey = nameof(MultipartSinglePart);
CreateMultipartUploadResponse createResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey).ConfigureAwait(false);
Assert.True(createResp.IsSuccess);
Assert.Equal(BucketName, createResp.Bucket);
Assert.Equal(objectKey, createResp.ObjectKey);
Assert.NotNull(createResp.UploadId);
//Test lifecycle expiration
Assert.Equal(DateTime.UtcNow.AddDays(2).Date, createResp.AbortsOn.Value.UtcDateTime.Date);
Assert.Equal("ExpireAll", createResp.AbortRuleId);
byte[] file = new byte[1024 * 1024 * 5];
file[0] = (byte)'a';
UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, createResp.UploadId, new MemoryStream(file)).ConfigureAwait(false);
Assert.True(uploadResp.IsSuccess);
Assert.Equal("\"10f74ef02085310ccd1f87150b83e537\"", uploadResp.ETag);
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.True(completeResp.IsSuccess);
Assert.Equal("\"bd74e21dfa8678d127240f76e518e9c2-1\"", completeResp.ETag);
Assert.NotNull(completeResp.VersionId);
//Test lifecycle expiration
Assert.Equal(DateTime.UtcNow.AddDays(2).Date, completeResp.LifeCycleExpiresOn.Value.UtcDateTime.Date);
Assert.Equal("ExpireAll", completeResp.LifeCycleRuleId);
}
public void TestOrdinary()
{
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.ContentType = "text/plain";
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
{
BucketName = this.bucketName,
Key = "test",
ObjectMetadata = objectMetadata
};
string uploadId = this.client.InitiateMultipartUpload(initRequest).UploadId;
List <PartETag> partETags = new List <PartETag>();
for (int i = 0; i < 1; ++i)
{
string eTag = this.client.UploadPart(new UploadPartRequest()
{
BucketName = this.bucketName,
Key = "test",
UploadId = uploadId,
PartNumber = i + 1,
PartSize = 1,
InputStream = new MemoryStream(Encoding.Default.GetBytes(i.ToString()))
}).ETag;
partETags.Add(new PartETag()
{
PartNumber = i + 1, ETag = eTag
});
}
objectMetadata = new ObjectMetadata();
Dictionary <string, string> userMetadata = new Dictionary <string, string>();
userMetadata["metakey"] = "metaValue";
objectMetadata.UserMetadata = userMetadata;
objectMetadata.ContentType = "text/json";
CompleteMultipartUploadRequest request =
new CompleteMultipartUploadRequest()
{
BucketName = this.bucketName,
Key = "test",
UploadId = uploadId,
PartETags = partETags,
ObjectMetadata = objectMetadata
};
CompleteMultipartUploadResponse response = this.client.CompleteMultipartUpload(request);
Assert.AreEqual(response.BucketName, this.bucketName);
Assert.AreEqual(response.Key, "test");
Assert.IsNotNull(response.ETag);
Assert.IsNotNull(response.Location);
ObjectMetadata metadata = this.client.GetObjectMetadata(bucketName, "test");
Assert.AreEqual(metadata.ContentType, "text/plain");
string resultUserMeta = metadata.UserMetadata["metakey"];
Assert.AreEqual(resultUserMeta, "metaValue");
}
private static void UnmarshallResult(XmlUnmarshallerContext context, CompleteMultipartUploadResponse response)
{
int currentDepth = context.get_CurrentDepth();
int num = currentDepth + 1;
if (context.get_IsStartOfDocument())
{
num += 2;
}
while (context.Read())
{
if (context.get_IsStartElement() || context.get_IsAttribute())
{
if (context.TestExpression("Location", num))
{
response.Location = StringUnmarshaller.GetInstance().Unmarshall(context);
}
else if (context.TestExpression("Bucket", num))
{
response.BucketName = StringUnmarshaller.GetInstance().Unmarshall(context);
}
else if (context.TestExpression("Key", num))
{
response.Key = StringUnmarshaller.GetInstance().Unmarshall(context);
}
else if (context.TestExpression("ETag", num))
{
response.ETag = StringUnmarshaller.GetInstance().Unmarshall(context);
}
}
else if (context.get_IsEndElement() && context.get_CurrentDepth() < currentDepth)
{
return;
}
}
IWebResponseData responseData = context.get_ResponseData();
if (responseData.IsHeaderPresent("x-amz-expiration"))
{
response.Expiration = new Expiration(responseData.GetHeaderValue("x-amz-expiration"));
}
if (responseData.IsHeaderPresent("x-amz-server-side-encryption"))
{
response.ServerSideEncryptionMethod = S3Transforms.ToString(responseData.GetHeaderValue("x-amz-server-side-encryption"));
}
if (responseData.IsHeaderPresent("x-amz-version-id"))
{
response.VersionId = S3Transforms.ToString(responseData.GetHeaderValue("x-amz-version-id"));
}
if (responseData.IsHeaderPresent("x-amz-server-side-encryption-aws-kms-key-id"))
{
response.ServerSideEncryptionKeyManagementServiceKeyId = S3Transforms.ToString(responseData.GetHeaderValue("x-amz-server-side-encryption-aws-kms-key-id"));
}
if (responseData.IsHeaderPresent(S3Constants.AmzHeaderRequestCharged))
{
response.RequestCharged = RequestCharged.FindValue(responseData.GetHeaderValue(S3Constants.AmzHeaderRequestCharged));
}
}
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
CompleteMultipartUploadResponse response = new CompleteMultipartUploadResponse();
context.Read();
UnmarshallResult(context, response);
return(response);
}
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
CompleteMultipartUploadResponse response = new CompleteMultipartUploadResponse();
context.Read();
response.CompleteMultipartUploadResult = CompleteMultipartUploadResultUnmarshaller.GetInstance().Unmarshall(context);
return(response);
}
public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext context)
{
CompleteMultipartUploadResponse completeMultipartUploadResponse = new CompleteMultipartUploadResponse();
while (context.Read())
{
if (context.get_IsStartElement())
{
UnmarshallResult(context, completeMultipartUploadResponse);
}
}
return(completeMultipartUploadResponse);
}
public string EndMultiPartUpload(long archiveSize, string checksum, string uploadId)
{
CompleteMultipartUploadRequest request = new CompleteMultipartUploadRequest()
{
ArchiveSize = archiveSize.ToString(),
Checksum = checksum,
UploadId = uploadId,
VaultName = _vault
};
CompleteMultipartUploadResponse response = _amazonGlacierClient.CompleteMultipartUpload(request);
return(response.CompleteMultipartUploadResult.ArchiveId);
}
// After each file is uploaded it will return an ArchiveID
static string CompleteMPU(string uploadID, AmazonGlacierClient client, List <string> partChecksumList, string archiveToUpload)
{
long fileLength = new FileInfo(archiveToUpload).Length;
CompleteMultipartUploadRequest completeMPUrequest = new CompleteMultipartUploadRequest()
{
UploadId = uploadID,
ArchiveSize = fileLength.ToString(),
Checksum = TreeHashGenerator.CalculateTreeHash(partChecksumList),
VaultName = vaultName
};
CompleteMultipartUploadResponse completeMPUresponse = client.CompleteMultipartUpload(completeMPUrequest);
return(completeMPUresponse.ArchiveId);
}
public string CompletePartToCloud()
{
// Step 3: complete.
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest();
compRequest.BucketName = this.DestinationBucket;
compRequest.Key = this.DestinationFilename;
compRequest.UploadId = initResponse.UploadId;
compRequest.PartETags = uploadPartETags;
string r = "Something went badly wrong";
using (CompleteMultipartUploadResponse completeUploadResponse = client.CompleteMultipartUpload(compRequest))
r = completeUploadResponse.ResponseXml;
return(r);
}
public async Task MultipartWithEncryption(S3Provider provider, string bucket, ISimpleClient client, SseAlgorithm algorithm)
{
string objectKey = nameof(MultipartWithEncryption);
CreateMultipartUploadResponse createResp = await client.CreateMultipartUploadAsync(bucket, objectKey, req => req.SseAlgorithm = algorithm).ConfigureAwait(false);
Assert.Equal(200, createResp.StatusCode);
if (provider == S3Provider.AmazonS3)
{
Assert.Equal(algorithm, createResp.SseAlgorithm);
}
if (algorithm == SseAlgorithm.AwsKms)
{
Assert.NotNull(createResp.SseKmsKeyId);
}
await using MemoryStream ms = new MemoryStream(new byte[1024 * 1024 * 5]);
UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, createResp.UploadId, ms).ConfigureAwait(false);
Assert.Equal(200, uploadResp.StatusCode);
if (provider == S3Provider.AmazonS3)
{
Assert.Equal(algorithm, uploadResp.SseAlgorithm);
}
if (algorithm == SseAlgorithm.AwsKms)
{
Assert.NotNull(uploadResp.SseKmsKeyId);
}
CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.Equal(200, completeResp.StatusCode);
if (provider == S3Provider.AmazonS3)
{
Assert.Equal(algorithm, completeResp.SseAlgorithm);
}
if (algorithm == SseAlgorithm.AwsKms)
{
Assert.NotNull(completeResp.SseKmsKeyId);
}
}
private static void UnmarshallResult(JsonUnmarshallerContext context, CompleteMultipartUploadResponse response)
{
if (context.ResponseData.GetHeaderValue("Location") != null)
{
response.Location = context.ResponseData.GetHeaderValue("Location");
}
if (context.ResponseData.GetHeaderValue("x-amz-sha256-tree-hash") != null)
{
response.Checksum = context.ResponseData.GetHeaderValue("x-amz-sha256-tree-hash");
}
if (context.ResponseData.GetHeaderValue("x-amz-archive-id") != null)
{
response.ArchiveId = context.ResponseData.GetHeaderValue("x-amz-archive-id");
}
return;
}
public async Task MultipartUpload()
{
string objectKey = nameof(MultipartUpload);
CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey).ConfigureAwait(false);
Assert.True(initResp.IsSuccess);
Assert.Equal(BucketName, initResp.Bucket);
Assert.Equal(objectKey, initResp.ObjectKey);
Assert.NotNull(initResp.UploadId);
byte[] file = new byte[1024 * 1024 * 10];
file[0] = (byte)'a';
file[file.Length - 1] = (byte)'b';
byte[][] parts = file.Chunk(file.Length / 2).Select(x => x.ToArray()).ToArray();
UploadPartResponse uploadResp1 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(parts[0])).ConfigureAwait(false);
Assert.True(uploadResp1.IsSuccess);
Assert.NotNull(uploadResp1.ETag);
UploadPartResponse uploadResp2 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 2, initResp.UploadId, new MemoryStream(parts[1])).ConfigureAwait(false);
Assert.True(uploadResp2.IsSuccess);
Assert.NotNull(uploadResp2.ETag);
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp1, uploadResp2 }).ConfigureAwait(false);
Assert.True(completeResp.IsSuccess);
Assert.NotNull(uploadResp2.ETag);
GetObjectResponse getResp = await ObjectClient.GetObjectAsync(BucketName, objectKey).ConfigureAwait(false);
//Provoke an 'InvalidArgument' error
GetObjectResponse gResp1 = await ObjectClient.GetObjectAsync(BucketName, nameof(MultipartUpload), req => req.PartNumber = 0).ConfigureAwait(false);
Assert.False(gResp1.IsSuccess);
Assert.IsType <InvalidArgumentError>(gResp1.Error);
GetObjectResponse gResp2 = await ObjectClient.GetObjectAsync(BucketName, nameof(MultipartUpload), req => req.PartNumber = 1).ConfigureAwait(false);
Assert.True(gResp2.IsSuccess);
Assert.Equal(file.Length / 2, (await gResp2.Content.AsDataAsync().ConfigureAwait(false)).Length);
Assert.Equal(file, await getResp.Content.AsDataAsync().ConfigureAwait(false));
}
/// <summary>
/// Send a CompleteMultipartUploadRequest request and return the response.
/// </summary>
/// <param name="existingBucketName"></param>
/// <param name="keyName"></param>
/// <param name="uploadID"></param>
/// <returns>Task<CompleteMultipartUploadRequest></returns>
public async Task <CompleteMultipartUploadRequest> CompleteMultipartUploadAsync(string existingBucketName, string keyName, string uploadId,
CancellationToken token)
{
token.ThrowIfCancellationRequested();
_log.Debug("Called CompleteMultipartUploadAsync with parameters keyName = \"" + keyName + "\" and uploadID = \"" + uploadId + "\".");
try
{
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = existingBucketName,
Key = keyName,
UploadId = uploadId,
};
// in any case request list parts to send etags for each part.
var uploadedParts = await this.ListPartsAsync(existingBucketName, keyName, uploadId, token).ConfigureAwait(false);
List <PartETag> eTags = new List <PartETag>();
foreach (var part in uploadedParts)
{
eTags.Add(new PartETag(part.PartNumber, part.ETag));
}
completeRequest.AddPartETags(eTags);
token.ThrowIfCancellationRequested();
CompleteMultipartUploadResponse completeResponse = await s3Client.CompleteMultipartUploadAsync(completeRequest, token).ConfigureAwait(false);
return(completeRequest);
}
catch (Exception e)
{
if (!(e is TaskCanceledException || e is OperationCanceledException))
{
string messagePart = " with parameters keyName = \"" + keyName + "\" and uploadID = \"" + uploadId + "\"";
this.LogAmazonException(messagePart, e);
}
throw;
}
}
/// <summary>
/// Unmarshaller the response from the service to the response class.
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public override AmazonWebServiceResponse Unmarshall(JsonUnmarshallerContext context)
{
CompleteMultipartUploadResponse response = new CompleteMultipartUploadResponse();
if (context.ResponseData.IsHeaderPresent("x-amz-archive-id"))
{
response.ArchiveId = context.ResponseData.GetHeaderValue("x-amz-archive-id");
}
if (context.ResponseData.IsHeaderPresent("x-amz-sha256-tree-hash"))
{
response.Checksum = context.ResponseData.GetHeaderValue("x-amz-sha256-tree-hash");
}
if (context.ResponseData.IsHeaderPresent("Location"))
{
response.Location = context.ResponseData.GetHeaderValue("Location");
}
return(response);
}
public string CompletePartialAndSaveResource(ReferenceRepository referenceRepository,
ResourceModel.User owner,
string uploadIdentifier,
String originalName,
String md5OfResource,
List <string> etags,
long totalSize)
{
IAmazonS3 s3Client = new AmazonS3Client();
ResourceModel.DigitalResource resource = null;
ResourceType imageRT = referenceRepository.AllResourceTypes().Where(x => x.Type == "Image").FirstOrDefault();
ResourceType otherRT = referenceRepository.AllResourceTypes().Where(x => x.Type == "Other").FirstOrDefault();
List <PartETag> partTags = new List <PartETag>();
int partNumber = 1;
foreach (string tag in etags)
{
partTags.Add(new PartETag {
ETag = tag, PartNumber = partNumber
});
partNumber++;
}
// complete the upload.
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest
{
BucketName = "piccoli",
Key = md5OfResource,
UploadId = uploadIdentifier,
PartETags = partTags
};
// Complete the upload.
CompleteMultipartUploadResponse completeUploadResponse = s3Client.CompleteMultipartUpload(completeRequest);
//create the resource object
resource = CreateNewResourceObject(owner, totalSize, originalName, imageRT, otherRT, md5OfResource);
SaveResource(resource);//update the database
return(completeUploadResponse.ETag);
}
public async Task MultipartSinglePart(S3Provider provider, string bucket, ISimpleClient client)
{
string objectKey = nameof(MultipartSinglePart);
CreateMultipartUploadResponse createResp = await client.CreateMultipartUploadAsync(bucket, objectKey).ConfigureAwait(false);
Assert.Equal(200, createResp.StatusCode);
Assert.Equal(bucket, createResp.BucketName);
Assert.Equal(objectKey, createResp.ObjectKey);
Assert.NotNull(createResp.UploadId);
if (provider == S3Provider.AmazonS3)
{
//Test lifecycle expiration
Assert.Equal(DateTime.UtcNow.AddDays(2).Date, createResp.AbortsOn !.Value.UtcDateTime.Date);
Assert.Equal("ExpireAll", createResp.AbortRuleId);
}
byte[] file = new byte[1024 * 1024 * 5];
file[0] = (byte)'a';
await using MemoryStream ms = new MemoryStream(file);
UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, createResp.UploadId, ms).ConfigureAwait(false);
Assert.Equal(200, uploadResp.StatusCode);
Assert.Equal("\"10f74ef02085310ccd1f87150b83e537\"", uploadResp.ETag);
CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.Equal(200, completeResp.StatusCode);
Assert.Equal("\"bd74e21dfa8678d127240f76e518e9c2-1\"", completeResp.ETag);
if (provider == S3Provider.AmazonS3)
{
//Test lifecycle expiration
Assert.Equal(DateTime.UtcNow.AddDays(2).Date, completeResp.LifeCycleExpiresOn !.Value.UtcDateTime.Date);
Assert.Equal("ExpireAll", completeResp.LifeCycleRuleId);
}
}
public async Task MultipartLockMode()
{
string objectKey = nameof(MultipartLockMode);
CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req =>
{
req.LockMode = LockMode.Governance;
req.LockRetainUntil = DateTimeOffset.UtcNow.AddMinutes(5);
}).ConfigureAwait(false);
Assert.True(initResp.IsSuccess);
byte[] file = new byte[1024 * 1024 * 5];
UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(file), req => req.ContentMd5 = CryptoHelper.Md5Hash(file)).ConfigureAwait(false);
Assert.True(uploadResp.IsSuccess);
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.True(completeResp.IsSuccess);
}
public async Task MultipartRequestPayer()
{
string objectKey = nameof(MultipartRequestPayer);
CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);
Assert.True(initResp.RequestCharged);
byte[] file = new byte[1024 * 1024 * 5];
UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(file), req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);
Assert.True(uploadResp.RequestCharged);
ListPartsResponse listResp = await MultipartClient.ListPartsAsync(BucketName, objectKey, initResp.UploadId, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);
Assert.True(listResp.RequestCharged);
CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp }, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);
Assert.True(completeResp.RequestCharged);
}
public async Task MultipartLockMode(S3Provider _, string bucket, ISimpleClient client)
{
string objectKey = nameof(MultipartLockMode);
CreateMultipartUploadResponse initResp = await client.CreateMultipartUploadAsync(bucket, objectKey, r =>
{
r.LockMode = LockMode.Governance;
r.LockRetainUntil = DateTimeOffset.UtcNow.AddMinutes(5);
}).ConfigureAwait(false);
Assert.Equal(200, initResp.StatusCode);
byte[] file = new byte[1024 * 1024 * 5];
await using MemoryStream ms = new MemoryStream(file);
UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, initResp.UploadId, ms, r => r.ContentMd5 = CryptoHelper.Md5Hash(file)).ConfigureAwait(false);
Assert.Equal(200, uploadResp.StatusCode);
CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);
Assert.Equal(200, completeResp.StatusCode);
}