private void Addobject(string filePath, string bucketKey)
{
try
{
var attribute = File.GetAttributes(filePath);
string relativePath = filePath.Replace(Utilities.Path + "\\", "").Replace("\\", "/");
if (attribute != FileAttributes.Directory && attribute != (FileAttributes.Archive | FileAttributes.Hidden))
{
if (File.Exists(filePath) && !Utilities.IsFileUsedbyAnotherProcess(filePath))
{
bool IsItSharedFile = false;
string Username = string.Empty;
if (relativePath.ToLower().IndexOf("shared") == 0)
{
IsItSharedFile = true;
var folders = relativePath.Split('/');
if (folders.Length > 1)
{
Username = folders[1];
relativePath = "";
for (int i = 2; i < folders.Length; i++)
relativePath += folders[i] + "/";
}
else
{
relativePath = "";
for (int i = 1; i < folders.Length; i++)
relativePath += folders[i] + "/";
}
relativePath = relativePath.Substring(0, relativePath.Length - 1);
}
var appUpdateInfo = new AppUpdateInfo
{
Key = relativePath.Replace("\\", "/"),
LastModifiedTime = new FileInfo(filePath).LastWriteTime,
Status = UpdateStatus.Update
};
_processingFiles.Add(filePath);
try
{
var uploadResponses = new List<UploadPartResponse>();
byte[] bytes;
long contentLength;
using (var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
contentLength = fileStream.Length;
bytes = new byte[contentLength];
fileStream.Read(bytes, 0, Convert.ToInt32(contentLength));
}
if (contentLength > 0)
{
var lastUploadedPartdetail = GetLastUploadedPartResponse(relativePath, bucketKey,
uploadResponses);
int alreadyUploadedParts = lastUploadedPartdetail.LastPartNumber;
string uploadId;
if (string.IsNullOrEmpty(lastUploadedPartdetail.UploadId))
{
InitiateMultipartUploadRequest initiateRequest =
new InitiateMultipartUploadRequest().WithBucketName(bucketKey).WithKey(
relativePath);
InitiateMultipartUploadResponse initResponse =
_amazons3.InitiateMultipartUpload(initiateRequest);
uploadId = initResponse.UploadId;
}
else
uploadId = lastUploadedPartdetail.UploadId;
try
{
long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB
long filePosition = partSize * alreadyUploadedParts;
for (int i = alreadyUploadedParts + 1; filePosition < contentLength; i++)
{
// Before upload the next set of upload part, need to check the last modified because user might modify it in the mean time
if (File.Exists(filePath) &&
appUpdateInfo.LastModifiedTime == new FileInfo(filePath).LastWriteTime)
{
byte[] bytesToStream;
if (filePosition + partSize < contentLength)
{
bytesToStream = new byte[partSize];
Array.Copy(bytes, filePosition, bytesToStream, 0, partSize);
}
else
{
bytesToStream = new byte[contentLength - filePosition];
Array.Copy(bytes, filePosition, bytesToStream, 0,
contentLength - filePosition);
}
Stream stream = new MemoryStream(bytesToStream);
UploadPartRequest uploadRequest = new UploadPartRequest()
.WithBucketName(bucketKey)
.WithKey(relativePath)
.WithUploadId(uploadId)
.WithPartNumber(i)
.WithPartSize(partSize)
.WithFilePosition(filePosition)
.WithTimeout(1000000000);
uploadRequest.WithInputStream(stream);
// Upload part and add response to our list.
var response = _amazons3.UploadPart(uploadRequest);
WriteResponseToFile(relativePath, bucketKey, uploadId,
appUpdateInfo.LastModifiedTime, response);
uploadResponses.Add(response);
filePosition += partSize;
ModifySyncStatus("Uploaded",
contentLength <= filePosition
? contentLength
: filePosition,
contentLength, relativePath);
}
else
{
// need to abort the upload process
_processingFiles.Remove(filePath);
RemoveConfig(relativePath, lastUploadedPartdetail.BucketKey);
_amazons3.AbortMultipartUpload(new AbortMultipartUploadRequest()
.WithBucketName(bucketKey)
.WithKey(relativePath)
.WithUploadId(uploadId));
return;
}
}
CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest()
.WithBucketName(bucketKey)
.WithKey(relativePath)
.WithUploadId(uploadId)
.WithPartETags(uploadResponses);
CompleteMultipartUploadResponse completeUploadResponse =
_amazons3.CompleteMultipartUpload(completeRequest);
RemoveConfig(relativePath, completeUploadResponse.BucketName);
//_service.AddObject(filePath, Utilities.MyConfig.BucketKey, relativePath);
SetAcltoObject(relativePath);
if (IsItSharedFile)
{
ProcessApplicationUpdates(appUpdateInfo, true, Username);
}
else
ProcessApplicationUpdates(appUpdateInfo, false, string.Empty);
_processingFiles.Remove(filePath);
UploadContentForSearch(filePath, relativePath);
}
catch (Exception)
{
_processingFiles.Remove(filePath);
_amazons3.AbortMultipartUpload(new AbortMultipartUploadRequest()
.WithBucketName(bucketKey)
.WithKey(relativePath)
.WithUploadId(uploadId));
RemoveConfig(relativePath, Utilities.MyConfig.BucketKey);
if (!_fileQueue.ContainsKey(filePath))
_fileQueue.Add(filePath,
new FileQueue
{
Type = WatcherChangeTypes.Created,
Name = Path.GetFileName(filePath)
});
else
_fileQueue.Add(filePath,
new FileQueue
{
Type = WatcherChangeTypes.Created,
Name = Path.GetFileName(filePath)
});
}
}
}
catch (Exception)
{
return;
}
}
}
}
catch (Exception)
{
return;
}
}
public override string InitiateMultiPartUpload(string keyName, NameValueCollection metaData)
{
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest();
request.CannedACL = S3CannedACL.PublicRead;
request.BucketName = BucketName;
request.Key = keyName;
foreach (string key in metaData)
{
request.Metadata.Add(key, metaData[key]);
}
InitiateMultipartUploadResponse response;
try
{
response = AmazonS3Client.InitiateMultipartUpload(request);
}
catch (AmazonS3Exception e)
{
throw new CloudUploadProviderException("Exception thrown for initiate multi part upload operation", e);
}
return response.UploadId;
}
/// <summary>
/// Sample code to contrast uploading a file using Amazon S3's Multi-Part Upload API
/// </summary>
/// <param name="s3Client"></param>
/// <param name="bucketName"></param>
/// <param name="fileName"></param>
static void UploadUsingMultiPartAPI(IAmazonS3 s3Client, string bucketName, string fileName)
{
const string objectKey = "multipart/myobject";
// tell S3 we're going to upload an object in multiple parts and receive an upload ID
// in return
var initializeUploadRequest = new InitiateMultipartUploadRequest
{
BucketName = bucketName,
Key = objectKey
};
var initializeUploadResponse = s3Client.InitiateMultipartUpload(initializeUploadRequest);
// this ID must accompany all parts and the final 'completed' call
var uploadID = initializeUploadResponse.UploadId;
// Send the file (synchronously) using 4*5MB parts - note we pass the upload id
// with each call. For each part we need to log the returned etag value to pass
// to the completion call
var partETags = new List<PartETag>();
var partSize = 5 * ONE_MEG; // this is the minimum part size allowed
for (var partNumber = 0; partNumber < 4; partNumber++)
{
// part numbers must be between 1 and 1000
var logicalPartNumber = partNumber + 1;
var uploadPartRequest = new UploadPartRequest
{
BucketName = bucketName,
Key = objectKey,
UploadId = uploadID,
PartNumber = logicalPartNumber,
PartSize = partSize,
FilePosition = partNumber * partSize,
FilePath = fileName
};
var partUploadResponse = s3Client.UploadPart(uploadPartRequest);
partETags.Add(new PartETag { PartNumber = logicalPartNumber, ETag = partUploadResponse.ETag });
}
var completeUploadRequest = new CompleteMultipartUploadRequest
{
BucketName = bucketName,
Key = objectKey,
UploadId = uploadID,
PartETags = partETags
};
s3Client.CompleteMultipartUpload(completeUploadRequest);
}
public AmazonMultiUpload(AmazonS3Client client, string bucket, string key, Metadata metadata, CancellationToken ct)
{
_client = client;
_bucket = bucket;
_key = key;
_blocks = new List<Task<UploadPartResponse>>();
_ct = ct;
// Fire up the multipart upload request...
var req = new InitiateMultipartUploadRequest { BucketName = _bucket, Key = key };
if (metadata != null)
{
foreach (var m in metadata)
{
req.Metadata.Add("x-amz-meta-" + m.Key, m.Value);
}
}
_uploadId = _client.InitiateMultipartUploadAsync(req, _ct).ContinueWith(r => r.Result.UploadId);
}
public async Task <string> MultipartUploadStartAsync(string bucket,
string key,
S3CannedACL s3CannedAcl = null,
CancellationToken cancellationToken = default)
{
this.Logger.LogDebug($"[{nameof(this.MultipartUploadStartAsync)}]");
this.Logger.LogTrace(JsonConvert.SerializeObject(new { bucket, key, s3CannedAcl }));
if (string.IsNullOrWhiteSpace(bucket))
{
throw new ArgumentNullException(nameof(bucket));
}
if (string.IsNullOrWhiteSpace(key))
{
throw new ArgumentNullException(nameof(key));
}
if (s3CannedAcl == null)
{
s3CannedAcl = S3CannedACL.BucketOwnerFullControl;
}
var request = new Amazon.S3.Model.InitiateMultipartUploadRequest
{
BucketName = bucket,
CannedACL = s3CannedAcl,
Key = key,
};
this.Logger.LogTrace(JsonConvert.SerializeObject(value: request));
var response = await this.Repository.InitiateMultipartUploadAsync(request : request,
cancellationToken : cancellationToken == default?this.CancellationToken.Token : cancellationToken);
this.Logger.LogTrace(JsonConvert.SerializeObject(value: response));
return(response.UploadId);
}
public static void mpuSerial(String fPath)
{
System.Console.WriteLine("\nhello,MPU!!");
NameValueCollection appConfig = ConfigurationManager.AppSettings;
AmazonS3 s3Client = AWSClientFactory.CreateAmazonS3Client(appConfig["AWSAccessKey"], appConfig["AWSSecretKey"]);
String bucketName = "region";
String objectNameA = "photos/2006/January/sample.jpg";
String objectNameB = "photos/2006/February/sample.jpg";
String objectNameC = "photos/2006/March/sample.jpg";
String objectNameD = "videos/2006/March/sample.wmv";
String objectNameE = "sample.jpg";
//PutBucket
//PutBucketResponse response = s3Client.PutBucket(new PutBucketRequest().WithBucketName(bucketName));
//********************************************************************************************************************************
//Initial with CannedACL
InitiateMultipartUploadRequest MPUCannedACL = new InitiateMultipartUploadRequest();
MPUCannedACL.WithBucketName(bucketName);
MPUCannedACL.WithKey(objectNameA);
MPUCannedACL.WithCannedACL(S3CannedACL.PublicRead);
MPUCannedACL.WithContentType("Content-Type: text/html");
MPUCannedACL.WithStorageClass(S3StorageClass.Standard);
//MPUCannedACL.WithServerSideEncryptionMethod(ServerSideEncryptionMethod.AES256);
InitiateMultipartUploadResponse MPUCannedACLResult = s3Client.InitiateMultipartUpload(MPUCannedACL);
String MPUCannedACLUID = MPUCannedACLResult.UploadId;
System.Console.WriteLine("\nInitial MPUCanned uploadID:{0}", MPUCannedACLUID);
AbortMultipartUploadResponse abortCannedACLResult = s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameA).WithUploadId(MPUCannedACLUID));
System.Console.WriteLine("\nAbort MPUCanned uploadID:{0}", MPUCannedACLUID);
//Initial with GrantACL
InitiateMultipartUploadRequest MPUACL = new InitiateMultipartUploadRequest();
MPUACL.WithBucketName(bucketName);
MPUACL.WithKey(objectNameA);
MPUACL.WithGrants(new S3Grant().WithGrantee(new S3Grantee().WithCanonicalUser("canonicalidhrchu", "hrchu")).WithPermission(S3Permission.FULL_CONTROL));
MPUACL.WithGrants(new S3Grant().WithGrantee(new S3Grantee().WithCanonicalUser("canonicalidannyren", "annyren")).WithPermission(S3Permission.WRITE_ACP));
InitiateMultipartUploadResponse MPUACLResult = s3Client.InitiateMultipartUpload(MPUACL);
String MPUACLUID = MPUACLResult.UploadId;
System.Console.WriteLine("\nInitial MPU Grants uploadID:{0}", MPUACLUID);
AbortMultipartUploadResponse abortACLResult = s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameA).WithUploadId(MPUACLUID));
System.Console.WriteLine("\nAbort MPU Grants uploadID:{0}", MPUACLUID);
//********************************************************************************************************************************
//Initial MPU * 5
InitiateMultipartUploadResponse InitialResultA = s3Client.InitiateMultipartUpload(new InitiateMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameA));
InitiateMultipartUploadResponse InitialResultB = s3Client.InitiateMultipartUpload(new InitiateMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameB));
InitiateMultipartUploadResponse InitialResultC = s3Client.InitiateMultipartUpload(new InitiateMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameC));
InitiateMultipartUploadResponse InitialResultD = s3Client.InitiateMultipartUpload(new InitiateMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameD));
InitiateMultipartUploadResponse InitialResultE = s3Client.InitiateMultipartUpload(new InitiateMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameE));
String objectAUID = InitialResultA.UploadId;
String objectBUID = InitialResultB.UploadId;
String objectCUID = InitialResultC.UploadId;
String objectDUID = InitialResultD.UploadId;
String objectEUID = InitialResultE.UploadId;
System.Console.WriteLine("\nInitial MPU:{0},uploadID:{1}",objectNameA,objectAUID);
System.Console.WriteLine("Initial MPU:{0},uploadID:{1}", objectNameB, objectBUID);
System.Console.WriteLine("\nInitial MPU:{0},uploadID:{1}", objectNameC, objectCUID);
System.Console.WriteLine("Initial MPU:{0},uploadID:{1}", objectNameD, objectDUID);
System.Console.WriteLine("Initial MPU:{0},uploadID:{1}", objectNameE, objectEUID);
//List MPUs
ListMultipartUploadsResponse listMPUsResult = s3Client.ListMultipartUploads(new ListMultipartUploadsRequest().WithBucketName(bucketName));
System.Console.WriteLine("\nList MPUs with WithMaxUploads Result:\n{0}", listMPUsResult.ResponseXml);
//**************List MPUs with MaxUploads=1*************************************************************************************
ListMultipartUploadsResponse listMPUsMaxUPsResult = s3Client.ListMultipartUploads(new ListMultipartUploadsRequest().WithBucketName(bucketName).WithMaxUploads(1));
System.Console.WriteLine("\nList MPUs with WithMaxUploads Result:\n{0}", listMPUsMaxUPsResult.ResponseXml);
//**************List MPUs with Delimiter /, result:<Key>sample.jpg</Key> only*************************************************************************************
ListMultipartUploadsResponse listMPUsDelimiterResult = s3Client.ListMultipartUploads(new ListMultipartUploadsRequest().WithBucketName(bucketName).WithDelimiter("/"));
System.Console.WriteLine("\nList MPUs with Delimiter Result:\n{0}", listMPUsDelimiterResult.ResponseXml);
//**************List MPUs with Prefix photos/2006/, result:<Prefix>photos/2006/February/</Prefix>, <Prefix>photos/2006/January/</Prefix> and <Prefix>photos/2006/March/</Prefix>*****************
ListMultipartUploadsResponse listMPUsPrefixResult = s3Client.ListMultipartUploads(new ListMultipartUploadsRequest().WithBucketName(bucketName).WithPrefix("photos/2006/"));
System.Console.WriteLine("\nList MPUs with Prefix Result:\n{0}", listMPUsPrefixResult.ResponseXml);
//**************List MPUs with Keymarker objectNameB, result:<Prefix>photos/2006/February/</Prefix>, <Prefix>photos/2006/January/</Prefix> and <Prefix>photos/2006/March/</Prefix>*****************
ListMultipartUploadsResponse listMPUsKeymarkerResult = s3Client.ListMultipartUploads(new ListMultipartUploadsRequest().WithBucketName(bucketName).WithKeyMarker(objectNameB));
System.Console.WriteLine("\nList MPUs with Keymarker Result:\n{0}", listMPUsKeymarkerResult.ResponseXml);
//**************List MPUs with UploadIDmarker objectNameB, result:<Prefix>photos/2006/February/</Prefix>, <Prefix>photos/2006/January/</Prefix> and <Prefix>photos/2006/March/</Prefix>*****************
ListMultipartUploadsResponse listMPUsUpIDmarkerResult = s3Client.ListMultipartUploads(new ListMultipartUploadsRequest().WithBucketName(bucketName).WithUploadIdMarker(objectBUID));
System.Console.WriteLine("\nList MPUs with UpIDmarker Result:\n{0}", listMPUsUpIDmarkerResult.ResponseXml);
//Upload Part * 2
UploadPartResponse part1Result = s3Client.UploadPart(new UploadPartRequest().WithBucketName(bucketName).WithKey(objectNameA).WithPartNumber(1).WithUploadId(objectAUID).WithFilePath(fPath).WithFilePosition(10).WithPartSize(6270544));
UploadPartResponse part2Result = s3Client.UploadPart(new UploadPartRequest().WithBucketName(bucketName).WithKey(objectNameA).WithPartNumber(2).WithUploadId(objectAUID).WithFilePath(fPath).WithFilePosition(10));
String part1Etag = part1Result.ETag;
String part2Etag = part2Result.ETag;
System.Console.WriteLine("\nUpload Part Result: part 1 requestID:{0} & Etag:{1}", part1Result.RequestId, part1Etag);
System.Console.WriteLine("\nUpload Part Result: part 1 requestID:{0} & Etag:{1}", part1Result.RequestId, part2Etag);
//List Upload Parts
ListPartsResponse listPartsResult = s3Client.ListParts(new ListPartsRequest().WithBucketName(bucketName).WithKey(objectNameA).WithUploadId(objectAUID));
System.Console.WriteLine("\nListParts Result:\n{0}", listPartsResult.ResponseXml);
//**************List Upload Parts with MaxParts(1), result:只列出Part 1資訊*************************************************************************************
ListPartsResponse listPartsMaxResult = s3Client.ListParts(new ListPartsRequest().WithBucketName(bucketName).WithKey(objectNameA).WithUploadId(objectAUID).WithMaxParts(1));
System.Console.WriteLine("\nListParts With Max Parts Result:\n{0}", listPartsMaxResult.ResponseXml);
//**************List Upload Parts with PartNumberMarker("1"), result:只列出Part 2資訊*************************************************************************************
ListPartsResponse listPartsKeymarkerResult = s3Client.ListParts(new ListPartsRequest().WithBucketName(bucketName).WithKey(objectNameA).WithUploadId(objectAUID).WithPartNumberMarker("1"));
System.Console.WriteLine("\nListParts With Key Marker Result:\n{0}", listPartsKeymarkerResult.ResponseXml);
//Complete MPU
List<UploadPartResponse> uploadResponses = new List<UploadPartResponse>();
uploadResponses.Add(part1Result);
uploadResponses.Add(part2Result);
CompleteMultipartUploadResponse completeResult = s3Client.CompleteMultipartUpload(new CompleteMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameA).WithUploadId(objectAUID).WithPartETags(uploadResponses));
System.Console.WriteLine("\nComplete MPUs Result:\n{0}", completeResult.ResponseXml);
System.Console.WriteLine("\nComplete MPUs Etag:\n{0}", completeResult.ETag);
//Upload Part Copy
CopyPartResponse copyPartResult = s3Client.CopyPart(new CopyPartRequest().WithSourceBucket(bucketName).WithDestinationBucket(bucketName)
.WithSourceKey(objectNameA).WithDestinationKey(objectNameB).WithPartNumber(1).WithUploadID(objectBUID));
System.Console.WriteLine("\nCopy Part Result: requestID:{0}", copyPartResult.RequestId);
//**************Upload Part Copy with ETagToMatch / etagsToNotMatch****************************************************
CopyPartResponse copyPartEtagmatchResult = s3Client.CopyPart(new CopyPartRequest().WithSourceBucket(bucketName).WithDestinationBucket(bucketName)
.WithSourceKey(objectNameA).WithDestinationKey(objectNameB).WithPartNumber(1).WithUploadID(objectBUID).WithETagsToMatch(completeResult.ETag));
System.Console.WriteLine("\nCopy Part with ETag Match Result: requestID:{0}", copyPartEtagmatchResult.RequestId);
CopyPartResponse copyPartEtagNOTmatchResult = s3Client.CopyPart(new CopyPartRequest().WithSourceBucket(bucketName).WithDestinationBucket(bucketName)
.WithSourceKey(objectNameA).WithDestinationKey(objectNameB).WithPartNumber(1).WithUploadID(objectBUID).WithETagsToNotMatch(part1Etag));
System.Console.WriteLine("\nCopy Part with ETag NOT Match Result: requestID:{0}", copyPartEtagNOTmatchResult.RequestId);
//**************Upload Part Copy with firstByte-lastByte，反向測試，<5GB檔案 error****************************************************
/* CopyPartResponse copyPartByterangeResult = s3Client.CopyPart(new CopyPartRequest().WithSourceBucket(bucketName).WithDestinationBucket(bucketName)
.WithSourceKey(objectNameA).WithDestinationKey(objectNameB).WithPartNumber(1).WithUploadID(objectBUID).WithFirstByte(500).WithLastByte(2000));
System.Console.WriteLine("\nCopy Part with ETagToMatch Result: requestID:{0}", copyPartByterangeResult.RequestId);*/
//*Bug**************Upload Part Copy with modifiedSinceDate / unmodifiedSinceDate**************************************************************
/* DateTime dt =DateTime.Now;
DateTime dt100 =dt.AddDays(100); //增加100天
DateTime dt30 = dt.AddDays(-30);//減少30天
CopyPartResponse copyPartDateResult = s3Client.CopyPart(new CopyPartRequest().WithSourceBucket(bucketName).WithDestinationBucket(bucketName)
.WithSourceKey(objectNameA).WithDestinationKey(objectNameB).WithPartNumber(1).WithUploadID(objectBUID).WithModifiedSinceDate(DateTime.Now.AddDays(-30)));
System.Console.WriteLine("\nCopy Part with modifiedSinceDate Result: requestID:{0}", copyPartDateResult.RequestId);*/
//**************Upload Part Copy with srcVersionId*************************************************************************************************
// 沿用 ACL Version test 環境bucket = netversion，內有兩筆 Object 資訊(Object 已刪除，帶有 delete marker)
// 若於佈板後刪除環境，請預先建立環境，執行ACL 程式 132~150行程式
//**************************************************************************************************************************************************
//versioning test
String vbucketName = "netversion";
String versionid = "784cca47e2a7423f97dedde6a72f9b3d";
CopyPartResponse copyPartVersionidResult = s3Client.CopyPart(new CopyPartRequest().WithSourceBucket(vbucketName).WithDestinationBucket(bucketName)
.WithSourceKey("hello").WithDestinationKey(objectNameB).WithPartNumber(1).WithUploadID(objectBUID).WithSourceVersionId(versionid));
System.Console.WriteLine("\nCopy Part with srcVersionId Result: requestID:{0}", copyPartVersionidResult.RequestId);
//Abort MPU
AbortMultipartUploadResponse abortobBResult = s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameB).WithUploadId(objectBUID));
System.Console.WriteLine("\nAbortMPU Result, requestID:{0}", abortobBResult.RequestId);
AbortMultipartUploadResponse abortobCResult = s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameC).WithUploadId(objectCUID));
System.Console.WriteLine("\nAbortMPU Result, requestID:{0}", abortobCResult.RequestId);
AbortMultipartUploadResponse abortobDResult = s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameD).WithUploadId(objectDUID));
System.Console.WriteLine("\nAbortMPU Result, requestID:{0}", abortobDResult.RequestId);
AbortMultipartUploadResponse abortobEResult = s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameE).WithUploadId(objectEUID));
System.Console.WriteLine("\nAbortMPU Result, requestID:{0}", abortobEResult.RequestId);
//DeleteObject
System.Console.WriteLine("\nDelete Object!\n");
s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey(objectNameA));
//DeleteBucket
System.Console.WriteLine("Delete Bucket!");
s3Client.DeleteBucket(new DeleteBucketRequest().WithBucketName(bucketName));
System.Console.WriteLine("END!");
}
/// <summary>
/// Initiates the asynchronous execution of the InitiateMultipartUpload operation.
/// <seealso cref="Amazon.S3.IAmazonS3.InitiateMultipartUpload"/>
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the InitiateMultipartUpload operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task<InitiateMultipartUploadResponse> InitiateMultipartUploadAsync(InitiateMultipartUploadRequest request, CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new InitiateMultipartUploadRequestMarshaller();
var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.GetInstance();
return Invoke<IRequest, InitiateMultipartUploadRequest, InitiateMultipartUploadResponse>(request, marshaller, unmarshaller, signer, cancellationToken);
}
/// <summary>
/// <para>Initiates a multipart upload and returns an upload ID.</para>
/// </summary>
/// <remarks>
/// <para>
/// After you initiate a multipart upload and upload one or more parts, you must either complete or abort
/// the multipart upload in order to stop getting charged for storage of the uploaded parts. Once you
/// complete or abort the multipart upload, Amazon S3 will release the stored parts and stop charging you
/// for their storage.
/// </para>
/// </remarks>
/// <param name="request">Container for the necessary parameters to execute the InitiateMultipartUpload service method on
/// AmazonS3.</param>
///
/// <returns>The response from the InitiateMultipartUpload service method, as returned by AmazonS3.</returns>
public InitiateMultipartUploadResponse InitiateMultipartUpload(InitiateMultipartUploadRequest request)
{
var task = InitiateMultipartUploadAsync(request);
try
{
return task.Result;
}
catch(AggregateException e)
{
throw e.InnerException;
}
}
/// <summary>
/// <para>Initiates a multipart upload and returns an upload ID.</para>
/// </summary>
///
/// <param name="initiateMultipartUploadRequest">Container for the necessary parameters to execute the InitiateMultipartUpload service method on
/// AmazonS3.</param>
///
/// <returns>The response from the InitiateMultipartUpload service method, as returned by AmazonS3.</returns>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
public async Task<InitiateMultipartUploadResponse> InitiateMultipartUploadAsync(InitiateMultipartUploadRequest initiateMultipartUploadRequest, CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new InitiateMultipartUploadRequestMarshaller();
var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.GetInstance();
var response = await Invoke<IRequest, InitiateMultipartUploadRequest, InitiateMultipartUploadResponse>(initiateMultipartUploadRequest, marshaller, unmarshaller, signer, cancellationToken)
.ConfigureAwait(continueOnCapturedContext: false);
return response;
}
/// <summary>
/// <para>Initiates a multipart upload and returns an upload ID.</para>
/// </summary>
/// <remarks>
/// <para>
/// After you initiate a multipart upload and upload one or more parts, you must either complete or abort
/// the multipart upload in order to stop getting charged for storage of the uploaded parts. Once you
/// complete or abort the multipart upload, Amazon S3 will release the stored parts and stop charging you
/// for their storage.
/// </para>
/// </remarks>
/// <param name="request">Container for the necessary parameters to execute the InitiateMultipartUpload service method on
/// AmazonS3.</param>
///
/// <returns>The response from the InitiateMultipartUpload service method, as returned by AmazonS3.</returns>
public InitiateMultipartUploadResponse InitiateMultipartUpload(InitiateMultipartUploadRequest request)
{
var task = InitiateMultipartUploadAsync(request);
try
{
return task.Result;
}
catch(AggregateException e)
{
ExceptionDispatchInfo.Capture(e.InnerException).Throw();
return null;
}
}
/// <summary>
/// Initiates a multipart upload and returns an upload ID.
///
///
/// <para>
/// <b>Note:</b> After you initiate multipart upload and upload one or more parts, you
/// must either complete or abort multipart upload in order to stop getting charged for
/// storage of the uploaded parts. Only after you either complete or abort multipart upload,
/// Amazon S3 frees up the parts storage and stops charging you for the parts storage.
/// </para>
/// </summary>
/// <param name="bucketName">A property of InitiateMultipartUploadRequest used to execute the InitiateMultipartUpload service method.</param>
/// <param name="key">A property of InitiateMultipartUploadRequest used to execute the InitiateMultipartUpload service method.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
///
/// <returns>The response from the InitiateMultipartUpload service method, as returned by S3.</returns>
public Task<InitiateMultipartUploadResponse> InitiateMultipartUploadAsync(string bucketName, string key, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var request = new InitiateMultipartUploadRequest();
request.BucketName = bucketName;
request.Key = key;
return InitiateMultipartUploadAsync(request, cancellationToken);
}
public static void Main(string[] args)
{
XmlSerializerFactory factory = new XmlSerializerFactory();
XmlSerializer serializer = factory.CreateSerializer(typeof(UploadState));
UploadState us;
if (args.Length == 2)
{
us = new UploadState(args[1], args[0]);
}
else if (args.Length == 0 && new FileInfo("upload.dat").Exists)
{
using (FileStream fs = new FileStream("upload.dat", FileMode.Open))
{
us = (UploadState)serializer.Deserialize(fs);
}
}
else
{
Console.Error.WriteLine("Usage: bucket filename");
return;
}
try
{
NameValueCollection appConfig = ConfigurationManager.AppSettings;
// Print the number of Amazon S3 Buckets.
AmazonS3 s3Client = AWSClientFactory.CreateAmazonS3Client(
appConfig["AWSAccessKey"],
appConfig["AWSSecretKey"]
);
if (string.IsNullOrEmpty(us.UploadId))
{
InitiateMultipartUploadRequest req =
new InitiateMultipartUploadRequest()
.WithBucketName(us.BucketName)
.WithKey(us.Key)
;
us.UploadId = s3Client.InitiateMultipartUpload(req).UploadId;
using (FileStream fs = new FileStream("upload.dat", FileMode.OpenOrCreate))
{
serializer.Serialize(fs, us);
}
}
while (us.FilePosition < us.FileLength)
{
try
{
Console.WriteLine("Uploading part {0} of {1}", us.PartNumber, us.NumChunks);
UploadPartRequest ureq = new UploadPartRequest()
.WithBucketName(us.BucketName)
.WithFilePath(us.FileName)
.WithFilePosition(us.FilePosition)
.WithPartNumber(us.PartNumber)
.WithPartSize(us.FileLength - us.FilePosition > us.ChunkSize ? us.ChunkSize : us.FileLength - us.FilePosition)
.WithGenerateChecksum(true)
.WithKey(us.Key)
.WithUploadId(us.UploadId)
.WithSubscriber(new EventHandler<UploadPartProgressArgs>(ShowProgress))
;
if (us.Responses.Count > us.PartNumber - 1)
{
us.Responses[us.PartNumber - 1] = new PartETag(us.PartNumber, s3Client.UploadPart(ureq).ETag);
}
else
{
us.Responses.Insert(us.PartNumber - 1, new PartETag(us.PartNumber, s3Client.UploadPart(ureq).ETag));
}
us.PartNumber++;
us.FilePosition += us.ChunkSize;
using (FileStream fs = new FileStream("upload.dat", FileMode.OpenOrCreate))
{
serializer.Serialize(fs, us);
}
}
catch (System.Net.WebException)
{
}
}
CompleteMultipartUploadRequest creq = new CompleteMultipartUploadRequest()
.WithPartETags(us.Responses)
.WithBucketName(us.BucketName)
.WithUploadId(us.UploadId)
.WithKey(us.Key)
;
CompleteMultipartUploadResponse cresp = s3Client.CompleteMultipartUpload(creq);
System.Console.WriteLine("File available at {0}", cresp.Location);
File.Delete("upload.dat");
}
catch (AmazonS3Exception e)
{
Console.Error.WriteLine(e);
}
//Console.Write(GetServiceOutput());
//Console.Read();
}
private void Init()
{
var initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = this.bucketName,
Key = this.fileName
};
this.initResponse = client.InitiateMultipartUpload(initiateRequest);
this.abortMPURequest = new AbortMultipartUploadRequest
{
BucketName = this.bucketName,
Key = this.fileName,
UploadId = this.initResponse.UploadId
};
this.uploadResponses = new List<UploadPartResponse>();
}
public override string InitiateChunkedUpload(string domain, string path)
{
var request = new InitiateMultipartUploadRequest
{
BucketName = _bucket,
Key = MakePath(domain, path),
ServerSideEncryptionMethod = ServerSideEncryptionMethod.AES256
};
using (var s3 = GetClient())
{
var response = s3.InitiateMultipartUpload(request);
return response.UploadId;
}
}
public void MultipartEncryptionTestInstructionFile()
{
string filePath = @"C:\temp\Upload15MegFileIn3PartsViaStream.txt";
string retrievedFilepath = @"C:\temp\Upload15MegFileIn3PartsViaStreamRetreived.txt";
int MEG_SIZE = (int)Math.Pow(2, 20) + 4001;
long totalSize = (long)(15 * MEG_SIZE);
UtilityMethods.GenerateFile(filePath, totalSize);
string key = "MultipartEncryptionTestInstrcutionFile" + random.Next();
s3EncryptionClientFileMode.PutBucket(new PutBucketRequest() { BucketName = bucketName });
Stream inputStream = File.OpenRead(filePath);
try
{
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
StorageClass = S3StorageClass.ReducedRedundancy,
ContentType = "text/html",
CannedACL = S3CannedACL.PublicRead
};
InitiateMultipartUploadResponse initResponse = s3EncryptionClientFileMode.InitiateMultipartUpload(initRequest);
// Upload part 1
UploadPartRequest uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 1,
PartSize = 5 * MEG_SIZE,
InputStream = inputStream
};
UploadPartResponse up1Response = s3EncryptionClientFileMode.UploadPart(uploadRequest);
// Upload part 2
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 2,
PartSize = 5 * MEG_SIZE + 4001,
InputStream = inputStream
};
UploadPartResponse up2Response = s3EncryptionClientFileMode.UploadPart(uploadRequest);
// Upload part 3
uploadRequest = new UploadPartRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId,
PartNumber = 3,
InputStream = inputStream,
IsLastPart = true
};
//uploadRequest.setLastPart();
UploadPartResponse up3Response = s3EncryptionClientFileMode.UploadPart(uploadRequest);
ListPartsRequest listPartRequest = new ListPartsRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
ListPartsResponse listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest);
Assert.AreEqual(3, listPartResponse.Parts.Count);
Assert.AreEqual(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber);
Assert.AreEqual(up1Response.ETag, listPartResponse.Parts[0].ETag);
Assert.AreEqual(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber);
Assert.AreEqual(up2Response.ETag, listPartResponse.Parts[1].ETag);
Assert.AreEqual(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber);
Assert.AreEqual(up3Response.ETag, listPartResponse.Parts[2].ETag);
listPartRequest.MaxParts = 1;
listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest);
Assert.AreEqual(1, listPartResponse.Parts.Count);
// Complete the response
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
{
BucketName = bucketName,
Key = key,
UploadId = initResponse.UploadId
};
compRequest.AddPartETags(up1Response, up2Response, up3Response);
CompleteMultipartUploadResponse compResponse = s3EncryptionClientFileMode.CompleteMultipartUpload(compRequest);
Assert.AreEqual(bucketName, compResponse.BucketName);
Assert.IsNotNull(compResponse.ETag);
Assert.AreEqual(key, compResponse.Key);
Assert.IsNotNull(compResponse.Location);
// Get the file back from S3 and make sure it is still the same.
GetObjectRequest getRequest = new GetObjectRequest()
{
BucketName = bucketName,
Key = key
};
GetObjectResponse getResponse = s3EncryptionClientFileMode.GetObject(getRequest);
getResponse.WriteResponseStreamToFile(retrievedFilepath);
UtilityMethods.CompareFiles(filePath, retrievedFilepath);
GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest()
{
BucketName = bucketName,
Key = key
};
GetObjectMetadataResponse metaDataResponse = s3EncryptionClientFileMode.GetObjectMetadata(metaDataRequest);
Assert.AreEqual("text/html", metaDataResponse.Headers.ContentType);
}
finally
{
inputStream.Close();
if (File.Exists(filePath))
File.Delete(filePath);
if (File.Exists(retrievedFilepath))
File.Delete(retrievedFilepath);
}
}
/// <summary>
/// Initiates the asynchronous execution of the InitiateMultipartUpload operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the InitiateMultipartUpload operation on AmazonS3Client.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndInitiateMultipartUpload
/// operation.</returns>
public IAsyncResult BeginInitiateMultipartUpload(InitiateMultipartUploadRequest request, AsyncCallback callback, object state)
{
var marshaller = new InitiateMultipartUploadRequestMarshaller();
var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.Instance;
return BeginInvoke<InitiateMultipartUploadRequest>(request, marshaller, unmarshaller,
callback, state);
}
/// <summary>
/// Initiates a multipart upload and returns an upload ID.
///
///
/// <para>
/// <b>Note:</b> After you initiate multipart upload and upload one or more parts, you
/// must either complete or abort multipart upload in order to stop getting charged for
/// storage of the uploaded parts. Only after you either complete or abort multipart upload,
/// Amazon S3 frees up the parts storage and stops charging you for the parts storage.
/// </para>
/// </summary>
/// <param name="bucketName">A property of InitiateMultipartUploadRequest used to execute the InitiateMultipartUpload service method.</param>
/// <param name="key">A property of InitiateMultipartUploadRequest used to execute the InitiateMultipartUpload service method.</param>
///
/// <returns>The response from the InitiateMultipartUpload service method, as returned by S3.</returns>
public InitiateMultipartUploadResponse InitiateMultipartUpload(string bucketName, string key)
{
var request = new InitiateMultipartUploadRequest();
request.BucketName = bucketName;
request.Key = key;
return InitiateMultipartUpload(request);
}
/// <summary>
/// Runs the multipart upload.
/// </summary>
public override void Execute()
{
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
{
BucketName = this._fileTransporterRequest.BucketName,
Key = this._fileTransporterRequest.Key,
CannedACL = this._fileTransporterRequest.CannedACL,
ContentType = determineContentType(),
StorageClass = this._fileTransporterRequest.StorageClass,
ServerSideEncryptionMethod = this._fileTransporterRequest.ServerSideEncryptionMethod
};
initRequest.BeforeRequestEvent += this.RequestEventHandler;
if (this._fileTransporterRequest.Metadata != null && this._fileTransporterRequest.Metadata.Count > 0)
initRequest.Metadata = this._fileTransporterRequest.Metadata;
if (this._fileTransporterRequest.Headers != null && this._fileTransporterRequest.Headers.Count > 0)
initRequest.Headers = this._fileTransporterRequest.Headers;
InitiateMultipartUploadResponse initResponse = this._s3Client.InitiateMultipartUpload(initRequest);
_logger.DebugFormat("Initiated upload: {0}", initResponse.UploadId);
try
{
_logger.DebugFormat("Queue up the UploadPartRequests to be executed");
long filePosition = 0;
for (int i = 1; filePosition < this._contentLength; i++)
{
UploadPartRequest uploadRequest = new UploadPartRequest()
{
BucketName = this._fileTransporterRequest.BucketName,
Key = this._fileTransporterRequest.Key,
UploadId = initResponse.UploadId,
PartNumber = i,
PartSize = this._partSize,
#if (BCL && !BCL45)
Timeout = ClientConfig.GetTimeoutValue(this._config.DefaultTimeout,this._fileTransporterRequest.Timeout)
#endif
};
if ((filePosition + this._partSize >= this._contentLength) && _s3Client is AmazonS3EncryptionClient)
{
uploadRequest.IsLastPart = true;
uploadRequest.PartSize = 0;
}
uploadRequest.StreamUploadProgressCallback += this.uploadPartProgressEventCallback;
uploadRequest.BeforeRequestEvent += this.RequestEventHandler;
if (this._fileTransporterRequest.IsSetFilePath())
{
uploadRequest.FilePosition = filePosition;
uploadRequest.FilePath = this._fileTransporterRequest.FilePath;
}
else
{
uploadRequest.InputStream = this._fileTransporterRequest.InputStream;
}
this._partsToUpload.Enqueue(uploadRequest);
filePosition += this._partSize;
}
this._totalNumberOfParts = this._partsToUpload.Count;
_logger.DebugFormat("Starting threads to execute the {0} UploadPartRequests in the queue", this._totalNumberOfParts);
startInvokerPool();
_logger.DebugFormat("Waiting for threads to complete. ({0})", initResponse.UploadId);
waitTillAllThreadsComplete();
_logger.DebugFormat("Beginning completing multipart. ({0})", initResponse.UploadId);
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
{
BucketName = this._fileTransporterRequest.BucketName,
Key = this._fileTransporterRequest.Key,
UploadId = initResponse.UploadId
};
compRequest.AddPartETags(this._uploadResponses);
compRequest.BeforeRequestEvent += this.RequestEventHandler;
this._s3Client.CompleteMultipartUpload(compRequest);
_logger.DebugFormat("Done completing multipart. ({0})", initResponse.UploadId);
}
catch (Exception e)
{
_logger.Error(e, "Exception while uploading. ({0})", initResponse.UploadId);
shutdown(initResponse.UploadId);
throw;
}
finally
{
if (this._fileTransporterRequest.InputStream != null && !this._fileTransporterRequest.IsSetFilePath() && this._fileTransporterRequest.AutoCloseStream)
{
this._fileTransporterRequest.InputStream.Close();
}
if (_logger != null)
{
_logger.Flush();
}
}
}
internal InitiateMultipartUploadResponse InitiateMultipartUpload(InitiateMultipartUploadRequest request)
{
var marshaller = new InitiateMultipartUploadRequestMarshaller();
var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.Instance;
return Invoke<InitiateMultipartUploadRequest,InitiateMultipartUploadResponse>(request, marshaller, unmarshaller);
}
internal static void CreateMultiPartS3Blob(AmazonS3Client client, string key, S3CopyMemoryStream stream)
{
if (stream.InitiatingPart) {
InitiateMultipartUploadRequest initiateMultipartUploadRequest =
new InitiateMultipartUploadRequest()
.WithBucketName("static.getbrickpile.com")
.WithCannedACL(S3CannedACL.PublicRead)
.WithKey(key);
InitiateMultipartUploadResponse initiateResponse =
client.InitiateMultipartUpload(initiateMultipartUploadRequest);
stream.UploadPartId = initiateResponse.UploadId;
}
stream.Position = 0;
UploadPartRequest uploadPartRequest =
new UploadPartRequest()
.WithBucketName("static.getbrickpile.com")
.WithKey(key)
.WithPartNumber(stream.WriteCount)
.WithPartSize(stream.Position)
.WithUploadId(stream.UploadPartId)
.WithInputStream(stream) as UploadPartRequest;
UploadPartResponse response = client.UploadPart(uploadPartRequest);
PartETag etag = new PartETag(response.PartNumber, response.ETag);
stream.PartETagCollection.Add(etag);
if (stream.EndOfPart) {
CompleteMultipartUploadRequest completeMultipartUploadRequest =
new CompleteMultipartUploadRequest()
.WithBucketName("static.getbrickpile.com")
.WithKey(key)
.WithPartETags(stream.PartETagCollection)
.WithUploadId(stream.UploadPartId);
CompleteMultipartUploadResponse completeMultipartUploadResponse = client.CompleteMultipartUpload(completeMultipartUploadRequest);
string loc = completeMultipartUploadResponse.Location;
}
}
private InitiateMultipartUploadRequest ConstructInitiateMultipartUploadRequest()
{
var initRequest = new InitiateMultipartUploadRequest()
{
BucketName = this._fileTransporterRequest.BucketName,
Key = this._fileTransporterRequest.Key,
CannedACL = this._fileTransporterRequest.CannedACL,
ContentType = determineContentType(),
StorageClass = this._fileTransporterRequest.StorageClass,
ServerSideEncryptionMethod = this._fileTransporterRequest.ServerSideEncryptionMethod,
ServerSideEncryptionCustomerMethod = this._fileTransporterRequest.ServerSideEncryptionCustomerMethod,
ServerSideEncryptionCustomerProvidedKey = this._fileTransporterRequest.ServerSideEncryptionCustomerProvidedKey,
ServerSideEncryptionCustomerProvidedKeyMD5 = this._fileTransporterRequest.ServerSideEncryptionCustomerProvidedKeyMD5
};
initRequest.BeforeRequestEvent += this.RequestEventHandler;
if (this._fileTransporterRequest.Metadata != null && this._fileTransporterRequest.Metadata.Count > 0)
initRequest.Metadata = this._fileTransporterRequest.Metadata;
if (this._fileTransporterRequest.Headers != null && this._fileTransporterRequest.Headers.Count > 0)
initRequest.Headers = this._fileTransporterRequest.Headers;
return initRequest;
}
/// <summary>
/// Initiates the asynchronous execution of the InitiateMultipartUpload operation.
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the InitiateMultipartUpload operation.</param>
/// <param name="cancellationToken">
/// A cancellation token that can be used by other objects or threads to receive notice of cancellation.
/// </param>
/// <returns>The task object representing the asynchronous operation.</returns>
public Task<InitiateMultipartUploadResponse> InitiateMultipartUploadAsync(InitiateMultipartUploadRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken))
{
var marshaller = new InitiateMultipartUploadRequestMarshaller();
var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.Instance;
return InvokeAsync<InitiateMultipartUploadRequest,InitiateMultipartUploadResponse>(request, marshaller,
unmarshaller, cancellationToken);
}
IAsyncResult invokeInitiateMultipartUpload(InitiateMultipartUploadRequest initiateMultipartUploadRequest, AsyncCallback callback, object state, bool synchronized)
{
var marshaller = new InitiateMultipartUploadRequestMarshaller();
var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.GetInstance();
var result = Invoke < IRequest, InitiateMultipartUploadRequest>(initiateMultipartUploadRequest, callback, state, synchronized, marshaller, unmarshaller, signer);
return result;
}
/// <summary>
/// Send a InitiateMultipartUploadRequest request and return the response.
/// </summary>
/// <param name="existingBucketName"></param>
/// <param name="keyName"></param>
/// <returns></returns>
public async Task<InitiateMultipartUploadResponse> InitiateMultipartUploadAsync(string existingBucketName, string keyName, CancellationToken token)
{
token.ThrowIfCancellationRequested();
_log.Debug("Called InitiateMultipartUploadAsync with parameter keyName = \"" + keyName + "\".");
try
{
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = existingBucketName,
Key = keyName
};
InitiateMultipartUploadResponse initResponse = await s3Client.InitiateMultipartUploadAsync(initiateRequest, token).ConfigureAwait(false);
return initResponse;
}
catch (Exception e)
{
string messagePart = " with parameter keyName = \"" + keyName + "\"";
this.LogAmazonException(messagePart, e);
throw;
}
}
/// <summary>
/// <para>Initiates a multipart upload and returns an upload ID.</para>
/// </summary>
/// <remarks>
/// <para>
/// After you initiate a multipart upload and upload one or more parts, you must either complete or abort
/// the multipart upload in order to stop getting charged for storage of the uploaded parts. Once you
/// complete or abort the multipart upload, Amazon S3 will release the stored parts and stop charging you
/// for their storage.
/// </para>
/// </remarks>
/// <param name="initiateMultipartUploadRequest">Container for the necessary parameters to execute the InitiateMultipartUpload service method on
/// AmazonS3.</param>
///
/// <returns>The response from the InitiateMultipartUpload service method, as returned by AmazonS3.</returns>
///
public InitiateMultipartUploadResponse InitiateMultipartUpload(InitiateMultipartUploadRequest initiateMultipartUploadRequest)
{
IAsyncResult asyncResult = invokeInitiateMultipartUpload(initiateMultipartUploadRequest, null, null, true);
return EndInitiateMultipartUpload(asyncResult);
}
/// <summary>
/// Initiates the asynchronous execution of the InitiateMultipartUpload operation.
/// <seealso cref="Amazon.S3.IAmazonS3.InitiateMultipartUpload"/>
/// </summary>
/// <remarks>
/// <para>
/// After you initiate a multipart upload and upload one or more parts, you must either complete or abort
/// the multipart upload in order to stop getting charged for storage of the uploaded parts. Once you
/// complete or abort the multipart upload, Amazon S3 will release the stored parts and stop charging you
/// for their storage.
/// </para>
/// </remarks>
/// <param name="initiateMultipartUploadRequest">Container for the necessary parameters to execute the InitiateMultipartUpload operation on
/// AmazonS3.</param>
/// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param>
/// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
///
/// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking
/// EndInitiateMultipartUpload operation.</returns>
public IAsyncResult BeginInitiateMultipartUpload(InitiateMultipartUploadRequest initiateMultipartUploadRequest, AsyncCallback callback, object state)
{
return invokeInitiateMultipartUpload(initiateMultipartUploadRequest, callback, state, false);
}
/// <summary>
/// Runs the multipart upload.
/// </summary>
public override void Execute()
{
int timeout = this._config.DefaultTimeout;
if (this._fileTransporterRequest.Timeout != 0)
timeout = this._fileTransporterRequest.Timeout;
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
.WithBucketName(this._fileTransporterRequest.BucketName)
.WithKey(this._fileTransporterRequest.Key)
.WithCannedACL(this._fileTransporterRequest.CannedACL)
.WithContentType(determineContentType())
.WithStorageClass(this._fileTransporterRequest.StorageClass)
.WithBeforeRequestHandler(RequestEventHandler) as InitiateMultipartUploadRequest;
if (this._fileTransporterRequest.metadata != null && this._fileTransporterRequest.metadata.Count > 0)
initRequest.WithMetaData(this._fileTransporterRequest.metadata);
if (this._fileTransporterRequest.Headers != null && this._fileTransporterRequest.Headers.Count > 0)
initRequest.AddHeaders(this._fileTransporterRequest.Headers);
InitiateMultipartUploadResponse initResponse = this._s3Client.InitiateMultipartUpload(initRequest);
this._logger.DebugFormat("Initiated upload: {0}", initResponse.UploadId);
try
{
this._logger.DebugFormat("Queue up the UploadPartRequests to be executed");
long filePosition = 0;
for (int i = 1; filePosition < this._contentLength; i++)
{
UploadPartRequest uploadRequest = new UploadPartRequest()
.WithBucketName(this._fileTransporterRequest.BucketName)
.WithKey(this._fileTransporterRequest.Key)
.WithUploadId(initResponse.UploadId)
.WithTimeout(timeout)
.WithPartNumber(i)
.WithPartSize(this._partSize)
.WithSubscriber(new EventHandler<UploadPartProgressArgs>(this.uploadPartProgressEventCallback))
.WithBeforeRequestHandler(RequestEventHandler) as UploadPartRequest;
if (this._fileTransporterRequest.IsSetFilePath())
{
uploadRequest
.WithFilePosition(filePosition)
.WithFilePath(this._fileTransporterRequest.FilePath);
}
else
{
uploadRequest.InputStream = this._fileTransporterRequest.InputStream;
}
this._partsToUpload.Enqueue(uploadRequest);
filePosition += this._partSize;
}
this._totalNumberOfParts = this._partsToUpload.Count;
this._logger.DebugFormat("Starting threads to execute the {0} UploadPartRequests in the queue", this._totalNumberOfParts);
startInvokerPool();
this._logger.DebugFormat("Waiting for threads to complete. ({0})", initResponse.UploadId);
waitTillAllThreadsComplete();
this._logger.DebugFormat("Beginning completing multipart. ({0})", initResponse.UploadId);
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
.WithBucketName(this._fileTransporterRequest.BucketName)
.WithKey(this._fileTransporterRequest.Key)
.WithUploadId(initResponse.UploadId)
.WithPartETags(this._uploadResponses)
.WithBeforeRequestHandler(RequestEventHandler) as CompleteMultipartUploadRequest;
this._s3Client.CompleteMultipartUpload(compRequest);
this._logger.DebugFormat("Done completing multipart. ({0})", initResponse.UploadId);
}
catch (Exception e)
{
this._logger.Error(string.Format("Exception while uploading. ({0})", initResponse.UploadId), e);
shutdown(initResponse.UploadId);
throw;
}
finally
{
if (this._fileTransporterRequest.InputStream != null && !this._fileTransporterRequest.IsSetFilePath())
{
this._fileTransporterRequest.InputStream.Close();
}
}
}
/// <summary>
/// Initiates the asynchronous execution of the InitiateMultipartUpload operation.
/// This API is supported only when AWSConfigs.HttpClient is set to AWSConfigs.HttpClientOption.UnityWebRequest, the default value for this configuration option is AWSConfigs.HttpClientOption.UnityWWW
/// </summary>
///
/// <param name="request">Container for the necessary parameters to execute the InitiateMultipartUpload operation on AmazonS3Client.</param>
/// <param name="callback">An Action delegate that is invoked when the operation completes.</param>
/// <param name="options">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.</param>
public void InitiateMultipartUploadAsync(InitiateMultipartUploadRequest request, AmazonServiceCallback<InitiateMultipartUploadRequest, InitiateMultipartUploadResponse> callback, AsyncOptions options = null)
{
if (AWSConfigs.HttpClient == AWSConfigs.HttpClientOption.UnityWWW)
{
throw new InvalidOperationException("InitiateMultipartUpload is only allowed with AWSConfigs.HttpClientOption.UnityWebRequest API option");
}
options = options == null?new AsyncOptions():options;
var marshaller = new InitiateMultipartUploadRequestMarshaller();
var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.Instance;
Action<AmazonWebServiceRequest, AmazonWebServiceResponse, Exception, AsyncOptions> callbackHelper = null;
if(callback !=null )
callbackHelper = (AmazonWebServiceRequest req, AmazonWebServiceResponse res, Exception ex, AsyncOptions ao) => {
AmazonServiceResult<InitiateMultipartUploadRequest,InitiateMultipartUploadResponse> responseObject
= new AmazonServiceResult<InitiateMultipartUploadRequest,InitiateMultipartUploadResponse>((InitiateMultipartUploadRequest)req, (InitiateMultipartUploadResponse)res, ex , ao.State);
callback(responseObject);
};
BeginInvoke<InitiateMultipartUploadRequest>(request, marshaller, unmarshaller, options, callbackHelper);
}
/// <summary>
/// Initiates a multipart upload and returns an upload ID.
///
///
/// <para>
/// <b>Note:</b> After you initiate multipart upload and upload one or more parts, you
/// must either complete or abort multipart upload in order to stop getting charged for
/// storage of the uploaded parts. Only after you either complete or abort multipart upload,
/// Amazon S3 frees up the parts storage and stops charging you for the parts storage.
/// </para>
/// This API is supported only when AWSConfigs.HttpClient is set to AWSConfigs.HttpClientOption.UnityWebRequest, the default value of this configuration option is AWSConfigs.HttpClientOption.UnityWWW
/// </summary>
/// <param name="bucketName">A property of InitiateMultipartUploadRequest used to execute the InitiateMultipartUpload service method.</param>
/// <param name="key">A property of InitiateMultipartUploadRequest used to execute the InitiateMultipartUpload service method.</param>
/// <param name="callback">An Action delegate that is invoked when the operation completes.</param>
/// <param name="options">
/// A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback
/// procedure using the AsyncState property.
/// </param>
///
/// <returns>The response from the InitiateMultipartUpload service method, as returned by S3.</returns>
public void InitiateMultipartUploadAsync(string bucketName, string key, AmazonServiceCallback<InitiateMultipartUploadRequest, InitiateMultipartUploadResponse> callback, AsyncOptions options = null)
{
var request = new InitiateMultipartUploadRequest();
request.BucketName = bucketName;
request.Key = key;
InitiateMultipartUploadAsync(request, callback, options);
}
static void Main()
{
_accessKeyId = Utilities.AwsAccessKey;
_secretAccessKey = Utilities.AwsSecretKey;
AmazonS3 s3Client = new AmazonS3Client(_accessKeyId, _secretAccessKey);
ListMultipartUploadsRequest allMultipartUploadsRequest = new ListMultipartUploadsRequest().WithBucketName(ExistingBucketName);
ListMultipartUploadsResponse mpUploadsResponse = s3Client.ListMultipartUploads(allMultipartUploadsRequest);
var objects = new List<Object>();
foreach (MultipartUpload multipartUpload in mpUploadsResponse.MultipartUploads)
{
bool isObjectdFound = false;
foreach (Object o in objects)
{
if (o.UploadId == multipartUpload.UploadId)
{
o.Parts.Add(new Part { PartId = o.Parts.Count, Etag = "" });
isObjectdFound = true;
}
}
if (!isObjectdFound)
{
objects.Add(new Object { Parts = new List<Part> { new Part() { Etag = "", PartId = 1 } }, UploadId = multipartUpload.UploadId });
}
}
var result = JsonConvert.SerializeObject(objects);
var objs = JsonConvert.DeserializeObject<List<Object>>(result);
//return;
// List to store upload part responses.
var uploadResponses = new List<UploadPartResponse>();
byte[] bytes;
long contentLength = 0;
using (var fileStream = new FileStream(FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
contentLength = fileStream.Length;
bytes = new byte[contentLength];
fileStream.Read(bytes, 0, Convert.ToInt32(contentLength));
}
// 1. Initialize.
InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest().WithBucketName(ExistingBucketName).WithKey(KeyName);
InitiateMultipartUploadResponse initResponse = s3Client.InitiateMultipartUpload(initiateRequest);
try
{
// 2. Upload Parts.
long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB
long filePosition = 0;
for (int i = 1; filePosition < contentLength; i++)
{
byte[] bytesToStream;
if (filePosition + partSize < contentLength)
{
bytesToStream = new byte[partSize];
Array.Copy(bytes, filePosition, bytesToStream, 0, partSize);
}
else
{
bytesToStream = new byte[contentLength - filePosition];
Array.Copy(bytes, filePosition, bytesToStream, 0, contentLength - filePosition);
}
Stream stream = new MemoryStream(bytesToStream);
// Create request to upload a part.
UploadPartRequest uploadRequest = new UploadPartRequest()
.WithBucketName(ExistingBucketName)
.WithKey(KeyName)
.WithUploadId(initResponse.UploadId)
.WithPartNumber(i)
.WithPartSize(partSize)
.WithFilePosition(filePosition)
.WithTimeout(1000000000)
.WithMD5Digest(Convert.ToBase64String(MD5.Create().ComputeHash(bytesToStream)));
uploadRequest.WithInputStream(stream);
// Upload part and add response to our list.
uploadResponses.Add(s3Client.UploadPart(uploadRequest));
filePosition += partSize;
}
// Step 3: complete.
CompleteMultipartUploadRequest completeRequest =
new CompleteMultipartUploadRequest()
.WithBucketName(ExistingBucketName)
.WithKey(KeyName)
.WithUploadId(initResponse.UploadId)
.WithPartETags(uploadResponses);
CompleteMultipartUploadResponse completeUploadResponse = s3Client.CompleteMultipartUpload(completeRequest);
Console.WriteLine(completeUploadResponse.ETag);
}
catch (Exception exception)
{
Console.WriteLine("Exception occurred: {0}", exception.Message);
s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest()
.WithBucketName(ExistingBucketName)
.WithKey(KeyName)
.WithUploadId(initResponse.UploadId));
}
}
public void ObjectSamples()
{
{
#region ListObjects Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// List all objects
ListObjectsRequest listRequest = new ListObjectsRequest
{
BucketName = "SampleBucket",
};
ListObjectsResponse listResponse;
do
{
// Get a list of objects
listResponse = client.ListObjects(listRequest);
foreach (S3Object obj in listResponse.S3Objects)
{
Console.WriteLine("Object - " + obj.Key);
Console.WriteLine(" Size - " + obj.Size);
Console.WriteLine(" LastModified - " + obj.LastModified);
Console.WriteLine(" Storage class - " + obj.StorageClass);
}
// Set the marker property
listRequest.Marker = listResponse.NextMarker;
} while (listResponse.IsTruncated);
#endregion
}
{
#region GetObject Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a GetObject request
GetObjectRequest request = new GetObjectRequest
{
BucketName = "SampleBucket",
Key = "Item1"
};
// Issue request and remember to dispose of the response
using (GetObjectResponse response = client.GetObject(request))
{
using (StreamReader reader = new StreamReader(response.ResponseStream))
{
string contents = reader.ReadToEnd();
Console.WriteLine("Object - " + response.Key);
Console.WriteLine(" Version Id - " + response.VersionId);
Console.WriteLine(" Contents - " + contents);
}
}
#endregion
}
{
#region GetObjectMetadata Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a GetObjectMetadata request
GetObjectMetadataRequest request = new GetObjectMetadataRequest
{
BucketName = "SampleBucket",
Key = "Item1"
};
// Issue request and view the response
GetObjectMetadataResponse response = client.GetObjectMetadata(request);
Console.WriteLine("Content Length - " + response.ContentLength);
Console.WriteLine("Content Type - " + response.Headers.ContentType);
if (response.Expiration != null)
{
Console.WriteLine("Expiration Date - " + response.Expiration.ExpiryDate);
Console.WriteLine("Expiration Rule Id - " + response.Expiration.RuleId);
}
#endregion
}
{
#region PutObject Sample 1
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a PutObject request
PutObjectRequest request = new PutObjectRequest
{
BucketName = "SampleBucket",
Key = "Item1",
ContentBody = "This is sample content..."
};
// Put object
PutObjectResponse response = client.PutObject(request);
#endregion
}
{
#region PutObject Sample 2
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a PutObject request
PutObjectRequest request = new PutObjectRequest
{
BucketName = "SampleBucket",
Key = "Item1",
FilePath = "contents.txt"
};
// Put object
PutObjectResponse response = client.PutObject(request);
#endregion
}
{
#region PutObject Sample 3
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a PutObject request
PutObjectRequest request = new PutObjectRequest
{
BucketName = "SampleBucket",
Key = "Item1",
};
using (FileStream stream = new FileStream("contents.txt", FileMode.Open))
{
request.InputStream = stream;
// Put object
PutObjectResponse response = client.PutObject(request);
}
#endregion
}
{
#region DeleteObject Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a DeleteObject request
DeleteObjectRequest request = new DeleteObjectRequest
{
BucketName = "SampleBucket",
Key = "Item1"
};
// Issue request
client.DeleteObject(request);
#endregion
}
{
#region DeleteObjects Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a DeleteObject request
DeleteObjectsRequest request = new DeleteObjectsRequest
{
BucketName = "SampleBucket",
Objects = new List<KeyVersion>
{
new KeyVersion() {Key = "Item1"},
// Versioned item
new KeyVersion() { Key = "Item2", VersionId = "Rej8CiBxcZKVK81cLr39j27Y5FVXghDK", },
// Item in subdirectory
new KeyVersion() { Key = "Logs/error.txt"}
}
};
try
{
// Issue request
DeleteObjectsResponse response = client.DeleteObjects(request);
}
catch (DeleteObjectsException doe)
{
// Catch error and list error details
DeleteObjectsResponse errorResponse = doe.Response;
foreach (DeletedObject deletedObject in errorResponse.DeletedObjects)
{
Console.WriteLine("Deleted item " + deletedObject.Key);
}
foreach (DeleteError deleteError in errorResponse.DeleteErrors)
{
Console.WriteLine("Error deleting item " + deleteError.Key);
Console.WriteLine(" Code - " + deleteError.Code);
Console.WriteLine(" Message - " + deleteError.Message);
}
}
#endregion
}
{
#region CopyObject Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a CopyObject request
CopyObjectRequest request = new CopyObjectRequest
{
SourceBucket = "SampleBucket",
SourceKey = "Item1",
DestinationBucket = "AnotherBucket",
DestinationKey = "Copy1",
CannedACL = S3CannedACL.PublicRead
};
// Issue request
client.CopyObject(request);
#endregion
}
{
#region CopyObject Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Create a CopyObject request
CopyObjectRequest request = new CopyObjectRequest
{
SourceBucket = "SampleBucket",
SourceKey = "Item1",
DestinationBucket = "AnotherBucket",
DestinationKey = "Copy1",
CannedACL = S3CannedACL.PublicRead
};
// Issue request
client.CopyObject(request);
#endregion
}
{
#region ListVersions Sample
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Turn versioning on for a bucket
client.PutBucketVersioning(new PutBucketVersioningRequest
{
BucketName = "SampleBucket",
VersioningConfig = new S3BucketVersioningConfig { Status = "Enable" }
});
// Populate bucket with multiple items, each with multiple versions
PopulateBucket(client, "SampleBucket");
// Get versions
ListVersionsRequest request = new ListVersionsRequest
{
BucketName = "SampleBucket"
};
// Make paged ListVersions calls
ListVersionsResponse response;
do
{
response = client.ListVersions(request);
// View information about versions
foreach (var version in response.Versions)
{
Console.WriteLine("Key = {0}, Version = {1}, IsLatest = {2}, LastModified = {3}, Size = {4}",
version.Key,
version.VersionId,
version.IsLatest,
version.LastModified,
version.Size);
}
request.KeyMarker = response.NextKeyMarker;
request.VersionIdMarker = response.NextVersionIdMarker;
} while (response.IsTruncated);
#endregion
}
{
#region Multipart Upload Sample
int MB = (int)Math.Pow(2, 20);
// Create a client
AmazonS3Client client = new AmazonS3Client();
// Define input stream
Stream inputStream = Create13MBDataStream();
// Initiate multipart upload
InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest
{
BucketName = "SampleBucket",
Key = "Item1"
};
InitiateMultipartUploadResponse initResponse = client.InitiateMultipartUpload(initRequest);
// Upload part 1
UploadPartRequest uploadRequest = new UploadPartRequest
{
BucketName = "SampleBucket",
Key = "Item1",
UploadId = initResponse.UploadId,
PartNumber = 1,
PartSize = 5 * MB,
InputStream = inputStream
};
UploadPartResponse up1Response = client.UploadPart(uploadRequest);
// Upload part 2
uploadRequest = new UploadPartRequest
{
BucketName = "SampleBucket",
Key = "Item1",
UploadId = initResponse.UploadId,
PartNumber = 2,
PartSize = 5 * MB,
InputStream = inputStream
};
UploadPartResponse up2Response = client.UploadPart(uploadRequest);
// Upload part 3
uploadRequest = new UploadPartRequest
{
BucketName = "SampleBucket",
Key = "Item1",
UploadId = initResponse.UploadId,
PartNumber = 3,
InputStream = inputStream
};
UploadPartResponse up3Response = client.UploadPart(uploadRequest);
// List parts for current upload
ListPartsRequest listPartRequest = new ListPartsRequest
{
BucketName = "SampleBucket",
Key = "Item1",
UploadId = initResponse.UploadId
};
ListPartsResponse listPartResponse = client.ListParts(listPartRequest);
Debug.Assert(listPartResponse.Parts.Count == 3);
// Complete the multipart upload
CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest
{
BucketName = "SampleBucket",
Key = "Item1",
UploadId = initResponse.UploadId,
PartETags = new List<PartETag>
{
new PartETag { ETag = up1Response.ETag, PartNumber = 1 },
new PartETag { ETag = up2Response.ETag, PartNumber = 2 },
new PartETag { ETag = up3Response.ETag, PartNumber = 3 }
}
};
CompleteMultipartUploadResponse compResponse = client.CompleteMultipartUpload(compRequest);
#endregion
}
}
