Initiates a multipart upload and returns an upload ID.
private void Addobject(string filePath, string bucketKey) { try { var attribute = File.GetAttributes(filePath); string relativePath = filePath.Replace(Utilities.Path + "\\", "").Replace("\\", "/"); if (attribute != FileAttributes.Directory && attribute != (FileAttributes.Archive | FileAttributes.Hidden)) { if (File.Exists(filePath) && !Utilities.IsFileUsedbyAnotherProcess(filePath)) { bool IsItSharedFile = false; string Username = string.Empty; if (relativePath.ToLower().IndexOf("shared") == 0) { IsItSharedFile = true; var folders = relativePath.Split('/'); if (folders.Length > 1) { Username = folders[1]; relativePath = ""; for (int i = 2; i < folders.Length; i++) relativePath += folders[i] + "/"; } else { relativePath = ""; for (int i = 1; i < folders.Length; i++) relativePath += folders[i] + "/"; } relativePath = relativePath.Substring(0, relativePath.Length - 1); } var appUpdateInfo = new AppUpdateInfo { Key = relativePath.Replace("\\", "/"), LastModifiedTime = new FileInfo(filePath).LastWriteTime, Status = UpdateStatus.Update }; _processingFiles.Add(filePath); try { var uploadResponses = new List<UploadPartResponse>(); byte[] bytes; long contentLength; using (var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { contentLength = fileStream.Length; bytes = new byte[contentLength]; fileStream.Read(bytes, 0, Convert.ToInt32(contentLength)); } if (contentLength > 0) { var lastUploadedPartdetail = GetLastUploadedPartResponse(relativePath, bucketKey, uploadResponses); int alreadyUploadedParts = lastUploadedPartdetail.LastPartNumber; string uploadId; if (string.IsNullOrEmpty(lastUploadedPartdetail.UploadId)) { InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest().WithBucketName(bucketKey).WithKey( relativePath); InitiateMultipartUploadResponse initResponse = _amazons3.InitiateMultipartUpload(initiateRequest); uploadId = initResponse.UploadId; } else uploadId = lastUploadedPartdetail.UploadId; try { long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB long filePosition = partSize * alreadyUploadedParts; for (int i = alreadyUploadedParts + 1; filePosition < contentLength; i++) { // Before upload the next set of upload part, need to check the last modified because user might modify it in the mean time if (File.Exists(filePath) && appUpdateInfo.LastModifiedTime == new FileInfo(filePath).LastWriteTime) { byte[] bytesToStream; if (filePosition + partSize < contentLength) { bytesToStream = new byte[partSize]; Array.Copy(bytes, filePosition, bytesToStream, 0, partSize); } else { bytesToStream = new byte[contentLength - filePosition]; Array.Copy(bytes, filePosition, bytesToStream, 0, contentLength - filePosition); } Stream stream = new MemoryStream(bytesToStream); UploadPartRequest uploadRequest = new UploadPartRequest() .WithBucketName(bucketKey) .WithKey(relativePath) .WithUploadId(uploadId) .WithPartNumber(i) .WithPartSize(partSize) .WithFilePosition(filePosition) .WithTimeout(1000000000); uploadRequest.WithInputStream(stream); // Upload part and add response to our list. var response = _amazons3.UploadPart(uploadRequest); WriteResponseToFile(relativePath, bucketKey, uploadId, appUpdateInfo.LastModifiedTime, response); uploadResponses.Add(response); filePosition += partSize; ModifySyncStatus("Uploaded", contentLength <= filePosition ? contentLength : filePosition, contentLength, relativePath); } else { // need to abort the upload process _processingFiles.Remove(filePath); RemoveConfig(relativePath, lastUploadedPartdetail.BucketKey); _amazons3.AbortMultipartUpload(new AbortMultipartUploadRequest() .WithBucketName(bucketKey) .WithKey(relativePath) .WithUploadId(uploadId)); return; } } CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest() .WithBucketName(bucketKey) .WithKey(relativePath) .WithUploadId(uploadId) .WithPartETags(uploadResponses); CompleteMultipartUploadResponse completeUploadResponse = _amazons3.CompleteMultipartUpload(completeRequest); RemoveConfig(relativePath, completeUploadResponse.BucketName); //_service.AddObject(filePath, Utilities.MyConfig.BucketKey, relativePath); SetAcltoObject(relativePath); if (IsItSharedFile) { ProcessApplicationUpdates(appUpdateInfo, true, Username); } else ProcessApplicationUpdates(appUpdateInfo, false, string.Empty); _processingFiles.Remove(filePath); UploadContentForSearch(filePath, relativePath); } catch (Exception) { _processingFiles.Remove(filePath); _amazons3.AbortMultipartUpload(new AbortMultipartUploadRequest() .WithBucketName(bucketKey) .WithKey(relativePath) .WithUploadId(uploadId)); RemoveConfig(relativePath, Utilities.MyConfig.BucketKey); if (!_fileQueue.ContainsKey(filePath)) _fileQueue.Add(filePath, new FileQueue { Type = WatcherChangeTypes.Created, Name = Path.GetFileName(filePath) }); else _fileQueue.Add(filePath, new FileQueue { Type = WatcherChangeTypes.Created, Name = Path.GetFileName(filePath) }); } } } catch (Exception) { return; } } } } catch (Exception) { return; } }
public override string InitiateMultiPartUpload(string keyName, NameValueCollection metaData) { InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(); request.CannedACL = S3CannedACL.PublicRead; request.BucketName = BucketName; request.Key = keyName; foreach (string key in metaData) { request.Metadata.Add(key, metaData[key]); } InitiateMultipartUploadResponse response; try { response = AmazonS3Client.InitiateMultipartUpload(request); } catch (AmazonS3Exception e) { throw new CloudUploadProviderException("Exception thrown for initiate multi part upload operation", e); } return response.UploadId; }
/// <summary> /// Sample code to contrast uploading a file using Amazon S3's Multi-Part Upload API /// </summary> /// <param name="s3Client"></param> /// <param name="bucketName"></param> /// <param name="fileName"></param> static void UploadUsingMultiPartAPI(IAmazonS3 s3Client, string bucketName, string fileName) { const string objectKey = "multipart/myobject"; // tell S3 we're going to upload an object in multiple parts and receive an upload ID // in return var initializeUploadRequest = new InitiateMultipartUploadRequest { BucketName = bucketName, Key = objectKey }; var initializeUploadResponse = s3Client.InitiateMultipartUpload(initializeUploadRequest); // this ID must accompany all parts and the final 'completed' call var uploadID = initializeUploadResponse.UploadId; // Send the file (synchronously) using 4*5MB parts - note we pass the upload id // with each call. For each part we need to log the returned etag value to pass // to the completion call var partETags = new List<PartETag>(); var partSize = 5 * ONE_MEG; // this is the minimum part size allowed for (var partNumber = 0; partNumber < 4; partNumber++) { // part numbers must be between 1 and 1000 var logicalPartNumber = partNumber + 1; var uploadPartRequest = new UploadPartRequest { BucketName = bucketName, Key = objectKey, UploadId = uploadID, PartNumber = logicalPartNumber, PartSize = partSize, FilePosition = partNumber * partSize, FilePath = fileName }; var partUploadResponse = s3Client.UploadPart(uploadPartRequest); partETags.Add(new PartETag { PartNumber = logicalPartNumber, ETag = partUploadResponse.ETag }); } var completeUploadRequest = new CompleteMultipartUploadRequest { BucketName = bucketName, Key = objectKey, UploadId = uploadID, PartETags = partETags }; s3Client.CompleteMultipartUpload(completeUploadRequest); }
public AmazonMultiUpload(AmazonS3Client client, string bucket, string key, Metadata metadata, CancellationToken ct) { _client = client; _bucket = bucket; _key = key; _blocks = new List<Task<UploadPartResponse>>(); _ct = ct; // Fire up the multipart upload request... var req = new InitiateMultipartUploadRequest { BucketName = _bucket, Key = key }; if (metadata != null) { foreach (var m in metadata) { req.Metadata.Add("x-amz-meta-" + m.Key, m.Value); } } _uploadId = _client.InitiateMultipartUploadAsync(req, _ct).ContinueWith(r => r.Result.UploadId); }
public async Task <string> MultipartUploadStartAsync(string bucket, string key, S3CannedACL s3CannedAcl = null, CancellationToken cancellationToken = default) { this.Logger.LogDebug($"[{nameof(this.MultipartUploadStartAsync)}]"); this.Logger.LogTrace(JsonConvert.SerializeObject(new { bucket, key, s3CannedAcl })); if (string.IsNullOrWhiteSpace(bucket)) { throw new ArgumentNullException(nameof(bucket)); } if (string.IsNullOrWhiteSpace(key)) { throw new ArgumentNullException(nameof(key)); } if (s3CannedAcl == null) { s3CannedAcl = S3CannedACL.BucketOwnerFullControl; } var request = new Amazon.S3.Model.InitiateMultipartUploadRequest { BucketName = bucket, CannedACL = s3CannedAcl, Key = key, }; this.Logger.LogTrace(JsonConvert.SerializeObject(value: request)); var response = await this.Repository.InitiateMultipartUploadAsync(request : request, cancellationToken : cancellationToken == default?this.CancellationToken.Token : cancellationToken); this.Logger.LogTrace(JsonConvert.SerializeObject(value: response)); return(response.UploadId); }
public static void mpuSerial(String fPath) { System.Console.WriteLine("\nhello,MPU!!"); NameValueCollection appConfig = ConfigurationManager.AppSettings; AmazonS3 s3Client = AWSClientFactory.CreateAmazonS3Client(appConfig["AWSAccessKey"], appConfig["AWSSecretKey"]); String bucketName = "region"; String objectNameA = "photos/2006/January/sample.jpg"; String objectNameB = "photos/2006/February/sample.jpg"; String objectNameC = "photos/2006/March/sample.jpg"; String objectNameD = "videos/2006/March/sample.wmv"; String objectNameE = "sample.jpg"; //PutBucket //PutBucketResponse response = s3Client.PutBucket(new PutBucketRequest().WithBucketName(bucketName)); //******************************************************************************************************************************** //Initial with CannedACL InitiateMultipartUploadRequest MPUCannedACL = new InitiateMultipartUploadRequest(); MPUCannedACL.WithBucketName(bucketName); MPUCannedACL.WithKey(objectNameA); MPUCannedACL.WithCannedACL(S3CannedACL.PublicRead); MPUCannedACL.WithContentType("Content-Type: text/html"); MPUCannedACL.WithStorageClass(S3StorageClass.Standard); //MPUCannedACL.WithServerSideEncryptionMethod(ServerSideEncryptionMethod.AES256); InitiateMultipartUploadResponse MPUCannedACLResult = s3Client.InitiateMultipartUpload(MPUCannedACL); String MPUCannedACLUID = MPUCannedACLResult.UploadId; System.Console.WriteLine("\nInitial MPUCanned uploadID:{0}", MPUCannedACLUID); AbortMultipartUploadResponse abortCannedACLResult = s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameA).WithUploadId(MPUCannedACLUID)); System.Console.WriteLine("\nAbort MPUCanned uploadID:{0}", MPUCannedACLUID); //Initial with GrantACL InitiateMultipartUploadRequest MPUACL = new InitiateMultipartUploadRequest(); MPUACL.WithBucketName(bucketName); MPUACL.WithKey(objectNameA); MPUACL.WithGrants(new S3Grant().WithGrantee(new S3Grantee().WithCanonicalUser("canonicalidhrchu", "hrchu")).WithPermission(S3Permission.FULL_CONTROL)); MPUACL.WithGrants(new S3Grant().WithGrantee(new S3Grantee().WithCanonicalUser("canonicalidannyren", "annyren")).WithPermission(S3Permission.WRITE_ACP)); InitiateMultipartUploadResponse MPUACLResult = s3Client.InitiateMultipartUpload(MPUACL); String MPUACLUID = MPUACLResult.UploadId; System.Console.WriteLine("\nInitial MPU Grants uploadID:{0}", MPUACLUID); AbortMultipartUploadResponse abortACLResult = s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameA).WithUploadId(MPUACLUID)); System.Console.WriteLine("\nAbort MPU Grants uploadID:{0}", MPUACLUID); //******************************************************************************************************************************** //Initial MPU * 5 InitiateMultipartUploadResponse InitialResultA = s3Client.InitiateMultipartUpload(new InitiateMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameA)); InitiateMultipartUploadResponse InitialResultB = s3Client.InitiateMultipartUpload(new InitiateMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameB)); InitiateMultipartUploadResponse InitialResultC = s3Client.InitiateMultipartUpload(new InitiateMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameC)); InitiateMultipartUploadResponse InitialResultD = s3Client.InitiateMultipartUpload(new InitiateMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameD)); InitiateMultipartUploadResponse InitialResultE = s3Client.InitiateMultipartUpload(new InitiateMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameE)); String objectAUID = InitialResultA.UploadId; String objectBUID = InitialResultB.UploadId; String objectCUID = InitialResultC.UploadId; String objectDUID = InitialResultD.UploadId; String objectEUID = InitialResultE.UploadId; System.Console.WriteLine("\nInitial MPU:{0},uploadID:{1}",objectNameA,objectAUID); System.Console.WriteLine("Initial MPU:{0},uploadID:{1}", objectNameB, objectBUID); System.Console.WriteLine("\nInitial MPU:{0},uploadID:{1}", objectNameC, objectCUID); System.Console.WriteLine("Initial MPU:{0},uploadID:{1}", objectNameD, objectDUID); System.Console.WriteLine("Initial MPU:{0},uploadID:{1}", objectNameE, objectEUID); //List MPUs ListMultipartUploadsResponse listMPUsResult = s3Client.ListMultipartUploads(new ListMultipartUploadsRequest().WithBucketName(bucketName)); System.Console.WriteLine("\nList MPUs with WithMaxUploads Result:\n{0}", listMPUsResult.ResponseXml); //**************List MPUs with MaxUploads=1************************************************************************************* ListMultipartUploadsResponse listMPUsMaxUPsResult = s3Client.ListMultipartUploads(new ListMultipartUploadsRequest().WithBucketName(bucketName).WithMaxUploads(1)); System.Console.WriteLine("\nList MPUs with WithMaxUploads Result:\n{0}", listMPUsMaxUPsResult.ResponseXml); //**************List MPUs with Delimiter /, result:<Key>sample.jpg</Key> only************************************************************************************* ListMultipartUploadsResponse listMPUsDelimiterResult = s3Client.ListMultipartUploads(new ListMultipartUploadsRequest().WithBucketName(bucketName).WithDelimiter("/")); System.Console.WriteLine("\nList MPUs with Delimiter Result:\n{0}", listMPUsDelimiterResult.ResponseXml); //**************List MPUs with Prefix photos/2006/, result:<Prefix>photos/2006/February/</Prefix>, <Prefix>photos/2006/January/</Prefix> and <Prefix>photos/2006/March/</Prefix>***************** ListMultipartUploadsResponse listMPUsPrefixResult = s3Client.ListMultipartUploads(new ListMultipartUploadsRequest().WithBucketName(bucketName).WithPrefix("photos/2006/")); System.Console.WriteLine("\nList MPUs with Prefix Result:\n{0}", listMPUsPrefixResult.ResponseXml); //**************List MPUs with Keymarker objectNameB, result:<Prefix>photos/2006/February/</Prefix>, <Prefix>photos/2006/January/</Prefix> and <Prefix>photos/2006/March/</Prefix>***************** ListMultipartUploadsResponse listMPUsKeymarkerResult = s3Client.ListMultipartUploads(new ListMultipartUploadsRequest().WithBucketName(bucketName).WithKeyMarker(objectNameB)); System.Console.WriteLine("\nList MPUs with Keymarker Result:\n{0}", listMPUsKeymarkerResult.ResponseXml); //**************List MPUs with UploadIDmarker objectNameB, result:<Prefix>photos/2006/February/</Prefix>, <Prefix>photos/2006/January/</Prefix> and <Prefix>photos/2006/March/</Prefix>***************** ListMultipartUploadsResponse listMPUsUpIDmarkerResult = s3Client.ListMultipartUploads(new ListMultipartUploadsRequest().WithBucketName(bucketName).WithUploadIdMarker(objectBUID)); System.Console.WriteLine("\nList MPUs with UpIDmarker Result:\n{0}", listMPUsUpIDmarkerResult.ResponseXml); //Upload Part * 2 UploadPartResponse part1Result = s3Client.UploadPart(new UploadPartRequest().WithBucketName(bucketName).WithKey(objectNameA).WithPartNumber(1).WithUploadId(objectAUID).WithFilePath(fPath).WithFilePosition(10).WithPartSize(6270544)); UploadPartResponse part2Result = s3Client.UploadPart(new UploadPartRequest().WithBucketName(bucketName).WithKey(objectNameA).WithPartNumber(2).WithUploadId(objectAUID).WithFilePath(fPath).WithFilePosition(10)); String part1Etag = part1Result.ETag; String part2Etag = part2Result.ETag; System.Console.WriteLine("\nUpload Part Result: part 1 requestID:{0} & Etag:{1}", part1Result.RequestId, part1Etag); System.Console.WriteLine("\nUpload Part Result: part 1 requestID:{0} & Etag:{1}", part1Result.RequestId, part2Etag); //List Upload Parts ListPartsResponse listPartsResult = s3Client.ListParts(new ListPartsRequest().WithBucketName(bucketName).WithKey(objectNameA).WithUploadId(objectAUID)); System.Console.WriteLine("\nListParts Result:\n{0}", listPartsResult.ResponseXml); //**************List Upload Parts with MaxParts(1), result:只列出Part 1資訊************************************************************************************* ListPartsResponse listPartsMaxResult = s3Client.ListParts(new ListPartsRequest().WithBucketName(bucketName).WithKey(objectNameA).WithUploadId(objectAUID).WithMaxParts(1)); System.Console.WriteLine("\nListParts With Max Parts Result:\n{0}", listPartsMaxResult.ResponseXml); //**************List Upload Parts with PartNumberMarker("1"), result:只列出Part 2資訊************************************************************************************* ListPartsResponse listPartsKeymarkerResult = s3Client.ListParts(new ListPartsRequest().WithBucketName(bucketName).WithKey(objectNameA).WithUploadId(objectAUID).WithPartNumberMarker("1")); System.Console.WriteLine("\nListParts With Key Marker Result:\n{0}", listPartsKeymarkerResult.ResponseXml); //Complete MPU List<UploadPartResponse> uploadResponses = new List<UploadPartResponse>(); uploadResponses.Add(part1Result); uploadResponses.Add(part2Result); CompleteMultipartUploadResponse completeResult = s3Client.CompleteMultipartUpload(new CompleteMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameA).WithUploadId(objectAUID).WithPartETags(uploadResponses)); System.Console.WriteLine("\nComplete MPUs Result:\n{0}", completeResult.ResponseXml); System.Console.WriteLine("\nComplete MPUs Etag:\n{0}", completeResult.ETag); //Upload Part Copy CopyPartResponse copyPartResult = s3Client.CopyPart(new CopyPartRequest().WithSourceBucket(bucketName).WithDestinationBucket(bucketName) .WithSourceKey(objectNameA).WithDestinationKey(objectNameB).WithPartNumber(1).WithUploadID(objectBUID)); System.Console.WriteLine("\nCopy Part Result: requestID:{0}", copyPartResult.RequestId); //**************Upload Part Copy with ETagToMatch / etagsToNotMatch**************************************************** CopyPartResponse copyPartEtagmatchResult = s3Client.CopyPart(new CopyPartRequest().WithSourceBucket(bucketName).WithDestinationBucket(bucketName) .WithSourceKey(objectNameA).WithDestinationKey(objectNameB).WithPartNumber(1).WithUploadID(objectBUID).WithETagsToMatch(completeResult.ETag)); System.Console.WriteLine("\nCopy Part with ETag Match Result: requestID:{0}", copyPartEtagmatchResult.RequestId); CopyPartResponse copyPartEtagNOTmatchResult = s3Client.CopyPart(new CopyPartRequest().WithSourceBucket(bucketName).WithDestinationBucket(bucketName) .WithSourceKey(objectNameA).WithDestinationKey(objectNameB).WithPartNumber(1).WithUploadID(objectBUID).WithETagsToNotMatch(part1Etag)); System.Console.WriteLine("\nCopy Part with ETag NOT Match Result: requestID:{0}", copyPartEtagNOTmatchResult.RequestId); //**************Upload Part Copy with firstByte-lastByte,反向測試,<5GB檔案 error**************************************************** /* CopyPartResponse copyPartByterangeResult = s3Client.CopyPart(new CopyPartRequest().WithSourceBucket(bucketName).WithDestinationBucket(bucketName) .WithSourceKey(objectNameA).WithDestinationKey(objectNameB).WithPartNumber(1).WithUploadID(objectBUID).WithFirstByte(500).WithLastByte(2000)); System.Console.WriteLine("\nCopy Part with ETagToMatch Result: requestID:{0}", copyPartByterangeResult.RequestId);*/ //*Bug**************Upload Part Copy with modifiedSinceDate / unmodifiedSinceDate************************************************************** /* DateTime dt =DateTime.Now; DateTime dt100 =dt.AddDays(100); //增加100天 DateTime dt30 = dt.AddDays(-30);//減少30天 CopyPartResponse copyPartDateResult = s3Client.CopyPart(new CopyPartRequest().WithSourceBucket(bucketName).WithDestinationBucket(bucketName) .WithSourceKey(objectNameA).WithDestinationKey(objectNameB).WithPartNumber(1).WithUploadID(objectBUID).WithModifiedSinceDate(DateTime.Now.AddDays(-30))); System.Console.WriteLine("\nCopy Part with modifiedSinceDate Result: requestID:{0}", copyPartDateResult.RequestId);*/ //**************Upload Part Copy with srcVersionId************************************************************************************************* // 沿用 ACL Version test 環境bucket = netversion,內有兩筆 Object 資訊(Object 已刪除,帶有 delete marker) // 若於佈板後刪除環境,請預先建立環境,執行ACL 程式 132~150行程式 //************************************************************************************************************************************************** //versioning test String vbucketName = "netversion"; String versionid = "784cca47e2a7423f97dedde6a72f9b3d"; CopyPartResponse copyPartVersionidResult = s3Client.CopyPart(new CopyPartRequest().WithSourceBucket(vbucketName).WithDestinationBucket(bucketName) .WithSourceKey("hello").WithDestinationKey(objectNameB).WithPartNumber(1).WithUploadID(objectBUID).WithSourceVersionId(versionid)); System.Console.WriteLine("\nCopy Part with srcVersionId Result: requestID:{0}", copyPartVersionidResult.RequestId); //Abort MPU AbortMultipartUploadResponse abortobBResult = s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameB).WithUploadId(objectBUID)); System.Console.WriteLine("\nAbortMPU Result, requestID:{0}", abortobBResult.RequestId); AbortMultipartUploadResponse abortobCResult = s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameC).WithUploadId(objectCUID)); System.Console.WriteLine("\nAbortMPU Result, requestID:{0}", abortobCResult.RequestId); AbortMultipartUploadResponse abortobDResult = s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameD).WithUploadId(objectDUID)); System.Console.WriteLine("\nAbortMPU Result, requestID:{0}", abortobDResult.RequestId); AbortMultipartUploadResponse abortobEResult = s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest().WithBucketName(bucketName).WithKey(objectNameE).WithUploadId(objectEUID)); System.Console.WriteLine("\nAbortMPU Result, requestID:{0}", abortobEResult.RequestId); //DeleteObject System.Console.WriteLine("\nDelete Object!\n"); s3Client.DeleteObject(new DeleteObjectRequest().WithBucketName(bucketName).WithKey(objectNameA)); //DeleteBucket System.Console.WriteLine("Delete Bucket!"); s3Client.DeleteBucket(new DeleteBucketRequest().WithBucketName(bucketName)); System.Console.WriteLine("END!"); }
/// <summary> /// Initiates the asynchronous execution of the InitiateMultipartUpload operation. /// <seealso cref="Amazon.S3.IAmazonS3.InitiateMultipartUpload"/> /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the InitiateMultipartUpload operation.</param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// <returns>The task object representing the asynchronous operation.</returns> public Task<InitiateMultipartUploadResponse> InitiateMultipartUploadAsync(InitiateMultipartUploadRequest request, CancellationToken cancellationToken = default(CancellationToken)) { var marshaller = new InitiateMultipartUploadRequestMarshaller(); var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.GetInstance(); return Invoke<IRequest, InitiateMultipartUploadRequest, InitiateMultipartUploadResponse>(request, marshaller, unmarshaller, signer, cancellationToken); }
/// <summary> /// <para>Initiates a multipart upload and returns an upload ID.</para> /// </summary> /// <remarks> /// <para> /// After you initiate a multipart upload and upload one or more parts, you must either complete or abort /// the multipart upload in order to stop getting charged for storage of the uploaded parts. Once you /// complete or abort the multipart upload, Amazon S3 will release the stored parts and stop charging you /// for their storage. /// </para> /// </remarks> /// <param name="request">Container for the necessary parameters to execute the InitiateMultipartUpload service method on /// AmazonS3.</param> /// /// <returns>The response from the InitiateMultipartUpload service method, as returned by AmazonS3.</returns> public InitiateMultipartUploadResponse InitiateMultipartUpload(InitiateMultipartUploadRequest request) { var task = InitiateMultipartUploadAsync(request); try { return task.Result; } catch(AggregateException e) { throw e.InnerException; } }
/// <summary> /// <para>Initiates a multipart upload and returns an upload ID.</para> /// </summary> /// /// <param name="initiateMultipartUploadRequest">Container for the necessary parameters to execute the InitiateMultipartUpload service method on /// AmazonS3.</param> /// /// <returns>The response from the InitiateMultipartUpload service method, as returned by AmazonS3.</returns> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> public async Task<InitiateMultipartUploadResponse> InitiateMultipartUploadAsync(InitiateMultipartUploadRequest initiateMultipartUploadRequest, CancellationToken cancellationToken = default(CancellationToken)) { var marshaller = new InitiateMultipartUploadRequestMarshaller(); var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.GetInstance(); var response = await Invoke<IRequest, InitiateMultipartUploadRequest, InitiateMultipartUploadResponse>(initiateMultipartUploadRequest, marshaller, unmarshaller, signer, cancellationToken) .ConfigureAwait(continueOnCapturedContext: false); return response; }
/// <summary> /// <para>Initiates a multipart upload and returns an upload ID.</para> /// </summary> /// <remarks> /// <para> /// After you initiate a multipart upload and upload one or more parts, you must either complete or abort /// the multipart upload in order to stop getting charged for storage of the uploaded parts. Once you /// complete or abort the multipart upload, Amazon S3 will release the stored parts and stop charging you /// for their storage. /// </para> /// </remarks> /// <param name="request">Container for the necessary parameters to execute the InitiateMultipartUpload service method on /// AmazonS3.</param> /// /// <returns>The response from the InitiateMultipartUpload service method, as returned by AmazonS3.</returns> public InitiateMultipartUploadResponse InitiateMultipartUpload(InitiateMultipartUploadRequest request) { var task = InitiateMultipartUploadAsync(request); try { return task.Result; } catch(AggregateException e) { ExceptionDispatchInfo.Capture(e.InnerException).Throw(); return null; } }
/// <summary> /// Initiates a multipart upload and returns an upload ID. /// /// /// <para> /// <b>Note:</b> After you initiate multipart upload and upload one or more parts, you /// must either complete or abort multipart upload in order to stop getting charged for /// storage of the uploaded parts. Only after you either complete or abort multipart upload, /// Amazon S3 frees up the parts storage and stops charging you for the parts storage. /// </para> /// </summary> /// <param name="bucketName">A property of InitiateMultipartUploadRequest used to execute the InitiateMultipartUpload service method.</param> /// <param name="key">A property of InitiateMultipartUploadRequest used to execute the InitiateMultipartUpload service method.</param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// /// <returns>The response from the InitiateMultipartUpload service method, as returned by S3.</returns> public Task<InitiateMultipartUploadResponse> InitiateMultipartUploadAsync(string bucketName, string key, System.Threading.CancellationToken cancellationToken = default(CancellationToken)) { var request = new InitiateMultipartUploadRequest(); request.BucketName = bucketName; request.Key = key; return InitiateMultipartUploadAsync(request, cancellationToken); }
public static void Main(string[] args) { XmlSerializerFactory factory = new XmlSerializerFactory(); XmlSerializer serializer = factory.CreateSerializer(typeof(UploadState)); UploadState us; if (args.Length == 2) { us = new UploadState(args[1], args[0]); } else if (args.Length == 0 && new FileInfo("upload.dat").Exists) { using (FileStream fs = new FileStream("upload.dat", FileMode.Open)) { us = (UploadState)serializer.Deserialize(fs); } } else { Console.Error.WriteLine("Usage: bucket filename"); return; } try { NameValueCollection appConfig = ConfigurationManager.AppSettings; // Print the number of Amazon S3 Buckets. AmazonS3 s3Client = AWSClientFactory.CreateAmazonS3Client( appConfig["AWSAccessKey"], appConfig["AWSSecretKey"] ); if (string.IsNullOrEmpty(us.UploadId)) { InitiateMultipartUploadRequest req = new InitiateMultipartUploadRequest() .WithBucketName(us.BucketName) .WithKey(us.Key) ; us.UploadId = s3Client.InitiateMultipartUpload(req).UploadId; using (FileStream fs = new FileStream("upload.dat", FileMode.OpenOrCreate)) { serializer.Serialize(fs, us); } } while (us.FilePosition < us.FileLength) { try { Console.WriteLine("Uploading part {0} of {1}", us.PartNumber, us.NumChunks); UploadPartRequest ureq = new UploadPartRequest() .WithBucketName(us.BucketName) .WithFilePath(us.FileName) .WithFilePosition(us.FilePosition) .WithPartNumber(us.PartNumber) .WithPartSize(us.FileLength - us.FilePosition > us.ChunkSize ? us.ChunkSize : us.FileLength - us.FilePosition) .WithGenerateChecksum(true) .WithKey(us.Key) .WithUploadId(us.UploadId) .WithSubscriber(new EventHandler<UploadPartProgressArgs>(ShowProgress)) ; if (us.Responses.Count > us.PartNumber - 1) { us.Responses[us.PartNumber - 1] = new PartETag(us.PartNumber, s3Client.UploadPart(ureq).ETag); } else { us.Responses.Insert(us.PartNumber - 1, new PartETag(us.PartNumber, s3Client.UploadPart(ureq).ETag)); } us.PartNumber++; us.FilePosition += us.ChunkSize; using (FileStream fs = new FileStream("upload.dat", FileMode.OpenOrCreate)) { serializer.Serialize(fs, us); } } catch (System.Net.WebException) { } } CompleteMultipartUploadRequest creq = new CompleteMultipartUploadRequest() .WithPartETags(us.Responses) .WithBucketName(us.BucketName) .WithUploadId(us.UploadId) .WithKey(us.Key) ; CompleteMultipartUploadResponse cresp = s3Client.CompleteMultipartUpload(creq); System.Console.WriteLine("File available at {0}", cresp.Location); File.Delete("upload.dat"); } catch (AmazonS3Exception e) { Console.Error.WriteLine(e); } //Console.Write(GetServiceOutput()); //Console.Read(); }
private void Init() { var initiateRequest = new InitiateMultipartUploadRequest { BucketName = this.bucketName, Key = this.fileName }; this.initResponse = client.InitiateMultipartUpload(initiateRequest); this.abortMPURequest = new AbortMultipartUploadRequest { BucketName = this.bucketName, Key = this.fileName, UploadId = this.initResponse.UploadId }; this.uploadResponses = new List<UploadPartResponse>(); }
public override string InitiateChunkedUpload(string domain, string path) { var request = new InitiateMultipartUploadRequest { BucketName = _bucket, Key = MakePath(domain, path), ServerSideEncryptionMethod = ServerSideEncryptionMethod.AES256 }; using (var s3 = GetClient()) { var response = s3.InitiateMultipartUpload(request); return response.UploadId; } }
public void MultipartEncryptionTestInstructionFile() { string filePath = @"C:\temp\Upload15MegFileIn3PartsViaStream.txt"; string retrievedFilepath = @"C:\temp\Upload15MegFileIn3PartsViaStreamRetreived.txt"; int MEG_SIZE = (int)Math.Pow(2, 20) + 4001; long totalSize = (long)(15 * MEG_SIZE); UtilityMethods.GenerateFile(filePath, totalSize); string key = "MultipartEncryptionTestInstrcutionFile" + random.Next(); s3EncryptionClientFileMode.PutBucket(new PutBucketRequest() { BucketName = bucketName }); Stream inputStream = File.OpenRead(filePath); try { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = bucketName, Key = key, StorageClass = S3StorageClass.ReducedRedundancy, ContentType = "text/html", CannedACL = S3CannedACL.PublicRead }; InitiateMultipartUploadResponse initResponse = s3EncryptionClientFileMode.InitiateMultipartUpload(initRequest); // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MEG_SIZE, InputStream = inputStream }; UploadPartResponse up1Response = s3EncryptionClientFileMode.UploadPart(uploadRequest); // Upload part 2 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MEG_SIZE + 4001, InputStream = inputStream }; UploadPartResponse up2Response = s3EncryptionClientFileMode.UploadPart(uploadRequest); // Upload part 3 uploadRequest = new UploadPartRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream, IsLastPart = true }; //uploadRequest.setLastPart(); UploadPartResponse up3Response = s3EncryptionClientFileMode.UploadPart(uploadRequest); ListPartsRequest listPartRequest = new ListPartsRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest); Assert.AreEqual(3, listPartResponse.Parts.Count); Assert.AreEqual(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber); Assert.AreEqual(up1Response.ETag, listPartResponse.Parts[0].ETag); Assert.AreEqual(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber); Assert.AreEqual(up2Response.ETag, listPartResponse.Parts[1].ETag); Assert.AreEqual(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber); Assert.AreEqual(up3Response.ETag, listPartResponse.Parts[2].ETag); listPartRequest.MaxParts = 1; listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest); Assert.AreEqual(1, listPartResponse.Parts.Count); // Complete the response CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = bucketName, Key = key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(up1Response, up2Response, up3Response); CompleteMultipartUploadResponse compResponse = s3EncryptionClientFileMode.CompleteMultipartUpload(compRequest); Assert.AreEqual(bucketName, compResponse.BucketName); Assert.IsNotNull(compResponse.ETag); Assert.AreEqual(key, compResponse.Key); Assert.IsNotNull(compResponse.Location); // Get the file back from S3 and make sure it is still the same. GetObjectRequest getRequest = new GetObjectRequest() { BucketName = bucketName, Key = key }; GetObjectResponse getResponse = s3EncryptionClientFileMode.GetObject(getRequest); getResponse.WriteResponseStreamToFile(retrievedFilepath); UtilityMethods.CompareFiles(filePath, retrievedFilepath); GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest() { BucketName = bucketName, Key = key }; GetObjectMetadataResponse metaDataResponse = s3EncryptionClientFileMode.GetObjectMetadata(metaDataRequest); Assert.AreEqual("text/html", metaDataResponse.Headers.ContentType); } finally { inputStream.Close(); if (File.Exists(filePath)) File.Delete(filePath); if (File.Exists(retrievedFilepath)) File.Delete(retrievedFilepath); } }
/// <summary> /// Initiates the asynchronous execution of the InitiateMultipartUpload operation. /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the InitiateMultipartUpload operation on AmazonS3Client.</param> /// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param> /// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property.</param> /// /// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking EndInitiateMultipartUpload /// operation.</returns> public IAsyncResult BeginInitiateMultipartUpload(InitiateMultipartUploadRequest request, AsyncCallback callback, object state) { var marshaller = new InitiateMultipartUploadRequestMarshaller(); var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.Instance; return BeginInvoke<InitiateMultipartUploadRequest>(request, marshaller, unmarshaller, callback, state); }
/// <summary> /// Initiates a multipart upload and returns an upload ID. /// /// /// <para> /// <b>Note:</b> After you initiate multipart upload and upload one or more parts, you /// must either complete or abort multipart upload in order to stop getting charged for /// storage of the uploaded parts. Only after you either complete or abort multipart upload, /// Amazon S3 frees up the parts storage and stops charging you for the parts storage. /// </para> /// </summary> /// <param name="bucketName">A property of InitiateMultipartUploadRequest used to execute the InitiateMultipartUpload service method.</param> /// <param name="key">A property of InitiateMultipartUploadRequest used to execute the InitiateMultipartUpload service method.</param> /// /// <returns>The response from the InitiateMultipartUpload service method, as returned by S3.</returns> public InitiateMultipartUploadResponse InitiateMultipartUpload(string bucketName, string key) { var request = new InitiateMultipartUploadRequest(); request.BucketName = bucketName; request.Key = key; return InitiateMultipartUpload(request); }
/// <summary> /// Runs the multipart upload. /// </summary> public override void Execute() { InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() { BucketName = this._fileTransporterRequest.BucketName, Key = this._fileTransporterRequest.Key, CannedACL = this._fileTransporterRequest.CannedACL, ContentType = determineContentType(), StorageClass = this._fileTransporterRequest.StorageClass, ServerSideEncryptionMethod = this._fileTransporterRequest.ServerSideEncryptionMethod }; initRequest.BeforeRequestEvent += this.RequestEventHandler; if (this._fileTransporterRequest.Metadata != null && this._fileTransporterRequest.Metadata.Count > 0) initRequest.Metadata = this._fileTransporterRequest.Metadata; if (this._fileTransporterRequest.Headers != null && this._fileTransporterRequest.Headers.Count > 0) initRequest.Headers = this._fileTransporterRequest.Headers; InitiateMultipartUploadResponse initResponse = this._s3Client.InitiateMultipartUpload(initRequest); _logger.DebugFormat("Initiated upload: {0}", initResponse.UploadId); try { _logger.DebugFormat("Queue up the UploadPartRequests to be executed"); long filePosition = 0; for (int i = 1; filePosition < this._contentLength; i++) { UploadPartRequest uploadRequest = new UploadPartRequest() { BucketName = this._fileTransporterRequest.BucketName, Key = this._fileTransporterRequest.Key, UploadId = initResponse.UploadId, PartNumber = i, PartSize = this._partSize, #if (BCL && !BCL45) Timeout = ClientConfig.GetTimeoutValue(this._config.DefaultTimeout,this._fileTransporterRequest.Timeout) #endif }; if ((filePosition + this._partSize >= this._contentLength) && _s3Client is AmazonS3EncryptionClient) { uploadRequest.IsLastPart = true; uploadRequest.PartSize = 0; } uploadRequest.StreamUploadProgressCallback += this.uploadPartProgressEventCallback; uploadRequest.BeforeRequestEvent += this.RequestEventHandler; if (this._fileTransporterRequest.IsSetFilePath()) { uploadRequest.FilePosition = filePosition; uploadRequest.FilePath = this._fileTransporterRequest.FilePath; } else { uploadRequest.InputStream = this._fileTransporterRequest.InputStream; } this._partsToUpload.Enqueue(uploadRequest); filePosition += this._partSize; } this._totalNumberOfParts = this._partsToUpload.Count; _logger.DebugFormat("Starting threads to execute the {0} UploadPartRequests in the queue", this._totalNumberOfParts); startInvokerPool(); _logger.DebugFormat("Waiting for threads to complete. ({0})", initResponse.UploadId); waitTillAllThreadsComplete(); _logger.DebugFormat("Beginning completing multipart. ({0})", initResponse.UploadId); CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() { BucketName = this._fileTransporterRequest.BucketName, Key = this._fileTransporterRequest.Key, UploadId = initResponse.UploadId }; compRequest.AddPartETags(this._uploadResponses); compRequest.BeforeRequestEvent += this.RequestEventHandler; this._s3Client.CompleteMultipartUpload(compRequest); _logger.DebugFormat("Done completing multipart. ({0})", initResponse.UploadId); } catch (Exception e) { _logger.Error(e, "Exception while uploading. ({0})", initResponse.UploadId); shutdown(initResponse.UploadId); throw; } finally { if (this._fileTransporterRequest.InputStream != null && !this._fileTransporterRequest.IsSetFilePath() && this._fileTransporterRequest.AutoCloseStream) { this._fileTransporterRequest.InputStream.Close(); } if (_logger != null) { _logger.Flush(); } } }
internal InitiateMultipartUploadResponse InitiateMultipartUpload(InitiateMultipartUploadRequest request) { var marshaller = new InitiateMultipartUploadRequestMarshaller(); var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.Instance; return Invoke<InitiateMultipartUploadRequest,InitiateMultipartUploadResponse>(request, marshaller, unmarshaller); }
internal static void CreateMultiPartS3Blob(AmazonS3Client client, string key, S3CopyMemoryStream stream) { if (stream.InitiatingPart) { InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest() .WithBucketName("static.getbrickpile.com") .WithCannedACL(S3CannedACL.PublicRead) .WithKey(key); InitiateMultipartUploadResponse initiateResponse = client.InitiateMultipartUpload(initiateMultipartUploadRequest); stream.UploadPartId = initiateResponse.UploadId; } stream.Position = 0; UploadPartRequest uploadPartRequest = new UploadPartRequest() .WithBucketName("static.getbrickpile.com") .WithKey(key) .WithPartNumber(stream.WriteCount) .WithPartSize(stream.Position) .WithUploadId(stream.UploadPartId) .WithInputStream(stream) as UploadPartRequest; UploadPartResponse response = client.UploadPart(uploadPartRequest); PartETag etag = new PartETag(response.PartNumber, response.ETag); stream.PartETagCollection.Add(etag); if (stream.EndOfPart) { CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest() .WithBucketName("static.getbrickpile.com") .WithKey(key) .WithPartETags(stream.PartETagCollection) .WithUploadId(stream.UploadPartId); CompleteMultipartUploadResponse completeMultipartUploadResponse = client.CompleteMultipartUpload(completeMultipartUploadRequest); string loc = completeMultipartUploadResponse.Location; } }
private InitiateMultipartUploadRequest ConstructInitiateMultipartUploadRequest() { var initRequest = new InitiateMultipartUploadRequest() { BucketName = this._fileTransporterRequest.BucketName, Key = this._fileTransporterRequest.Key, CannedACL = this._fileTransporterRequest.CannedACL, ContentType = determineContentType(), StorageClass = this._fileTransporterRequest.StorageClass, ServerSideEncryptionMethod = this._fileTransporterRequest.ServerSideEncryptionMethod, ServerSideEncryptionCustomerMethod = this._fileTransporterRequest.ServerSideEncryptionCustomerMethod, ServerSideEncryptionCustomerProvidedKey = this._fileTransporterRequest.ServerSideEncryptionCustomerProvidedKey, ServerSideEncryptionCustomerProvidedKeyMD5 = this._fileTransporterRequest.ServerSideEncryptionCustomerProvidedKeyMD5 }; initRequest.BeforeRequestEvent += this.RequestEventHandler; if (this._fileTransporterRequest.Metadata != null && this._fileTransporterRequest.Metadata.Count > 0) initRequest.Metadata = this._fileTransporterRequest.Metadata; if (this._fileTransporterRequest.Headers != null && this._fileTransporterRequest.Headers.Count > 0) initRequest.Headers = this._fileTransporterRequest.Headers; return initRequest; }
/// <summary> /// Initiates the asynchronous execution of the InitiateMultipartUpload operation. /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the InitiateMultipartUpload operation.</param> /// <param name="cancellationToken"> /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// </param> /// <returns>The task object representing the asynchronous operation.</returns> public Task<InitiateMultipartUploadResponse> InitiateMultipartUploadAsync(InitiateMultipartUploadRequest request, System.Threading.CancellationToken cancellationToken = default(CancellationToken)) { var marshaller = new InitiateMultipartUploadRequestMarshaller(); var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.Instance; return InvokeAsync<InitiateMultipartUploadRequest,InitiateMultipartUploadResponse>(request, marshaller, unmarshaller, cancellationToken); }
IAsyncResult invokeInitiateMultipartUpload(InitiateMultipartUploadRequest initiateMultipartUploadRequest, AsyncCallback callback, object state, bool synchronized) { var marshaller = new InitiateMultipartUploadRequestMarshaller(); var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.GetInstance(); var result = Invoke < IRequest, InitiateMultipartUploadRequest>(initiateMultipartUploadRequest, callback, state, synchronized, marshaller, unmarshaller, signer); return result; }
/// <summary> /// Send a InitiateMultipartUploadRequest request and return the response. /// </summary> /// <param name="existingBucketName"></param> /// <param name="keyName"></param> /// <returns></returns> public async Task<InitiateMultipartUploadResponse> InitiateMultipartUploadAsync(string existingBucketName, string keyName, CancellationToken token) { token.ThrowIfCancellationRequested(); _log.Debug("Called InitiateMultipartUploadAsync with parameter keyName = \"" + keyName + "\"."); try { InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest { BucketName = existingBucketName, Key = keyName }; InitiateMultipartUploadResponse initResponse = await s3Client.InitiateMultipartUploadAsync(initiateRequest, token).ConfigureAwait(false); return initResponse; } catch (Exception e) { string messagePart = " with parameter keyName = \"" + keyName + "\""; this.LogAmazonException(messagePart, e); throw; } }
/// <summary> /// <para>Initiates a multipart upload and returns an upload ID.</para> /// </summary> /// <remarks> /// <para> /// After you initiate a multipart upload and upload one or more parts, you must either complete or abort /// the multipart upload in order to stop getting charged for storage of the uploaded parts. Once you /// complete or abort the multipart upload, Amazon S3 will release the stored parts and stop charging you /// for their storage. /// </para> /// </remarks> /// <param name="initiateMultipartUploadRequest">Container for the necessary parameters to execute the InitiateMultipartUpload service method on /// AmazonS3.</param> /// /// <returns>The response from the InitiateMultipartUpload service method, as returned by AmazonS3.</returns> /// public InitiateMultipartUploadResponse InitiateMultipartUpload(InitiateMultipartUploadRequest initiateMultipartUploadRequest) { IAsyncResult asyncResult = invokeInitiateMultipartUpload(initiateMultipartUploadRequest, null, null, true); return EndInitiateMultipartUpload(asyncResult); }
/// <summary> /// Initiates the asynchronous execution of the InitiateMultipartUpload operation. /// <seealso cref="Amazon.S3.IAmazonS3.InitiateMultipartUpload"/> /// </summary> /// <remarks> /// <para> /// After you initiate a multipart upload and upload one or more parts, you must either complete or abort /// the multipart upload in order to stop getting charged for storage of the uploaded parts. Once you /// complete or abort the multipart upload, Amazon S3 will release the stored parts and stop charging you /// for their storage. /// </para> /// </remarks> /// <param name="initiateMultipartUploadRequest">Container for the necessary parameters to execute the InitiateMultipartUpload operation on /// AmazonS3.</param> /// <param name="callback">An AsyncCallback delegate that is invoked when the operation completes.</param> /// <param name="state">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property.</param> /// /// <returns>An IAsyncResult that can be used to poll or wait for results, or both; this value is also needed when invoking /// EndInitiateMultipartUpload operation.</returns> public IAsyncResult BeginInitiateMultipartUpload(InitiateMultipartUploadRequest initiateMultipartUploadRequest, AsyncCallback callback, object state) { return invokeInitiateMultipartUpload(initiateMultipartUploadRequest, callback, state, false); }
/// <summary> /// Runs the multipart upload. /// </summary> public override void Execute() { int timeout = this._config.DefaultTimeout; if (this._fileTransporterRequest.Timeout != 0) timeout = this._fileTransporterRequest.Timeout; InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest() .WithBucketName(this._fileTransporterRequest.BucketName) .WithKey(this._fileTransporterRequest.Key) .WithCannedACL(this._fileTransporterRequest.CannedACL) .WithContentType(determineContentType()) .WithStorageClass(this._fileTransporterRequest.StorageClass) .WithBeforeRequestHandler(RequestEventHandler) as InitiateMultipartUploadRequest; if (this._fileTransporterRequest.metadata != null && this._fileTransporterRequest.metadata.Count > 0) initRequest.WithMetaData(this._fileTransporterRequest.metadata); if (this._fileTransporterRequest.Headers != null && this._fileTransporterRequest.Headers.Count > 0) initRequest.AddHeaders(this._fileTransporterRequest.Headers); InitiateMultipartUploadResponse initResponse = this._s3Client.InitiateMultipartUpload(initRequest); this._logger.DebugFormat("Initiated upload: {0}", initResponse.UploadId); try { this._logger.DebugFormat("Queue up the UploadPartRequests to be executed"); long filePosition = 0; for (int i = 1; filePosition < this._contentLength; i++) { UploadPartRequest uploadRequest = new UploadPartRequest() .WithBucketName(this._fileTransporterRequest.BucketName) .WithKey(this._fileTransporterRequest.Key) .WithUploadId(initResponse.UploadId) .WithTimeout(timeout) .WithPartNumber(i) .WithPartSize(this._partSize) .WithSubscriber(new EventHandler<UploadPartProgressArgs>(this.uploadPartProgressEventCallback)) .WithBeforeRequestHandler(RequestEventHandler) as UploadPartRequest; if (this._fileTransporterRequest.IsSetFilePath()) { uploadRequest .WithFilePosition(filePosition) .WithFilePath(this._fileTransporterRequest.FilePath); } else { uploadRequest.InputStream = this._fileTransporterRequest.InputStream; } this._partsToUpload.Enqueue(uploadRequest); filePosition += this._partSize; } this._totalNumberOfParts = this._partsToUpload.Count; this._logger.DebugFormat("Starting threads to execute the {0} UploadPartRequests in the queue", this._totalNumberOfParts); startInvokerPool(); this._logger.DebugFormat("Waiting for threads to complete. ({0})", initResponse.UploadId); waitTillAllThreadsComplete(); this._logger.DebugFormat("Beginning completing multipart. ({0})", initResponse.UploadId); CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest() .WithBucketName(this._fileTransporterRequest.BucketName) .WithKey(this._fileTransporterRequest.Key) .WithUploadId(initResponse.UploadId) .WithPartETags(this._uploadResponses) .WithBeforeRequestHandler(RequestEventHandler) as CompleteMultipartUploadRequest; this._s3Client.CompleteMultipartUpload(compRequest); this._logger.DebugFormat("Done completing multipart. ({0})", initResponse.UploadId); } catch (Exception e) { this._logger.Error(string.Format("Exception while uploading. ({0})", initResponse.UploadId), e); shutdown(initResponse.UploadId); throw; } finally { if (this._fileTransporterRequest.InputStream != null && !this._fileTransporterRequest.IsSetFilePath()) { this._fileTransporterRequest.InputStream.Close(); } } }
/// <summary> /// Initiates the asynchronous execution of the InitiateMultipartUpload operation. /// This API is supported only when AWSConfigs.HttpClient is set to AWSConfigs.HttpClientOption.UnityWebRequest, the default value for this configuration option is AWSConfigs.HttpClientOption.UnityWWW /// </summary> /// /// <param name="request">Container for the necessary parameters to execute the InitiateMultipartUpload operation on AmazonS3Client.</param> /// <param name="callback">An Action delegate that is invoked when the operation completes.</param> /// <param name="options">A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property.</param> public void InitiateMultipartUploadAsync(InitiateMultipartUploadRequest request, AmazonServiceCallback<InitiateMultipartUploadRequest, InitiateMultipartUploadResponse> callback, AsyncOptions options = null) { if (AWSConfigs.HttpClient == AWSConfigs.HttpClientOption.UnityWWW) { throw new InvalidOperationException("InitiateMultipartUpload is only allowed with AWSConfigs.HttpClientOption.UnityWebRequest API option"); } options = options == null?new AsyncOptions():options; var marshaller = new InitiateMultipartUploadRequestMarshaller(); var unmarshaller = InitiateMultipartUploadResponseUnmarshaller.Instance; Action<AmazonWebServiceRequest, AmazonWebServiceResponse, Exception, AsyncOptions> callbackHelper = null; if(callback !=null ) callbackHelper = (AmazonWebServiceRequest req, AmazonWebServiceResponse res, Exception ex, AsyncOptions ao) => { AmazonServiceResult<InitiateMultipartUploadRequest,InitiateMultipartUploadResponse> responseObject = new AmazonServiceResult<InitiateMultipartUploadRequest,InitiateMultipartUploadResponse>((InitiateMultipartUploadRequest)req, (InitiateMultipartUploadResponse)res, ex , ao.State); callback(responseObject); }; BeginInvoke<InitiateMultipartUploadRequest>(request, marshaller, unmarshaller, options, callbackHelper); }
/// <summary> /// Initiates a multipart upload and returns an upload ID. /// /// /// <para> /// <b>Note:</b> After you initiate multipart upload and upload one or more parts, you /// must either complete or abort multipart upload in order to stop getting charged for /// storage of the uploaded parts. Only after you either complete or abort multipart upload, /// Amazon S3 frees up the parts storage and stops charging you for the parts storage. /// </para> /// This API is supported only when AWSConfigs.HttpClient is set to AWSConfigs.HttpClientOption.UnityWebRequest, the default value of this configuration option is AWSConfigs.HttpClientOption.UnityWWW /// </summary> /// <param name="bucketName">A property of InitiateMultipartUploadRequest used to execute the InitiateMultipartUpload service method.</param> /// <param name="key">A property of InitiateMultipartUploadRequest used to execute the InitiateMultipartUpload service method.</param> /// <param name="callback">An Action delegate that is invoked when the operation completes.</param> /// <param name="options"> /// A user-defined state object that is passed to the callback procedure. Retrieve this object from within the callback /// procedure using the AsyncState property. /// </param> /// /// <returns>The response from the InitiateMultipartUpload service method, as returned by S3.</returns> public void InitiateMultipartUploadAsync(string bucketName, string key, AmazonServiceCallback<InitiateMultipartUploadRequest, InitiateMultipartUploadResponse> callback, AsyncOptions options = null) { var request = new InitiateMultipartUploadRequest(); request.BucketName = bucketName; request.Key = key; InitiateMultipartUploadAsync(request, callback, options); }
static void Main() { _accessKeyId = Utilities.AwsAccessKey; _secretAccessKey = Utilities.AwsSecretKey; AmazonS3 s3Client = new AmazonS3Client(_accessKeyId, _secretAccessKey); ListMultipartUploadsRequest allMultipartUploadsRequest = new ListMultipartUploadsRequest().WithBucketName(ExistingBucketName); ListMultipartUploadsResponse mpUploadsResponse = s3Client.ListMultipartUploads(allMultipartUploadsRequest); var objects = new List<Object>(); foreach (MultipartUpload multipartUpload in mpUploadsResponse.MultipartUploads) { bool isObjectdFound = false; foreach (Object o in objects) { if (o.UploadId == multipartUpload.UploadId) { o.Parts.Add(new Part { PartId = o.Parts.Count, Etag = "" }); isObjectdFound = true; } } if (!isObjectdFound) { objects.Add(new Object { Parts = new List<Part> { new Part() { Etag = "", PartId = 1 } }, UploadId = multipartUpload.UploadId }); } } var result = JsonConvert.SerializeObject(objects); var objs = JsonConvert.DeserializeObject<List<Object>>(result); //return; // List to store upload part responses. var uploadResponses = new List<UploadPartResponse>(); byte[] bytes; long contentLength = 0; using (var fileStream = new FileStream(FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { contentLength = fileStream.Length; bytes = new byte[contentLength]; fileStream.Read(bytes, 0, Convert.ToInt32(contentLength)); } // 1. Initialize. InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest().WithBucketName(ExistingBucketName).WithKey(KeyName); InitiateMultipartUploadResponse initResponse = s3Client.InitiateMultipartUpload(initiateRequest); try { // 2. Upload Parts. long partSize = 5 * (long)Math.Pow(2, 20); // 5 MB long filePosition = 0; for (int i = 1; filePosition < contentLength; i++) { byte[] bytesToStream; if (filePosition + partSize < contentLength) { bytesToStream = new byte[partSize]; Array.Copy(bytes, filePosition, bytesToStream, 0, partSize); } else { bytesToStream = new byte[contentLength - filePosition]; Array.Copy(bytes, filePosition, bytesToStream, 0, contentLength - filePosition); } Stream stream = new MemoryStream(bytesToStream); // Create request to upload a part. UploadPartRequest uploadRequest = new UploadPartRequest() .WithBucketName(ExistingBucketName) .WithKey(KeyName) .WithUploadId(initResponse.UploadId) .WithPartNumber(i) .WithPartSize(partSize) .WithFilePosition(filePosition) .WithTimeout(1000000000) .WithMD5Digest(Convert.ToBase64String(MD5.Create().ComputeHash(bytesToStream))); uploadRequest.WithInputStream(stream); // Upload part and add response to our list. uploadResponses.Add(s3Client.UploadPart(uploadRequest)); filePosition += partSize; } // Step 3: complete. CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest() .WithBucketName(ExistingBucketName) .WithKey(KeyName) .WithUploadId(initResponse.UploadId) .WithPartETags(uploadResponses); CompleteMultipartUploadResponse completeUploadResponse = s3Client.CompleteMultipartUpload(completeRequest); Console.WriteLine(completeUploadResponse.ETag); } catch (Exception exception) { Console.WriteLine("Exception occurred: {0}", exception.Message); s3Client.AbortMultipartUpload(new AbortMultipartUploadRequest() .WithBucketName(ExistingBucketName) .WithKey(KeyName) .WithUploadId(initResponse.UploadId)); } }
public void ObjectSamples() { { #region ListObjects Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // List all objects ListObjectsRequest listRequest = new ListObjectsRequest { BucketName = "SampleBucket", }; ListObjectsResponse listResponse; do { // Get a list of objects listResponse = client.ListObjects(listRequest); foreach (S3Object obj in listResponse.S3Objects) { Console.WriteLine("Object - " + obj.Key); Console.WriteLine(" Size - " + obj.Size); Console.WriteLine(" LastModified - " + obj.LastModified); Console.WriteLine(" Storage class - " + obj.StorageClass); } // Set the marker property listRequest.Marker = listResponse.NextMarker; } while (listResponse.IsTruncated); #endregion } { #region GetObject Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a GetObject request GetObjectRequest request = new GetObjectRequest { BucketName = "SampleBucket", Key = "Item1" }; // Issue request and remember to dispose of the response using (GetObjectResponse response = client.GetObject(request)) { using (StreamReader reader = new StreamReader(response.ResponseStream)) { string contents = reader.ReadToEnd(); Console.WriteLine("Object - " + response.Key); Console.WriteLine(" Version Id - " + response.VersionId); Console.WriteLine(" Contents - " + contents); } } #endregion } { #region GetObjectMetadata Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a GetObjectMetadata request GetObjectMetadataRequest request = new GetObjectMetadataRequest { BucketName = "SampleBucket", Key = "Item1" }; // Issue request and view the response GetObjectMetadataResponse response = client.GetObjectMetadata(request); Console.WriteLine("Content Length - " + response.ContentLength); Console.WriteLine("Content Type - " + response.Headers.ContentType); if (response.Expiration != null) { Console.WriteLine("Expiration Date - " + response.Expiration.ExpiryDate); Console.WriteLine("Expiration Rule Id - " + response.Expiration.RuleId); } #endregion } { #region PutObject Sample 1 // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a PutObject request PutObjectRequest request = new PutObjectRequest { BucketName = "SampleBucket", Key = "Item1", ContentBody = "This is sample content..." }; // Put object PutObjectResponse response = client.PutObject(request); #endregion } { #region PutObject Sample 2 // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a PutObject request PutObjectRequest request = new PutObjectRequest { BucketName = "SampleBucket", Key = "Item1", FilePath = "contents.txt" }; // Put object PutObjectResponse response = client.PutObject(request); #endregion } { #region PutObject Sample 3 // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a PutObject request PutObjectRequest request = new PutObjectRequest { BucketName = "SampleBucket", Key = "Item1", }; using (FileStream stream = new FileStream("contents.txt", FileMode.Open)) { request.InputStream = stream; // Put object PutObjectResponse response = client.PutObject(request); } #endregion } { #region DeleteObject Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a DeleteObject request DeleteObjectRequest request = new DeleteObjectRequest { BucketName = "SampleBucket", Key = "Item1" }; // Issue request client.DeleteObject(request); #endregion } { #region DeleteObjects Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a DeleteObject request DeleteObjectsRequest request = new DeleteObjectsRequest { BucketName = "SampleBucket", Objects = new List<KeyVersion> { new KeyVersion() {Key = "Item1"}, // Versioned item new KeyVersion() { Key = "Item2", VersionId = "Rej8CiBxcZKVK81cLr39j27Y5FVXghDK", }, // Item in subdirectory new KeyVersion() { Key = "Logs/error.txt"} } }; try { // Issue request DeleteObjectsResponse response = client.DeleteObjects(request); } catch (DeleteObjectsException doe) { // Catch error and list error details DeleteObjectsResponse errorResponse = doe.Response; foreach (DeletedObject deletedObject in errorResponse.DeletedObjects) { Console.WriteLine("Deleted item " + deletedObject.Key); } foreach (DeleteError deleteError in errorResponse.DeleteErrors) { Console.WriteLine("Error deleting item " + deleteError.Key); Console.WriteLine(" Code - " + deleteError.Code); Console.WriteLine(" Message - " + deleteError.Message); } } #endregion } { #region CopyObject Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a CopyObject request CopyObjectRequest request = new CopyObjectRequest { SourceBucket = "SampleBucket", SourceKey = "Item1", DestinationBucket = "AnotherBucket", DestinationKey = "Copy1", CannedACL = S3CannedACL.PublicRead }; // Issue request client.CopyObject(request); #endregion } { #region CopyObject Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Create a CopyObject request CopyObjectRequest request = new CopyObjectRequest { SourceBucket = "SampleBucket", SourceKey = "Item1", DestinationBucket = "AnotherBucket", DestinationKey = "Copy1", CannedACL = S3CannedACL.PublicRead }; // Issue request client.CopyObject(request); #endregion } { #region ListVersions Sample // Create a client AmazonS3Client client = new AmazonS3Client(); // Turn versioning on for a bucket client.PutBucketVersioning(new PutBucketVersioningRequest { BucketName = "SampleBucket", VersioningConfig = new S3BucketVersioningConfig { Status = "Enable" } }); // Populate bucket with multiple items, each with multiple versions PopulateBucket(client, "SampleBucket"); // Get versions ListVersionsRequest request = new ListVersionsRequest { BucketName = "SampleBucket" }; // Make paged ListVersions calls ListVersionsResponse response; do { response = client.ListVersions(request); // View information about versions foreach (var version in response.Versions) { Console.WriteLine("Key = {0}, Version = {1}, IsLatest = {2}, LastModified = {3}, Size = {4}", version.Key, version.VersionId, version.IsLatest, version.LastModified, version.Size); } request.KeyMarker = response.NextKeyMarker; request.VersionIdMarker = response.NextVersionIdMarker; } while (response.IsTruncated); #endregion } { #region Multipart Upload Sample int MB = (int)Math.Pow(2, 20); // Create a client AmazonS3Client client = new AmazonS3Client(); // Define input stream Stream inputStream = Create13MBDataStream(); // Initiate multipart upload InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest { BucketName = "SampleBucket", Key = "Item1" }; InitiateMultipartUploadResponse initResponse = client.InitiateMultipartUpload(initRequest); // Upload part 1 UploadPartRequest uploadRequest = new UploadPartRequest { BucketName = "SampleBucket", Key = "Item1", UploadId = initResponse.UploadId, PartNumber = 1, PartSize = 5 * MB, InputStream = inputStream }; UploadPartResponse up1Response = client.UploadPart(uploadRequest); // Upload part 2 uploadRequest = new UploadPartRequest { BucketName = "SampleBucket", Key = "Item1", UploadId = initResponse.UploadId, PartNumber = 2, PartSize = 5 * MB, InputStream = inputStream }; UploadPartResponse up2Response = client.UploadPart(uploadRequest); // Upload part 3 uploadRequest = new UploadPartRequest { BucketName = "SampleBucket", Key = "Item1", UploadId = initResponse.UploadId, PartNumber = 3, InputStream = inputStream }; UploadPartResponse up3Response = client.UploadPart(uploadRequest); // List parts for current upload ListPartsRequest listPartRequest = new ListPartsRequest { BucketName = "SampleBucket", Key = "Item1", UploadId = initResponse.UploadId }; ListPartsResponse listPartResponse = client.ListParts(listPartRequest); Debug.Assert(listPartResponse.Parts.Count == 3); // Complete the multipart upload CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest { BucketName = "SampleBucket", Key = "Item1", UploadId = initResponse.UploadId, PartETags = new List<PartETag> { new PartETag { ETag = up1Response.ETag, PartNumber = 1 }, new PartETag { ETag = up2Response.ETag, PartNumber = 2 }, new PartETag { ETag = up3Response.ETag, PartNumber = 3 } } }; CompleteMultipartUploadResponse compResponse = client.CompleteMultipartUpload(compRequest); #endregion } }