Пример #1
0
 private void addResponse(UploadPartResponse response)
 {
     lock (this.WAIT_FOR_COMPLETION_LOCK)
     {
         this._uploadResponses.Add(response);
     }
 }
        static void UploadPart()
        {
            try
            {
                UploadPartRequest request = new UploadPartRequest()
                {
                    BucketName = bucketName,
                    ObjectKey  = objectName,
                    FilePath   = filePath,
                    PartNumber = 1,
                    PartSize   = partSize,
                    UploadId   = uploadId,
                    Offset     = 100,
                };
                UploadPartResponse response = client.UploadPart(request);

                Console.WriteLine("UploadPart response: {0}", response.StatusCode);
                Console.WriteLine("ETag: {0}", response.ETag);
                etag = response.ETag;
            }
            catch (ObsException ex)
            {
                Console.WriteLine("Exception errorcode: {0}, when upload part.", ex.ErrorCode);
                Console.WriteLine("Exception errormessage: {0}", ex.ErrorMessage);
            }
        }
        public override AmazonWebServiceResponse Unmarshall(XmlUnmarshallerContext context)
        {
            UploadPartResponse response = new UploadPartResponse();

            UnmarshallResult(context, response);

            return(response);
        }
Пример #4
0
 /// <summary>
 /// Helper to assert that uploaded parts have the same checksum as listed parts.
 /// Genearlly only one checksum is expected to be set.
 /// </summary>
 /// <param name="uploadPartResponse">Response after uploading a part</param>
 /// <param name="partDetail">Response for a single part after listing parts</param>
 private static void AssertPartsAreEqual(UploadPartResponse uploadPartResponse, PartDetail partDetail)
 {
     Assert.AreEqual(uploadPartResponse.PartNumber, partDetail.PartNumber);
     Assert.AreEqual(uploadPartResponse.ETag, partDetail.ETag);
     Assert.AreEqual(uploadPartResponse.ChecksumCRC32C, partDetail.ChecksumCRC32C);
     Assert.AreEqual(uploadPartResponse.ChecksumCRC32, partDetail.ChecksumCRC32);
     Assert.AreEqual(uploadPartResponse.ChecksumSHA1, partDetail.ChecksumSHA1);
     Assert.AreEqual(uploadPartResponse.ChecksumSHA256, partDetail.ChecksumSHA256);
 }
Пример #5
0
        private static async Task <PartETag> ProcessChunk(UploadPartRequest upr)
        {
            Console.WriteLine(string.Format("Sending chunk {0} starting at position {1}", upr.PartNumber, upr.FilePosition));

            // upload the chucnk and return a new PartETag when upload completes
            UploadPartResponse response = await AWSS3Factory.getS3Client().UploadPartAsync(upr, new System.Threading.CancellationToken());

            return(new PartETag(response.PartNumber, response.ETag));
        }
Пример #6
0
        /// <summary>
        /// 结束对上传段的异步请求。
        /// </summary>
        /// <param name="ar">异步请求的响应结果。</param>
        /// <returns>上传段的响应结果。</returns>
        public UploadPartResponse EndUploadPart(IAsyncResult ar)
        {
            UploadPartResponse response = this.EndDoRequest <UploadPartRequest, UploadPartResponse>(ar);
            HttpObsAsyncResult result   = ar as HttpObsAsyncResult;
            UploadPartRequest  request  = result.AdditionalState as UploadPartRequest;

            response.PartNumber = request.PartNumber;
            return(response);
        }
Пример #7
0
        public void RegisterSuccess(int partNumber, UploadPartResponse response)
        {
            var part = _parts[partNumber];

            part.Details = new CommitMultipartUploadPartDetails()
            {
                Etag    = response.ETag,
                PartNum = partNumber
            };
            part.Complete = true;
        }
Пример #8
0
    public async Task MultipartWithEncryption(S3Provider provider, string bucket, ISimpleClient client, SseAlgorithm algorithm)
    {
        string objectKey = nameof(MultipartWithEncryption);

        CreateMultipartUploadResponse createResp = await client.CreateMultipartUploadAsync(bucket, objectKey, req => req.SseAlgorithm = algorithm).ConfigureAwait(false);

        Assert.Equal(200, createResp.StatusCode);

        if (provider == S3Provider.AmazonS3)
        {
            Assert.Equal(algorithm, createResp.SseAlgorithm);
        }

        if (algorithm == SseAlgorithm.AwsKms)
        {
            Assert.NotNull(createResp.SseKmsKeyId);
        }

        await using MemoryStream ms = new MemoryStream(new byte[1024 * 1024 * 5]);

        UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, createResp.UploadId, ms).ConfigureAwait(false);

        Assert.Equal(200, uploadResp.StatusCode);

        if (provider == S3Provider.AmazonS3)
        {
            Assert.Equal(algorithm, uploadResp.SseAlgorithm);
        }

        if (algorithm == SseAlgorithm.AwsKms)
        {
            Assert.NotNull(uploadResp.SseKmsKeyId);
        }

        CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);

        Assert.Equal(200, completeResp.StatusCode);

        if (provider == S3Provider.AmazonS3)
        {
            Assert.Equal(algorithm, completeResp.SseAlgorithm);
        }

        if (algorithm == SseAlgorithm.AwsKms)
        {
            Assert.NotNull(completeResp.SseKmsKeyId);
        }
    }
        private static void UnmarshallResult(XmlUnmarshallerContext context, UploadPartResponse response)
        {
            IWebResponseData responseData = context.ResponseData;

            if (responseData.IsHeaderPresent("x-amz-server-side-encryption"))
            {
                response.ServerSideEncryptionMethod = S3Transforms.ToString(responseData.GetHeaderValue("x-amz-server-side-encryption"));
            }
            if (responseData.IsHeaderPresent("ETag"))
            {
                response.ETag = S3Transforms.ToString(responseData.GetHeaderValue("ETag"));
            }


            return;
        }
Пример #10
0
        public async Task MultipartUpload()
        {
            string objectKey = nameof(MultipartUpload);

            CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey).ConfigureAwait(false);

            Assert.True(initResp.IsSuccess);
            Assert.Equal(BucketName, initResp.Bucket);
            Assert.Equal(objectKey, initResp.ObjectKey);
            Assert.NotNull(initResp.UploadId);

            byte[] file = new byte[1024 * 1024 * 10];
            file[0] = (byte)'a';
            file[file.Length - 1] = (byte)'b';

            byte[][] parts = file.Chunk(file.Length / 2).Select(x => x.ToArray()).ToArray();

            UploadPartResponse uploadResp1 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(parts[0])).ConfigureAwait(false);

            Assert.True(uploadResp1.IsSuccess);
            Assert.NotNull(uploadResp1.ETag);

            UploadPartResponse uploadResp2 = await MultipartClient.UploadPartAsync(BucketName, objectKey, 2, initResp.UploadId, new MemoryStream(parts[1])).ConfigureAwait(false);

            Assert.True(uploadResp2.IsSuccess);
            Assert.NotNull(uploadResp2.ETag);

            CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp1, uploadResp2 }).ConfigureAwait(false);

            Assert.True(completeResp.IsSuccess);

            Assert.NotNull(uploadResp2.ETag);

            GetObjectResponse getResp = await ObjectClient.GetObjectAsync(BucketName, objectKey).ConfigureAwait(false);

            //Provoke an 'InvalidArgument' error
            GetObjectResponse gResp1 = await ObjectClient.GetObjectAsync(BucketName, nameof(MultipartUpload), req => req.PartNumber = 0).ConfigureAwait(false);

            Assert.False(gResp1.IsSuccess);
            Assert.IsType <InvalidArgumentError>(gResp1.Error);

            GetObjectResponse gResp2 = await ObjectClient.GetObjectAsync(BucketName, nameof(MultipartUpload), req => req.PartNumber = 1).ConfigureAwait(false);

            Assert.True(gResp2.IsSuccess);
            Assert.Equal(file.Length / 2, (await gResp2.Content.AsDataAsync().ConfigureAwait(false)).Length);
            Assert.Equal(file, await getResp.Content.AsDataAsync().ConfigureAwait(false));
        }
Пример #11
0
        private static void UnmarshallResult(XmlUnmarshallerContext context, UploadPartResponse response)
        {
            IWebResponseData responseData = context.get_ResponseData();

            if (responseData.IsHeaderPresent("x-amz-server-side-encryption"))
            {
                response.ServerSideEncryptionMethod = S3Transforms.ToString(responseData.GetHeaderValue("x-amz-server-side-encryption"));
            }
            if (responseData.IsHeaderPresent("ETag"))
            {
                response.ETag = S3Transforms.ToString(responseData.GetHeaderValue("ETag"));
            }
            if (responseData.IsHeaderPresent(S3Constants.AmzHeaderRequestCharged))
            {
                response.RequestCharged = RequestCharged.FindValue(responseData.GetHeaderValue(S3Constants.AmzHeaderRequestCharged));
            }
        }
        private static async Task UploadPartAsync(TaskCompletionSource <UploadPartResponse> completionSource, IMultipartOperations operations, string bucketName, string objectKey, byte[] data, int length, int partNumber, string uploadId, SemaphoreSlim semaphore, CancellationToken token)
        {
            try
            {
                using (MemoryStream ms = new MemoryStream(data, 0, length))
                {
                    UploadPartResponse result = await operations.UploadPartAsync(new UploadPartRequest(bucketName, objectKey, partNumber, uploadId, ms), token)
                                                .ConfigureAwait(false);

                    completionSource.SetResult(result);
                }
            }
            catch (Exception ex)
            {
                completionSource.SetException(ex);
            }
            finally
            {
                semaphore.Release();
            }
        }
Пример #13
0
    public async Task MultipartSinglePart(S3Provider provider, string bucket, ISimpleClient client)
    {
        string objectKey = nameof(MultipartSinglePart);

        CreateMultipartUploadResponse createResp = await client.CreateMultipartUploadAsync(bucket, objectKey).ConfigureAwait(false);

        Assert.Equal(200, createResp.StatusCode);
        Assert.Equal(bucket, createResp.BucketName);
        Assert.Equal(objectKey, createResp.ObjectKey);
        Assert.NotNull(createResp.UploadId);

        if (provider == S3Provider.AmazonS3)
        {
            //Test lifecycle expiration
            Assert.Equal(DateTime.UtcNow.AddDays(2).Date, createResp.AbortsOn !.Value.UtcDateTime.Date);
            Assert.Equal("ExpireAll", createResp.AbortRuleId);
        }

        byte[] file = new byte[1024 * 1024 * 5];
        file[0] = (byte)'a';

        await using MemoryStream ms = new MemoryStream(file);

        UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, createResp.UploadId, ms).ConfigureAwait(false);

        Assert.Equal(200, uploadResp.StatusCode);
        Assert.Equal("\"10f74ef02085310ccd1f87150b83e537\"", uploadResp.ETag);

        CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, createResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);

        Assert.Equal(200, completeResp.StatusCode);
        Assert.Equal("\"bd74e21dfa8678d127240f76e518e9c2-1\"", completeResp.ETag);

        if (provider == S3Provider.AmazonS3)
        {
            //Test lifecycle expiration
            Assert.Equal(DateTime.UtcNow.AddDays(2).Date, completeResp.LifeCycleExpiresOn !.Value.UtcDateTime.Date);
            Assert.Equal("ExpireAll", completeResp.LifeCycleRuleId);
        }
    }
Пример #14
0
        public async Task MultipartLockMode()
        {
            string objectKey = nameof(MultipartLockMode);

            CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req =>
            {
                req.LockMode        = LockMode.Governance;
                req.LockRetainUntil = DateTimeOffset.UtcNow.AddMinutes(5);
            }).ConfigureAwait(false);

            Assert.True(initResp.IsSuccess);

            byte[] file = new byte[1024 * 1024 * 5];

            UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(file), req => req.ContentMd5 = CryptoHelper.Md5Hash(file)).ConfigureAwait(false);

            Assert.True(uploadResp.IsSuccess);

            CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);

            Assert.True(completeResp.IsSuccess);
        }
Пример #15
0
        public async Task MultipartRequestPayer()
        {
            string objectKey = nameof(MultipartRequestPayer);

            CreateMultipartUploadResponse initResp = await MultipartClient.CreateMultipartUploadAsync(BucketName, objectKey, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);

            Assert.True(initResp.RequestCharged);

            byte[] file = new byte[1024 * 1024 * 5];

            UploadPartResponse uploadResp = await MultipartClient.UploadPartAsync(BucketName, objectKey, 1, initResp.UploadId, new MemoryStream(file), req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);

            Assert.True(uploadResp.RequestCharged);

            ListPartsResponse listResp = await MultipartClient.ListPartsAsync(BucketName, objectKey, initResp.UploadId, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);

            Assert.True(listResp.RequestCharged);

            CompleteMultipartUploadResponse completeResp = await MultipartClient.CompleteMultipartUploadAsync(BucketName, objectKey, initResp.UploadId, new[] { uploadResp }, req => req.RequestPayer = Payer.Requester).ConfigureAwait(false);

            Assert.True(completeResp.RequestCharged);
        }
Пример #16
0
        private static void UnmarshallResult(XmlUnmarshallerContext context, UploadPartResponse response)
        {
            IWebResponseData responseData = context.ResponseData;

            if (responseData.IsHeaderPresent("x-amz-server-side-encryption"))
            {
                response.ServerSideEncryptionMethod = S3Transforms.ToString(responseData.GetHeaderValue("x-amz-server-side-encryption"));
            }
            if (responseData.IsHeaderPresent("ETag"))
            {
                response.ETag = S3Transforms.ToString(responseData.GetHeaderValue("ETag"));
            }
            if (responseData.IsHeaderPresent(S3Constants.AmzHeaderRequestCharged))
            {
                response.RequestCharged = RequestCharged.FindValue(responseData.GetHeaderValue(S3Constants.AmzHeaderRequestCharged));
            }
            if (responseData.IsHeaderPresent(S3Constants.AmzHeaderBucketKeyEnabled))
            {
                response.BucketKeyEnabled = S3Transforms.ToBool(responseData.GetHeaderValue(S3Constants.AmzHeaderBucketKeyEnabled));
            }
            if (context.ResponseData.IsHeaderPresent("x-amz-checksum-crc32"))
            {
                response.ChecksumCRC32 = S3Transforms.ToString(context.ResponseData.GetHeaderValue("x-amz-checksum-crc32"));
            }
            if (context.ResponseData.IsHeaderPresent("x-amz-checksum-crc32c"))
            {
                response.ChecksumCRC32C = S3Transforms.ToString(context.ResponseData.GetHeaderValue("x-amz-checksum-crc32c"));
            }
            if (context.ResponseData.IsHeaderPresent("x-amz-checksum-sha1"))
            {
                response.ChecksumSHA1 = S3Transforms.ToString(context.ResponseData.GetHeaderValue("x-amz-checksum-sha1"));
            }
            if (context.ResponseData.IsHeaderPresent("x-amz-checksum-sha256"))
            {
                response.ChecksumSHA256 = S3Transforms.ToString(context.ResponseData.GetHeaderValue("x-amz-checksum-sha256"));
            }
            return;
        }
Пример #17
0
        /// <summary>
        /// Upload a part.
        /// </summary>
        /// <param name="request">Parameters in a request for uploading a part</param>
        /// <returns>Response to a part upload request</returns>
        public UploadPartResponse UploadPart(UploadPartRequest request)
        {
            UploadPartResponse response = this.DoRequest <UploadPartRequest, UploadPartResponse>(request, delegate()
            {
                if (request.ObjectKey == null)
                {
                    throw new ObsException(Constants.InvalidObjectKeyMessage, ErrorType.Sender, Constants.InvalidObjectKey, "");
                }
                if (string.IsNullOrEmpty(request.UploadId))
                {
                    throw new ObsException(Constants.InvalidUploadIdMessage, ErrorType.Sender, Constants.InvalidUploadId, "");
                }

                if (request.PartNumber <= 0)
                {
                    throw new ObsException(Constants.InvalidPartNumberMessage, ErrorType.Sender, Constants.InvalidPartNumber, "");
                }
            });

            response.PartNumber = request.PartNumber;

            return(response);
        }
Пример #18
0
    public async Task MultipartLockMode(S3Provider _, string bucket, ISimpleClient client)
    {
        string objectKey = nameof(MultipartLockMode);

        CreateMultipartUploadResponse initResp = await client.CreateMultipartUploadAsync(bucket, objectKey, r =>
        {
            r.LockMode        = LockMode.Governance;
            r.LockRetainUntil = DateTimeOffset.UtcNow.AddMinutes(5);
        }).ConfigureAwait(false);

        Assert.Equal(200, initResp.StatusCode);

        byte[] file = new byte[1024 * 1024 * 5];
        await using MemoryStream ms = new MemoryStream(file);

        UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, initResp.UploadId, ms, r => r.ContentMd5 = CryptoHelper.Md5Hash(file)).ConfigureAwait(false);

        Assert.Equal(200, uploadResp.StatusCode);

        CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, new[] { uploadResp }).ConfigureAwait(false);

        Assert.Equal(200, completeResp.StatusCode);
    }
Пример #19
0
            public void TestOrdinary()
            {
                String             uploadId = this.client.InitiateMultipartUpload(this.bucketName, "test").UploadId;
                UploadPartResponse response = this.client.UploadPart(new UploadPartRequest()
                {
                    BucketName  = this.bucketName,
                    Key         = "test",
                    UploadId    = uploadId,
                    PartNumber  = 1,
                    PartSize    = 4,
                    InputStream = new MemoryStream(Encoding.Default.GetBytes("data"))
                });

                Assert.AreEqual(response.PartNumber, 1);
                Assert.IsNotNull(response.ETag);
                List <PartSummary> parts = this.client.ListParts(this.bucketName, "test", uploadId).Parts;

                Assert.AreEqual(parts.Count, 1);
                PartSummary part = parts[0];

                Assert.IsNotNull(part);
                Assert.AreEqual(part.ETag, response.ETag);
                Assert.AreEqual(part.Size, 4L);
            }
Пример #20
0
        public void BasicUploadLargeFile()
        {
            byte[] part1 = new byte[0x500000]; // 5MB
            byte[] part2 = new byte[0x500000]; // 5MB
            byte[] part3 = new byte[0x10000];  // 64K

            FillByteArray(part1);
            FillByteArray(part2);
            FillByteArray(part3);

            string[] sha1Array = new string[3];

            using (SHA1Cng sha1 = new SHA1Cng())
            {
                sha1Array[0] = BitConverter.ToString(sha1.ComputeHash(part1)).Replace("-", "").ToLowerInvariant();
                sha1Array[1] = BitConverter.ToString(sha1.ComputeHash(part2)).Replace("-", "").ToLowerInvariant();
                sha1Array[2] = BitConverter.ToString(sha1.ComputeHash(part3)).Replace("-", "").ToLowerInvariant();
            }

            string filename = Guid.NewGuid().ToString("N") + ".txt";

            using (BackblazeB2Client client = CreateClient())
            {
                StartLargeFileResponse startLargeFileResponse =
                    client.StartLargeUpload(accountInfo.BucketId, filename).Result;

                GetUploadPartUrlResponse getPartUploadResponse =
                    client.GetUploadPartUrl(startLargeFileResponse.FileId).Result;

                using (MemoryStream memoryStream = new MemoryStream(part1))
                {
                    UploadPartResponse partUploadResponse =
                        client.UploadPart(
                            getPartUploadResponse.UploadUrl,
                            getPartUploadResponse.AuthorizationToken,
                            1,
                            sha1Array[0],
                            part1.Length,
                            memoryStream)
                        .Result;

                    Assert.AreEqual(5242880, int.Parse(partUploadResponse.ContentLength));
                    Assert.AreEqual(1, Int32.Parse(partUploadResponse.PartNumber));
                    Assert.AreEqual(sha1Array[0], partUploadResponse.ContentSha1);
                }

                using (MemoryStream memoryStream = new MemoryStream(part2))
                {
                    UploadPartResponse partUploadResponse =
                        client.UploadPart(
                            getPartUploadResponse.UploadUrl,
                            getPartUploadResponse.AuthorizationToken,
                            2,
                            sha1Array[1],
                            part2.Length,
                            memoryStream)
                        .Result;

                    Assert.AreEqual(5242880, int.Parse(partUploadResponse.ContentLength));
                    Assert.AreEqual(2, int.Parse(partUploadResponse.PartNumber));
                    Assert.AreEqual(sha1Array[1], partUploadResponse.ContentSha1);
                }

                using (MemoryStream memoryStream = new MemoryStream(part3))
                {
                    UploadPartResponse partUploadResponse =
                        client.UploadPart(
                            getPartUploadResponse.UploadUrl,
                            getPartUploadResponse.AuthorizationToken,
                            3,
                            sha1Array[2],
                            part3.Length,
                            memoryStream)
                        .Result;

                    Assert.AreEqual(65536, int.Parse(partUploadResponse.ContentLength));
                    Assert.AreEqual(3, int.Parse(partUploadResponse.PartNumber));
                    Assert.AreEqual(sha1Array[2], partUploadResponse.ContentSha1);
                }

                FinishLargeFileResponse finishLargeFileResponse =
                    client.FinishLargeFile(
                        startLargeFileResponse.FileId,
                        sha1Array)
                    .Result;

                Assert.AreEqual(accountInfo.BucketId, finishLargeFileResponse.BucketId);
                Assert.AreEqual(10551296, finishLargeFileResponse.ContentLength);
            }
        }
        /// <summary>
        /// Saves a NEW resource. Only saves if resource with the MD5Sum has not previously been added.
        /// </summary>
        /// <param name="referenceRepository"></param>
        /// <param name="fileStream"></param>
        /// <param name="originalName"></param>
        /// <returns>the AWS Upload ID</returns>
        public async Task <string> UploadPartial(ResourceModel.User owner,
                                                 string md5OfResource,
                                                 string uploadIdentifier,
                                                 Stream fileStream,
                                                 int partNumber,
                                                 int numberOfParts)
        {
            ResourceModel.DigitalResource resource = null;
            //User user = reference

            try
            {
                //bool exists = Exists(md5Sum);
                DigitalResource existingResource = Get(md5OfResource);
                if (existingResource == null)
                {
                    //fileStream.Position = 0;
                    //TransferUtilityUploadRequest tuu = new TransferUtilityUploadRequest
                    //{
                    //    InputStream = fileStream,
                    //    BucketName = "piccoli",
                    //    Key = "belvedere"
                    //};
                    //tr.UploadAsync(tuu);

                    //upload the file
                    IAmazonS3 s3Client = new AmazonS3Client();
                    //ListBucketsResponse response = s3Client.ListBuckets();

                    //https://docs.aws.amazon.com/sdkfornet/v3/apidocs/items/S3/TUploadPartResponse.html
                    //https://docs.aws.amazon.com/AmazonS3/latest/dev/LLuploadFileDotNet.html
                    string uploadID = uploadIdentifier;
                    if (partNumber == 1)
                    {
                        InitiateMultipartUploadRequest initiateRequest = new InitiateMultipartUploadRequest
                        {
                            BucketName = "piccoli",
                            Key        = md5OfResource,
                        };

                        // Initiate the upload.
                        InitiateMultipartUploadResponse initResponse =
                            await s3Client.InitiateMultipartUploadAsync(initiateRequest);

                        uploadID = initResponse.UploadId;
                    }
                    // Upload part X
                    UploadPartRequest uploadRequest = new UploadPartRequest
                    {
                        BucketName  = "piccoli",
                        Key         = md5OfResource,
                        UploadId    = uploadID,
                        PartNumber  = partNumber,
                        PartSize    = fileStream.Length, //5242880,//5MB ?
                        InputStream = fileStream
                    };
                    UploadPartResponse response = s3Client.UploadPart(uploadRequest);
                    return(uploadID);
                }
                return("resource already exists");
            }
            catch (AmazonS3Exception ex)
            {
                throw ex;
            }
        }
Пример #22
0
    public async Task <CompleteMultipartUploadResponse> MultipartUploadAsync(CreateMultipartUploadRequest req, Stream data, int partSize = 16777216, int numParallelParts = 4, Action <UploadPartResponse>?onPartResponse = null, CancellationToken token = default)
    {
        Validator.RequireNotNull(req, nameof(req));
        Validator.RequireNotNull(data, nameof(data));

        foreach (IRequestWrapper wrapper in _requestWrappers)
        {
            if (wrapper.IsSupported(req))
            {
                data = wrapper.Wrap(data, req);
            }
        }

        string bucket    = req.BucketName;
        string objectKey = req.ObjectKey;

        byte[]? encryptionKey = null;

        try
        {
            if (req.SseCustomerKey != null)
            {
                encryptionKey = new byte[req.SseCustomerKey.Length];
                Array.Copy(req.SseCustomerKey, 0, encryptionKey, 0, encryptionKey.Length);
            }

            CreateMultipartUploadResponse initResp = await _multipartOperations.CreateMultipartUploadAsync(req, token).ConfigureAwait(false);

            if (token.IsCancellationRequested)
            {
                return new CompleteMultipartUploadResponse {
                           BucketName = bucket, ObjectKey = objectKey
                }
            }
            ;

            if (!initResp.IsSuccess)
            {
                throw new S3RequestException(initResp, "CreateMultipartUploadRequest was unsuccessful");
            }

            IEnumerable <ArraySegment <byte> > chunks = ReadChunks(data, partSize);

            int partNumber = 0;

            IEnumerable <UploadPartResponse> responses = await ParallelHelper.ExecuteAsync(chunks, async (bytes, innerToken) =>
            {
                Interlocked.Increment(ref partNumber);

                using (MemoryStream ms = new MemoryStream(bytes.Array !, 0, bytes.Count))
                {
                    UploadPartResponse resp = await _multipartClient.UploadPartAsync(bucket, objectKey, partNumber, initResp.UploadId, ms, uploadPart =>
                    {
                        uploadPart.SseCustomerAlgorithm = req.SseCustomerAlgorithm;
                        uploadPart.SseCustomerKey       = encryptionKey;
                        uploadPart.SseCustomerKeyMd5    = req.SseCustomerKeyMd5;
                    }, innerToken).ConfigureAwait(false);
                    onPartResponse?.Invoke(resp);
                    return(resp);
                }
            }, numParallelParts, token);

            CompleteMultipartUploadResponse completeResp = await _multipartClient.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, responses.OrderBy(x => x.PartNumber), null, token).ConfigureAwait(false);

            return(completeResp);
        }
        finally
        {
            if (encryptionKey != null)
            {
                Array.Clear(encryptionKey, 0, encryptionKey.Length);
            }
        }
    }
        public static void MultipartEncryptionTestAPM(IAmazonS3 s3EncryptionClient, IAmazonS3 s3DecryptionClient, string bucketName)
        {
            var guid              = Guid.NewGuid();
            var filePath          = Path.Combine(Path.GetTempPath(), $"multi-{guid}.txt");
            var retrievedFilepath = Path.Combine(Path.GetTempPath(), $"retrieved-{guid}.txt");
            var totalSize         = MegaByteSize * 15;

            UtilityMethods.GenerateFile(filePath, totalSize);
            string key = $"key-{guid}";

            Stream inputStream = File.OpenRead(filePath);

            try
            {
                InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
                {
                    BucketName   = bucketName,
                    Key          = key,
                    StorageClass = S3StorageClass.OneZoneInfrequentAccess,
                    ContentType  = "text/html"
                };

                InitiateMultipartUploadResponse initResponse = null;
                if (IsKMSEncryptionClient(s3EncryptionClient))
                {
                    initResponse = s3EncryptionClient.InitiateMultipartUpload(initRequest);
                }
                else
                {
                    initResponse = s3EncryptionClient.EndInitiateMultipartUpload(
                        s3EncryptionClient.BeginInitiateMultipartUpload(initRequest, null, null));
                }

                // Upload part 1
                UploadPartRequest uploadRequest = new UploadPartRequest()
                {
                    BucketName  = bucketName,
                    Key         = key,
                    UploadId    = initResponse.UploadId,
                    PartNumber  = 1,
                    PartSize    = 5 * MegaByteSize,
                    InputStream = inputStream,
                };

                UploadPartResponse up1Response = s3EncryptionClient.EndUploadPart(
                    s3EncryptionClient.BeginUploadPart(uploadRequest, null, null));

                // Upload part 2
                uploadRequest = new UploadPartRequest()
                {
                    BucketName  = bucketName,
                    Key         = key,
                    UploadId    = initResponse.UploadId,
                    PartNumber  = 2,
                    PartSize    = 5 * MegaByteSize,
                    InputStream = inputStream,
                };

                UploadPartResponse up2Response = s3EncryptionClient.EndUploadPart(
                    s3EncryptionClient.BeginUploadPart(uploadRequest, null, null));

                // Upload part 3
                uploadRequest = new UploadPartRequest()
                {
                    BucketName  = bucketName,
                    Key         = key,
                    UploadId    = initResponse.UploadId,
                    PartNumber  = 3,
                    InputStream = inputStream,
                    IsLastPart  = true
                };

                UploadPartResponse up3Response = s3EncryptionClient.EndUploadPart(
                    s3EncryptionClient.BeginUploadPart(uploadRequest, null, null));

                ListPartsRequest listPartRequest = new ListPartsRequest()
                {
                    BucketName = bucketName,
                    Key        = key,
                    UploadId   = initResponse.UploadId
                };

                ListPartsResponse listPartResponse = s3EncryptionClient.EndListParts(
                    s3EncryptionClient.BeginListParts(listPartRequest, null, null));
                Assert.Equal(3, listPartResponse.Parts.Count);
                Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber);
                Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag);
                Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber);
                Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag);
                Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber);
                Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag);

                listPartRequest.MaxParts = 1;
                listPartResponse         = s3EncryptionClient.EndListParts(
                    s3EncryptionClient.BeginListParts(listPartRequest, null, null));
                Assert.Equal(1, listPartResponse.Parts.Count);

                // Complete the response
                CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
                {
                    BucketName = bucketName,
                    Key        = key,
                    UploadId   = initResponse.UploadId
                };
                compRequest.AddPartETags(up1Response, up2Response, up3Response);

                CompleteMultipartUploadResponse compResponse = s3EncryptionClient.EndCompleteMultipartUpload(
                    s3EncryptionClient.BeginCompleteMultipartUpload(compRequest, null, null));

                Assert.Equal(bucketName, compResponse.BucketName);
                Assert.NotNull(compResponse.ETag);
                Assert.Equal(key, compResponse.Key);
                Assert.NotNull(compResponse.Location);

                // Get the file back from S3 and make sure it is still the same.
                GetObjectRequest getRequest = new GetObjectRequest()
                {
                    BucketName = bucketName,
                    Key        = key
                };

                GetObjectResponse getResponse = null;
                if (IsKMSEncryptionClient(s3EncryptionClient))
                {
                    getResponse = s3DecryptionClient.GetObject(getRequest);
                }
                else
                {
                    getResponse = s3DecryptionClient.EndGetObject(
                        s3DecryptionClient.BeginGetObject(getRequest, null, null));
                }

                getResponse.WriteResponseStreamToFile(retrievedFilepath);

                UtilityMethods.CompareFiles(filePath, retrievedFilepath);

                GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest()
                {
                    BucketName = bucketName,
                    Key        = key
                };
                GetObjectMetadataResponse metaDataResponse = s3DecryptionClient.EndGetObjectMetadata(
                    s3DecryptionClient.BeginGetObjectMetadata(metaDataRequest, null, null));
                Assert.Equal("text/html", metaDataResponse.Headers.ContentType);
            }
            finally
            {
                inputStream.Close();
                if (File.Exists(filePath))
                {
                    File.Delete(filePath);
                }
                if (File.Exists(retrievedFilepath))
                {
                    File.Delete(retrievedFilepath);
                }
            }
        }
Пример #24
0
        public static void Main(string[] args)
        {
            // create the AWS S3 client
            AmazonS3Client s3 = AWSS3Factory.getS3Client();

            // retrieve the object key/value from user
            Console.Write("Enter the object key: ");
            string key = Console.ReadLine();

            Console.Write("Enter the file location: ");
            string filePath = Console.ReadLine();

            // grab the start time of upload
            DateTime startDate = DateTime.Now;

            // part size for chunking in multi-part
            long partSize = 1024 * 1024 * 2; // 2 MB

            // list of upload part response objects for each part that is uploaded
            List <PartETag> partETags = new List <PartETag>();

            // Step 1: Initialize
            InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
            {
                BucketName = AWSS3Factory.S3_BUCKET,
                Key        = key,
            };

            InitiateMultipartUploadResponse initResponse = s3.InitiateMultipartUpload(initRequest);

            // get the file and file length
            long contentLength = new FileInfo(filePath).Length;

            Console.WriteLine(string.Format("Starting multi-part upload for object {0}/{1} with file path {2} and size {3} in {4} MB size chunks",
                                            AWSS3Factory.S3_BUCKET, key, filePath, Convert.ToString(contentLength), partSize / 1024 / 1024));

            try
            {
                // Step 2: Upload parts
                long filePosition = 0;
                for (int i = 1; filePosition < contentLength; i++)
                {
                    // get the size of the chunk. Note - the last part can be less than the chunk size
                    partSize = Math.Min(partSize, (contentLength - filePosition));

                    Console.WriteLine(string.Format("Sending chunk {0} starting at position {1}", i, filePosition));

                    // create request to upload  a part
                    UploadPartRequest uploadRequest = new UploadPartRequest()
                    {
                        BucketName   = AWSS3Factory.S3_BUCKET,
                        Key          = key,
                        UploadId     = initResponse.UploadId,
                        PartNumber   = i,
                        FilePosition = filePosition,
                        FilePath     = filePath,
                        PartSize     = partSize
                    };

                    UploadPartResponse partResponse = s3.UploadPart(uploadRequest);
                    PartETag           eTagPart     = new PartETag(partResponse.PartNumber, partResponse.ETag);
                    partETags.Add(eTagPart);

                    filePosition = filePosition += partSize;
                }

                // Step 3: complete
                Console.WriteLine("Waiting for completion of multi-part upload");
                CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
                {
                    BucketName = AWSS3Factory.S3_BUCKET,
                    Key        = key,
                    UploadId   = initResponse.UploadId,
                    PartETags  = partETags
                };

                s3.CompleteMultipartUpload(compRequest);
            }
            catch (Exception e)
            {
                s3.AbortMultipartUpload(new AbortMultipartUploadRequest()
                {
                    BucketName = AWSS3Factory.S3_BUCKET,
                    Key        = key,
                    UploadId   = initResponse.UploadId
                });
            }

            // grab the end time of upload
            DateTime endDate = DateTime.Now;

            Console.WriteLine(string.Format("Completed multi-part upload for object {0}/{1} with file path: {2}", AWSS3Factory.S3_BUCKET, key, filePath));
            Console.WriteLine(string.Format("Process took: {0} seconds.", (endDate - startDate).TotalSeconds.ToString()));
            Console.ReadLine();
        }
Пример #25
0
        private async Task <bool> CompressFile(ILambdaContext context, string inputBucketName, string inputKey)
        {
            //Check if file is already zipped
            if (inputKey.EndsWith(_ZippedFileExtension))
            {
                logger.LogLineWithId($"File already appears to be zipped, file extension is: {_ZippedFileExtension}. Function complete");
                return(false);
            }

            //Retrieve input stream
            var obj = await S3Client.GetObjectAsync(new GetObjectRequest { BucketName = inputBucketName, Key = inputKey });

            var inputStream = obj.ResponseStream;

            logger.LogLineWithId("Object response stream obtained");

            //Initiate multipart upload
            string outputBucketName = !string.IsNullOrEmpty(this.variables.OutputBucket) ? this.variables.OutputBucket : inputBucketName;
            string outputKey        = this.GetOutputKey(inputKey);
            InitiateMultipartUploadRequest uploadRequest = new InitiateMultipartUploadRequest {
                BucketName = outputBucketName, Key = outputKey
            };
            InitiateMultipartUploadResponse uploadResponse = await S3Client.InitiateMultipartUploadAsync(uploadRequest);

            var uploadPartResponses = new List <PartETag>();

            logger.LogLineWithId($"Upload initiated. Output object: {outputBucketName}/{outputKey}");

            var partNumber        = 1;
            var filePartReadSize  = this.variables.FilePartReadSize;
            var minimumUploadSize = this.variables.MinimumUploadSize;
            var streamEnded       = false;

            //Process input file
            while (!streamEnded)
            {
                using (MemoryStream memoryStream = new MemoryStream())
                {
                    //Read chunks of the input file, compressing as we go, until we hit the minimum upload size or the end of the stream
                    using (GZipStream zipStream = new GZipStream(memoryStream, CompressionLevel.Fastest, true))
                    {
                        while (memoryStream.Length < minimumUploadSize)
                        {
                            //Copy a set number of bytes from the input stream to the compressed memory stream
                            var bytesCopied = CopyBytes(filePartReadSize, inputStream, zipStream);
                            if (bytesCopied == 0)
                            {
                                streamEnded = true;
                                break;
                            }
                        }
                    }
                    logger.LogLineWithId($"Part {partNumber} read and compressed");

                    //Check to make sure that we have actually read something in before proceeding
                    if (memoryStream.Length == 0)
                    {
                        break;
                    }

                    //Upload memory stream to S3
                    memoryStream.Position = 0;
                    UploadPartRequest  uploadPartRequest  = GetUploadPartRequest(outputBucketName, outputKey, uploadResponse.UploadId, partNumber, streamEnded, memoryStream);
                    UploadPartResponse uploadPartResponse = await S3Client.UploadPartAsync(uploadPartRequest);

                    uploadPartResponses.Add(new PartETag {
                        ETag = uploadPartResponse.ETag, PartNumber = partNumber
                    });
                    logger.LogLineWithId($"Part {partNumber} uploaded.");

                    partNumber++;
                }

                //Run the garbage collector so that Lambda doesn't run out of memory for large files
                GC.Collect();
                GC.WaitForPendingFinalizers();
            }

            //Complete multipart upload request
            logger.LogLineWithId($"Completing upload request");
            CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest
            {
                BucketName = outputBucketName,
                Key        = outputKey,
                UploadId   = uploadResponse.UploadId,
                PartETags  = uploadPartResponses
            };
            CompleteMultipartUploadResponse compResponse = await S3Client.CompleteMultipartUploadAsync(compRequest);

            logger.LogLineWithId($"Upload request completed");

            inputStream.Dispose();
            return(true);
        }
Пример #26
0
        public async Task ManualMultiplePartUpload()
        {
            string resourceName = nameof(ManualMultiplePartUpload);

            byte[] key    = { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 };
            byte[] keyMd5 = CryptoHelper.Md5Hash(key);

            InitiateMultipartUploadResponse initResp = await ObjectClient.InitiateMultipartUploadAsync(BucketName, resourceName, request =>
            {
                request.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
                request.SseCustomerKey       = key;
                request.SseCustomerKeyMd5    = keyMd5;
            }).ConfigureAwait(false);

            Assert.True(initResp.IsSuccess);
            Assert.Equal(BucketName, initResp.Bucket);
            Assert.Equal(resourceName, initResp.Key);
            Assert.NotNull(initResp.UploadId);

            byte[] file = new byte[1024 * 1024 * 10];
            file[0] = (byte)'a';
            file[file.Length - 1] = (byte)'b';

            byte[][] parts = file.Chunk(file.Length / 2).Select(x => x.ToArray()).ToArray();

            UploadPartResponse uploadResp1 = await ObjectClient.UploadPartAsync(BucketName, resourceName, 1, initResp.UploadId, new MemoryStream(parts[0]), request =>
            {
                request.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
                request.SseCustomerKey       = key;
                request.SseCustomerKeyMd5    = keyMd5;
            }).ConfigureAwait(false);

            Assert.Equal(SseCustomerAlgorithm.Aes256, uploadResp1.SseCustomerAlgorithm);
            Assert.Equal(keyMd5, uploadResp1.SseCustomerKeyMd5);

            Assert.True(uploadResp1.IsSuccess);
            Assert.NotNull(uploadResp1.ETag);

            UploadPartResponse uploadResp2 = await ObjectClient.UploadPartAsync(BucketName, resourceName, 2, initResp.UploadId, new MemoryStream(parts[1]), request =>
            {
                request.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
                request.SseCustomerKey       = key;
                request.SseCustomerKeyMd5    = keyMd5;
            }).ConfigureAwait(false);

            Assert.True(uploadResp2.IsSuccess);
            Assert.NotNull(uploadResp2.ETag);

            CompleteMultipartUploadResponse completeResp = await ObjectClient.CompleteMultipartUploadAsync(BucketName, resourceName, initResp.UploadId, new[] { uploadResp1, uploadResp2 }).ConfigureAwait(false);

            Assert.True(completeResp.IsSuccess);
            Assert.NotNull(uploadResp2.ETag);

            GetObjectResponse getResp = await ObjectClient.GetObjectAsync(BucketName, resourceName, request =>
            {
                request.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
                request.SseCustomerKey       = key;
                request.SseCustomerKeyMd5    = keyMd5;
            }).ConfigureAwait(false);

            //Provoke an 'InvalidArgument' error
            GetObjectResponse gResp1 = await ObjectClient.GetObjectAsync(BucketName, nameof(ManualMultiplePartUpload), request =>
            {
                request.PartNumber           = 0;
                request.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
                request.SseCustomerKey       = key;
                request.SseCustomerKeyMd5    = keyMd5;
            }).ConfigureAwait(false);

            Assert.False(gResp1.IsSuccess);
            Assert.IsType <InvalidArgumentError>(gResp1.Error);

            GetObjectResponse gResp2 = await ObjectClient.GetObjectAsync(BucketName, nameof(ManualMultiplePartUpload), request =>
            {
                request.PartNumber           = 1;
                request.SseCustomerAlgorithm = SseCustomerAlgorithm.Aes256;
                request.SseCustomerKey       = key;
                request.SseCustomerKeyMd5    = keyMd5;
            }).ConfigureAwait(false);

            Assert.True(gResp2.IsSuccess);
            Assert.Equal(file.Length / 2, gResp2.Content.AsStream().Length);
            Assert.Equal(file, await getResp.Content.AsDataAsync().ConfigureAwait(false));
        }
Пример #27
0
        public async IAsyncEnumerable <UploadPartResponse> MultipartUploadAsync(InitiateMultipartUploadRequest req, Stream data, int partSize = 16777216, int numParallelParts = 4, [EnumeratorCancellation] CancellationToken token = default)
        {
            Validator.RequireNotNull(req);
            Validator.RequireNotNull(data);

            if (RequestWrappers != null)
            {
                foreach (IRequestWrapper wrapper in RequestWrappers)
                {
                    if (wrapper.IsSupported(req))
                    {
                        data = wrapper.Wrap(data, req);
                    }
                }
            }

            string bucket   = req.BucketName;
            string resource = req.Resource;

            InitiateMultipartUploadResponse initResp = await InitiateMultipartUploadAsync(req, token).ConfigureAwait(false);

            if (token.IsCancellationRequested)
            {
                yield break;
            }

            if (!initResp.IsSuccess)
            {
                throw new RequestException(initResp.StatusCode, "InitiateMultipartUploadRequest was unsuccessful");
            }

            Queue <Task <UploadPartResponse> > uploads = new Queue <Task <UploadPartResponse> >();

            using (SemaphoreSlim semaphore = new SemaphoreSlim(numParallelParts))
            {
                long offset = 0;

                for (int i = 1; offset < data.Length; i++)
                {
                    await semaphore.WaitAsync(token).ConfigureAwait(false);

                    if (token.IsCancellationRequested)
                    {
                        break;
                    }

                    long remaining  = data.Length - offset;
                    long bufferSize = Math.Min(remaining, partSize);

                    byte[] partData = new byte[bufferSize];
                    await data.ReadAsync(partData, 0, partData.Length, token).ConfigureAwait(false);

                    uploads.Enqueue(UploadPartAsync(bucket, resource, partData, i, initResp.UploadId, semaphore, token));

                    offset += partSize;
                }

                Queue <UploadPartResponse> responses = new Queue <UploadPartResponse>(uploads.Count);

                while (uploads.TryDequeue(out Task <UploadPartResponse> task))
                {
                    if (token.IsCancellationRequested)
                    {
                        yield break;
                    }

                    UploadPartResponse response = await task.ConfigureAwait(false);

                    responses.Enqueue(response);

                    yield return(response);
                }

                CompleteMultipartUploadRequest  completeReq  = new CompleteMultipartUploadRequest(bucket, resource, initResp.UploadId, responses);
                CompleteMultipartUploadResponse completeResp = await CompleteMultipartUploadAsync(completeReq, token).ConfigureAwait(false);

                if (!completeResp.IsSuccess)
                {
                    throw new RequestException(completeResp.StatusCode, "CompleteMultipartUploadRequest was unsuccessful");
                }
            }
        }
Пример #28
0
        public void MultipartEncryptionTestInstructionFile()
        {
            string filePath          = @"C:\temp\Upload15MegFileIn3PartsViaStream.txt";
            string retrievedFilepath = @"C:\temp\Upload15MegFileIn3PartsViaStreamRetreived.txt";
            int    MEG_SIZE          = (int)Math.Pow(2, 20) + 4001;
            long   totalSize         = (long)(15 * MEG_SIZE);

            UtilityMethods.GenerateFile(filePath, totalSize);

            string key = "MultipartEncryptionTestInstrcutionFile" + random.Next();

            s3EncryptionClientFileMode.PutBucket(new PutBucketRequest()
            {
                BucketName = bucketName
            });

            Stream inputStream = File.OpenRead(filePath);

            try
            {
                InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
                {
                    BucketName   = bucketName,
                    Key          = key,
                    StorageClass = S3StorageClass.ReducedRedundancy,
                    ContentType  = "text/html",
                    CannedACL    = S3CannedACL.PublicRead
                };

                InitiateMultipartUploadResponse initResponse = s3EncryptionClientFileMode.InitiateMultipartUpload(initRequest);

                // Upload part 1
                UploadPartRequest uploadRequest = new UploadPartRequest()
                {
                    BucketName  = bucketName,
                    Key         = key,
                    UploadId    = initResponse.UploadId,
                    PartNumber  = 1,
                    PartSize    = 5 * MEG_SIZE,
                    InputStream = inputStream
                };

                UploadPartResponse up1Response = s3EncryptionClientFileMode.UploadPart(uploadRequest);

                // Upload part 2
                uploadRequest = new UploadPartRequest()
                {
                    BucketName  = bucketName,
                    Key         = key,
                    UploadId    = initResponse.UploadId,
                    PartNumber  = 2,
                    PartSize    = 5 * MEG_SIZE + 4001,
                    InputStream = inputStream
                };

                UploadPartResponse up2Response = s3EncryptionClientFileMode.UploadPart(uploadRequest);

                // Upload part 3
                uploadRequest = new UploadPartRequest()
                {
                    BucketName  = bucketName,
                    Key         = key,
                    UploadId    = initResponse.UploadId,
                    PartNumber  = 3,
                    InputStream = inputStream,
                    IsLastPart  = true
                };

                //uploadRequest.setLastPart();
                UploadPartResponse up3Response = s3EncryptionClientFileMode.UploadPart(uploadRequest);


                ListPartsRequest listPartRequest = new ListPartsRequest()
                {
                    BucketName = bucketName,
                    Key        = key,
                    UploadId   = initResponse.UploadId
                };

                ListPartsResponse listPartResponse = s3EncryptionClientFileMode.ListParts(listPartRequest);
                Assert.AreEqual(3, listPartResponse.Parts.Count);
                Assert.AreEqual(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber);
                Assert.AreEqual(up1Response.ETag, listPartResponse.Parts[0].ETag);
                Assert.AreEqual(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber);
                Assert.AreEqual(up2Response.ETag, listPartResponse.Parts[1].ETag);
                Assert.AreEqual(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber);
                Assert.AreEqual(up3Response.ETag, listPartResponse.Parts[2].ETag);

                listPartRequest.MaxParts = 1;
                listPartResponse         = s3EncryptionClientFileMode.ListParts(listPartRequest);
                Assert.AreEqual(1, listPartResponse.Parts.Count);

                // Complete the response
                CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
                {
                    BucketName = bucketName,
                    Key        = key,
                    UploadId   = initResponse.UploadId
                };
                compRequest.AddPartETags(up1Response, up2Response, up3Response);

                CompleteMultipartUploadResponse compResponse = s3EncryptionClientFileMode.CompleteMultipartUpload(compRequest);
                Assert.AreEqual(bucketName, compResponse.BucketName);
                Assert.IsNotNull(compResponse.ETag);
                Assert.AreEqual(key, compResponse.Key);
                Assert.IsNotNull(compResponse.Location);

                // Get the file back from S3 and make sure it is still the same.
                GetObjectRequest getRequest = new GetObjectRequest()
                {
                    BucketName = bucketName,
                    Key        = key
                };

                GetObjectResponse getResponse = s3EncryptionClientFileMode.GetObject(getRequest);
                getResponse.WriteResponseStreamToFile(retrievedFilepath);

                UtilityMethods.CompareFiles(filePath, retrievedFilepath);

                GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest()
                {
                    BucketName = bucketName,
                    Key        = key
                };
                GetObjectMetadataResponse metaDataResponse = s3EncryptionClientFileMode.GetObjectMetadata(metaDataRequest);
                Assert.AreEqual("text/html", metaDataResponse.Headers.ContentType);
            }
            finally
            {
                inputStream.Close();
                if (File.Exists(filePath))
                {
                    File.Delete(filePath);
                }
                if (File.Exists(retrievedFilepath))
                {
                    File.Delete(retrievedFilepath);
                }
            }
        }
Пример #29
0
        public void ObjectSamples()
        {
            {
                #region ListObjects Sample

                // Create a client
                AmazonS3Client client = new AmazonS3Client();

                // List all objects
                ListObjectsRequest listRequest = new ListObjectsRequest
                {
                    BucketName = "SampleBucket",
                };

                ListObjectsResponse listResponse;
                do
                {
                    // Get a list of objects
                    listResponse = client.ListObjects(listRequest);
                    foreach (S3Object obj in listResponse.S3Objects)
                    {
                        Console.WriteLine("Object - " + obj.Key);
                        Console.WriteLine(" Size - " + obj.Size);
                        Console.WriteLine(" LastModified - " + obj.LastModified);
                        Console.WriteLine(" Storage class - " + obj.StorageClass);
                    }

                    // Set the marker property
                    listRequest.Marker = listResponse.NextMarker;
                } while (listResponse.IsTruncated);

                #endregion
            }

            {
                #region GetObject Sample

                // Create a client
                AmazonS3Client client = new AmazonS3Client();

                // Create a GetObject request
                GetObjectRequest request = new GetObjectRequest
                {
                    BucketName = "SampleBucket",
                    Key        = "Item1"
                };

                // Issue request and remember to dispose of the response
                using (GetObjectResponse response = client.GetObject(request))
                {
                    using (StreamReader reader = new StreamReader(response.ResponseStream))
                    {
                        string contents = reader.ReadToEnd();
                        Console.WriteLine("Object - " + response.Key);
                        Console.WriteLine(" Version Id - " + response.VersionId);
                        Console.WriteLine(" Contents - " + contents);
                    }
                }

                #endregion
            }

            {
                #region GetObjectMetadata Sample

                // Create a client
                AmazonS3Client client = new AmazonS3Client();


                // Create a GetObjectMetadata request
                GetObjectMetadataRequest request = new GetObjectMetadataRequest
                {
                    BucketName = "SampleBucket",
                    Key        = "Item1"
                };

                // Issue request and view the response
                GetObjectMetadataResponse response = client.GetObjectMetadata(request);
                Console.WriteLine("Content Length - " + response.ContentLength);
                Console.WriteLine("Content Type - " + response.Headers.ContentType);
                if (response.Expiration != null)
                {
                    Console.WriteLine("Expiration Date - " + response.Expiration.ExpiryDate);
                    Console.WriteLine("Expiration Rule Id - " + response.Expiration.RuleId);
                }

                #endregion
            }

            {
                #region PutObject Sample 1

                // Create a client
                AmazonS3Client client = new AmazonS3Client();

                // Create a PutObject request
                PutObjectRequest request = new PutObjectRequest
                {
                    BucketName  = "SampleBucket",
                    Key         = "Item1",
                    ContentBody = "This is sample content..."
                };

                // Put object
                PutObjectResponse response = client.PutObject(request);

                #endregion
            }

            {
                #region PutObject Sample 2

                // Create a client
                AmazonS3Client client = new AmazonS3Client();

                // Create a PutObject request
                PutObjectRequest request = new PutObjectRequest
                {
                    BucketName = "SampleBucket",
                    Key        = "Item1",
                    FilePath   = "contents.txt"
                };

                // Put object
                PutObjectResponse response = client.PutObject(request);

                #endregion
            }

            {
                #region PutObject Sample 3

                // Create a client
                AmazonS3Client client = new AmazonS3Client();

                // Create a PutObject request
                PutObjectRequest request = new PutObjectRequest
                {
                    BucketName = "SampleBucket",
                    Key        = "Item1",
                };
                using (FileStream stream = new FileStream("contents.txt", FileMode.Open))
                {
                    request.InputStream = stream;

                    // Put object
                    PutObjectResponse response = client.PutObject(request);
                }

                #endregion
            }

            {
                #region DeleteObject Sample

                // Create a client
                AmazonS3Client client = new AmazonS3Client();

                // Create a DeleteObject request
                DeleteObjectRequest request = new DeleteObjectRequest
                {
                    BucketName = "SampleBucket",
                    Key        = "Item1"
                };

                // Issue request
                client.DeleteObject(request);

                #endregion
            }

            {
                #region DeleteObjects Sample

                // Create a client
                AmazonS3Client client = new AmazonS3Client();

                // Create a DeleteObject request
                DeleteObjectsRequest request = new DeleteObjectsRequest
                {
                    BucketName = "SampleBucket",
                    Objects    = new List <KeyVersion>
                    {
                        new KeyVersion()
                        {
                            Key = "Item1"
                        },
                        // Versioned item
                        new KeyVersion()
                        {
                            Key = "Item2", VersionId = "Rej8CiBxcZKVK81cLr39j27Y5FVXghDK",
                        },
                        // Item in subdirectory
                        new KeyVersion()
                        {
                            Key = "Logs/error.txt"
                        }
                    }
                };

                try
                {
                    // Issue request
                    DeleteObjectsResponse response = client.DeleteObjects(request);
                }
                catch (DeleteObjectsException doe)
                {
                    // Catch error and list error details
                    DeleteObjectsResponse errorResponse = doe.Response;

                    foreach (DeletedObject deletedObject in errorResponse.DeletedObjects)
                    {
                        Console.WriteLine("Deleted item " + deletedObject.Key);
                    }
                    foreach (DeleteError deleteError in errorResponse.DeleteErrors)
                    {
                        Console.WriteLine("Error deleting item " + deleteError.Key);
                        Console.WriteLine(" Code - " + deleteError.Code);
                        Console.WriteLine(" Message - " + deleteError.Message);
                    }
                }

                #endregion
            }

            {
                #region CopyObject Sample

                // Create a client
                AmazonS3Client client = new AmazonS3Client();

                // Create a CopyObject request
                CopyObjectRequest request = new CopyObjectRequest
                {
                    SourceBucket      = "SampleBucket",
                    SourceKey         = "Item1",
                    DestinationBucket = "AnotherBucket",
                    DestinationKey    = "Copy1",
                    CannedACL         = S3CannedACL.PublicRead
                };

                // Issue request
                client.CopyObject(request);

                #endregion
            }

            {
                #region CopyObject Sample

                // Create a client
                AmazonS3Client client = new AmazonS3Client();

                // Create a CopyObject request
                CopyObjectRequest request = new CopyObjectRequest
                {
                    SourceBucket      = "SampleBucket",
                    SourceKey         = "Item1",
                    DestinationBucket = "AnotherBucket",
                    DestinationKey    = "Copy1",
                    CannedACL         = S3CannedACL.PublicRead
                };

                // Issue request
                client.CopyObject(request);

                #endregion
            }

            {
                #region ListVersions Sample

                // Create a client
                AmazonS3Client client = new AmazonS3Client();

                // Turn versioning on for a bucket
                client.PutBucketVersioning(new PutBucketVersioningRequest
                {
                    BucketName       = "SampleBucket",
                    VersioningConfig = new S3BucketVersioningConfig {
                        Status = "Enable"
                    }
                });

                // Populate bucket with multiple items, each with multiple versions
                PopulateBucket(client, "SampleBucket");

                // Get versions
                ListVersionsRequest request = new ListVersionsRequest
                {
                    BucketName = "SampleBucket"
                };

                // Make paged ListVersions calls
                ListVersionsResponse response;
                do
                {
                    response = client.ListVersions(request);
                    // View information about versions
                    foreach (var version in response.Versions)
                    {
                        Console.WriteLine("Key = {0}, Version = {1}, IsLatest = {2}, LastModified = {3}, Size = {4}",
                                          version.Key,
                                          version.VersionId,
                                          version.IsLatest,
                                          version.LastModified,
                                          version.Size);
                    }

                    request.KeyMarker       = response.NextKeyMarker;
                    request.VersionIdMarker = response.NextVersionIdMarker;
                } while (response.IsTruncated);

                #endregion
            }

            {
                #region Multipart Upload Sample

                int MB = (int)Math.Pow(2, 20);

                // Create a client
                AmazonS3Client client = new AmazonS3Client();

                // Define input stream
                Stream inputStream = Create13MBDataStream();

                // Initiate multipart upload
                InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest
                {
                    BucketName = "SampleBucket",
                    Key        = "Item1"
                };
                InitiateMultipartUploadResponse initResponse = client.InitiateMultipartUpload(initRequest);

                // Upload part 1
                UploadPartRequest uploadRequest = new UploadPartRequest
                {
                    BucketName  = "SampleBucket",
                    Key         = "Item1",
                    UploadId    = initResponse.UploadId,
                    PartNumber  = 1,
                    PartSize    = 5 * MB,
                    InputStream = inputStream
                };
                UploadPartResponse up1Response = client.UploadPart(uploadRequest);

                // Upload part 2
                uploadRequest = new UploadPartRequest
                {
                    BucketName  = "SampleBucket",
                    Key         = "Item1",
                    UploadId    = initResponse.UploadId,
                    PartNumber  = 2,
                    PartSize    = 5 * MB,
                    InputStream = inputStream
                };
                UploadPartResponse up2Response = client.UploadPart(uploadRequest);

                // Upload part 3
                uploadRequest = new UploadPartRequest
                {
                    BucketName  = "SampleBucket",
                    Key         = "Item1",
                    UploadId    = initResponse.UploadId,
                    PartNumber  = 3,
                    InputStream = inputStream
                };
                UploadPartResponse up3Response = client.UploadPart(uploadRequest);

                // List parts for current upload
                ListPartsRequest listPartRequest = new ListPartsRequest
                {
                    BucketName = "SampleBucket",
                    Key        = "Item1",
                    UploadId   = initResponse.UploadId
                };
                ListPartsResponse listPartResponse = client.ListParts(listPartRequest);
                Debug.Assert(listPartResponse.Parts.Count == 3);

                // Complete the multipart upload
                CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest
                {
                    BucketName = "SampleBucket",
                    Key        = "Item1",
                    UploadId   = initResponse.UploadId,
                    PartETags  = new List <PartETag>
                    {
                        new PartETag {
                            ETag = up1Response.ETag, PartNumber = 1
                        },
                        new PartETag {
                            ETag = up2Response.ETag, PartNumber = 2
                        },
                        new PartETag {
                            ETag = up3Response.ETag, PartNumber = 3
                        }
                    }
                };
                CompleteMultipartUploadResponse compResponse = client.CompleteMultipartUpload(compRequest);

                #endregion
            }
        }
Пример #30
0
        public async Task MultipartEncryptionTestInstructionFile()
        {
            string filePath          = Path.Combine(Path.GetTempPath(), "MulitpartEncryptionTestInstructionFile_upload.txt");
            string retrievedFilepath = Path.Combine(Path.GetTempPath(), "MulitpartEncryptionTestInstructionFile_download.txt");
            int    MEG_SIZE          = (int)Math.Pow(2, 20);
            long   totalSize         = (long)(15 * MEG_SIZE) + 4001;

            UtilityMethods.GenerateFile(filePath, totalSize);
            _filesToDelete.Add(filePath);

            string key = "MultipartEncryptionTestInstrcutionFile" + random.Next();

            using (Stream inputStream = File.OpenRead(filePath))
            {
                InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest()
                {
                    BucketName   = bucketName,
                    Key          = key,
                    StorageClass = S3StorageClass.ReducedRedundancy,
                    ContentType  = "text/html",
                    CannedACL    = S3CannedACL.PublicRead
                };

                InitiateMultipartUploadResponse initResponse = await s3EncryptionClientFileMode.InitiateMultipartUploadAsync(initRequest);

                // Upload part 1
                UploadPartRequest uploadRequest = new UploadPartRequest()
                {
                    BucketName  = bucketName,
                    Key         = key,
                    UploadId    = initResponse.UploadId,
                    PartNumber  = 1,
                    PartSize    = 5 * MEG_SIZE,
                    InputStream = inputStream
                };

                UploadPartResponse up1Response = await s3EncryptionClientFileMode.UploadPartAsync(uploadRequest);

                // Upload part 2
                uploadRequest = new UploadPartRequest()
                {
                    BucketName  = bucketName,
                    Key         = key,
                    UploadId    = initResponse.UploadId,
                    PartNumber  = 2,
                    PartSize    = 5 * MEG_SIZE + 4001,
                    InputStream = inputStream
                };

                UploadPartResponse up2Response = await s3EncryptionClientFileMode.UploadPartAsync(uploadRequest);

                // Upload part 3
                uploadRequest = new UploadPartRequest()
                {
                    BucketName  = bucketName,
                    Key         = key,
                    UploadId    = initResponse.UploadId,
                    PartNumber  = 3,
                    InputStream = inputStream,
                    IsLastPart  = true
                };

                //uploadRequest.setLastPart();
                UploadPartResponse up3Response = await s3EncryptionClientFileMode.UploadPartAsync(uploadRequest);


                ListPartsRequest listPartRequest = new ListPartsRequest()
                {
                    BucketName = bucketName,
                    Key        = key,
                    UploadId   = initResponse.UploadId
                };

                ListPartsResponse listPartResponse = await s3EncryptionClientFileMode.ListPartsAsync(listPartRequest);

                Assert.Equal(3, listPartResponse.Parts.Count);
                Assert.Equal(up1Response.PartNumber, listPartResponse.Parts[0].PartNumber);
                Assert.Equal(up1Response.ETag, listPartResponse.Parts[0].ETag);
                Assert.Equal(up2Response.PartNumber, listPartResponse.Parts[1].PartNumber);
                Assert.Equal(up2Response.ETag, listPartResponse.Parts[1].ETag);
                Assert.Equal(up3Response.PartNumber, listPartResponse.Parts[2].PartNumber);
                Assert.Equal(up3Response.ETag, listPartResponse.Parts[2].ETag);

                listPartRequest.MaxParts = 1;
                listPartResponse         = await s3EncryptionClientFileMode.ListPartsAsync(listPartRequest);

                Assert.Equal(1, listPartResponse.Parts.Count);

                // Complete the response
                CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest()
                {
                    BucketName = bucketName,
                    Key        = key,
                    UploadId   = initResponse.UploadId
                };
                compRequest.AddPartETags(up1Response, up2Response, up3Response);

                CompleteMultipartUploadResponse compResponse = await s3EncryptionClientFileMode.CompleteMultipartUploadAsync(compRequest);

                Assert.Equal(bucketName, compResponse.BucketName);
                Assert.NotNull(compResponse.ETag);
                Assert.Equal(key, compResponse.Key);
                Assert.NotNull(compResponse.Location);

                // Get the file back from S3 and make sure it is still the same.
                GetObjectRequest getRequest = new GetObjectRequest()
                {
                    BucketName = bucketName,
                    Key        = key
                };

                GetObjectResponse getResponse = await s3EncryptionClientFileMode.GetObjectAsync(getRequest);

                await getResponse.WriteResponseStreamToFileAsync(retrievedFilepath, false, System.Threading.CancellationToken.None);

                _filesToDelete.Add(retrievedFilepath);

                UtilityMethods.CompareFiles(filePath, retrievedFilepath);

                GetObjectMetadataRequest metaDataRequest = new GetObjectMetadataRequest()
                {
                    BucketName = bucketName,
                    Key        = key
                };
                GetObjectMetadataResponse metaDataResponse = await s3EncryptionClientFileMode.GetObjectMetadataAsync(metaDataRequest);

                Assert.Equal("text/html", metaDataResponse.Headers.ContentType);
            }
        }