public IRequest Marshall(UploadPartRequest uploadPartRequest) { IRequest request = new DefaultRequest(uploadPartRequest, "AmazonS3"); request.HttpMethod = "PUT"; if (uploadPartRequest.IsSetMD5Digest()) { request.Headers[HeaderKeys.ContentMD5Header] = uploadPartRequest.MD5Digest; } if (uploadPartRequest.IsSetServerSideEncryptionCustomerMethod()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerAlgorithmHeader, uploadPartRequest.ServerSideEncryptionCustomerMethod); } if (uploadPartRequest.IsSetServerSideEncryptionCustomerProvidedKey()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyHeader, uploadPartRequest.ServerSideEncryptionCustomerProvidedKey); if (uploadPartRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, uploadPartRequest.ServerSideEncryptionCustomerProvidedKeyMD5); } else { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, AmazonS3Util.ComputeEncodedMD5FromEncodedString(uploadPartRequest.ServerSideEncryptionCustomerProvidedKey)); } } if (uploadPartRequest.IsSetRequestPayer()) { request.Headers.Add(S3Constants.AmzHeaderRequestPayer, S3Transforms.ToStringValue(uploadPartRequest.RequestPayer.ToString())); } if (uploadPartRequest.IsSetExpectedBucketOwner()) { request.Headers.Add(S3Constants.AmzHeaderExpectedBucketOwner, S3Transforms.ToStringValue(uploadPartRequest.ExpectedBucketOwner)); } if (string.IsNullOrEmpty(uploadPartRequest.BucketName)) { throw new System.ArgumentException("BucketName is a required property and must be set before making this call.", "UploadPartRequest.BucketName"); } if (string.IsNullOrEmpty(uploadPartRequest.Key)) { throw new System.ArgumentException("Key is a required property and must be set before making this call.", "UploadPartRequest.Key"); } request.ResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}", S3Transforms.ToStringValue(uploadPartRequest.BucketName), S3Transforms.ToStringValue(uploadPartRequest.Key)); if (uploadPartRequest.IsSetPartNumber()) { request.AddSubResource("partNumber", S3Transforms.ToStringValue(uploadPartRequest.PartNumber)); } if (uploadPartRequest.IsSetUploadId()) { request.AddSubResource("uploadId", S3Transforms.ToStringValue(uploadPartRequest.UploadId)); } if (uploadPartRequest.InputStream != null) { // Wrap input stream in partial wrapper (to upload only part of the stream) var partialStream = new PartialWrapperStream(uploadPartRequest.InputStream, uploadPartRequest.PartSize); if (partialStream.Length > 0 && !(uploadPartRequest.DisablePayloadSigning ?? false)) { request.UseChunkEncoding = uploadPartRequest.UseChunkEncoding; } if (!request.Headers.ContainsKey(HeaderKeys.ContentLengthHeader)) { request.Headers.Add(HeaderKeys.ContentLengthHeader, partialStream.Length.ToString(CultureInfo.InvariantCulture)); } request.DisablePayloadSigning = uploadPartRequest.DisablePayloadSigning; if (!(uploadPartRequest.DisableMD5Stream ?? AWSConfigsS3.DisableMD5Stream)) { // Wrap input stream in MD5Stream; after this we can no longer seek or position the stream var hashStream = new MD5Stream(partialStream, null, partialStream.Length); uploadPartRequest.InputStream = hashStream; } else { uploadPartRequest.InputStream = partialStream; } } request.ContentStream = uploadPartRequest.InputStream; if (!request.Headers.ContainsKey(HeaderKeys.ContentTypeHeader)) { request.Headers.Add(HeaderKeys.ContentTypeHeader, "text/plain"); } return(request); }
public IRequest Marshall(UploadPartRequest uploadPartRequest) { IRequest request = new DefaultRequest(uploadPartRequest, "AmazonS3"); request.HttpMethod = "PUT"; if (uploadPartRequest.IsSetMD5Digest()) { request.Headers[HeaderKeys.ContentMD5Header] = uploadPartRequest.MD5Digest; } if (uploadPartRequest.IsSetServerSideEncryptionCustomerMethod()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerAlgorithmHeader, uploadPartRequest.ServerSideEncryptionCustomerMethod); } if (uploadPartRequest.IsSetServerSideEncryptionCustomerProvidedKey()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyHeader, uploadPartRequest.ServerSideEncryptionCustomerProvidedKey); if (uploadPartRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, uploadPartRequest.ServerSideEncryptionCustomerProvidedKeyMD5); } else { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, AmazonS3Util.ComputeEncodedMD5FromEncodedString(uploadPartRequest.ServerSideEncryptionCustomerProvidedKey)); } } if (uploadPartRequest.IsSetRequestPayer()) { request.Headers.Add(S3Constants.AmzHeaderRequestPayer, S3Transforms.ToStringValue(uploadPartRequest.RequestPayer.ToString())); } request.ResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}", S3Transforms.ToStringValue(uploadPartRequest.BucketName), S3Transforms.ToStringValue(uploadPartRequest.Key)); if (uploadPartRequest.IsSetPartNumber()) { request.AddSubResource("partNumber", S3Transforms.ToStringValue(uploadPartRequest.PartNumber)); } if (uploadPartRequest.IsSetUploadId()) { request.AddSubResource("uploadId", S3Transforms.ToStringValue(uploadPartRequest.UploadId)); } if (uploadPartRequest.InputStream != null) { // Wrap input stream in partial wrapper (to upload only part of the stream) var partialStream = new PartialWrapperStream(uploadPartRequest.InputStream, uploadPartRequest.PartSize); if (partialStream.Length > 0) { request.UseChunkEncoding = uploadPartRequest.UseChunkEncoding; } if (!request.Headers.ContainsKey(HeaderKeys.ContentLengthHeader)) { request.Headers.Add(HeaderKeys.ContentLengthHeader, partialStream.Length.ToString(CultureInfo.InvariantCulture)); } // Wrap input stream in MD5Stream; after this we can no longer seek or position the stream var hashStream = new MD5Stream(partialStream, null, partialStream.Length); uploadPartRequest.InputStream = hashStream; } request.ContentStream = uploadPartRequest.InputStream; if (!request.Headers.ContainsKey(HeaderKeys.ContentTypeHeader)) { request.Headers.Add(HeaderKeys.ContentTypeHeader, "text/plain"); } return(request); }
public IRequest Marshall(PutObjectRequest putObjectRequest) { IRequest request = new DefaultRequest(putObjectRequest, "AmazonS3"); request.HttpMethod = "PUT"; if (putObjectRequest.IsSetCannedACL()) { request.Headers.Add(HeaderKeys.XAmzAclHeader, S3Transforms.ToStringValue(putObjectRequest.CannedACL)); } var headers = putObjectRequest.Headers; foreach (var key in headers.Keys) { request.Headers[key] = headers[key]; } if (putObjectRequest.IsSetMD5Digest()) { request.Headers[HeaderKeys.ContentMD5Header] = putObjectRequest.MD5Digest; } HeaderACLRequestMarshaller.Marshall(request, putObjectRequest); if (putObjectRequest.IsSetServerSideEncryptionMethod()) { request.Headers.Add(HeaderKeys.XAmzServerSideEncryptionHeader, S3Transforms.ToStringValue(putObjectRequest.ServerSideEncryptionMethod)); } if (putObjectRequest.IsSetStorageClass()) { request.Headers.Add(HeaderKeys.XAmzStorageClassHeader, S3Transforms.ToStringValue(putObjectRequest.StorageClass)); } if (putObjectRequest.IsSetWebsiteRedirectLocation()) { request.Headers.Add(HeaderKeys.XAmzWebsiteRedirectLocationHeader, S3Transforms.ToStringValue(putObjectRequest.WebsiteRedirectLocation)); } if (putObjectRequest.IsSetServerSideEncryptionCustomerMethod()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerAlgorithmHeader, putObjectRequest.ServerSideEncryptionCustomerMethod); } if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKey()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyHeader, putObjectRequest.ServerSideEncryptionCustomerProvidedKey); if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, putObjectRequest.ServerSideEncryptionCustomerProvidedKeyMD5); } else { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, AmazonS3Util.ComputeEncodedMD5FromEncodedString(putObjectRequest.ServerSideEncryptionCustomerProvidedKey)); } } if (putObjectRequest.IsSetServerSideEncryptionKeyManagementServiceKeyId()) { request.Headers.Add(HeaderKeys.XAmzServerSideEncryptionAwsKmsKeyIdHeader, putObjectRequest.ServerSideEncryptionKeyManagementServiceKeyId); } if (putObjectRequest.IsSetObjectLockLegalHoldStatus()) { request.Headers.Add("x-amz-object-lock-legal-hold", S3Transforms.ToStringValue(putObjectRequest.ObjectLockLegalHoldStatus)); } if (putObjectRequest.IsSetObjectLockMode()) { request.Headers.Add("x-amz-object-lock-mode", S3Transforms.ToStringValue(putObjectRequest.ObjectLockMode)); } if (putObjectRequest.IsSetObjectLockRetainUntilDate()) { request.Headers.Add("x-amz-object-lock-retain-until-date", S3Transforms.ToStringValue(putObjectRequest.ObjectLockRetainUntilDate, AWSSDKUtils.ISO8601DateFormat)); } if (putObjectRequest.IsSetRequestPayer()) { request.Headers.Add(S3Constants.AmzHeaderRequestPayer, S3Transforms.ToStringValue(putObjectRequest.RequestPayer.ToString())); } if (putObjectRequest.IsSetTagSet()) { request.Headers.Add(S3Constants.AmzHeaderTagging, AmazonS3Util.TagSetToQueryString(putObjectRequest.TagSet)); } AmazonS3Util.SetMetadataHeaders(request, putObjectRequest.Metadata); if (string.IsNullOrEmpty(putObjectRequest.BucketName)) { throw new System.ArgumentException("BucketName is a required property and must be set before making this call.", "PutObjectRequest.BucketName"); } if (string.IsNullOrEmpty(putObjectRequest.Key)) { throw new System.ArgumentException("Key is a required property and must be set before making this call.", "PutObjectRequest.Key"); } request.ResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}", S3Transforms.ToStringValue(putObjectRequest.BucketName), S3Transforms.ToStringValue(putObjectRequest.Key)); if (putObjectRequest.InputStream != null) { // Wrap the stream in a stream that has a length var streamWithLength = GetStreamWithLength(putObjectRequest.InputStream, putObjectRequest.Headers.ContentLength); if (streamWithLength.Length > 0) { request.UseChunkEncoding = putObjectRequest.UseChunkEncoding; } var length = streamWithLength.Length - streamWithLength.Position; if (!request.Headers.ContainsKey(HeaderKeys.ContentLengthHeader)) { request.Headers.Add(HeaderKeys.ContentLengthHeader, length.ToString(CultureInfo.InvariantCulture)); } // Wrap input stream in MD5Stream var hashStream = new MD5Stream(streamWithLength, null, length); putObjectRequest.InputStream = hashStream; } request.ContentStream = putObjectRequest.InputStream; if (!request.Headers.ContainsKey(HeaderKeys.ContentTypeHeader)) { request.Headers.Add(HeaderKeys.ContentTypeHeader, "text/plain"); } return(request); }
protected override void ProcessResponseHandlers(AmazonWebServiceResponse response, IRequest request, IWebResponseData webResponseData) { base.ProcessResponseHandlers(response, request, webResponseData); var getObjectResponse = response as GetObjectResponse; if (getObjectResponse != null) { GetObjectRequest getObjectRequest = request.OriginalRequest as GetObjectRequest; getObjectResponse.BucketName = getObjectRequest.BucketName; getObjectResponse.Key = getObjectRequest.Key; // If ETag is present and is an MD5 hash (not a multi-part upload ETag), and no byte range is specified, // wrap the response stream in an MD5Stream. if (!string.IsNullOrEmpty(getObjectResponse.ETag) && !getObjectResponse.ETag.Contains("-") && getObjectRequest.ByteRange == null) { string etag = getObjectResponse.ETag.Trim(etagTrimChars); byte[] expectedHash = AWSSDKUtils.HexStringToBytes(etag); HashStream hashStream = new MD5Stream(getObjectResponse.ResponseStream, expectedHash, getObjectResponse.ContentLength); getObjectResponse.ResponseStream = hashStream; } } var deleteObjectsResponse = response as DeleteObjectsResponse; if (deleteObjectsResponse != null) { if (deleteObjectsResponse.DeleteErrors != null && deleteObjectsResponse.DeleteErrors.Count > 0) { throw new DeleteObjectsException(deleteObjectsResponse as DeleteObjectsResponse); } } var putObjectResponse = response as PutObjectResponse; var putObjectRequest = request.OriginalRequest as PutObjectRequest; if (putObjectRequest != null) { // If InputStream was a HashStream, compare calculated hash to returned etag HashStream hashStream = putObjectRequest.InputStream as HashStream; if (hashStream != null) { if (putObjectResponse != null) { // Stream may not have been closed, so force calculation of hash hashStream.CalculateHash(); CompareHashes(putObjectResponse.ETag, hashStream.CalculatedHash); } // Set InputStream to its original value putObjectRequest.InputStream = hashStream.GetNonWrapperBaseStream(); } } var listObjectsResponse = response as ListObjectsResponse; if (listObjectsResponse != null) { if (listObjectsResponse.IsTruncated && string.IsNullOrEmpty(listObjectsResponse.NextMarker) && listObjectsResponse.S3Objects.Count > 0) { listObjectsResponse.NextMarker = listObjectsResponse.S3Objects.Last().Key; } } var uploadPartRequest = request.OriginalRequest as UploadPartRequest; var uploadPartResponse = response as UploadPartResponse; if (uploadPartRequest != null) { if (uploadPartResponse != null) { uploadPartResponse.PartNumber = uploadPartRequest.PartNumber; } // If InputStream was a HashStream, compare calculated hash to returned etag HashStream hashStream = uploadPartRequest.InputStream as HashStream; if (hashStream != null) { if (uploadPartResponse != null) { // Stream may not have been closed, so force calculation of hash hashStream.CalculateHash(); CompareHashes(uploadPartResponse.ETag, hashStream.CalculatedHash); } // Set InputStream to its original value uploadPartRequest.InputStream = hashStream.GetNonWrapperBaseStream(); } } var copyPartResponse = response as CopyPartResponse; if (copyPartResponse != null) { copyPartResponse.PartNumber = ((CopyPartRequest)request.OriginalRequest).PartNumber; } CleanupRequest(request); }
// ReSharper disable once UnusedMember.Global public AnnotationResult Run(AnnotationConfig config, ILambdaContext context) { var result = new AnnotationResult { id = config.id }; string snsTopicArn = null; var runLog = new StringBuilder(); try { LogUtilities.UpdateLogger(context.Logger, runLog); LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion); LogUtilities.LogObject("Config", config); LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey }); LambdaUtilities.GarbageCollect(); LambdaUtilities.DeleteTempOutput(); snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey); string vcfUrl = config.vcfUrl; using (var annotationResources = GetAnnotationResources(config)) { if (annotationResources.InputStartVirtualPosition == -1) { return(GetSuccessOutput(result)); } long fileOffset = VirtualPosition.From(annotationResources.InputStartVirtualPosition).FileOffset; using (var preloadVcfStream = PersistentStreamUtils.GetReadStream(vcfUrl, fileOffset)) { annotationResources.GetVariantPositions(new BlockGZipStream(preloadVcfStream, CompressionMode.Decompress), config.annotationRange.ToGenomicRange(annotationResources.SequenceProvider.RefNameToChromosome)); } Logger.WriteLine("Scan for positions to preload complete."); using (var aes = new AesCryptoServiceProvider()) { FileMetadata jsonMetadata, jasixMetadata; string jsonPath = Path.GetTempPath() + LambdaUrlHelper.JsonSuffix; string jasixPath = jsonPath + LambdaUrlHelper.JsonIndexSuffix; using (var inputVcfStream = new BlockGZipStream(PersistentStreamUtils.GetReadStream(vcfUrl, fileOffset), CompressionMode.Decompress)) using (var headerStream = config.annotationRange == null ? null : new BlockGZipStream(PersistentStreamUtils.GetReadStream(vcfUrl), CompressionMode.Decompress)) // using (var jsonFileStream = FileUtilities.GetCreateStream(jsonPath)) using (var jsonCryptoStream = new CryptoStream(jsonFileStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var jsonMd5Stream = new MD5Stream(jsonCryptoStream)) // using (var jasixFileStream = FileUtilities.GetCreateStream(jasixPath)) using (var jasixCryptoStream = new CryptoStream(jasixFileStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var jasixMd5Stream = new MD5Stream(jasixCryptoStream)) { IVcfFilter vcfFilter = config.annotationRange == null ? new NullVcfFilter() as IVcfFilter : new VcfFilter(config.annotationRange.ToGenomicRange(annotationResources.SequenceProvider.RefNameToChromosome)); using (var jsonCompressStream = new BlockGZipStream(jsonMd5Stream, CompressionMode.Compress)) { StreamAnnotation.Annotate(headerStream, inputVcfStream, jsonCompressStream, jasixMd5Stream, annotationResources, vcfFilter, true); } Logger.WriteLine("Annotation done."); jsonMetadata = jsonMd5Stream.GetFileMetadata(); jasixMetadata = jasixMd5Stream.GetFileMetadata(); } result.filePath = S3Utilities.GetKey(config.outputDir.path, config.outputPrefix + LambdaUrlHelper.JsonSuffix); string jasixKey = result.filePath + LambdaUrlHelper.JsonIndexSuffix; var s3Client = config.outputDir.GetS3Client(context.RemainingTime); s3Client.DecryptUpload(config.outputDir.bucketName, jasixKey, jasixPath, aes, jasixMetadata); s3Client.DecryptUpload(config.outputDir.bucketName, result.filePath, jsonPath, aes, jsonMetadata); Logger.WriteLine("Nirvana result files uploaded."); } } LambdaUtilities.DeleteTempOutput(); if (string.IsNullOrEmpty(result.filePath)) { throw new FileNotFoundException(); } return(GetSuccessOutput(result)); } catch (Exception exception) { LambdaUtilities.DeleteTempOutput(); return(HandleException(runLog, result, exception, snsTopicArn)); } }
public IRequest Marshall(PutObjectRequest putObjectRequest) { IRequest request = new DefaultRequest(putObjectRequest, "AmazonS3"); request.HttpMethod = "PUT"; if (putObjectRequest.IsSetCannedACL()) { request.Headers.Add(HeaderKeys.XAmzAclHeader, S3Transforms.ToStringValue(putObjectRequest.CannedACL)); } var headers = putObjectRequest.Headers; foreach (var key in headers.Keys) { request.Headers[key] = headers[key]; } if (putObjectRequest.IsSetMD5Digest()) { request.Headers[HeaderKeys.ContentMD5Header] = putObjectRequest.MD5Digest; } HeaderACLRequestMarshaller.Marshall(request, putObjectRequest); if (putObjectRequest.IsSetServerSideEncryptionMethod()) { request.Headers.Add(HeaderKeys.XAmzServerSideEncryptionHeader, S3Transforms.ToStringValue(putObjectRequest.ServerSideEncryptionMethod)); } if (putObjectRequest.IsSetStorageClass()) { request.Headers.Add(HeaderKeys.XAmzStorageClassHeader, S3Transforms.ToStringValue(putObjectRequest.StorageClass)); } if (putObjectRequest.IsSetWebsiteRedirectLocation()) { request.Headers.Add(HeaderKeys.XAmzWebsiteRedirectLocationHeader, S3Transforms.ToStringValue(putObjectRequest.WebsiteRedirectLocation)); } if (putObjectRequest.IsSetServerSideEncryptionCustomerMethod()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerAlgorithmHeader, putObjectRequest.ServerSideEncryptionCustomerMethod); } if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKey()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyHeader, putObjectRequest.ServerSideEncryptionCustomerProvidedKey); if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, putObjectRequest.ServerSideEncryptionCustomerProvidedKeyMD5); } else { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, AmazonS3Util.ComputeEncodedMD5FromEncodedString(putObjectRequest.ServerSideEncryptionCustomerProvidedKey)); } } AmazonS3Util.SetMetadataHeaders(request, putObjectRequest.Metadata); request.ResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}", S3Transforms.ToStringValue(putObjectRequest.BucketName), S3Transforms.ToStringValue(putObjectRequest.Key)); if (putObjectRequest.InputStream != null) { // Wrap the stream in a stream that has a length var streamWithLength = GetStreamWithLength(putObjectRequest.InputStream, putObjectRequest.Headers.ContentLength); if (streamWithLength.Length > 0) { request.UseChunkEncoding = true; } var length = streamWithLength.Length - streamWithLength.Position; if (!request.Headers.ContainsKey(HeaderKeys.ContentLengthHeader)) { request.Headers.Add(HeaderKeys.ContentLengthHeader, length.ToString(CultureInfo.InvariantCulture)); } // Wrap input stream in MD5Stream var hashStream = new MD5Stream(streamWithLength, null, length); putObjectRequest.InputStream = hashStream; } request.ContentStream = putObjectRequest.InputStream; if (!request.Headers.ContainsKey(HeaderKeys.ContentTypeHeader)) { request.Headers.Add(HeaderKeys.ContentTypeHeader, "text/plain"); } return(request); }
private static void ProcessResponseHandlers(IExecutionContext executionContext) { AmazonWebServiceResponse response = executionContext.ResponseContext.Response; IRequest request = executionContext.RequestContext.Request; IWebResponseData webResponseData = executionContext.ResponseContext.HttpResponse; bool isSse = HasSSEHeaders(webResponseData); var getObjectResponse = response as GetObjectResponse; if (getObjectResponse != null) { GetObjectRequest getObjectRequest = request.OriginalRequest as GetObjectRequest; getObjectResponse.BucketName = getObjectRequest.BucketName; getObjectResponse.Key = getObjectRequest.Key; // If ETag is present and is an MD5 hash (not a multi-part upload ETag), and no byte range is specified, // wrap the response stream in an MD5Stream. // If there is a customer encryption algorithm the etag is not an MD5. if (!string.IsNullOrEmpty(getObjectResponse.ETag) && !getObjectResponse.ETag.Contains("-") && !isSse && getObjectRequest.ByteRange == null) { string etag = getObjectResponse.ETag.Trim(etagTrimChars); byte[] expectedHash = AWSSDKUtils.HexStringToBytes(etag); HashStream hashStream = new MD5Stream(getObjectResponse.ResponseStream, expectedHash, getObjectResponse.ContentLength); getObjectResponse.ResponseStream = hashStream; } } var deleteObjectsResponse = response as DeleteObjectsResponse; if (deleteObjectsResponse != null) { if (deleteObjectsResponse.DeleteErrors != null && deleteObjectsResponse.DeleteErrors.Count > 0) { throw new DeleteObjectsException(deleteObjectsResponse as DeleteObjectsResponse); } } var putObjectResponse = response as PutObjectResponse; var putObjectRequest = request.OriginalRequest as PutObjectRequest; if (putObjectRequest != null) { // If InputStream was a HashStream, compare calculated hash to returned etag HashStream hashStream = putObjectRequest.InputStream as HashStream; if (hashStream != null) { if (putObjectResponse != null && !isSse) { // Stream may not have been closed, so force calculation of hash hashStream.CalculateHash(); CompareHashes(putObjectResponse.ETag, hashStream.CalculatedHash); } // Set InputStream to its original value putObjectRequest.InputStream = hashStream.GetNonWrapperBaseStream(); } } var listObjectsResponse = response as ListObjectsResponse; if (listObjectsResponse != null) { if (listObjectsResponse.IsTruncated && string.IsNullOrEmpty(listObjectsResponse.NextMarker) && listObjectsResponse.S3Objects.Count > 0) { listObjectsResponse.NextMarker = listObjectsResponse.S3Objects.Last().Key; } } var uploadPartRequest = request.OriginalRequest as UploadPartRequest; var uploadPartResponse = response as UploadPartResponse; if (uploadPartRequest != null) { if (uploadPartResponse != null) uploadPartResponse.PartNumber = uploadPartRequest.PartNumber; // If InputStream was a HashStream, compare calculated hash to returned etag HashStream hashStream = uploadPartRequest.InputStream as HashStream; if (hashStream != null) { if (uploadPartResponse != null && !isSse) { // Stream may not have been closed, so force calculation of hash hashStream.CalculateHash(); CompareHashes(uploadPartResponse.ETag, hashStream.CalculatedHash); } // Set InputStream to its original value uploadPartRequest.InputStream = hashStream.GetNonWrapperBaseStream(); } } var copyPartResponse = response as CopyPartResponse; if (copyPartResponse != null) { copyPartResponse.PartNumber = ((CopyPartRequest)request.OriginalRequest).PartNumber; } AmazonS3Client.CleanupRequest(request); }
public IRequest Marshall(UploadPartRequest uploadPartRequest) { IRequest request = new DefaultRequest(uploadPartRequest, "AmazonS3"); request.HttpMethod = "PUT"; if (uploadPartRequest.IsSetMD5Digest()) { request.Headers["Content-MD5"] = uploadPartRequest.MD5Digest; } var uriResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}", S3Transforms.ToStringValue(uploadPartRequest.BucketName), S3Transforms.ToStringValue(uploadPartRequest.Key)); if (uploadPartRequest.IsSetPartNumber()) { request.Parameters.Add("partNumber", S3Transforms.ToStringValue(uploadPartRequest.PartNumber)); } if (uploadPartRequest.IsSetUploadId()) { request.Parameters.Add("uploadId", S3Transforms.ToStringValue(uploadPartRequest.UploadId)); } request.CanonicalResource = S3Transforms.GetCanonicalResource(uriResourcePath, request.Parameters); request.ResourcePath = S3Transforms.FormatResourcePath(uriResourcePath, request.Parameters); if (uploadPartRequest.InputStream != null) { // Wrap input stream in partial wrapper (to upload only part of the stream) var partialStream = new PartialWrapperStream(uploadPartRequest.InputStream, uploadPartRequest.PartSize); if (partialStream.Length > 0) { request.UseChunkEncoding = true; } if (!request.Headers.ContainsKey("Content-Length")) { request.Headers.Add("Content-Length", partialStream.Length.ToString(CultureInfo.InvariantCulture)); } // Wrap input stream in MD5Stream; after this we can no longer seek or position the stream var hashStream = new MD5Stream(partialStream, null, partialStream.Length); uploadPartRequest.InputStream = hashStream; } request.ContentStream = uploadPartRequest.InputStream; if (!request.Headers.ContainsKey("Content-Type")) { request.Headers.Add("Content-Type", "text/plain"); } if (!request.UseQueryString) { var queryString = AWSSDKUtils.GetParametersAsString(request.Parameters); if (!string.IsNullOrEmpty(queryString)) { request.ResourcePath = string.Concat(request.ResourcePath, request.ResourcePath.Contains("?") ? "&" : "?", queryString); } } return(request); }
public static UploadPartCommand Create(IServiceClient client, Uri endpoint, ExecutionContext context, UploadPartRequest uploadPartRequest) { OssUtils.CheckBucketName(uploadPartRequest.BucketName); OssUtils.CheckObjectKey(uploadPartRequest.Key); if (string.IsNullOrEmpty(uploadPartRequest.UploadId)) { throw new ArgumentException("uploadId should be specified"); } if (!uploadPartRequest.PartNumber.HasValue) { throw new ArgumentException("partNumber should be specified"); } if (!uploadPartRequest.PartSize.HasValue) { throw new ArgumentException("partSize should be specified"); } if (uploadPartRequest.InputStream == null) { throw new ArgumentException("inputStream should be specified"); } if (uploadPartRequest.PartSize < 0 || uploadPartRequest.PartSize > OssUtils.MaxFileSize) { throw new ArgumentException("partSize not live in valid range"); } if (!OssUtils.IsPartNumberInRange(uploadPartRequest.PartNumber)) { throw new ArgumentException("partNumber not live in valid range"); } var conf = OssUtils.GetClientConfiguration(client); var originalStream = uploadPartRequest.InputStream; var streamLength = uploadPartRequest.PartSize.Value; // wrap input stream in PartialWrapperStream originalStream = new PartialWrapperStream(originalStream, streamLength); // setup progress var callback = uploadPartRequest.StreamTransferProgress; if (callback != null) { originalStream = OssUtils.SetupProgressListeners(originalStream, conf.ProgressUpdateInterval, client, callback); uploadPartRequest.InputStream = originalStream; } // wrap input stream in MD5Stream if (conf.EnalbeMD5Check) { var hashStream = new MD5Stream(originalStream, null, streamLength); uploadPartRequest.InputStream = hashStream; context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream)); } else if (conf.EnableCrcCheck) { var hashStream = new Crc64Stream(originalStream, null, streamLength); uploadPartRequest.InputStream = hashStream; context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream)); } return(new UploadPartCommand(client, endpoint, context, DeserializerFactory.GetFactory().CreateUploadPartResultDeserializer(uploadPartRequest.PartNumber.Value, streamLength), uploadPartRequest)); }
public IRequest Marshall(UploadPartRequest uploadPartRequest) { //IL_0006: Unknown result type (might be due to invalid IL or missing references) //IL_000c: Expected O, but got Unknown //IL_0160: Unknown result type (might be due to invalid IL or missing references) //IL_0166: Expected O, but got Unknown //IL_01b4: Unknown result type (might be due to invalid IL or missing references) //IL_01ba: Expected O, but got Unknown IRequest val = new DefaultRequest(uploadPartRequest, "AmazonS3"); val.set_HttpMethod("PUT"); if (uploadPartRequest.IsSetMD5Digest()) { val.get_Headers()["Content-MD5"] = uploadPartRequest.MD5Digest; } if (uploadPartRequest.IsSetServerSideEncryptionCustomerMethod()) { val.get_Headers().Add("x-amz-server-side-encryption-customer-algorithm", ConstantClass.op_Implicit(uploadPartRequest.ServerSideEncryptionCustomerMethod)); } if (uploadPartRequest.IsSetServerSideEncryptionCustomerProvidedKey()) { val.get_Headers().Add("x-amz-server-side-encryption-customer-key", uploadPartRequest.ServerSideEncryptionCustomerProvidedKey); if (uploadPartRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5()) { val.get_Headers().Add("x-amz-server-side-encryption-customer-key-MD5", uploadPartRequest.ServerSideEncryptionCustomerProvidedKeyMD5); } else { val.get_Headers().Add("x-amz-server-side-encryption-customer-key-MD5", AmazonS3Util.ComputeEncodedMD5FromEncodedString(uploadPartRequest.ServerSideEncryptionCustomerProvidedKey)); } } if (uploadPartRequest.IsSetRequestPayer()) { val.get_Headers().Add(S3Constants.AmzHeaderRequestPayer, S3Transforms.ToStringValue(((object)uploadPartRequest.RequestPayer).ToString())); } val.set_ResourcePath(string.Format(CultureInfo.InvariantCulture, "/{0}/{1}", S3Transforms.ToStringValue(uploadPartRequest.BucketName), S3Transforms.ToStringValue(uploadPartRequest.Key))); if (uploadPartRequest.IsSetPartNumber()) { val.AddSubResource("partNumber", S3Transforms.ToStringValue(uploadPartRequest.PartNumber)); } if (uploadPartRequest.IsSetUploadId()) { val.AddSubResource("uploadId", S3Transforms.ToStringValue(uploadPartRequest.UploadId)); } if (uploadPartRequest.InputStream != null) { PartialWrapperStream val2 = new PartialWrapperStream(uploadPartRequest.InputStream, uploadPartRequest.PartSize); if (((Stream)val2).Length > 0) { val.set_UseChunkEncoding(true); } if (!val.get_Headers().ContainsKey("Content-Length")) { val.get_Headers().Add("Content-Length", ((Stream)val2).Length.ToString(CultureInfo.InvariantCulture)); } MD5Stream val3 = uploadPartRequest.InputStream = (Stream) new MD5Stream((Stream)val2, (byte[])null, ((Stream)val2).Length); } val.set_ContentStream(uploadPartRequest.InputStream); if (!val.get_Headers().ContainsKey("Content-Type")) { val.get_Headers().Add("Content-Type", "text/plain"); } return(val); }
public IRequest Marshall(PutObjectRequest putObjectRequest) { IRequest request = new DefaultRequest(putObjectRequest, "AmazonS3"); request.HttpMethod = "PUT"; if (putObjectRequest.IsSetCannedACL()) { request.Headers.Add("x-amz-acl", S3Transforms.ToStringValue(putObjectRequest.CannedACL)); } var headers = putObjectRequest.Headers; foreach (var key in headers.Keys) { request.Headers[key] = headers[key]; } if (putObjectRequest.IsSetMD5Digest()) { request.Headers["Content-MD5"] = putObjectRequest.MD5Digest; } HeaderACLRequestMarshaller.Marshall(request, putObjectRequest); if (putObjectRequest.IsSetServerSideEncryptionMethod()) { request.Headers.Add("x-amz-server-side-encryption", S3Transforms.ToStringValue(putObjectRequest.ServerSideEncryptionMethod)); } if (putObjectRequest.IsSetStorageClass()) { request.Headers.Add("x-amz-storage-class", S3Transforms.ToStringValue(putObjectRequest.StorageClass)); } if (putObjectRequest.IsSetWebsiteRedirectLocation()) { request.Headers.Add("x-amz-website-redirect-location", S3Transforms.ToStringValue(putObjectRequest.WebsiteRedirectLocation)); } AmazonS3Util.SetMetadataHeaders(request, putObjectRequest.Metadata); var uriResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}", S3Transforms.ToStringValue(putObjectRequest.BucketName), S3Transforms.ToStringValue(putObjectRequest.Key)); request.CanonicalResource = S3Transforms.GetCanonicalResource(uriResourcePath, request.Parameters); request.ResourcePath = S3Transforms.FormatResourcePath(uriResourcePath, request.Parameters); if (putObjectRequest.InputStream != null) { // Wrap the stream in a stream that has a length var streamWithLength = GetStreamWithLength(putObjectRequest.InputStream, putObjectRequest.Headers.ContentLength); if (streamWithLength.Length > 0) { request.UseChunkEncoding = true; } var length = streamWithLength.Length - streamWithLength.Position; if (!request.Headers.ContainsKey("Content-Length")) { request.Headers.Add("Content-Length", length.ToString(CultureInfo.InvariantCulture)); } // Wrap input stream in MD5Stream var hashStream = new MD5Stream(streamWithLength, null, length); putObjectRequest.InputStream = hashStream; } request.ContentStream = putObjectRequest.InputStream; if (!request.Headers.ContainsKey("Content-Type")) { request.Headers.Add("Content-Type", "text/plain"); } if (!request.UseQueryString) { var queryString = AWSSDKUtils.GetParametersAsString(request.Parameters); if (!string.IsNullOrEmpty(queryString)) { request.ResourcePath = string.Concat(request.ResourcePath, request.ResourcePath.Contains("?") ? "&" : "?", queryString); } } return(request); }
public static CustomResult Create(CustomConfig config, string inputFileName, CustomResult result, IS3Client s3Client) { string tempPath = Path.GetTempPath(); string inputBaseName = inputFileName.TrimEndFromFirst(".tsv"); string nsaFileName = inputBaseName + SaCommon.SaFileSuffix; string localNsaPath = Path.Combine(tempPath, nsaFileName); string localIndexPath = localNsaPath + SaCommon.IndexSufix; string localSchemaPath = localNsaPath + SaCommon.JsonSchemaSuffix; var outputFiles = new List <string>(); using (var aes = new AesCryptoServiceProvider()) { FileMetadata nsaMetadata, indexMetadata, schemaMetadata; List <CustomInterval> intervals; string jsonTag; SaJsonSchema intervalJsonSchema; DataSourceVersion version; GenomeAssembly genomeAssembly; int nsaItemsCount; ReportFor reportFor; using (var customTsvStream = (PersistentStream)PersistentStreamUtils.GetReadStream(config.tsvUrl)) using (var parser = GetVariantAnnotationsParserFromCustomTsvStream(customTsvStream)) // using (var nsaStream = FileUtilities.GetCreateStream(localNsaPath)) using (var nsaCryptoStream = new CryptoStream(nsaStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var nsaMd5Stream = new MD5Stream(nsaCryptoStream)) // using (var indexStream = FileUtilities.GetCreateStream(localIndexPath)) using (var indexCryptoStream = new CryptoStream(indexStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var indexMd5Stream = new MD5Stream(indexCryptoStream)) // using (var schemaStream = FileUtilities.GetCreateStream(localSchemaPath)) using (var schemaCryptoStream = new CryptoStream(schemaStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var schemaMd5Stream = new MD5Stream(schemaCryptoStream)) { genomeAssembly = parser.Assembly; result.genomeAssembly = genomeAssembly.ToString(); reportFor = parser.ReportFor; using (var nsaWriter = CaUtilities.GetNsaWriter(nsaMd5Stream, indexMd5Stream, parser, inputFileName, parser.SequenceProvider, out version)) using (var schemaWriter = new StreamWriter(schemaMd5Stream)) { (jsonTag, nsaItemsCount, intervalJsonSchema, intervals) = CaUtilities.WriteSmallVariants(parser, nsaWriter, schemaWriter); } nsaMetadata = nsaMd5Stream.GetFileMetadata(); indexMetadata = indexMd5Stream.GetFileMetadata(); schemaMetadata = schemaMd5Stream.GetFileMetadata(); } if (nsaItemsCount > 0) { string nsaS3Path = string.Join('/', config.outputDir.path.Trim('/'), nsaFileName); string indexS3Path = nsaS3Path + SaCommon.IndexSufix; string schemaS3Path = nsaS3Path + SaCommon.JsonSchemaSuffix; s3Client.DecryptUpload(config.outputDir.bucketName, nsaS3Path, localNsaPath, aes, nsaMetadata); s3Client.DecryptUpload(config.outputDir.bucketName, indexS3Path, localIndexPath, aes, indexMetadata); s3Client.DecryptUpload(config.outputDir.bucketName, schemaS3Path, localSchemaPath, aes, schemaMetadata); outputFiles.Add(nsaFileName); outputFiles.Add(nsaFileName + SaCommon.IndexSufix); outputFiles.Add(nsaFileName + SaCommon.JsonSchemaSuffix); } if (intervals == null) { return(CustomAnnotationLambda.GetSuccessResult(config, result, outputFiles)); } FileMetadata nsiMetadata, nsiSchemaMetadata; string nsiFileName = inputBaseName + SaCommon.SiFileSuffix; string localNsiPath = Path.Combine(tempPath, nsiFileName); string localNsiSchemaPath = localNsiPath + SaCommon.JsonSchemaSuffix; // using (var nsiStream = FileUtilities.GetCreateStream(localNsiPath)) using (var nsiCryptoStream = new CryptoStream(nsiStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var nsiMd5Stream = new MD5Stream(nsiCryptoStream)) // using (var nsiSchemaSteam = FileUtilities.GetCreateStream(localNsiSchemaPath)) using (var nsiSchemaCryptoStream = new CryptoStream(nsiSchemaSteam, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var nsiSchemaMd5Stream = new MD5Stream(nsiSchemaCryptoStream)) { using (var nsiWriter = CaUtilities.GetNsiWriter(nsiMd5Stream, version, genomeAssembly, jsonTag, reportFor)) using (var schemaWriter = new StreamWriter(nsiSchemaMd5Stream)) { nsiWriter.Write(intervals); schemaWriter.Write(intervalJsonSchema); } nsiMetadata = nsiMd5Stream.GetFileMetadata(); nsiSchemaMetadata = nsiSchemaMd5Stream.GetFileMetadata(); } string nsiS3Path = string.Join('/', config.outputDir.path.Trim('/'), nsiFileName); string nsiSchemaS3PathFile = nsiS3Path + SaCommon.JsonSchemaSuffix; s3Client.DecryptUpload(config.outputDir.bucketName, nsiS3Path, localNsiPath, aes, nsiMetadata); s3Client.DecryptUpload(config.outputDir.bucketName, nsiSchemaS3PathFile, localNsiSchemaPath, aes, nsiSchemaMetadata); outputFiles.Add(nsiFileName); outputFiles.Add(nsiFileName + SaCommon.JsonSchemaSuffix); } LambdaUtilities.DeleteTempOutput(); return(CustomAnnotationLambda.GetSuccessResult(config, result, outputFiles)); }
public IRequest Marshall(PutObjectRequest putObjectRequest) { //IL_0006: Unknown result type (might be due to invalid IL or missing references) //IL_000c: Expected O, but got Unknown //IL_02b8: Unknown result type (might be due to invalid IL or missing references) //IL_02bf: Expected O, but got Unknown IRequest val = new DefaultRequest(putObjectRequest, "AmazonS3"); val.set_HttpMethod("PUT"); if (putObjectRequest.IsSetCannedACL()) { val.get_Headers().Add("x-amz-acl", S3Transforms.ToStringValue(ConstantClass.op_Implicit(putObjectRequest.CannedACL))); } HeadersCollection headers = putObjectRequest.Headers; foreach (string key in headers.Keys) { val.get_Headers()[key] = headers[key]; } if (putObjectRequest.IsSetMD5Digest()) { val.get_Headers()["Content-MD5"] = putObjectRequest.MD5Digest; } HeaderACLRequestMarshaller.Marshall(val, putObjectRequest); if (putObjectRequest.IsSetServerSideEncryptionMethod()) { val.get_Headers().Add("x-amz-server-side-encryption", S3Transforms.ToStringValue(ConstantClass.op_Implicit(putObjectRequest.ServerSideEncryptionMethod))); } if (putObjectRequest.IsSetStorageClass()) { val.get_Headers().Add("x-amz-storage-class", S3Transforms.ToStringValue(ConstantClass.op_Implicit(putObjectRequest.StorageClass))); } if (putObjectRequest.IsSetWebsiteRedirectLocation()) { val.get_Headers().Add("x-amz-website-redirect-location", S3Transforms.ToStringValue(putObjectRequest.WebsiteRedirectLocation)); } if (putObjectRequest.IsSetServerSideEncryptionCustomerMethod()) { val.get_Headers().Add("x-amz-server-side-encryption-customer-algorithm", ConstantClass.op_Implicit(putObjectRequest.ServerSideEncryptionCustomerMethod)); } if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKey()) { val.get_Headers().Add("x-amz-server-side-encryption-customer-key", putObjectRequest.ServerSideEncryptionCustomerProvidedKey); if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5()) { val.get_Headers().Add("x-amz-server-side-encryption-customer-key-MD5", putObjectRequest.ServerSideEncryptionCustomerProvidedKeyMD5); } else { val.get_Headers().Add("x-amz-server-side-encryption-customer-key-MD5", AmazonS3Util.ComputeEncodedMD5FromEncodedString(putObjectRequest.ServerSideEncryptionCustomerProvidedKey)); } } if (putObjectRequest.IsSetServerSideEncryptionKeyManagementServiceKeyId()) { val.get_Headers().Add("x-amz-server-side-encryption-aws-kms-key-id", putObjectRequest.ServerSideEncryptionKeyManagementServiceKeyId); } if (putObjectRequest.IsSetRequestPayer()) { val.get_Headers().Add(S3Constants.AmzHeaderRequestPayer, S3Transforms.ToStringValue(((object)putObjectRequest.RequestPayer).ToString())); } if (putObjectRequest.IsSetTagSet()) { val.get_Headers().Add(S3Constants.AmzHeaderTagging, AmazonS3Util.TagSetToQueryString(putObjectRequest.TagSet)); } AmazonS3Util.SetMetadataHeaders(val, putObjectRequest.Metadata); val.set_ResourcePath(string.Format(CultureInfo.InvariantCulture, "/{0}/{1}", S3Transforms.ToStringValue(putObjectRequest.BucketName), S3Transforms.ToStringValue(putObjectRequest.Key))); if (putObjectRequest.InputStream != null) { Stream streamWithLength = GetStreamWithLength(putObjectRequest.InputStream, putObjectRequest.Headers.ContentLength); if (streamWithLength.Length > 0) { val.set_UseChunkEncoding(true); } long num = streamWithLength.Length - streamWithLength.Position; if (!val.get_Headers().ContainsKey("Content-Length")) { val.get_Headers().Add("Content-Length", num.ToString(CultureInfo.InvariantCulture)); } MD5Stream val2 = putObjectRequest.InputStream = (Stream) new MD5Stream(streamWithLength, (byte[])null, num); } val.set_ContentStream(putObjectRequest.InputStream); if (!val.get_Headers().ContainsKey("Content-Type")) { val.get_Headers().Add("Content-Type", "text/plain"); } return(val); }
public static CustomResult Create(CustomConfig config, string inputBaseName, CustomResult result, IS3Client s3Client) { string ngaFileName = inputBaseName + SaCommon.NgaFileSuffix; string localNgaPath = Path.Combine(Path.GetTempPath(), ngaFileName); string localSchemaPath = localNgaPath + SaCommon.JsonSchemaSuffix; string localLogPath = Path.Combine(Path.GetTempPath(), LogFileName); HttpUtilities.ValidateUrl(LambdaUrlHelper.GetUgaUrl()); var outputFiles = new List <string>(); using (var aes = new AesCryptoServiceProvider()) { FileMetadata ngaMetadata, schemaMetadata, logMetaData; using (var logStream = FileUtilities.GetCreateStream(localLogPath)) using (var logCryptoStream = new CryptoStream(logStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var logMd5Stream = new MD5Stream(logCryptoStream)) // using (var customTsvStream = (PersistentStream)PersistentStreamUtils.GetReadStream(config.tsvUrl)) using (var parser = GetGeneAnnotationsParserFromCustomTsvStream(customTsvStream)) // using (var ngaStream = FileUtilities.GetCreateStream(localNgaPath)) using (var ngaCryptoStream = new CryptoStream(ngaStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var ngaMd5Stream = new MD5Stream(ngaCryptoStream)) // using (var schemaStream = FileUtilities.GetCreateStream(localSchemaPath)) using (var schemaCryptoStream = new CryptoStream(schemaStream, aes.CreateEncryptor(), CryptoStreamMode.Write)) using (var schemaMd5Stream = new MD5Stream(schemaCryptoStream)) { using (var ngaWriter = CaUtilities.GetNgaWriter(ngaMd5Stream, parser, config.tsvUrl)) using (var schemaWriter = new StreamWriter(schemaMd5Stream)) using (var logWriter = new StreamWriter(logMd5Stream)) { ngaWriter.Write(parser.GetItems(config.skipGeneIdValidation, logWriter)); schemaWriter.Write(parser.JsonSchema); } //all the writers have to be disposed before GetFileMetaData is called ngaMetadata = ngaMd5Stream.GetFileMetadata(); schemaMetadata = schemaMd5Stream.GetFileMetadata(); logMetaData = logMd5Stream.GetFileMetadata(); } if (config.skipGeneIdValidation) { string logS3Key = string.Join('/', config.outputDir.path.Trim('/'), LogFileName); Logger.WriteLine("uploading log file to " + logS3Key); s3Client.DecryptUpload(config.outputDir.bucketName, logS3Key, localLogPath, aes, logMetaData); } string nsaS3Path = string.Join('/', config.outputDir.path.Trim('/'), ngaFileName); string schemaS3Path = nsaS3Path + SaCommon.JsonSchemaSuffix; s3Client.DecryptUpload(config.outputDir.bucketName, nsaS3Path, localNgaPath, aes, ngaMetadata); s3Client.DecryptUpload(config.outputDir.bucketName, schemaS3Path, localSchemaPath, aes, schemaMetadata); outputFiles.Add(ngaFileName); outputFiles.Add(ngaFileName + SaCommon.JsonSchemaSuffix); LambdaUtilities.DeleteTempOutput(); return(CustomAnnotationLambda.GetSuccessResult(config, result, outputFiles)); } }
public IRequest Marshall(PutObjectRequest putObjectRequest) { IRequest request = new DefaultRequest(putObjectRequest, "AmazonS3"); request.HttpMethod = "PUT"; if (putObjectRequest.IsSetCannedACL()) { request.Headers.Add(HeaderKeys.XAmzAclHeader, S3Transforms.ToStringValue(putObjectRequest.CannedACL)); } var headers = putObjectRequest.Headers; foreach (var key in headers.Keys) { request.Headers[key] = headers[key]; } if (putObjectRequest.IsSetMD5Digest()) { request.Headers[HeaderKeys.ContentMD5Header] = putObjectRequest.MD5Digest; } HeaderACLRequestMarshaller.Marshall(request, putObjectRequest); if (putObjectRequest.IsSetServerSideEncryptionMethod()) { request.Headers.Add(HeaderKeys.XAmzServerSideEncryptionHeader, S3Transforms.ToStringValue(putObjectRequest.ServerSideEncryptionMethod)); } if (putObjectRequest.IsSetStorageClass()) { request.Headers.Add(HeaderKeys.XAmzStorageClassHeader, S3Transforms.ToStringValue(putObjectRequest.StorageClass)); } if (putObjectRequest.IsSetWebsiteRedirectLocation()) { request.Headers.Add(HeaderKeys.XAmzWebsiteRedirectLocationHeader, S3Transforms.ToStringValue(putObjectRequest.WebsiteRedirectLocation)); } if (putObjectRequest.IsSetServerSideEncryptionCustomerMethod()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerAlgorithmHeader, putObjectRequest.ServerSideEncryptionCustomerMethod); } if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKey()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyHeader, putObjectRequest.ServerSideEncryptionCustomerProvidedKey); if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5()) { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, putObjectRequest.ServerSideEncryptionCustomerProvidedKeyMD5); } else { request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, AmazonS3Util.ComputeEncodedMD5FromEncodedString(putObjectRequest.ServerSideEncryptionCustomerProvidedKey)); } } if (putObjectRequest.IsSetServerSideEncryptionKeyManagementServiceKeyId()) { request.Headers.Add(HeaderKeys.XAmzServerSideEncryptionAwsKmsKeyIdHeader, putObjectRequest.ServerSideEncryptionKeyManagementServiceKeyId); } if (putObjectRequest.IsSetServerSideEncryptionKeyManagementServiceEncryptionContext()) { request.Headers.Add("x-amz-server-side-encryption-context", putObjectRequest.ServerSideEncryptionKeyManagementServiceEncryptionContext); } if (putObjectRequest.IsSetObjectLockLegalHoldStatus()) { request.Headers.Add("x-amz-object-lock-legal-hold", S3Transforms.ToStringValue(putObjectRequest.ObjectLockLegalHoldStatus)); } if (putObjectRequest.IsSetObjectLockMode()) { request.Headers.Add("x-amz-object-lock-mode", S3Transforms.ToStringValue(putObjectRequest.ObjectLockMode)); } if (putObjectRequest.IsSetObjectLockRetainUntilDate()) { request.Headers.Add("x-amz-object-lock-retain-until-date", S3Transforms.ToStringValue(putObjectRequest.ObjectLockRetainUntilDate, AWSSDKUtils.ISO8601DateFormat)); } if (putObjectRequest.IsSetRequestPayer()) { request.Headers.Add(S3Constants.AmzHeaderRequestPayer, S3Transforms.ToStringValue(putObjectRequest.RequestPayer.ToString())); } if (putObjectRequest.IsSetTagSet()) { request.Headers.Add(S3Constants.AmzHeaderTagging, AmazonS3Util.TagSetToQueryString(putObjectRequest.TagSet)); } if (putObjectRequest.IsSetExpectedBucketOwner()) { request.Headers.Add(S3Constants.AmzHeaderExpectedBucketOwner, S3Transforms.ToStringValue(putObjectRequest.ExpectedBucketOwner)); } if (putObjectRequest.IsSetBucketKeyEnabled()) { request.Headers.Add(S3Constants.AmzHeaderBucketKeyEnabled, S3Transforms.ToStringValue(putObjectRequest.BucketKeyEnabled)); } if (putObjectRequest.IsSetChecksumAlgorithm()) { request.Headers.Add(S3Constants.AmzHeaderSdkChecksumAlgorithm, S3Transforms.ToStringValue(putObjectRequest.ChecksumAlgorithm)); } if (putObjectRequest.IsSetChecksumCRC32()) { request.Headers.Add("x-amz-checksum-crc32", S3Transforms.ToStringValue(putObjectRequest.ChecksumCRC32)); } if (putObjectRequest.IsSetChecksumCRC32C()) { request.Headers.Add("x-amz-checksum-crc32c", S3Transforms.ToStringValue(putObjectRequest.ChecksumCRC32C)); } if (putObjectRequest.IsSetChecksumSHA1()) { request.Headers.Add("x-amz-checksum-sha1", S3Transforms.ToStringValue(putObjectRequest.ChecksumSHA1)); } if (putObjectRequest.IsSetChecksumSHA256()) { request.Headers.Add("x-amz-checksum-sha256", S3Transforms.ToStringValue(putObjectRequest.ChecksumSHA256)); } AmazonS3Util.SetMetadataHeaders(request, putObjectRequest.Metadata); if (string.IsNullOrEmpty(putObjectRequest.BucketName)) { throw new System.ArgumentException("BucketName is a required property and must be set before making this call.", "PutObjectRequest.BucketName"); } if (string.IsNullOrEmpty(putObjectRequest.Key)) { throw new System.ArgumentException("Key is a required property and must be set before making this call.", "PutObjectRequest.Key"); } request.ResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}", S3Transforms.ToStringValue(putObjectRequest.BucketName), S3Transforms.ToStringValue(putObjectRequest.Key)); if (putObjectRequest.InputStream != null) { // Wrap the stream in a stream that has a length var streamWithLength = GetStreamWithLength(putObjectRequest.InputStream, putObjectRequest.Headers.ContentLength); if (streamWithLength.Length > 0 && !(putObjectRequest.DisablePayloadSigning ?? false)) { request.UseChunkEncoding = putObjectRequest.UseChunkEncoding; } var length = streamWithLength.Length - streamWithLength.Position; if (!request.Headers.ContainsKey(HeaderKeys.ContentLengthHeader)) { request.Headers.Add(HeaderKeys.ContentLengthHeader, length.ToString(CultureInfo.InvariantCulture)); } request.DisablePayloadSigning = putObjectRequest.DisablePayloadSigning; // Calculate Content-MD5 if not already set if (!putObjectRequest.IsSetMD5Digest() && putObjectRequest.CalculateContentMD5Header) { string md5 = AmazonS3Util.GenerateMD5ChecksumForStream(putObjectRequest.InputStream); if (!string.IsNullOrEmpty(md5)) { request.Headers[HeaderKeys.ContentMD5Header] = md5; } } if (!(putObjectRequest.DisableMD5Stream ?? AWSConfigsS3.DisableMD5Stream)) { // Wrap input stream in MD5Stream var hashStream = new MD5Stream(streamWithLength, null, length); putObjectRequest.InputStream = hashStream; } else { putObjectRequest.InputStream = streamWithLength; } } request.ContentStream = putObjectRequest.InputStream; ChecksumUtils.SetRequestChecksum(request, putObjectRequest.ChecksumAlgorithm, fallbackToMD5: false); if (!request.Headers.ContainsKey(HeaderKeys.ContentTypeHeader)) { request.Headers.Add(HeaderKeys.ContentTypeHeader, "text/plain"); } return(request); }