public IRequest Marshall(UploadPartRequest uploadPartRequest)
{
IRequest request = new DefaultRequest(uploadPartRequest, "AmazonS3");
request.HttpMethod = "PUT";
if (uploadPartRequest.IsSetMD5Digest())
{
request.Headers[HeaderKeys.ContentMD5Header] = uploadPartRequest.MD5Digest;
}
if (uploadPartRequest.IsSetServerSideEncryptionCustomerMethod())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerAlgorithmHeader, uploadPartRequest.ServerSideEncryptionCustomerMethod);
}
if (uploadPartRequest.IsSetServerSideEncryptionCustomerProvidedKey())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyHeader, uploadPartRequest.ServerSideEncryptionCustomerProvidedKey);
if (uploadPartRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, uploadPartRequest.ServerSideEncryptionCustomerProvidedKeyMD5);
}
else
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, AmazonS3Util.ComputeEncodedMD5FromEncodedString(uploadPartRequest.ServerSideEncryptionCustomerProvidedKey));
}
}
if (uploadPartRequest.IsSetRequestPayer())
{
request.Headers.Add(S3Constants.AmzHeaderRequestPayer, S3Transforms.ToStringValue(uploadPartRequest.RequestPayer.ToString()));
}
if (uploadPartRequest.IsSetExpectedBucketOwner())
{
request.Headers.Add(S3Constants.AmzHeaderExpectedBucketOwner, S3Transforms.ToStringValue(uploadPartRequest.ExpectedBucketOwner));
}
if (string.IsNullOrEmpty(uploadPartRequest.BucketName))
{
throw new System.ArgumentException("BucketName is a required property and must be set before making this call.", "UploadPartRequest.BucketName");
}
if (string.IsNullOrEmpty(uploadPartRequest.Key))
{
throw new System.ArgumentException("Key is a required property and must be set before making this call.", "UploadPartRequest.Key");
}
request.ResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}",
S3Transforms.ToStringValue(uploadPartRequest.BucketName),
S3Transforms.ToStringValue(uploadPartRequest.Key));
if (uploadPartRequest.IsSetPartNumber())
{
request.AddSubResource("partNumber", S3Transforms.ToStringValue(uploadPartRequest.PartNumber));
}
if (uploadPartRequest.IsSetUploadId())
{
request.AddSubResource("uploadId", S3Transforms.ToStringValue(uploadPartRequest.UploadId));
}
if (uploadPartRequest.InputStream != null)
{
// Wrap input stream in partial wrapper (to upload only part of the stream)
var partialStream = new PartialWrapperStream(uploadPartRequest.InputStream, uploadPartRequest.PartSize);
if (partialStream.Length > 0 && !(uploadPartRequest.DisablePayloadSigning ?? false))
{
request.UseChunkEncoding = uploadPartRequest.UseChunkEncoding;
}
if (!request.Headers.ContainsKey(HeaderKeys.ContentLengthHeader))
{
request.Headers.Add(HeaderKeys.ContentLengthHeader, partialStream.Length.ToString(CultureInfo.InvariantCulture));
}
request.DisablePayloadSigning = uploadPartRequest.DisablePayloadSigning;
if (!(uploadPartRequest.DisableMD5Stream ?? AWSConfigsS3.DisableMD5Stream))
{
// Wrap input stream in MD5Stream; after this we can no longer seek or position the stream
var hashStream = new MD5Stream(partialStream, null, partialStream.Length);
uploadPartRequest.InputStream = hashStream;
}
else
{
uploadPartRequest.InputStream = partialStream;
}
}
request.ContentStream = uploadPartRequest.InputStream;
if (!request.Headers.ContainsKey(HeaderKeys.ContentTypeHeader))
{
request.Headers.Add(HeaderKeys.ContentTypeHeader, "text/plain");
}
return(request);
}
public IRequest Marshall(UploadPartRequest uploadPartRequest)
{
IRequest request = new DefaultRequest(uploadPartRequest, "AmazonS3");
request.HttpMethod = "PUT";
if (uploadPartRequest.IsSetMD5Digest())
{
request.Headers[HeaderKeys.ContentMD5Header] = uploadPartRequest.MD5Digest;
}
if (uploadPartRequest.IsSetServerSideEncryptionCustomerMethod())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerAlgorithmHeader, uploadPartRequest.ServerSideEncryptionCustomerMethod);
}
if (uploadPartRequest.IsSetServerSideEncryptionCustomerProvidedKey())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyHeader, uploadPartRequest.ServerSideEncryptionCustomerProvidedKey);
if (uploadPartRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, uploadPartRequest.ServerSideEncryptionCustomerProvidedKeyMD5);
}
else
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, AmazonS3Util.ComputeEncodedMD5FromEncodedString(uploadPartRequest.ServerSideEncryptionCustomerProvidedKey));
}
}
if (uploadPartRequest.IsSetRequestPayer())
{
request.Headers.Add(S3Constants.AmzHeaderRequestPayer, S3Transforms.ToStringValue(uploadPartRequest.RequestPayer.ToString()));
}
request.ResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}",
S3Transforms.ToStringValue(uploadPartRequest.BucketName),
S3Transforms.ToStringValue(uploadPartRequest.Key));
if (uploadPartRequest.IsSetPartNumber())
{
request.AddSubResource("partNumber", S3Transforms.ToStringValue(uploadPartRequest.PartNumber));
}
if (uploadPartRequest.IsSetUploadId())
{
request.AddSubResource("uploadId", S3Transforms.ToStringValue(uploadPartRequest.UploadId));
}
if (uploadPartRequest.InputStream != null)
{
// Wrap input stream in partial wrapper (to upload only part of the stream)
var partialStream = new PartialWrapperStream(uploadPartRequest.InputStream, uploadPartRequest.PartSize);
if (partialStream.Length > 0)
{
request.UseChunkEncoding = uploadPartRequest.UseChunkEncoding;
}
if (!request.Headers.ContainsKey(HeaderKeys.ContentLengthHeader))
{
request.Headers.Add(HeaderKeys.ContentLengthHeader, partialStream.Length.ToString(CultureInfo.InvariantCulture));
}
// Wrap input stream in MD5Stream; after this we can no longer seek or position the stream
var hashStream = new MD5Stream(partialStream, null, partialStream.Length);
uploadPartRequest.InputStream = hashStream;
}
request.ContentStream = uploadPartRequest.InputStream;
if (!request.Headers.ContainsKey(HeaderKeys.ContentTypeHeader))
{
request.Headers.Add(HeaderKeys.ContentTypeHeader, "text/plain");
}
return(request);
}
public IRequest Marshall(PutObjectRequest putObjectRequest)
{
IRequest request = new DefaultRequest(putObjectRequest, "AmazonS3");
request.HttpMethod = "PUT";
if (putObjectRequest.IsSetCannedACL())
{
request.Headers.Add(HeaderKeys.XAmzAclHeader, S3Transforms.ToStringValue(putObjectRequest.CannedACL));
}
var headers = putObjectRequest.Headers;
foreach (var key in headers.Keys)
{
request.Headers[key] = headers[key];
}
if (putObjectRequest.IsSetMD5Digest())
{
request.Headers[HeaderKeys.ContentMD5Header] = putObjectRequest.MD5Digest;
}
HeaderACLRequestMarshaller.Marshall(request, putObjectRequest);
if (putObjectRequest.IsSetServerSideEncryptionMethod())
{
request.Headers.Add(HeaderKeys.XAmzServerSideEncryptionHeader, S3Transforms.ToStringValue(putObjectRequest.ServerSideEncryptionMethod));
}
if (putObjectRequest.IsSetStorageClass())
{
request.Headers.Add(HeaderKeys.XAmzStorageClassHeader, S3Transforms.ToStringValue(putObjectRequest.StorageClass));
}
if (putObjectRequest.IsSetWebsiteRedirectLocation())
{
request.Headers.Add(HeaderKeys.XAmzWebsiteRedirectLocationHeader, S3Transforms.ToStringValue(putObjectRequest.WebsiteRedirectLocation));
}
if (putObjectRequest.IsSetServerSideEncryptionCustomerMethod())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerAlgorithmHeader, putObjectRequest.ServerSideEncryptionCustomerMethod);
}
if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKey())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyHeader, putObjectRequest.ServerSideEncryptionCustomerProvidedKey);
if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, putObjectRequest.ServerSideEncryptionCustomerProvidedKeyMD5);
}
else
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, AmazonS3Util.ComputeEncodedMD5FromEncodedString(putObjectRequest.ServerSideEncryptionCustomerProvidedKey));
}
}
if (putObjectRequest.IsSetServerSideEncryptionKeyManagementServiceKeyId())
{
request.Headers.Add(HeaderKeys.XAmzServerSideEncryptionAwsKmsKeyIdHeader, putObjectRequest.ServerSideEncryptionKeyManagementServiceKeyId);
}
if (putObjectRequest.IsSetObjectLockLegalHoldStatus())
{
request.Headers.Add("x-amz-object-lock-legal-hold", S3Transforms.ToStringValue(putObjectRequest.ObjectLockLegalHoldStatus));
}
if (putObjectRequest.IsSetObjectLockMode())
{
request.Headers.Add("x-amz-object-lock-mode", S3Transforms.ToStringValue(putObjectRequest.ObjectLockMode));
}
if (putObjectRequest.IsSetObjectLockRetainUntilDate())
{
request.Headers.Add("x-amz-object-lock-retain-until-date", S3Transforms.ToStringValue(putObjectRequest.ObjectLockRetainUntilDate, AWSSDKUtils.ISO8601DateFormat));
}
if (putObjectRequest.IsSetRequestPayer())
{
request.Headers.Add(S3Constants.AmzHeaderRequestPayer, S3Transforms.ToStringValue(putObjectRequest.RequestPayer.ToString()));
}
if (putObjectRequest.IsSetTagSet())
{
request.Headers.Add(S3Constants.AmzHeaderTagging, AmazonS3Util.TagSetToQueryString(putObjectRequest.TagSet));
}
AmazonS3Util.SetMetadataHeaders(request, putObjectRequest.Metadata);
if (string.IsNullOrEmpty(putObjectRequest.BucketName))
{
throw new System.ArgumentException("BucketName is a required property and must be set before making this call.", "PutObjectRequest.BucketName");
}
if (string.IsNullOrEmpty(putObjectRequest.Key))
{
throw new System.ArgumentException("Key is a required property and must be set before making this call.", "PutObjectRequest.Key");
}
request.ResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}",
S3Transforms.ToStringValue(putObjectRequest.BucketName),
S3Transforms.ToStringValue(putObjectRequest.Key));
if (putObjectRequest.InputStream != null)
{
// Wrap the stream in a stream that has a length
var streamWithLength = GetStreamWithLength(putObjectRequest.InputStream, putObjectRequest.Headers.ContentLength);
if (streamWithLength.Length > 0)
{
request.UseChunkEncoding = putObjectRequest.UseChunkEncoding;
}
var length = streamWithLength.Length - streamWithLength.Position;
if (!request.Headers.ContainsKey(HeaderKeys.ContentLengthHeader))
{
request.Headers.Add(HeaderKeys.ContentLengthHeader, length.ToString(CultureInfo.InvariantCulture));
}
// Wrap input stream in MD5Stream
var hashStream = new MD5Stream(streamWithLength, null, length);
putObjectRequest.InputStream = hashStream;
}
request.ContentStream = putObjectRequest.InputStream;
if (!request.Headers.ContainsKey(HeaderKeys.ContentTypeHeader))
{
request.Headers.Add(HeaderKeys.ContentTypeHeader, "text/plain");
}
return(request);
}
protected override void ProcessResponseHandlers(AmazonWebServiceResponse response, IRequest request, IWebResponseData webResponseData)
{
base.ProcessResponseHandlers(response, request, webResponseData);
var getObjectResponse = response as GetObjectResponse;
if (getObjectResponse != null)
{
GetObjectRequest getObjectRequest = request.OriginalRequest as GetObjectRequest;
getObjectResponse.BucketName = getObjectRequest.BucketName;
getObjectResponse.Key = getObjectRequest.Key;
// If ETag is present and is an MD5 hash (not a multi-part upload ETag), and no byte range is specified,
// wrap the response stream in an MD5Stream.
if (!string.IsNullOrEmpty(getObjectResponse.ETag) &&
!getObjectResponse.ETag.Contains("-") &&
getObjectRequest.ByteRange == null)
{
string etag = getObjectResponse.ETag.Trim(etagTrimChars);
byte[] expectedHash = AWSSDKUtils.HexStringToBytes(etag);
HashStream hashStream = new MD5Stream(getObjectResponse.ResponseStream, expectedHash, getObjectResponse.ContentLength);
getObjectResponse.ResponseStream = hashStream;
}
}
var deleteObjectsResponse = response as DeleteObjectsResponse;
if (deleteObjectsResponse != null)
{
if (deleteObjectsResponse.DeleteErrors != null && deleteObjectsResponse.DeleteErrors.Count > 0)
{
throw new DeleteObjectsException(deleteObjectsResponse as DeleteObjectsResponse);
}
}
var putObjectResponse = response as PutObjectResponse;
var putObjectRequest = request.OriginalRequest as PutObjectRequest;
if (putObjectRequest != null)
{
// If InputStream was a HashStream, compare calculated hash to returned etag
HashStream hashStream = putObjectRequest.InputStream as HashStream;
if (hashStream != null)
{
if (putObjectResponse != null)
{
// Stream may not have been closed, so force calculation of hash
hashStream.CalculateHash();
CompareHashes(putObjectResponse.ETag, hashStream.CalculatedHash);
}
// Set InputStream to its original value
putObjectRequest.InputStream = hashStream.GetNonWrapperBaseStream();
}
}
var listObjectsResponse = response as ListObjectsResponse;
if (listObjectsResponse != null)
{
if (listObjectsResponse.IsTruncated &&
string.IsNullOrEmpty(listObjectsResponse.NextMarker) &&
listObjectsResponse.S3Objects.Count > 0)
{
listObjectsResponse.NextMarker = listObjectsResponse.S3Objects.Last().Key;
}
}
var uploadPartRequest = request.OriginalRequest as UploadPartRequest;
var uploadPartResponse = response as UploadPartResponse;
if (uploadPartRequest != null)
{
if (uploadPartResponse != null)
{
uploadPartResponse.PartNumber = uploadPartRequest.PartNumber;
}
// If InputStream was a HashStream, compare calculated hash to returned etag
HashStream hashStream = uploadPartRequest.InputStream as HashStream;
if (hashStream != null)
{
if (uploadPartResponse != null)
{
// Stream may not have been closed, so force calculation of hash
hashStream.CalculateHash();
CompareHashes(uploadPartResponse.ETag, hashStream.CalculatedHash);
}
// Set InputStream to its original value
uploadPartRequest.InputStream = hashStream.GetNonWrapperBaseStream();
}
}
var copyPartResponse = response as CopyPartResponse;
if (copyPartResponse != null)
{
copyPartResponse.PartNumber = ((CopyPartRequest)request.OriginalRequest).PartNumber;
}
CleanupRequest(request);
}
// ReSharper disable once UnusedMember.Global
public AnnotationResult Run(AnnotationConfig config, ILambdaContext context)
{
var result = new AnnotationResult {
id = config.id
};
string snsTopicArn = null;
var runLog = new StringBuilder();
try
{
LogUtilities.UpdateLogger(context.Logger, runLog);
LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion);
LogUtilities.LogObject("Config", config);
LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey });
LambdaUtilities.GarbageCollect();
LambdaUtilities.DeleteTempOutput();
snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey);
string vcfUrl = config.vcfUrl;
using (var annotationResources = GetAnnotationResources(config))
{
if (annotationResources.InputStartVirtualPosition == -1)
{
return(GetSuccessOutput(result));
}
long fileOffset = VirtualPosition.From(annotationResources.InputStartVirtualPosition).FileOffset;
using (var preloadVcfStream = PersistentStreamUtils.GetReadStream(vcfUrl, fileOffset))
{
annotationResources.GetVariantPositions(new BlockGZipStream(preloadVcfStream, CompressionMode.Decompress), config.annotationRange.ToGenomicRange(annotationResources.SequenceProvider.RefNameToChromosome));
}
Logger.WriteLine("Scan for positions to preload complete.");
using (var aes = new AesCryptoServiceProvider())
{
FileMetadata jsonMetadata, jasixMetadata;
string jsonPath = Path.GetTempPath() + LambdaUrlHelper.JsonSuffix;
string jasixPath = jsonPath + LambdaUrlHelper.JsonIndexSuffix;
using (var inputVcfStream = new BlockGZipStream(PersistentStreamUtils.GetReadStream(vcfUrl, fileOffset), CompressionMode.Decompress))
using (var headerStream = config.annotationRange == null ? null : new BlockGZipStream(PersistentStreamUtils.GetReadStream(vcfUrl), CompressionMode.Decompress))
//
using (var jsonFileStream = FileUtilities.GetCreateStream(jsonPath))
using (var jsonCryptoStream = new CryptoStream(jsonFileStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
using (var jsonMd5Stream = new MD5Stream(jsonCryptoStream))
//
using (var jasixFileStream = FileUtilities.GetCreateStream(jasixPath))
using (var jasixCryptoStream = new CryptoStream(jasixFileStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
using (var jasixMd5Stream = new MD5Stream(jasixCryptoStream))
{
IVcfFilter vcfFilter = config.annotationRange == null
? new NullVcfFilter() as IVcfFilter
: new VcfFilter(config.annotationRange.ToGenomicRange(annotationResources.SequenceProvider.RefNameToChromosome));
using (var jsonCompressStream = new BlockGZipStream(jsonMd5Stream, CompressionMode.Compress))
{
StreamAnnotation.Annotate(headerStream, inputVcfStream, jsonCompressStream, jasixMd5Stream, annotationResources, vcfFilter, true);
}
Logger.WriteLine("Annotation done.");
jsonMetadata = jsonMd5Stream.GetFileMetadata();
jasixMetadata = jasixMd5Stream.GetFileMetadata();
}
result.filePath = S3Utilities.GetKey(config.outputDir.path, config.outputPrefix + LambdaUrlHelper.JsonSuffix);
string jasixKey = result.filePath + LambdaUrlHelper.JsonIndexSuffix;
var s3Client = config.outputDir.GetS3Client(context.RemainingTime);
s3Client.DecryptUpload(config.outputDir.bucketName, jasixKey, jasixPath, aes, jasixMetadata);
s3Client.DecryptUpload(config.outputDir.bucketName, result.filePath, jsonPath, aes, jsonMetadata);
Logger.WriteLine("Nirvana result files uploaded.");
}
}
LambdaUtilities.DeleteTempOutput();
if (string.IsNullOrEmpty(result.filePath))
{
throw new FileNotFoundException();
}
return(GetSuccessOutput(result));
}
catch (Exception exception)
{
LambdaUtilities.DeleteTempOutput();
return(HandleException(runLog, result, exception, snsTopicArn));
}
}
public IRequest Marshall(PutObjectRequest putObjectRequest)
{
IRequest request = new DefaultRequest(putObjectRequest, "AmazonS3");
request.HttpMethod = "PUT";
if (putObjectRequest.IsSetCannedACL())
{
request.Headers.Add(HeaderKeys.XAmzAclHeader, S3Transforms.ToStringValue(putObjectRequest.CannedACL));
}
var headers = putObjectRequest.Headers;
foreach (var key in headers.Keys)
{
request.Headers[key] = headers[key];
}
if (putObjectRequest.IsSetMD5Digest())
{
request.Headers[HeaderKeys.ContentMD5Header] = putObjectRequest.MD5Digest;
}
HeaderACLRequestMarshaller.Marshall(request, putObjectRequest);
if (putObjectRequest.IsSetServerSideEncryptionMethod())
{
request.Headers.Add(HeaderKeys.XAmzServerSideEncryptionHeader, S3Transforms.ToStringValue(putObjectRequest.ServerSideEncryptionMethod));
}
if (putObjectRequest.IsSetStorageClass())
{
request.Headers.Add(HeaderKeys.XAmzStorageClassHeader, S3Transforms.ToStringValue(putObjectRequest.StorageClass));
}
if (putObjectRequest.IsSetWebsiteRedirectLocation())
{
request.Headers.Add(HeaderKeys.XAmzWebsiteRedirectLocationHeader, S3Transforms.ToStringValue(putObjectRequest.WebsiteRedirectLocation));
}
if (putObjectRequest.IsSetServerSideEncryptionCustomerMethod())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerAlgorithmHeader, putObjectRequest.ServerSideEncryptionCustomerMethod);
}
if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKey())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyHeader, putObjectRequest.ServerSideEncryptionCustomerProvidedKey);
if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, putObjectRequest.ServerSideEncryptionCustomerProvidedKeyMD5);
}
else
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, AmazonS3Util.ComputeEncodedMD5FromEncodedString(putObjectRequest.ServerSideEncryptionCustomerProvidedKey));
}
}
AmazonS3Util.SetMetadataHeaders(request, putObjectRequest.Metadata);
request.ResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}",
S3Transforms.ToStringValue(putObjectRequest.BucketName),
S3Transforms.ToStringValue(putObjectRequest.Key));
if (putObjectRequest.InputStream != null)
{
// Wrap the stream in a stream that has a length
var streamWithLength = GetStreamWithLength(putObjectRequest.InputStream, putObjectRequest.Headers.ContentLength);
if (streamWithLength.Length > 0)
{
request.UseChunkEncoding = true;
}
var length = streamWithLength.Length - streamWithLength.Position;
if (!request.Headers.ContainsKey(HeaderKeys.ContentLengthHeader))
{
request.Headers.Add(HeaderKeys.ContentLengthHeader, length.ToString(CultureInfo.InvariantCulture));
}
// Wrap input stream in MD5Stream
var hashStream = new MD5Stream(streamWithLength, null, length);
putObjectRequest.InputStream = hashStream;
}
request.ContentStream = putObjectRequest.InputStream;
if (!request.Headers.ContainsKey(HeaderKeys.ContentTypeHeader))
{
request.Headers.Add(HeaderKeys.ContentTypeHeader, "text/plain");
}
return(request);
}
private static void ProcessResponseHandlers(IExecutionContext executionContext)
{
AmazonWebServiceResponse response = executionContext.ResponseContext.Response;
IRequest request = executionContext.RequestContext.Request;
IWebResponseData webResponseData = executionContext.ResponseContext.HttpResponse;
bool isSse = HasSSEHeaders(webResponseData);
var getObjectResponse = response as GetObjectResponse;
if (getObjectResponse != null)
{
GetObjectRequest getObjectRequest = request.OriginalRequest as GetObjectRequest;
getObjectResponse.BucketName = getObjectRequest.BucketName;
getObjectResponse.Key = getObjectRequest.Key;
// If ETag is present and is an MD5 hash (not a multi-part upload ETag), and no byte range is specified,
// wrap the response stream in an MD5Stream.
// If there is a customer encryption algorithm the etag is not an MD5.
if (!string.IsNullOrEmpty(getObjectResponse.ETag)
&& !getObjectResponse.ETag.Contains("-")
&& !isSse
&& getObjectRequest.ByteRange == null)
{
string etag = getObjectResponse.ETag.Trim(etagTrimChars);
byte[] expectedHash = AWSSDKUtils.HexStringToBytes(etag);
HashStream hashStream = new MD5Stream(getObjectResponse.ResponseStream, expectedHash, getObjectResponse.ContentLength);
getObjectResponse.ResponseStream = hashStream;
}
}
var deleteObjectsResponse = response as DeleteObjectsResponse;
if (deleteObjectsResponse != null)
{
if (deleteObjectsResponse.DeleteErrors != null && deleteObjectsResponse.DeleteErrors.Count > 0)
{
throw new DeleteObjectsException(deleteObjectsResponse as DeleteObjectsResponse);
}
}
var putObjectResponse = response as PutObjectResponse;
var putObjectRequest = request.OriginalRequest as PutObjectRequest;
if (putObjectRequest != null)
{
// If InputStream was a HashStream, compare calculated hash to returned etag
HashStream hashStream = putObjectRequest.InputStream as HashStream;
if (hashStream != null)
{
if (putObjectResponse != null && !isSse)
{
// Stream may not have been closed, so force calculation of hash
hashStream.CalculateHash();
CompareHashes(putObjectResponse.ETag, hashStream.CalculatedHash);
}
// Set InputStream to its original value
putObjectRequest.InputStream = hashStream.GetNonWrapperBaseStream();
}
}
var listObjectsResponse = response as ListObjectsResponse;
if (listObjectsResponse != null)
{
if (listObjectsResponse.IsTruncated &&
string.IsNullOrEmpty(listObjectsResponse.NextMarker) &&
listObjectsResponse.S3Objects.Count > 0)
{
listObjectsResponse.NextMarker = listObjectsResponse.S3Objects.Last().Key;
}
}
var uploadPartRequest = request.OriginalRequest as UploadPartRequest;
var uploadPartResponse = response as UploadPartResponse;
if (uploadPartRequest != null)
{
if (uploadPartResponse != null)
uploadPartResponse.PartNumber = uploadPartRequest.PartNumber;
// If InputStream was a HashStream, compare calculated hash to returned etag
HashStream hashStream = uploadPartRequest.InputStream as HashStream;
if (hashStream != null)
{
if (uploadPartResponse != null && !isSse)
{
// Stream may not have been closed, so force calculation of hash
hashStream.CalculateHash();
CompareHashes(uploadPartResponse.ETag, hashStream.CalculatedHash);
}
// Set InputStream to its original value
uploadPartRequest.InputStream = hashStream.GetNonWrapperBaseStream();
}
}
var copyPartResponse = response as CopyPartResponse;
if (copyPartResponse != null)
{
copyPartResponse.PartNumber = ((CopyPartRequest)request.OriginalRequest).PartNumber;
}
AmazonS3Client.CleanupRequest(request);
}
public IRequest Marshall(UploadPartRequest uploadPartRequest)
{
IRequest request = new DefaultRequest(uploadPartRequest, "AmazonS3");
request.HttpMethod = "PUT";
if (uploadPartRequest.IsSetMD5Digest())
{
request.Headers["Content-MD5"] = uploadPartRequest.MD5Digest;
}
var uriResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}",
S3Transforms.ToStringValue(uploadPartRequest.BucketName),
S3Transforms.ToStringValue(uploadPartRequest.Key));
if (uploadPartRequest.IsSetPartNumber())
{
request.Parameters.Add("partNumber", S3Transforms.ToStringValue(uploadPartRequest.PartNumber));
}
if (uploadPartRequest.IsSetUploadId())
{
request.Parameters.Add("uploadId", S3Transforms.ToStringValue(uploadPartRequest.UploadId));
}
request.CanonicalResource = S3Transforms.GetCanonicalResource(uriResourcePath, request.Parameters);
request.ResourcePath = S3Transforms.FormatResourcePath(uriResourcePath, request.Parameters);
if (uploadPartRequest.InputStream != null)
{
// Wrap input stream in partial wrapper (to upload only part of the stream)
var partialStream = new PartialWrapperStream(uploadPartRequest.InputStream, uploadPartRequest.PartSize);
if (partialStream.Length > 0)
{
request.UseChunkEncoding = true;
}
if (!request.Headers.ContainsKey("Content-Length"))
{
request.Headers.Add("Content-Length", partialStream.Length.ToString(CultureInfo.InvariantCulture));
}
// Wrap input stream in MD5Stream; after this we can no longer seek or position the stream
var hashStream = new MD5Stream(partialStream, null, partialStream.Length);
uploadPartRequest.InputStream = hashStream;
}
request.ContentStream = uploadPartRequest.InputStream;
if (!request.Headers.ContainsKey("Content-Type"))
{
request.Headers.Add("Content-Type", "text/plain");
}
if (!request.UseQueryString)
{
var queryString = AWSSDKUtils.GetParametersAsString(request.Parameters);
if (!string.IsNullOrEmpty(queryString))
{
request.ResourcePath = string.Concat(request.ResourcePath, request.ResourcePath.Contains("?") ? "&" : "?", queryString);
}
}
return(request);
}
public static UploadPartCommand Create(IServiceClient client, Uri endpoint, ExecutionContext context,
UploadPartRequest uploadPartRequest)
{
OssUtils.CheckBucketName(uploadPartRequest.BucketName);
OssUtils.CheckObjectKey(uploadPartRequest.Key);
if (string.IsNullOrEmpty(uploadPartRequest.UploadId))
{
throw new ArgumentException("uploadId should be specified");
}
if (!uploadPartRequest.PartNumber.HasValue)
{
throw new ArgumentException("partNumber should be specified");
}
if (!uploadPartRequest.PartSize.HasValue)
{
throw new ArgumentException("partSize should be specified");
}
if (uploadPartRequest.InputStream == null)
{
throw new ArgumentException("inputStream should be specified");
}
if (uploadPartRequest.PartSize < 0 || uploadPartRequest.PartSize > OssUtils.MaxFileSize)
{
throw new ArgumentException("partSize not live in valid range");
}
if (!OssUtils.IsPartNumberInRange(uploadPartRequest.PartNumber))
{
throw new ArgumentException("partNumber not live in valid range");
}
var conf = OssUtils.GetClientConfiguration(client);
var originalStream = uploadPartRequest.InputStream;
var streamLength = uploadPartRequest.PartSize.Value;
// wrap input stream in PartialWrapperStream
originalStream = new PartialWrapperStream(originalStream, streamLength);
// setup progress
var callback = uploadPartRequest.StreamTransferProgress;
if (callback != null)
{
originalStream = OssUtils.SetupProgressListeners(originalStream, conf.ProgressUpdateInterval, client, callback);
uploadPartRequest.InputStream = originalStream;
}
// wrap input stream in MD5Stream
if (conf.EnalbeMD5Check)
{
var hashStream = new MD5Stream(originalStream, null, streamLength);
uploadPartRequest.InputStream = hashStream;
context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream));
}
else if (conf.EnableCrcCheck)
{
var hashStream = new Crc64Stream(originalStream, null, streamLength);
uploadPartRequest.InputStream = hashStream;
context.ResponseHandlers.Add(new Crc64CheckHandler(hashStream));
}
return(new UploadPartCommand(client, endpoint, context,
DeserializerFactory.GetFactory().CreateUploadPartResultDeserializer(uploadPartRequest.PartNumber.Value, streamLength),
uploadPartRequest));
}
public IRequest Marshall(UploadPartRequest uploadPartRequest)
{
//IL_0006: Unknown result type (might be due to invalid IL or missing references)
//IL_000c: Expected O, but got Unknown
//IL_0160: Unknown result type (might be due to invalid IL or missing references)
//IL_0166: Expected O, but got Unknown
//IL_01b4: Unknown result type (might be due to invalid IL or missing references)
//IL_01ba: Expected O, but got Unknown
IRequest val = new DefaultRequest(uploadPartRequest, "AmazonS3");
val.set_HttpMethod("PUT");
if (uploadPartRequest.IsSetMD5Digest())
{
val.get_Headers()["Content-MD5"] = uploadPartRequest.MD5Digest;
}
if (uploadPartRequest.IsSetServerSideEncryptionCustomerMethod())
{
val.get_Headers().Add("x-amz-server-side-encryption-customer-algorithm", ConstantClass.op_Implicit(uploadPartRequest.ServerSideEncryptionCustomerMethod));
}
if (uploadPartRequest.IsSetServerSideEncryptionCustomerProvidedKey())
{
val.get_Headers().Add("x-amz-server-side-encryption-customer-key", uploadPartRequest.ServerSideEncryptionCustomerProvidedKey);
if (uploadPartRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5())
{
val.get_Headers().Add("x-amz-server-side-encryption-customer-key-MD5", uploadPartRequest.ServerSideEncryptionCustomerProvidedKeyMD5);
}
else
{
val.get_Headers().Add("x-amz-server-side-encryption-customer-key-MD5", AmazonS3Util.ComputeEncodedMD5FromEncodedString(uploadPartRequest.ServerSideEncryptionCustomerProvidedKey));
}
}
if (uploadPartRequest.IsSetRequestPayer())
{
val.get_Headers().Add(S3Constants.AmzHeaderRequestPayer, S3Transforms.ToStringValue(((object)uploadPartRequest.RequestPayer).ToString()));
}
val.set_ResourcePath(string.Format(CultureInfo.InvariantCulture, "/{0}/{1}", S3Transforms.ToStringValue(uploadPartRequest.BucketName), S3Transforms.ToStringValue(uploadPartRequest.Key)));
if (uploadPartRequest.IsSetPartNumber())
{
val.AddSubResource("partNumber", S3Transforms.ToStringValue(uploadPartRequest.PartNumber));
}
if (uploadPartRequest.IsSetUploadId())
{
val.AddSubResource("uploadId", S3Transforms.ToStringValue(uploadPartRequest.UploadId));
}
if (uploadPartRequest.InputStream != null)
{
PartialWrapperStream val2 = new PartialWrapperStream(uploadPartRequest.InputStream, uploadPartRequest.PartSize);
if (((Stream)val2).Length > 0)
{
val.set_UseChunkEncoding(true);
}
if (!val.get_Headers().ContainsKey("Content-Length"))
{
val.get_Headers().Add("Content-Length", ((Stream)val2).Length.ToString(CultureInfo.InvariantCulture));
}
MD5Stream val3 = uploadPartRequest.InputStream = (Stream) new MD5Stream((Stream)val2, (byte[])null, ((Stream)val2).Length);
}
val.set_ContentStream(uploadPartRequest.InputStream);
if (!val.get_Headers().ContainsKey("Content-Type"))
{
val.get_Headers().Add("Content-Type", "text/plain");
}
return(val);
}
public IRequest Marshall(PutObjectRequest putObjectRequest)
{
IRequest request = new DefaultRequest(putObjectRequest, "AmazonS3");
request.HttpMethod = "PUT";
if (putObjectRequest.IsSetCannedACL())
{
request.Headers.Add("x-amz-acl", S3Transforms.ToStringValue(putObjectRequest.CannedACL));
}
var headers = putObjectRequest.Headers;
foreach (var key in headers.Keys)
{
request.Headers[key] = headers[key];
}
if (putObjectRequest.IsSetMD5Digest())
{
request.Headers["Content-MD5"] = putObjectRequest.MD5Digest;
}
HeaderACLRequestMarshaller.Marshall(request, putObjectRequest);
if (putObjectRequest.IsSetServerSideEncryptionMethod())
{
request.Headers.Add("x-amz-server-side-encryption", S3Transforms.ToStringValue(putObjectRequest.ServerSideEncryptionMethod));
}
if (putObjectRequest.IsSetStorageClass())
{
request.Headers.Add("x-amz-storage-class", S3Transforms.ToStringValue(putObjectRequest.StorageClass));
}
if (putObjectRequest.IsSetWebsiteRedirectLocation())
{
request.Headers.Add("x-amz-website-redirect-location", S3Transforms.ToStringValue(putObjectRequest.WebsiteRedirectLocation));
}
AmazonS3Util.SetMetadataHeaders(request, putObjectRequest.Metadata);
var uriResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}",
S3Transforms.ToStringValue(putObjectRequest.BucketName),
S3Transforms.ToStringValue(putObjectRequest.Key));
request.CanonicalResource = S3Transforms.GetCanonicalResource(uriResourcePath, request.Parameters);
request.ResourcePath = S3Transforms.FormatResourcePath(uriResourcePath, request.Parameters);
if (putObjectRequest.InputStream != null)
{
// Wrap the stream in a stream that has a length
var streamWithLength = GetStreamWithLength(putObjectRequest.InputStream, putObjectRequest.Headers.ContentLength);
if (streamWithLength.Length > 0)
{
request.UseChunkEncoding = true;
}
var length = streamWithLength.Length - streamWithLength.Position;
if (!request.Headers.ContainsKey("Content-Length"))
{
request.Headers.Add("Content-Length", length.ToString(CultureInfo.InvariantCulture));
}
// Wrap input stream in MD5Stream
var hashStream = new MD5Stream(streamWithLength, null, length);
putObjectRequest.InputStream = hashStream;
}
request.ContentStream = putObjectRequest.InputStream;
if (!request.Headers.ContainsKey("Content-Type"))
{
request.Headers.Add("Content-Type", "text/plain");
}
if (!request.UseQueryString)
{
var queryString = AWSSDKUtils.GetParametersAsString(request.Parameters);
if (!string.IsNullOrEmpty(queryString))
{
request.ResourcePath = string.Concat(request.ResourcePath, request.ResourcePath.Contains("?") ? "&" : "?", queryString);
}
}
return(request);
}
public static CustomResult Create(CustomConfig config, string inputFileName, CustomResult result, IS3Client s3Client)
{
string tempPath = Path.GetTempPath();
string inputBaseName = inputFileName.TrimEndFromFirst(".tsv");
string nsaFileName = inputBaseName + SaCommon.SaFileSuffix;
string localNsaPath = Path.Combine(tempPath, nsaFileName);
string localIndexPath = localNsaPath + SaCommon.IndexSufix;
string localSchemaPath = localNsaPath + SaCommon.JsonSchemaSuffix;
var outputFiles = new List <string>();
using (var aes = new AesCryptoServiceProvider())
{
FileMetadata nsaMetadata, indexMetadata, schemaMetadata;
List <CustomInterval> intervals;
string jsonTag;
SaJsonSchema intervalJsonSchema;
DataSourceVersion version;
GenomeAssembly genomeAssembly;
int nsaItemsCount;
ReportFor reportFor;
using (var customTsvStream = (PersistentStream)PersistentStreamUtils.GetReadStream(config.tsvUrl))
using (var parser = GetVariantAnnotationsParserFromCustomTsvStream(customTsvStream))
//
using (var nsaStream = FileUtilities.GetCreateStream(localNsaPath))
using (var nsaCryptoStream = new CryptoStream(nsaStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
using (var nsaMd5Stream = new MD5Stream(nsaCryptoStream))
//
using (var indexStream = FileUtilities.GetCreateStream(localIndexPath))
using (var indexCryptoStream = new CryptoStream(indexStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
using (var indexMd5Stream = new MD5Stream(indexCryptoStream))
//
using (var schemaStream = FileUtilities.GetCreateStream(localSchemaPath))
using (var schemaCryptoStream = new CryptoStream(schemaStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
using (var schemaMd5Stream = new MD5Stream(schemaCryptoStream))
{
genomeAssembly = parser.Assembly;
result.genomeAssembly = genomeAssembly.ToString();
reportFor = parser.ReportFor;
using (var nsaWriter = CaUtilities.GetNsaWriter(nsaMd5Stream, indexMd5Stream, parser, inputFileName, parser.SequenceProvider, out version))
using (var schemaWriter = new StreamWriter(schemaMd5Stream))
{
(jsonTag, nsaItemsCount, intervalJsonSchema, intervals) = CaUtilities.WriteSmallVariants(parser, nsaWriter, schemaWriter);
}
nsaMetadata = nsaMd5Stream.GetFileMetadata();
indexMetadata = indexMd5Stream.GetFileMetadata();
schemaMetadata = schemaMd5Stream.GetFileMetadata();
}
if (nsaItemsCount > 0)
{
string nsaS3Path = string.Join('/', config.outputDir.path.Trim('/'), nsaFileName);
string indexS3Path = nsaS3Path + SaCommon.IndexSufix;
string schemaS3Path = nsaS3Path + SaCommon.JsonSchemaSuffix;
s3Client.DecryptUpload(config.outputDir.bucketName, nsaS3Path, localNsaPath, aes, nsaMetadata);
s3Client.DecryptUpload(config.outputDir.bucketName, indexS3Path, localIndexPath, aes,
indexMetadata);
s3Client.DecryptUpload(config.outputDir.bucketName, schemaS3Path, localSchemaPath, aes,
schemaMetadata);
outputFiles.Add(nsaFileName);
outputFiles.Add(nsaFileName + SaCommon.IndexSufix);
outputFiles.Add(nsaFileName + SaCommon.JsonSchemaSuffix);
}
if (intervals == null)
{
return(CustomAnnotationLambda.GetSuccessResult(config, result, outputFiles));
}
FileMetadata nsiMetadata, nsiSchemaMetadata;
string nsiFileName = inputBaseName + SaCommon.SiFileSuffix;
string localNsiPath = Path.Combine(tempPath, nsiFileName);
string localNsiSchemaPath = localNsiPath + SaCommon.JsonSchemaSuffix;
//
using (var nsiStream = FileUtilities.GetCreateStream(localNsiPath))
using (var nsiCryptoStream = new CryptoStream(nsiStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
using (var nsiMd5Stream = new MD5Stream(nsiCryptoStream))
//
using (var nsiSchemaSteam = FileUtilities.GetCreateStream(localNsiSchemaPath))
using (var nsiSchemaCryptoStream =
new CryptoStream(nsiSchemaSteam, aes.CreateEncryptor(), CryptoStreamMode.Write))
using (var nsiSchemaMd5Stream = new MD5Stream(nsiSchemaCryptoStream))
{
using (var nsiWriter = CaUtilities.GetNsiWriter(nsiMd5Stream, version, genomeAssembly, jsonTag, reportFor))
using (var schemaWriter = new StreamWriter(nsiSchemaMd5Stream))
{
nsiWriter.Write(intervals);
schemaWriter.Write(intervalJsonSchema);
}
nsiMetadata = nsiMd5Stream.GetFileMetadata();
nsiSchemaMetadata = nsiSchemaMd5Stream.GetFileMetadata();
}
string nsiS3Path = string.Join('/', config.outputDir.path.Trim('/'), nsiFileName);
string nsiSchemaS3PathFile = nsiS3Path + SaCommon.JsonSchemaSuffix;
s3Client.DecryptUpload(config.outputDir.bucketName, nsiS3Path, localNsiPath, aes, nsiMetadata);
s3Client.DecryptUpload(config.outputDir.bucketName, nsiSchemaS3PathFile, localNsiSchemaPath, aes,
nsiSchemaMetadata);
outputFiles.Add(nsiFileName);
outputFiles.Add(nsiFileName + SaCommon.JsonSchemaSuffix);
}
LambdaUtilities.DeleteTempOutput();
return(CustomAnnotationLambda.GetSuccessResult(config, result, outputFiles));
}
public IRequest Marshall(PutObjectRequest putObjectRequest)
{
//IL_0006: Unknown result type (might be due to invalid IL or missing references)
//IL_000c: Expected O, but got Unknown
//IL_02b8: Unknown result type (might be due to invalid IL or missing references)
//IL_02bf: Expected O, but got Unknown
IRequest val = new DefaultRequest(putObjectRequest, "AmazonS3");
val.set_HttpMethod("PUT");
if (putObjectRequest.IsSetCannedACL())
{
val.get_Headers().Add("x-amz-acl", S3Transforms.ToStringValue(ConstantClass.op_Implicit(putObjectRequest.CannedACL)));
}
HeadersCollection headers = putObjectRequest.Headers;
foreach (string key in headers.Keys)
{
val.get_Headers()[key] = headers[key];
}
if (putObjectRequest.IsSetMD5Digest())
{
val.get_Headers()["Content-MD5"] = putObjectRequest.MD5Digest;
}
HeaderACLRequestMarshaller.Marshall(val, putObjectRequest);
if (putObjectRequest.IsSetServerSideEncryptionMethod())
{
val.get_Headers().Add("x-amz-server-side-encryption", S3Transforms.ToStringValue(ConstantClass.op_Implicit(putObjectRequest.ServerSideEncryptionMethod)));
}
if (putObjectRequest.IsSetStorageClass())
{
val.get_Headers().Add("x-amz-storage-class", S3Transforms.ToStringValue(ConstantClass.op_Implicit(putObjectRequest.StorageClass)));
}
if (putObjectRequest.IsSetWebsiteRedirectLocation())
{
val.get_Headers().Add("x-amz-website-redirect-location", S3Transforms.ToStringValue(putObjectRequest.WebsiteRedirectLocation));
}
if (putObjectRequest.IsSetServerSideEncryptionCustomerMethod())
{
val.get_Headers().Add("x-amz-server-side-encryption-customer-algorithm", ConstantClass.op_Implicit(putObjectRequest.ServerSideEncryptionCustomerMethod));
}
if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKey())
{
val.get_Headers().Add("x-amz-server-side-encryption-customer-key", putObjectRequest.ServerSideEncryptionCustomerProvidedKey);
if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5())
{
val.get_Headers().Add("x-amz-server-side-encryption-customer-key-MD5", putObjectRequest.ServerSideEncryptionCustomerProvidedKeyMD5);
}
else
{
val.get_Headers().Add("x-amz-server-side-encryption-customer-key-MD5", AmazonS3Util.ComputeEncodedMD5FromEncodedString(putObjectRequest.ServerSideEncryptionCustomerProvidedKey));
}
}
if (putObjectRequest.IsSetServerSideEncryptionKeyManagementServiceKeyId())
{
val.get_Headers().Add("x-amz-server-side-encryption-aws-kms-key-id", putObjectRequest.ServerSideEncryptionKeyManagementServiceKeyId);
}
if (putObjectRequest.IsSetRequestPayer())
{
val.get_Headers().Add(S3Constants.AmzHeaderRequestPayer, S3Transforms.ToStringValue(((object)putObjectRequest.RequestPayer).ToString()));
}
if (putObjectRequest.IsSetTagSet())
{
val.get_Headers().Add(S3Constants.AmzHeaderTagging, AmazonS3Util.TagSetToQueryString(putObjectRequest.TagSet));
}
AmazonS3Util.SetMetadataHeaders(val, putObjectRequest.Metadata);
val.set_ResourcePath(string.Format(CultureInfo.InvariantCulture, "/{0}/{1}", S3Transforms.ToStringValue(putObjectRequest.BucketName), S3Transforms.ToStringValue(putObjectRequest.Key)));
if (putObjectRequest.InputStream != null)
{
Stream streamWithLength = GetStreamWithLength(putObjectRequest.InputStream, putObjectRequest.Headers.ContentLength);
if (streamWithLength.Length > 0)
{
val.set_UseChunkEncoding(true);
}
long num = streamWithLength.Length - streamWithLength.Position;
if (!val.get_Headers().ContainsKey("Content-Length"))
{
val.get_Headers().Add("Content-Length", num.ToString(CultureInfo.InvariantCulture));
}
MD5Stream val2 = putObjectRequest.InputStream = (Stream) new MD5Stream(streamWithLength, (byte[])null, num);
}
val.set_ContentStream(putObjectRequest.InputStream);
if (!val.get_Headers().ContainsKey("Content-Type"))
{
val.get_Headers().Add("Content-Type", "text/plain");
}
return(val);
}
public static CustomResult Create(CustomConfig config, string inputBaseName, CustomResult result, IS3Client s3Client)
{
string ngaFileName = inputBaseName + SaCommon.NgaFileSuffix;
string localNgaPath = Path.Combine(Path.GetTempPath(), ngaFileName);
string localSchemaPath = localNgaPath + SaCommon.JsonSchemaSuffix;
string localLogPath = Path.Combine(Path.GetTempPath(), LogFileName);
HttpUtilities.ValidateUrl(LambdaUrlHelper.GetUgaUrl());
var outputFiles = new List <string>();
using (var aes = new AesCryptoServiceProvider())
{
FileMetadata ngaMetadata, schemaMetadata, logMetaData;
using (var logStream = FileUtilities.GetCreateStream(localLogPath))
using (var logCryptoStream = new CryptoStream(logStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
using (var logMd5Stream = new MD5Stream(logCryptoStream))
//
using (var customTsvStream = (PersistentStream)PersistentStreamUtils.GetReadStream(config.tsvUrl))
using (var parser = GetGeneAnnotationsParserFromCustomTsvStream(customTsvStream))
//
using (var ngaStream = FileUtilities.GetCreateStream(localNgaPath))
using (var ngaCryptoStream = new CryptoStream(ngaStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
using (var ngaMd5Stream = new MD5Stream(ngaCryptoStream))
//
using (var schemaStream = FileUtilities.GetCreateStream(localSchemaPath))
using (var schemaCryptoStream = new CryptoStream(schemaStream, aes.CreateEncryptor(), CryptoStreamMode.Write))
using (var schemaMd5Stream = new MD5Stream(schemaCryptoStream))
{
using (var ngaWriter = CaUtilities.GetNgaWriter(ngaMd5Stream, parser, config.tsvUrl))
using (var schemaWriter = new StreamWriter(schemaMd5Stream))
using (var logWriter = new StreamWriter(logMd5Stream))
{
ngaWriter.Write(parser.GetItems(config.skipGeneIdValidation, logWriter));
schemaWriter.Write(parser.JsonSchema);
}
//all the writers have to be disposed before GetFileMetaData is called
ngaMetadata = ngaMd5Stream.GetFileMetadata();
schemaMetadata = schemaMd5Stream.GetFileMetadata();
logMetaData = logMd5Stream.GetFileMetadata();
}
if (config.skipGeneIdValidation)
{
string logS3Key = string.Join('/', config.outputDir.path.Trim('/'), LogFileName);
Logger.WriteLine("uploading log file to " + logS3Key);
s3Client.DecryptUpload(config.outputDir.bucketName, logS3Key, localLogPath, aes, logMetaData);
}
string nsaS3Path = string.Join('/', config.outputDir.path.Trim('/'), ngaFileName);
string schemaS3Path = nsaS3Path + SaCommon.JsonSchemaSuffix;
s3Client.DecryptUpload(config.outputDir.bucketName, nsaS3Path, localNgaPath, aes, ngaMetadata);
s3Client.DecryptUpload(config.outputDir.bucketName, schemaS3Path, localSchemaPath, aes, schemaMetadata);
outputFiles.Add(ngaFileName);
outputFiles.Add(ngaFileName + SaCommon.JsonSchemaSuffix);
LambdaUtilities.DeleteTempOutput();
return(CustomAnnotationLambda.GetSuccessResult(config, result, outputFiles));
}
}
public IRequest Marshall(PutObjectRequest putObjectRequest)
{
IRequest request = new DefaultRequest(putObjectRequest, "AmazonS3");
request.HttpMethod = "PUT";
if (putObjectRequest.IsSetCannedACL())
{
request.Headers.Add(HeaderKeys.XAmzAclHeader, S3Transforms.ToStringValue(putObjectRequest.CannedACL));
}
var headers = putObjectRequest.Headers;
foreach (var key in headers.Keys)
{
request.Headers[key] = headers[key];
}
if (putObjectRequest.IsSetMD5Digest())
{
request.Headers[HeaderKeys.ContentMD5Header] = putObjectRequest.MD5Digest;
}
HeaderACLRequestMarshaller.Marshall(request, putObjectRequest);
if (putObjectRequest.IsSetServerSideEncryptionMethod())
{
request.Headers.Add(HeaderKeys.XAmzServerSideEncryptionHeader, S3Transforms.ToStringValue(putObjectRequest.ServerSideEncryptionMethod));
}
if (putObjectRequest.IsSetStorageClass())
{
request.Headers.Add(HeaderKeys.XAmzStorageClassHeader, S3Transforms.ToStringValue(putObjectRequest.StorageClass));
}
if (putObjectRequest.IsSetWebsiteRedirectLocation())
{
request.Headers.Add(HeaderKeys.XAmzWebsiteRedirectLocationHeader, S3Transforms.ToStringValue(putObjectRequest.WebsiteRedirectLocation));
}
if (putObjectRequest.IsSetServerSideEncryptionCustomerMethod())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerAlgorithmHeader, putObjectRequest.ServerSideEncryptionCustomerMethod);
}
if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKey())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyHeader, putObjectRequest.ServerSideEncryptionCustomerProvidedKey);
if (putObjectRequest.IsSetServerSideEncryptionCustomerProvidedKeyMD5())
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, putObjectRequest.ServerSideEncryptionCustomerProvidedKeyMD5);
}
else
{
request.Headers.Add(HeaderKeys.XAmzSSECustomerKeyMD5Header, AmazonS3Util.ComputeEncodedMD5FromEncodedString(putObjectRequest.ServerSideEncryptionCustomerProvidedKey));
}
}
if (putObjectRequest.IsSetServerSideEncryptionKeyManagementServiceKeyId())
{
request.Headers.Add(HeaderKeys.XAmzServerSideEncryptionAwsKmsKeyIdHeader, putObjectRequest.ServerSideEncryptionKeyManagementServiceKeyId);
}
if (putObjectRequest.IsSetServerSideEncryptionKeyManagementServiceEncryptionContext())
{
request.Headers.Add("x-amz-server-side-encryption-context", putObjectRequest.ServerSideEncryptionKeyManagementServiceEncryptionContext);
}
if (putObjectRequest.IsSetObjectLockLegalHoldStatus())
{
request.Headers.Add("x-amz-object-lock-legal-hold", S3Transforms.ToStringValue(putObjectRequest.ObjectLockLegalHoldStatus));
}
if (putObjectRequest.IsSetObjectLockMode())
{
request.Headers.Add("x-amz-object-lock-mode", S3Transforms.ToStringValue(putObjectRequest.ObjectLockMode));
}
if (putObjectRequest.IsSetObjectLockRetainUntilDate())
{
request.Headers.Add("x-amz-object-lock-retain-until-date", S3Transforms.ToStringValue(putObjectRequest.ObjectLockRetainUntilDate, AWSSDKUtils.ISO8601DateFormat));
}
if (putObjectRequest.IsSetRequestPayer())
{
request.Headers.Add(S3Constants.AmzHeaderRequestPayer, S3Transforms.ToStringValue(putObjectRequest.RequestPayer.ToString()));
}
if (putObjectRequest.IsSetTagSet())
{
request.Headers.Add(S3Constants.AmzHeaderTagging, AmazonS3Util.TagSetToQueryString(putObjectRequest.TagSet));
}
if (putObjectRequest.IsSetExpectedBucketOwner())
{
request.Headers.Add(S3Constants.AmzHeaderExpectedBucketOwner, S3Transforms.ToStringValue(putObjectRequest.ExpectedBucketOwner));
}
if (putObjectRequest.IsSetBucketKeyEnabled())
{
request.Headers.Add(S3Constants.AmzHeaderBucketKeyEnabled, S3Transforms.ToStringValue(putObjectRequest.BucketKeyEnabled));
}
if (putObjectRequest.IsSetChecksumAlgorithm())
{
request.Headers.Add(S3Constants.AmzHeaderSdkChecksumAlgorithm, S3Transforms.ToStringValue(putObjectRequest.ChecksumAlgorithm));
}
if (putObjectRequest.IsSetChecksumCRC32())
{
request.Headers.Add("x-amz-checksum-crc32", S3Transforms.ToStringValue(putObjectRequest.ChecksumCRC32));
}
if (putObjectRequest.IsSetChecksumCRC32C())
{
request.Headers.Add("x-amz-checksum-crc32c", S3Transforms.ToStringValue(putObjectRequest.ChecksumCRC32C));
}
if (putObjectRequest.IsSetChecksumSHA1())
{
request.Headers.Add("x-amz-checksum-sha1", S3Transforms.ToStringValue(putObjectRequest.ChecksumSHA1));
}
if (putObjectRequest.IsSetChecksumSHA256())
{
request.Headers.Add("x-amz-checksum-sha256", S3Transforms.ToStringValue(putObjectRequest.ChecksumSHA256));
}
AmazonS3Util.SetMetadataHeaders(request, putObjectRequest.Metadata);
if (string.IsNullOrEmpty(putObjectRequest.BucketName))
{
throw new System.ArgumentException("BucketName is a required property and must be set before making this call.", "PutObjectRequest.BucketName");
}
if (string.IsNullOrEmpty(putObjectRequest.Key))
{
throw new System.ArgumentException("Key is a required property and must be set before making this call.", "PutObjectRequest.Key");
}
request.ResourcePath = string.Format(CultureInfo.InvariantCulture, "/{0}/{1}",
S3Transforms.ToStringValue(putObjectRequest.BucketName),
S3Transforms.ToStringValue(putObjectRequest.Key));
if (putObjectRequest.InputStream != null)
{
// Wrap the stream in a stream that has a length
var streamWithLength = GetStreamWithLength(putObjectRequest.InputStream, putObjectRequest.Headers.ContentLength);
if (streamWithLength.Length > 0 && !(putObjectRequest.DisablePayloadSigning ?? false))
{
request.UseChunkEncoding = putObjectRequest.UseChunkEncoding;
}
var length = streamWithLength.Length - streamWithLength.Position;
if (!request.Headers.ContainsKey(HeaderKeys.ContentLengthHeader))
{
request.Headers.Add(HeaderKeys.ContentLengthHeader, length.ToString(CultureInfo.InvariantCulture));
}
request.DisablePayloadSigning = putObjectRequest.DisablePayloadSigning;
// Calculate Content-MD5 if not already set
if (!putObjectRequest.IsSetMD5Digest() && putObjectRequest.CalculateContentMD5Header)
{
string md5 = AmazonS3Util.GenerateMD5ChecksumForStream(putObjectRequest.InputStream);
if (!string.IsNullOrEmpty(md5))
{
request.Headers[HeaderKeys.ContentMD5Header] = md5;
}
}
if (!(putObjectRequest.DisableMD5Stream ?? AWSConfigsS3.DisableMD5Stream))
{
// Wrap input stream in MD5Stream
var hashStream = new MD5Stream(streamWithLength, null, length);
putObjectRequest.InputStream = hashStream;
}
else
{
putObjectRequest.InputStream = streamWithLength;
}
}
request.ContentStream = putObjectRequest.InputStream;
ChecksumUtils.SetRequestChecksum(request, putObjectRequest.ChecksumAlgorithm, fallbackToMD5: false);
if (!request.Headers.ContainsKey(HeaderKeys.ContentTypeHeader))
{
request.Headers.Add(HeaderKeys.ContentTypeHeader, "text/plain");
}
return(request);
}
