/// <summary> /// Detect a bucket region mismatch based on the x-amz-bucket-region header, and the status code provided. /// </summary> /// <param name="requestedBucketUri"></param> /// <param name="headBucketStatusCode"></param> /// <param name="xAmzBucketRegionHeaderValue"></param> /// <returns></returns> internal static string GetCorrectRegion(AmazonS3Uri requestedBucketUri, HttpStatusCode headBucketStatusCode, string xAmzBucketRegionHeaderValue) { if (xAmzBucketRegionHeaderValue != null && headBucketStatusCode == HttpStatusCode.BadRequest) { return(CheckRegionAndUpdateCache(requestedBucketUri, xAmzBucketRegionHeaderValue)); } return(null); }
/// <summary> /// Detect a bucket region mismatch based on the x-amz-bucket-region header, and the status code provided. /// </summary> /// <param name="requestedBucketUri"></param> /// <param name="headBucketStatusCode"></param> /// <param name="xAmzBucketRegionHeaderValue"></param> /// <returns></returns> internal static string GetCorrectRegion(AmazonS3Uri requestedBucketUri, HttpStatusCode headBucketStatusCode, string xAmzBucketRegionHeaderValue) { if (xAmzBucketRegionHeaderValue != null && headBucketStatusCode == HttpStatusCode.BadRequest) { return CheckRegionAndUpdateCache(requestedBucketUri, xAmzBucketRegionHeaderValue); } return null; }
/// <summary> /// If the given string is an AmazonS3Endpoint return true and set the AmazonS3Uri out parameter. /// </summary> /// <param name="uri"></param> /// <param name="decode">Flag indicating if URI string should be preprocessed to decode certain characters.</param> /// <param name="amazonS3Uri"></param> /// <returns>true if the string is an AmazonS3Endpoint, and the out paramter has been filled in, false otherwise</returns> public static bool TryParseAmazonS3Uri(string uri, bool decode, out AmazonS3Uri amazonS3Uri) { if (decode) { uri = EscapeSpecialControlCharacters(uri); } return(TryParseAmazonS3Uri(new Uri(uri), out amazonS3Uri)); }
public static bool TryParseAmazonS3Uri(Uri uri, out AmazonS3Uri amazonS3Uri) { if (IsAmazonS3Endpoint(uri)) { amazonS3Uri = new AmazonS3Uri(uri); return(true); } amazonS3Uri = null; return(false); }
internal static void EvaluateIfSigV4Required(IRequest request) { if (request.OriginalRequest is S3.Model.GetObjectRequest) { var region = new AmazonS3Uri(request.Endpoint.OriginalString).Region; if(region != RegionEndpoint.USEast1) request.UseSigV4 = true; } }
private static string CheckRegionAndUpdateCache(AmazonS3Uri requestedBucketUri, string actualRegion) { var requestedRegion = requestedBucketUri.Region == null ? null : requestedBucketUri.Region.SystemName; if (actualRegion != null && !string.Equals(requestedRegion, actualRegion, StringComparison.Ordinal)) { BucketRegionCache.AddOrUpdate(requestedBucketUri.Bucket, RegionEndpoint.GetBySystemName(actualRegion)); return(actualRegion); } return(null); }
/// <summary> /// If the given string is an AmazonS3Endpoint return true and set the AmazonS3Uri out parameter. /// </summary> /// <param name="uri"></param> /// <param name="amazonS3Uri"></param> /// <returns>true if the string is an AmazonS3Endpoint, and the out paramter has been filled in, false otherwise</returns> public static bool TryParseAmazonS3Uri(string uri, out AmazonS3Uri amazonS3Uri) { try { return(TryParseAmazonS3Uri(new Uri(uri), out amazonS3Uri)); } catch (Exception) { amazonS3Uri = null; return(false); } }
/// <summary> /// If the given Uri is an AmazonS3Endpoint return true and set the AmazonS3Uri out parameter. /// </summary> /// <param name="uri"></param> /// <param name="amazonS3Uri"></param> /// <returns>true if the Uri is an AmazonS3Endpoint, and the out paramter has been filled in, false otherwise</returns> public static bool TryParseAmazonS3Uri(Uri uri, out AmazonS3Uri amazonS3Uri) { try { if (IsAmazonS3Endpoint(uri)) { amazonS3Uri = new AmazonS3Uri(uri); return(true); } } // intentionally suppress all exceptions, because this is a try method #pragma warning disable CA1031 // Do not catch general exception types catch { } #pragma warning restore CA1031 // Do not catch general exception types amazonS3Uri = null; return(false); }
/// <summary> /// Determines whether an S3 bucket exists or not. /// This is done by: /// 1. Creating a PreSigned Url for the bucket. To work with Signature V4 only regions, as /// well as Signature V4-optional regions, we keep the expiry to within the maximum for V4 /// (which is one week). /// 2. Making a HEAD request to the Url /// </summary> /// <param name="bucketName">The name of the bucket to check.</param> /// <param name="s3Client">The Amazon S3 Client to use for S3 specific operations.</param> /// <returns></returns> public static bool DoesS3BucketExist(IAmazonS3 s3Client, string bucketName) { if (s3Client == null) { throw new ArgumentNullException("s3Client", "The s3Client cannot be null!"); } if (String.IsNullOrEmpty(bucketName)) { throw new ArgumentNullException("bucketName", "The bucketName cannot be null or the empty string!"); } var request = new GetPreSignedUrlRequest { BucketName = bucketName, Expires = DateTime.Now.AddDays(1), Verb = HttpVerb.HEAD, Protocol = Protocol.HTTP }; var url = s3Client.GetPreSignedURL(request); var uri = new Uri(url); var config = s3Client.Config; var response = AmazonS3HttpUtil.GetHead(s3Client, config, url, HeaderKeys.XAmzBucketRegion); if (response.StatusCode == null) { // there was a problem with the request and we weren't able // conclusively determine if the bucket exists return(false); } else { AmazonS3Uri s3Uri; var mismatchDetected = AmazonS3Uri.TryParseAmazonS3Uri(uri, out s3Uri) && BucketRegionDetector.GetCorrectRegion(s3Uri, response.StatusCode.Value, response.HeaderValue) != null; var statusCodeAcceptable = response.StatusCode != HttpStatusCode.NotFound && response.StatusCode != HttpStatusCode.BadRequest; return(statusCodeAcceptable || mismatchDetected); } }
protected override void FinalizeForRedirect(IExecutionContext executionContext, string redirectedLocation) { var request = executionContext.RequestContext.Request; if (request.UseChunkEncoding) { if (request.Headers.ContainsKey(HeaderKeys.XAmzDecodedContentLengthHeader)) { request.Headers[HeaderKeys.ContentLengthHeader] = request.Headers[HeaderKeys.XAmzDecodedContentLengthHeader]; } } if (request.Headers.ContainsKey(HeaderKeys.HostHeader)) { request.Headers.Remove(HeaderKeys.HostHeader); } // FinalizeForRedirect() sets the correct endpoint as per the redirected location. base.FinalizeForRedirect(executionContext, redirectedLocation); // Evaluate if this request requires SigV4. The endpoint set by FinalizeForRedirect() // is one of the inputs to decide if SigV4 is required. AmazonS3KmsHandler.EvaluateIfSigV4Required(executionContext.RequestContext.Request); var redirect = new AmazonS3Uri(redirectedLocation); if (AWSConfigsS3.UseSignatureVersion4 || request.UseSigV4 || redirect.Region.GetEndpointForService("s3").SignatureVersionOverride == "4" || redirect.Region.GetEndpointForService("s3").SignatureVersionOverride == null) { // Resign if sigV4 is enabled, the request explicitly requires SigV4 or if the redirected region mandates sigV4. // resign appropriately for the redirected region, re-instating the user's client // config to original state when done request.AuthenticationRegion = redirect.Region.SystemName; Signer.SignRequest(executionContext.RequestContext); } }
private static string GetCorrectRegion(AmazonS3Uri requestedBucketUri, AmazonServiceException serviceException) { string text = null; string text2 = null; AmazonS3Exception ex = serviceException as AmazonS3Exception; if (ex != null) { if (string.Equals(ex.get_ErrorCode(), "AuthorizationHeaderMalformed", StringComparison.Ordinal)) { text = CheckRegionAndUpdateCache(requestedBucketUri, ex.Region); } if (text == null) { HttpErrorResponseException val = ((Exception)ex).InnerException as HttpErrorResponseException; if (val != null && val.get_Response() != null && val.get_Response().IsHeaderPresent("x-amz-bucket-region")) { text2 = CheckRegionAndUpdateCache(requestedBucketUri, val.get_Response().GetHeaderValue("x-amz-bucket-region")); } } } return(text ?? text2); }
/// <summary> /// Detects if the signature is malformed, and the requested bucket is in a Region /// different from the Region of the request. /// </summary> /// <param name="requestedBucketUri"></param> /// <param name="serviceException"></param> /// <returns>the correct region if a mismatch was detected, null otherwise</returns> private static string GetCorrectRegion(AmazonS3Uri requestedBucketUri, AmazonServiceException serviceException) { string regionFromExceptionBody = null; string regionFromExceptionHeader = null; var s3Exception = serviceException as AmazonS3Exception; if (s3Exception != null) { if (string.Equals(s3Exception.ErrorCode, AuthorizationHeaderMalformedErrorCode, StringComparison.Ordinal)) { regionFromExceptionBody = CheckRegionAndUpdateCache(requestedBucketUri, s3Exception.Region); } if (regionFromExceptionBody == null) { var innerException = s3Exception.InnerException as HttpErrorResponseException; if (innerException != null && innerException.Response != null && innerException.Response.IsHeaderPresent(HeaderKeys.XAmzBucketRegion)) { regionFromExceptionHeader = CheckRegionAndUpdateCache(requestedBucketUri, innerException.Response.GetHeaderValue(HeaderKeys.XAmzBucketRegion)); } } } return regionFromExceptionBody ?? regionFromExceptionHeader; }
/// <summary> /// Detects if the signature is malformed, and the requested bucket is in a Region /// different from the Region of the request. /// </summary> /// <param name="requestedBucketUri"></param> /// <param name="serviceException"></param> /// <returns>the correct region if a mismatch was detected, null otherwise</returns> private static string GetCorrectRegion(AmazonS3Uri requestedBucketUri, AmazonServiceException serviceException) { string regionFromExceptionBody = null; string regionFromExceptionHeader = null; var s3Exception = serviceException as AmazonS3Exception; if (s3Exception != null) { if (string.Equals(s3Exception.ErrorCode, AuthorizationHeaderMalformedErrorCode, StringComparison.Ordinal)) { regionFromExceptionBody = CheckRegionAndUpdateCache(requestedBucketUri, s3Exception.Region); } if (regionFromExceptionBody == null) { var innerException = s3Exception.InnerException as HttpErrorResponseException; if (innerException != null && innerException.Response != null && innerException.Response.IsHeaderPresent(HeaderKeys.XAmzBucketRegion)) { regionFromExceptionHeader = CheckRegionAndUpdateCache(requestedBucketUri, innerException.Response.GetHeaderValue(HeaderKeys.XAmzBucketRegion)); } } } return(regionFromExceptionBody ?? regionFromExceptionHeader); }
/// <summary> /// Deletes the artifacts associated with an import task using a presigned /// url to address the manifest for the import. No check is performed to /// determine whether the associated conversion task is in progress. /// </summary> /// <param name="s3Client"> /// An Amazon S3 client for the operation to use. This should have been constructed /// using credentials that have access to the bucket containing the image file /// artifacts and be scoped to the region containing the bucket. /// </param> /// <param name="manifestUrl"> /// Presigned URL to the import manifest file /// </param> /// <param name="progressCallback">Optional progress callback</param> public static void DeleteImageArtifacts(IAmazonS3 s3Client, string manifestUrl, CleanupProgressCallback progressCallback) { if (string.IsNullOrEmpty(manifestUrl)) throw new ArgumentException("Expected valid presigned url to the import manifest."); var s3Uri = new AmazonS3Uri(manifestUrl); // strip the manifest object name away from the key to get the overall key prefix // to the objects var lastSlashPos = s3Uri.Key.LastIndexOf('/'); DeleteImageArtifacts(s3Client, s3Uri.Bucket, s3Uri.Key.Substring(0, lastSlashPos), progressCallback); }
/// <summary> /// If the given string is an AmazonS3Endpoint return true and set the AmazonS3Uri out parameter. /// </summary> /// <param name="uri"></param> /// <param name="amazonS3Uri"></param> /// <returns>true if the string is an AmazonS3Endpoint, and the out paramter has been filled in, false otherwise</returns> public static bool TryParseAmazonS3Uri(string uri, out AmazonS3Uri amazonS3Uri) { return TryParseAmazonS3Uri(new Uri(uri), out amazonS3Uri); }
/// <summary> /// If the given Uri is an AmazonS3Endpoint return true and set the AmazonS3Uri out parameter. /// </summary> /// <param name="uri"></param> /// <param name="amazonS3Uri"></param> /// <returns>true if the Uri is an AmazonS3Endpoint, and the out paramter has been filled in, false otherwise</returns> public static bool TryParseAmazonS3Uri(Uri uri, out AmazonS3Uri amazonS3Uri) { if (IsAmazonS3Endpoint(uri)) { amazonS3Uri = new AmazonS3Uri(uri); return true; } else { amazonS3Uri = null; return false; } }
/// <summary> /// Return true if the request should be retried. Implements additional checks /// specific to S3 on top of the checks in DefaultRetryPolicy. /// </summary> /// <param name="executionContext">Request context containing the state of the request.</param> /// <param name="exception">The exception thrown by the previous request.</param> /// <returns>Return true if the request should be retried.</returns> public override bool RetryForException(Runtime.IExecutionContext executionContext, Exception exception) { var serviceException = exception as AmazonServiceException; if (serviceException != null) { if (serviceException.StatusCode == HttpStatusCode.OK) { var requestType = executionContext.RequestContext.OriginalRequest.GetType(); if (AmazonS3RetryPolicy.RequestsWith200Error.Contains(requestType)) { // Retry on HTTP 200 responses which contain an error response. // CopyObject, CopyPart and CompleteMultipartUpload operations can return this // response. return true; } } if (serviceException.StatusCode == HttpStatusCode.BadRequest) { var configuredUri = new Uri(executionContext.RequestContext.ClientConfig.DetermineServiceURL()); if (configuredUri.Host.Equals(S3Constants.S3DefaultEndpoint) && (serviceException.Message.Contains(AWS4Signer.AWS4AlgorithmTag) || serviceException.Message.Contains(AWS_KMS_Signature_Error)) ) { // If the response message indicates AWS4 signing should have been used, // we've attempted to access a bucket in an AWS4-only region (e.g. EU Central (Frankfurt)) with an AWS2 // signature and/or client not configured with the correct region. // Retry the request to the s3-external endpoint to yield a 307 redirect // that we can then follow to the correct bucket location with the expected // signing algorithm. Logger.InfoFormat("Request {0}: the bucket you are attempting to access should be addressed using a region-specific endpoint." + " Additional calls will be made to attempt to determine the correct region to be used." + " For better performance configure your client to use the correct region.", executionContext.RequestContext.RequestName); var r = executionContext.RequestContext.Request; var s3Uri = new AmazonS3Uri(r.Endpoint); // since DNS resolved, yielding an auth error from the service, // we're assuming we do not need to test (again) for dns compatibility // on the bucket name var tempEndpoint = string.Format(CultureInfo.InvariantCulture, "https://{0}.{1}", s3Uri.Bucket, S3Constants.S3AlternateDefaultEndpoint); r.Endpoint = new Uri(tempEndpoint); if (serviceException.Message.Contains(AWS_KMS_Signature_Error)) { r.UseSigV4 = true; r.AuthenticationRegion = RegionEndpoint.USEast1.SystemName; executionContext.RequestContext.IsSigned = false; } return true; } } } return base.RetryForException(executionContext, exception); }
public static bool TryParseAmazonS3Uri(string uri, out AmazonS3Uri amazonS3Uri) { return(TryParseAmazonS3Uri(new Uri(uri), out amazonS3Uri)); }
/// <summary> /// Detects if the signature is malformed, and the requested bucket is in a Region /// different from the Region of the request. /// </summary> /// <param name="requestedBucketUri"></param> /// <param name="serviceException"></param> /// <param name="credentials"></param> /// <returns>the correct region if a mismatch was detected, null otherwise</returns> internal static async Task<string> DetectMismatchWithHeadBucketFallbackAsync(AmazonS3Uri requestedBucketUri, AmazonServiceException serviceException, ImmutableCredentials credentials) { return GetCorrectRegion(requestedBucketUri, serviceException) ?? CheckRegionAndUpdateCache(requestedBucketUri, await GetBucketRegionNoPipelineAsync(requestedBucketUri.Bucket, credentials)); }
/// <summary> /// Detects if the signature is malformed, and the requested bucket is in a Region /// different from the Region of the request. /// </summary> /// <param name="requestedBucketUri"></param> /// <param name="serviceException"></param> /// <param name="credentials"></param> /// <returns>the correct region if a mismatch was detected, null otherwise</returns> internal static async Task <string> DetectMismatchWithHeadBucketFallbackAsync(AmazonS3Uri requestedBucketUri, AmazonServiceException serviceException, ImmutableCredentials credentials) { return(GetCorrectRegion(requestedBucketUri, serviceException) ?? CheckRegionAndUpdateCache(requestedBucketUri, await GetBucketRegionNoPipelineAsync(requestedBucketUri.Bucket, credentials))); }
private static string CheckRegionAndUpdateCache(AmazonS3Uri requestedBucketUri, string actualRegion) { var requestedRegion = requestedBucketUri.Region == null ? null : requestedBucketUri.Region.SystemName; if (actualRegion != null && !string.Equals(requestedRegion, actualRegion, StringComparison.Ordinal)) { BucketRegionCache.AddOrUpdate(requestedBucketUri.Bucket, RegionEndpoint.GetBySystemName(actualRegion)); return actualRegion; } return null; }