/// <summary>
/// Detect a bucket region mismatch based on the x-amz-bucket-region header, and the status code provided.
/// </summary>
/// <param name="requestedBucketUri"></param>
/// <param name="headBucketStatusCode"></param>
/// <param name="xAmzBucketRegionHeaderValue"></param>
/// <returns></returns>
internal static string GetCorrectRegion(AmazonS3Uri requestedBucketUri, HttpStatusCode headBucketStatusCode, string xAmzBucketRegionHeaderValue)
{
if (xAmzBucketRegionHeaderValue != null && headBucketStatusCode == HttpStatusCode.BadRequest)
{
return(CheckRegionAndUpdateCache(requestedBucketUri, xAmzBucketRegionHeaderValue));
}
return(null);
}
/// <summary>
/// Detect a bucket region mismatch based on the x-amz-bucket-region header, and the status code provided.
/// </summary>
/// <param name="requestedBucketUri"></param>
/// <param name="headBucketStatusCode"></param>
/// <param name="xAmzBucketRegionHeaderValue"></param>
/// <returns></returns>
internal static string GetCorrectRegion(AmazonS3Uri requestedBucketUri, HttpStatusCode headBucketStatusCode, string xAmzBucketRegionHeaderValue)
{
if (xAmzBucketRegionHeaderValue != null && headBucketStatusCode == HttpStatusCode.BadRequest)
{
return CheckRegionAndUpdateCache(requestedBucketUri, xAmzBucketRegionHeaderValue);
}
return null;
}
/// <summary>
/// If the given string is an AmazonS3Endpoint return true and set the AmazonS3Uri out parameter.
/// </summary>
/// <param name="uri"></param>
/// <param name="decode">Flag indicating if URI string should be preprocessed to decode certain characters.</param>
/// <param name="amazonS3Uri"></param>
/// <returns>true if the string is an AmazonS3Endpoint, and the out paramter has been filled in, false otherwise</returns>
public static bool TryParseAmazonS3Uri(string uri, bool decode, out AmazonS3Uri amazonS3Uri)
{
if (decode)
{
uri = EscapeSpecialControlCharacters(uri);
}
return(TryParseAmazonS3Uri(new Uri(uri), out amazonS3Uri));
}
public static bool TryParseAmazonS3Uri(Uri uri, out AmazonS3Uri amazonS3Uri)
{
if (IsAmazonS3Endpoint(uri))
{
amazonS3Uri = new AmazonS3Uri(uri);
return(true);
}
amazonS3Uri = null;
return(false);
}
internal static void EvaluateIfSigV4Required(IRequest request)
{
if (request.OriginalRequest is S3.Model.GetObjectRequest)
{
var region = new AmazonS3Uri(request.Endpoint.OriginalString).Region;
if(region != RegionEndpoint.USEast1)
request.UseSigV4 = true;
}
}
private static string CheckRegionAndUpdateCache(AmazonS3Uri requestedBucketUri, string actualRegion)
{
var requestedRegion = requestedBucketUri.Region == null ? null : requestedBucketUri.Region.SystemName;
if (actualRegion != null && !string.Equals(requestedRegion, actualRegion, StringComparison.Ordinal))
{
BucketRegionCache.AddOrUpdate(requestedBucketUri.Bucket, RegionEndpoint.GetBySystemName(actualRegion));
return(actualRegion);
}
return(null);
}
/// <summary>
/// If the given string is an AmazonS3Endpoint return true and set the AmazonS3Uri out parameter.
/// </summary>
/// <param name="uri"></param>
/// <param name="amazonS3Uri"></param>
/// <returns>true if the string is an AmazonS3Endpoint, and the out paramter has been filled in, false otherwise</returns>
public static bool TryParseAmazonS3Uri(string uri, out AmazonS3Uri amazonS3Uri)
{
try
{
return(TryParseAmazonS3Uri(new Uri(uri), out amazonS3Uri));
}
catch (Exception)
{
amazonS3Uri = null;
return(false);
}
}
/// <summary>
/// If the given Uri is an AmazonS3Endpoint return true and set the AmazonS3Uri out parameter.
/// </summary>
/// <param name="uri"></param>
/// <param name="amazonS3Uri"></param>
/// <returns>true if the Uri is an AmazonS3Endpoint, and the out paramter has been filled in, false otherwise</returns>
public static bool TryParseAmazonS3Uri(Uri uri, out AmazonS3Uri amazonS3Uri)
{
try
{
if (IsAmazonS3Endpoint(uri))
{
amazonS3Uri = new AmazonS3Uri(uri);
return(true);
}
}
// intentionally suppress all exceptions, because this is a try method
#pragma warning disable CA1031 // Do not catch general exception types
catch { }
#pragma warning restore CA1031 // Do not catch general exception types
amazonS3Uri = null;
return(false);
}
/// <summary>
/// Determines whether an S3 bucket exists or not.
/// This is done by:
/// 1. Creating a PreSigned Url for the bucket. To work with Signature V4 only regions, as
/// well as Signature V4-optional regions, we keep the expiry to within the maximum for V4
/// (which is one week).
/// 2. Making a HEAD request to the Url
/// </summary>
/// <param name="bucketName">The name of the bucket to check.</param>
/// <param name="s3Client">The Amazon S3 Client to use for S3 specific operations.</param>
/// <returns></returns>
public static bool DoesS3BucketExist(IAmazonS3 s3Client, string bucketName)
{
if (s3Client == null)
{
throw new ArgumentNullException("s3Client", "The s3Client cannot be null!");
}
if (String.IsNullOrEmpty(bucketName))
{
throw new ArgumentNullException("bucketName", "The bucketName cannot be null or the empty string!");
}
var request = new GetPreSignedUrlRequest
{
BucketName = bucketName,
Expires = DateTime.Now.AddDays(1),
Verb = HttpVerb.HEAD,
Protocol = Protocol.HTTP
};
var url = s3Client.GetPreSignedURL(request);
var uri = new Uri(url);
var config = s3Client.Config;
var response = AmazonS3HttpUtil.GetHead(s3Client, config, url, HeaderKeys.XAmzBucketRegion);
if (response.StatusCode == null)
{
// there was a problem with the request and we weren't able
// conclusively determine if the bucket exists
return(false);
}
else
{
AmazonS3Uri s3Uri;
var mismatchDetected = AmazonS3Uri.TryParseAmazonS3Uri(uri, out s3Uri) &&
BucketRegionDetector.GetCorrectRegion(s3Uri, response.StatusCode.Value, response.HeaderValue) != null;
var statusCodeAcceptable = response.StatusCode != HttpStatusCode.NotFound && response.StatusCode != HttpStatusCode.BadRequest;
return(statusCodeAcceptable || mismatchDetected);
}
}
protected override void FinalizeForRedirect(IExecutionContext executionContext, string redirectedLocation)
{
var request = executionContext.RequestContext.Request;
if (request.UseChunkEncoding)
{
if (request.Headers.ContainsKey(HeaderKeys.XAmzDecodedContentLengthHeader))
{
request.Headers[HeaderKeys.ContentLengthHeader] =
request.Headers[HeaderKeys.XAmzDecodedContentLengthHeader];
}
}
if (request.Headers.ContainsKey(HeaderKeys.HostHeader))
{
request.Headers.Remove(HeaderKeys.HostHeader);
}
// FinalizeForRedirect() sets the correct endpoint as per the redirected location.
base.FinalizeForRedirect(executionContext, redirectedLocation);
// Evaluate if this request requires SigV4. The endpoint set by FinalizeForRedirect()
// is one of the inputs to decide if SigV4 is required.
AmazonS3KmsHandler.EvaluateIfSigV4Required(executionContext.RequestContext.Request);
var redirect = new AmazonS3Uri(redirectedLocation);
if (AWSConfigsS3.UseSignatureVersion4 ||
request.UseSigV4 ||
redirect.Region.GetEndpointForService("s3").SignatureVersionOverride == "4" ||
redirect.Region.GetEndpointForService("s3").SignatureVersionOverride == null)
{
// Resign if sigV4 is enabled, the request explicitly requires SigV4 or if the redirected region mandates sigV4.
// resign appropriately for the redirected region, re-instating the user's client
// config to original state when done
request.AuthenticationRegion = redirect.Region.SystemName;
Signer.SignRequest(executionContext.RequestContext);
}
}
private static string GetCorrectRegion(AmazonS3Uri requestedBucketUri, AmazonServiceException serviceException)
{
string text = null;
string text2 = null;
AmazonS3Exception ex = serviceException as AmazonS3Exception;
if (ex != null)
{
if (string.Equals(ex.get_ErrorCode(), "AuthorizationHeaderMalformed", StringComparison.Ordinal))
{
text = CheckRegionAndUpdateCache(requestedBucketUri, ex.Region);
}
if (text == null)
{
HttpErrorResponseException val = ((Exception)ex).InnerException as HttpErrorResponseException;
if (val != null && val.get_Response() != null && val.get_Response().IsHeaderPresent("x-amz-bucket-region"))
{
text2 = CheckRegionAndUpdateCache(requestedBucketUri, val.get_Response().GetHeaderValue("x-amz-bucket-region"));
}
}
}
return(text ?? text2);
}
/// <summary>
/// Detects if the signature is malformed, and the requested bucket is in a Region
/// different from the Region of the request.
/// </summary>
/// <param name="requestedBucketUri"></param>
/// <param name="serviceException"></param>
/// <returns>the correct region if a mismatch was detected, null otherwise</returns>
private static string GetCorrectRegion(AmazonS3Uri requestedBucketUri, AmazonServiceException serviceException)
{
string regionFromExceptionBody = null;
string regionFromExceptionHeader = null;
var s3Exception = serviceException as AmazonS3Exception;
if (s3Exception != null)
{
if (string.Equals(s3Exception.ErrorCode, AuthorizationHeaderMalformedErrorCode, StringComparison.Ordinal))
{
regionFromExceptionBody = CheckRegionAndUpdateCache(requestedBucketUri, s3Exception.Region);
}
if (regionFromExceptionBody == null)
{
var innerException = s3Exception.InnerException as HttpErrorResponseException;
if (innerException != null && innerException.Response != null && innerException.Response.IsHeaderPresent(HeaderKeys.XAmzBucketRegion))
{
regionFromExceptionHeader = CheckRegionAndUpdateCache(requestedBucketUri, innerException.Response.GetHeaderValue(HeaderKeys.XAmzBucketRegion));
}
}
}
return regionFromExceptionBody ?? regionFromExceptionHeader;
}
/// <summary>
/// Detects if the signature is malformed, and the requested bucket is in a Region
/// different from the Region of the request.
/// </summary>
/// <param name="requestedBucketUri"></param>
/// <param name="serviceException"></param>
/// <returns>the correct region if a mismatch was detected, null otherwise</returns>
private static string GetCorrectRegion(AmazonS3Uri requestedBucketUri, AmazonServiceException serviceException)
{
string regionFromExceptionBody = null;
string regionFromExceptionHeader = null;
var s3Exception = serviceException as AmazonS3Exception;
if (s3Exception != null)
{
if (string.Equals(s3Exception.ErrorCode, AuthorizationHeaderMalformedErrorCode, StringComparison.Ordinal))
{
regionFromExceptionBody = CheckRegionAndUpdateCache(requestedBucketUri, s3Exception.Region);
}
if (regionFromExceptionBody == null)
{
var innerException = s3Exception.InnerException as HttpErrorResponseException;
if (innerException != null && innerException.Response != null && innerException.Response.IsHeaderPresent(HeaderKeys.XAmzBucketRegion))
{
regionFromExceptionHeader = CheckRegionAndUpdateCache(requestedBucketUri, innerException.Response.GetHeaderValue(HeaderKeys.XAmzBucketRegion));
}
}
}
return(regionFromExceptionBody ?? regionFromExceptionHeader);
}
/// <summary>
/// Deletes the artifacts associated with an import task using a presigned
/// url to address the manifest for the import. No check is performed to
/// determine whether the associated conversion task is in progress.
/// </summary>
/// <param name="s3Client">
/// An Amazon S3 client for the operation to use. This should have been constructed
/// using credentials that have access to the bucket containing the image file
/// artifacts and be scoped to the region containing the bucket.
/// </param>
/// <param name="manifestUrl">
/// Presigned URL to the import manifest file
/// </param>
/// <param name="progressCallback">Optional progress callback</param>
public static void DeleteImageArtifacts(IAmazonS3 s3Client,
string manifestUrl,
CleanupProgressCallback progressCallback)
{
if (string.IsNullOrEmpty(manifestUrl))
throw new ArgumentException("Expected valid presigned url to the import manifest.");
var s3Uri = new AmazonS3Uri(manifestUrl);
// strip the manifest object name away from the key to get the overall key prefix
// to the objects
var lastSlashPos = s3Uri.Key.LastIndexOf('/');
DeleteImageArtifacts(s3Client, s3Uri.Bucket, s3Uri.Key.Substring(0, lastSlashPos), progressCallback);
}
/// <summary>
/// If the given string is an AmazonS3Endpoint return true and set the AmazonS3Uri out parameter.
/// </summary>
/// <param name="uri"></param>
/// <param name="amazonS3Uri"></param>
/// <returns>true if the string is an AmazonS3Endpoint, and the out paramter has been filled in, false otherwise</returns>
public static bool TryParseAmazonS3Uri(string uri, out AmazonS3Uri amazonS3Uri)
{
return TryParseAmazonS3Uri(new Uri(uri), out amazonS3Uri);
}
/// <summary>
/// If the given Uri is an AmazonS3Endpoint return true and set the AmazonS3Uri out parameter.
/// </summary>
/// <param name="uri"></param>
/// <param name="amazonS3Uri"></param>
/// <returns>true if the Uri is an AmazonS3Endpoint, and the out paramter has been filled in, false otherwise</returns>
public static bool TryParseAmazonS3Uri(Uri uri, out AmazonS3Uri amazonS3Uri)
{
if (IsAmazonS3Endpoint(uri))
{
amazonS3Uri = new AmazonS3Uri(uri);
return true;
}
else
{
amazonS3Uri = null;
return false;
}
}
/// <summary>
/// Return true if the request should be retried. Implements additional checks
/// specific to S3 on top of the checks in DefaultRetryPolicy.
/// </summary>
/// <param name="executionContext">Request context containing the state of the request.</param>
/// <param name="exception">The exception thrown by the previous request.</param>
/// <returns>Return true if the request should be retried.</returns>
public override bool RetryForException(Runtime.IExecutionContext executionContext, Exception exception)
{
var serviceException = exception as AmazonServiceException;
if (serviceException != null)
{
if (serviceException.StatusCode == HttpStatusCode.OK)
{
var requestType = executionContext.RequestContext.OriginalRequest.GetType();
if (AmazonS3RetryPolicy.RequestsWith200Error.Contains(requestType))
{
// Retry on HTTP 200 responses which contain an error response.
// CopyObject, CopyPart and CompleteMultipartUpload operations can return this
// response.
return true;
}
}
if (serviceException.StatusCode == HttpStatusCode.BadRequest)
{
var configuredUri = new Uri(executionContext.RequestContext.ClientConfig.DetermineServiceURL());
if (configuredUri.Host.Equals(S3Constants.S3DefaultEndpoint) &&
(serviceException.Message.Contains(AWS4Signer.AWS4AlgorithmTag) ||
serviceException.Message.Contains(AWS_KMS_Signature_Error))
)
{
// If the response message indicates AWS4 signing should have been used,
// we've attempted to access a bucket in an AWS4-only region (e.g. EU Central (Frankfurt)) with an AWS2
// signature and/or client not configured with the correct region.
// Retry the request to the s3-external endpoint to yield a 307 redirect
// that we can then follow to the correct bucket location with the expected
// signing algorithm.
Logger.InfoFormat("Request {0}: the bucket you are attempting to access should be addressed using a region-specific endpoint."
+ " Additional calls will be made to attempt to determine the correct region to be used."
+ " For better performance configure your client to use the correct region.",
executionContext.RequestContext.RequestName);
var r = executionContext.RequestContext.Request;
var s3Uri = new AmazonS3Uri(r.Endpoint);
// since DNS resolved, yielding an auth error from the service,
// we're assuming we do not need to test (again) for dns compatibility
// on the bucket name
var tempEndpoint = string.Format(CultureInfo.InvariantCulture,
"https://{0}.{1}", s3Uri.Bucket, S3Constants.S3AlternateDefaultEndpoint);
r.Endpoint = new Uri(tempEndpoint);
if (serviceException.Message.Contains(AWS_KMS_Signature_Error))
{
r.UseSigV4 = true;
r.AuthenticationRegion = RegionEndpoint.USEast1.SystemName;
executionContext.RequestContext.IsSigned = false;
}
return true;
}
}
}
return base.RetryForException(executionContext, exception);
}
public static bool TryParseAmazonS3Uri(string uri, out AmazonS3Uri amazonS3Uri)
{
return(TryParseAmazonS3Uri(new Uri(uri), out amazonS3Uri));
}
/// <summary>
/// Detects if the signature is malformed, and the requested bucket is in a Region
/// different from the Region of the request.
/// </summary>
/// <param name="requestedBucketUri"></param>
/// <param name="serviceException"></param>
/// <param name="credentials"></param>
/// <returns>the correct region if a mismatch was detected, null otherwise</returns>
internal static async Task<string> DetectMismatchWithHeadBucketFallbackAsync(AmazonS3Uri requestedBucketUri, AmazonServiceException serviceException, ImmutableCredentials credentials)
{
return GetCorrectRegion(requestedBucketUri, serviceException) ??
CheckRegionAndUpdateCache(requestedBucketUri, await GetBucketRegionNoPipelineAsync(requestedBucketUri.Bucket, credentials));
}
/// <summary>
/// Detects if the signature is malformed, and the requested bucket is in a Region
/// different from the Region of the request.
/// </summary>
/// <param name="requestedBucketUri"></param>
/// <param name="serviceException"></param>
/// <param name="credentials"></param>
/// <returns>the correct region if a mismatch was detected, null otherwise</returns>
internal static async Task <string> DetectMismatchWithHeadBucketFallbackAsync(AmazonS3Uri requestedBucketUri, AmazonServiceException serviceException, ImmutableCredentials credentials)
{
return(GetCorrectRegion(requestedBucketUri, serviceException) ??
CheckRegionAndUpdateCache(requestedBucketUri, await GetBucketRegionNoPipelineAsync(requestedBucketUri.Bucket, credentials)));
}
private static string CheckRegionAndUpdateCache(AmazonS3Uri requestedBucketUri, string actualRegion)
{
var requestedRegion = requestedBucketUri.Region == null ? null : requestedBucketUri.Region.SystemName;
if (actualRegion != null && !string.Equals(requestedRegion, actualRegion, StringComparison.Ordinal))
{
BucketRegionCache.AddOrUpdate(requestedBucketUri.Bucket, RegionEndpoint.GetBySystemName(actualRegion));
return actualRegion;
}
return null;
}
