public static AmazonS3Uri GetAmazonS3Uri(Uri uri) { AmazonS3Uri amazonS3Uri = null; var success = AmazonS3Uri.TryParseAmazonS3Uri(uri, out amazonS3Uri); return(success ? amazonS3Uri : null); }
#pragma warning disable CA1031 // warning CA1031: Modify '***' to catch a more specific exception type, or rethrow the exception. #pragma warning disable CA1303 // warning CA1303: Method '***' passes a literal string as parameter 'value' internal static Outcome <bool, string> Validate(this CommandLine commandLine) { Contract.Requires(null != commandLine); // we must have staging folder if (string.IsNullOrWhiteSpace(commandLine.StagingFolder)) { return(new Outcome <bool, string>(false).AddErrors("Staging folder is not specified, you may want to set TEMP or HOME environment variable")); } if (!commandLine.StagingFolder.EndsWith(Path.DirectorySeparatorChar)) { commandLine.StagingFolder += Path.DirectorySeparatorChar; } // validate var outcome = new Outcome <bool, string>(true); foreach (var a in commandLine.Arguments) { try { // this one throws... thank you, Amazon var uri = new Uri(a); if (!AmazonS3Uri.TryParseAmazonS3Uri(uri, out var s3uri)) { outcome.AddErrors($"Invalid AWS S3 Uri: {a}"); } } catch (Exception x) { outcome.AddErrors($"{x.GetType().Name} {x.Message}: {a}"); } } return(outcome); }
public void TestNonS3EndpointDetection() { string[] thirdPartyProviderUriExamples = { "http://storage.googleapis.com", "http://bucket.storage.googleapis.com", }; string[] s3UriExamples = { "http://s3.amazonaws.com", "http://s3-external-1.amazonaws.com", "http://s3-us-west-2.amazonaws.com", "http://bucketname.s3-us-west-2.amazonaws.com", "http://s3.eu-central-1.amazonaws.com", "http://bucketname.s3.eu-central-1.amazonaws.com", }; foreach (var uri in thirdPartyProviderUriExamples) { Assert.IsFalse(AmazonS3Uri.IsAmazonS3Endpoint(uri)); } foreach (var uri in s3UriExamples) { Assert.IsTrue(AmazonS3Uri.IsAmazonS3Endpoint(uri)); } }
public Stream RetrieveFile(string path) { try { AmazonS3Uri amazonS3Uri = new AmazonS3Uri(path); var uploadRequest = new GetObjectRequest() { Key = amazonS3Uri.Key, BucketName = amazonS3Uri.Bucket, }; var getFileResponse = _client.GetObjectAsync(uploadRequest); var file = getFileResponse.Result.ResponseStream; return(file); } catch (Exception e) { var exception = cheekAggregateException(e); if (exception is AmazonS3Exception) { if ((exception as AmazonS3Exception).StatusCode == HttpStatusCode.NotFound) { return(null); } } throw; } }
internal static void EvaluateIfSigV4Required(IRequest request) { if (request.get_OriginalRequest() is GetObjectRequest && AmazonS3Uri.IsAmazonS3Endpoint(request.get_Endpoint()) && new AmazonS3Uri(request.get_Endpoint().OriginalString).Region != RegionEndpoint.USEast1) { request.set_UseSigV4(true); } }
internal static async Task <bool> SharedRetryForExceptionAsync(Runtime.IExecutionContext executionContext, Exception exception, Func <Runtime.IExecutionContext, Exception, bool?> retryForExceptionSync, Func <Runtime.IExecutionContext, Exception, bool> baseRetryForException) { var syncResult = retryForExceptionSync(executionContext, exception); if (syncResult.HasValue) { return(syncResult.Value); } else { var serviceException = exception as AmazonServiceException; string correctedRegion = null; AmazonS3Uri s3BucketUri; if (AmazonS3Uri.TryParseAmazonS3Uri(executionContext.RequestContext.Request.Endpoint, out s3BucketUri)) { var credentials = executionContext.RequestContext.ImmutableCredentials; correctedRegion = await BucketRegionDetector.DetectMismatchWithHeadBucketFallbackAsync(s3BucketUri, serviceException, credentials).ConfigureAwait(false); } if (correctedRegion == null) { return(baseRetryForException(executionContext, exception)); } else { // change authentication region of request and signal the handler to sign again with the new region executionContext.RequestContext.Request.AuthenticationRegion = correctedRegion; executionContext.RequestContext.IsSigned = false; return(true); } } }
/// <summary> /// Return true if the request should be retried. Implements additional checks /// specific to S3 on top of the checks in DefaultRetryPolicy. /// </summary> /// <param name="executionContext">Request context containing the state of the request.</param> /// <param name="exception">The exception thrown by the previous request.</param> /// <returns>Return true if the request should be retried.</returns> public override bool RetryForException(Runtime.IExecutionContext executionContext, Exception exception) { var serviceException = exception as AmazonServiceException; if (serviceException != null) { if (serviceException.StatusCode == HttpStatusCode.OK) { var requestType = executionContext.RequestContext.OriginalRequest.GetType(); if (AmazonS3RetryPolicy.RequestsWith200Error.Contains(requestType)) { // Retry on HTTP 200 responses which contain an error response. // CopyObject, CopyPart and CompleteMultipartUpload operations can return this // response. return(true); } } if (serviceException.StatusCode == HttpStatusCode.BadRequest) { var configuredUri = new Uri(executionContext.RequestContext.ClientConfig.DetermineServiceURL()); if (configuredUri.Host.Equals(S3Constants.S3DefaultEndpoint) && (serviceException.Message.Contains(AWS4Signer.AWS4AlgorithmTag) || serviceException.Message.Contains(AWS_KMS_Signature_Error)) ) { // If the response message indicates AWS4 signing should have been used, // we've attempted to access a bucket in an AWS4-only region (e.g. EU Central (Frankfurt)) with an AWS2 // signature and/or client not configured with the correct region. // Retry the request to the s3-external endpoint to yield a 307 redirect // that we can then follow to the correct bucket location with the expected // signing algorithm. Logger.InfoFormat("Request {0}: the bucket you are attempting to access should be addressed using a region-specific endpoint." + " Additional calls will be made to attempt to determine the correct region to be used." + " For better performance configure your client to use the correct region.", executionContext.RequestContext.RequestName); var r = executionContext.RequestContext.Request; var s3Uri = new AmazonS3Uri(r.Endpoint); // since DNS resolved, yielding an auth error from the service, // we're assuming we do not need to test (again) for dns compatibility // on the bucket name var tempEndpoint = string.Format(CultureInfo.InvariantCulture, "https://{0}.{1}", s3Uri.Bucket, S3Constants.S3AlternateDefaultEndpoint); r.Endpoint = new Uri(tempEndpoint); if (serviceException.Message.Contains(AWS_KMS_Signature_Error)) { r.UseSigV4 = true; r.AuthenticationRegion = RegionEndpoint.USEast1.SystemName; executionContext.RequestContext.IsSigned = false; } return(true); } } } return(base.RetryForException(executionContext, exception)); }
public override void Sign(IRequest request, IClientConfig clientConfig, RequestMetrics metrics, string awsAccessKeyId, string awsSecretAccessKey) { var signer = SelectSigner(this, _useSigV4, request, clientConfig); var aws4Signer = signer as AWS4Signer; var useV4 = aws4Signer != null; if (useV4) { AmazonS3Uri s3Uri; if (AmazonS3Uri.TryParseAmazonS3Uri(request.Endpoint, out s3Uri)) { if (s3Uri.Bucket != null) { RegionEndpoint cachedRegion; if (BucketRegionDetector.BucketRegionCache.TryGetValue(s3Uri.Bucket, out cachedRegion)) { request.AlternateEndpoint = cachedRegion; } } } var signingResult = aws4Signer.SignRequest(request, clientConfig, metrics, awsAccessKeyId, awsSecretAccessKey); request.Headers[HeaderKeys.AuthorizationHeader] = signingResult.ForAuthorizationHeader; if (request.UseChunkEncoding) { request.AWS4SignerResult = signingResult; } } else { SignRequest(request, metrics, awsAccessKeyId, awsSecretAccessKey); } }
/// <summary> /// Return true if the request should be retried. Implements additional checks /// specific to S3 on top of the checks in DefaultRetryPolicy. /// </summary> /// <param name="executionContext">Request context containing the state of the request.</param> /// <param name="exception">The exception thrown by the previous request.</param> /// <returns>Return true if the request should be retried.</returns> public override bool RetryForException(Runtime.IExecutionContext executionContext, Exception exception) { var syncResult = RetryForExceptionSync(executionContext, exception); if (syncResult.HasValue) { return(syncResult.Value); } else { var serviceException = exception as AmazonServiceException; string correctedRegion = null; AmazonS3Uri s3BucketUri; if (AmazonS3Uri.TryParseAmazonS3Uri(executionContext.RequestContext.Request.Endpoint, out s3BucketUri)) { var credentials = executionContext.RequestContext.ImmutableCredentials; if (credentials != null) { correctedRegion = BucketRegionDetector.DetectMismatchWithHeadBucketFallback(s3BucketUri, serviceException, credentials); } } if (correctedRegion == null) { return(base.RetryForException(executionContext, exception)); } else { // change authentication region of request and signal the handler to sign again with the new region executionContext.RequestContext.Request.AuthenticationRegion = correctedRegion; executionContext.RequestContext.IsSigned = false; return(true); } } }
// See https://github.com/aws/aws-sdk-net/issues/1426 public static bool TryParse(Uri uri, out AmazonS3Uri s3uri) { Contract.Requires(null != uri); #pragma warning disable CA1031 // Modify '***' to catch a more specific exception type, or rethrow the exception. try { return(AmazonS3Uri.TryParseAmazonS3Uri(uri, out s3uri)); } catch (Exception) { s3uri = null; return(false); } #pragma warning restore CA1031 }
/// <summary> /// Initializes a new instance of the <see cref="AwsS3ReadonlyStream" /> class. /// </summary> /// <param name="s3">The <see cref="IAmazonS3"/>.</param> /// <param name="s3Uri">The <see cref="AmazonS3Uri"/>.</param> /// <param name="s3ObjectMetadata">The <see cref="GetObjectMetadataResponse"/>.</param> public AwsS3ReadonlyStream(IAmazonS3 s3, AmazonS3Uri s3Uri, GetObjectMetadataResponse s3ObjectMetadata) { this.s3 = s3 ?? throw new ArgumentNullException(nameof(s3)); this.s3Uri = s3Uri ?? throw new ArgumentNullException(nameof(s3Uri)); Length = s3ObjectMetadata.ContentLength; s3ObjectEtag = s3ObjectMetadata.ETag; }
public void S3UriWithPlusForSpaceDecodeTryParseTest() { AmazonS3Uri s3Uri; bool result = AmazonS3Uri.TryParseAmazonS3Uri("https://testbucket.s3.us-east-2.amazonaws.com/C8624825821_S3+Log", true, out s3Uri); Assert.IsTrue(result); Assert.AreEqual(s3Uri.Key, "C8624825821_S3 Log"); }
internal static void EvaluateIfSigV4Required(IRequest request) { // Skip this for S3-compatible storage provider endpoints if (request.OriginalRequest is S3.Model.GetObjectRequest && AmazonS3Uri.TryParseAmazonS3Uri(request.Endpoint, out var amazonS3Uri) && amazonS3Uri.Region?.SystemName != RegionEndpoint.USEast1.SystemName) { request.UseSigV4 = true; } }
internal static void EvaluateIfSigV4Required(IRequest request) { if (request.OriginalRequest is S3.Model.GetObjectRequest) { var region = new AmazonS3Uri(request.Endpoint.OriginalString).Region; if (region != RegionEndpoint.USEast1) { request.UseSigV4 = true; } } }
internal static void EvaluateIfSigV4Required(IRequest request) { // Skip this for S3-compatible storage provider endpoints if ((request.OriginalRequest is S3.Model.GetObjectRequest) && (AmazonS3Uri.IsAmazonS3Endpoint(request.Endpoint))) { var region = new AmazonS3Uri(request.Endpoint.OriginalString).Region; if (region != RegionEndpoint.USEast1) { request.UseSigV4 = true; } } }
internal static void EvaluateIfSigV4Required(IRequest request) { // Skip this if the request has already been "upgraded" to SigV4a if (request.SignatureVersion == SignatureVersion.SigV4a) { return; } // Skip this for S3-compatible storage provider endpoints if (request.OriginalRequest is S3.Model.GetObjectRequest && AmazonS3Uri.TryParseAmazonS3Uri(request.Endpoint, out var amazonS3Uri) && amazonS3Uri.Region?.SystemName != RegionEndpoint.USEast1.SystemName) { request.SignatureVersion = SignatureVersion.SigV4; } }
private static void RegionDetectionUpdater(IRequest request) { AmazonS3Uri s3Uri; if (AmazonS3Uri.TryParseAmazonS3Uri(request.Endpoint, out s3Uri)) { if (s3Uri.Bucket != null) { RegionEndpoint cachedRegion; if (BucketRegionDetector.BucketRegionCache.TryGetValue(s3Uri.Bucket, out cachedRegion)) { request.AlternateEndpoint = cachedRegion; } } } }
/// <summary> /// Deletes the artifacts associated with an import task using a presigned /// url to address the manifest for the import. No check is performed to /// determine whether the associated conversion task is in progress. /// </summary> /// <param name="s3Client"> /// An Amazon S3 client for the operation to use. This should have been constructed /// using credentials that have access to the bucket containing the image file /// artifacts and be scoped to the region containing the bucket. /// </param> /// <param name="manifestUrl"> /// Presigned URL to the import manifest file /// </param> /// <param name="progressCallback">Optional progress callback</param> public static void DeleteImageArtifacts(IAmazonS3 s3Client, string manifestUrl, CleanupProgressCallback progressCallback) { if (string.IsNullOrEmpty(manifestUrl)) { throw new ArgumentException("Expected valid presigned url to the import manifest."); } var s3Uri = new AmazonS3Uri(manifestUrl); // strip the manifest object name away from the key to get the overall key prefix // to the objects var lastSlashPos = s3Uri.Key.LastIndexOf('/'); DeleteImageArtifacts(s3Client, s3Uri.Bucket, s3Uri.Key.Substring(0, lastSlashPos), progressCallback); }
public static string GetKey(Uri s3Uri) { AmazonS3Uri amazonS3Uri = GetAmazonS3Uri(s3Uri); if (amazonS3Uri != null) { return(amazonS3Uri.Key); } Match match = PathStyleRegEx.Match(s3Uri.OriginalString); if (!match.Success) { throw new FormatException("Cannot parse S3 Uri"); } return(match.Groups["key"].Value); }
public static Stream GetUriStream(IUriOptions Options) { switch (Options.Uri.Scheme) { case "file": return(new FileStream(Options.Uri.AbsolutePath, FileMode.Open, FileAccess.Read, FileShare.None, 1048576)); case "http": case "https": if (Options.Uri.Host.EndsWith("amazonaws.com")) { TransferUtility util = null; var uri = new AmazonS3Uri(Options.Uri); if (!string.IsNullOrEmpty(Options.AWSProfileName)) { var sharedFile = new SharedCredentialsFile(); if (sharedFile.TryGetProfile(Options.AWSProfileName, out CredentialProfile profile)) { if (AWSCredentialsFactory.TryGetAWSCredentials(profile, sharedFile, out AWSCredentials credentials)) { var s3client = new AmazonS3Client(credentials, uri.Region); util = new TransferUtility(s3client); } } } if (util == null) { util = new TransferUtility(uri.Region); } return(util.OpenStream(uri.Bucket, uri.Key)); } else { var request = WebRequest.Create(Options.Uri); return(request.GetRequestStream()); } default: throw new FileNotFoundException("Cannot parse Uri", Options.Uri.AbsoluteUri); } }
protected override void InitializeAndValidate(Uri uri, out UriFormatException parsingError) { AmazonS3Uri amazonS3Uri = GetAmazonS3Uri(uri); if (amazonS3Uri != null) { parsingError = null; return; } Match match = PathStyleRegEx.Match(uri.OriginalString); if (!match.Success) { parsingError = new UriFormatException("S3 Uri could not be parsed"); return; } parsingError = null; }
public static bool TryParseAmazonS3Uri(string uriString, out AmazonS3Uri amazonS3Uri) { try { if (AmazonS3Uri.TryParseAmazonS3Uri(uriString, out amazonS3Uri)) { return(true); } // "s3://bucket/key.txt" var uri = new Uri(uriString); amazonS3Uri = CreateAmazonS3Uri(uri.Host, uri.PathAndQuery); return(true); } catch (Exception) { amazonS3Uri = null; return(false); } }
protected override void FinalizeForRedirect(IExecutionContext executionContext, string redirectedLocation) { var request = executionContext.RequestContext.Request; if (request.UseChunkEncoding) { if (request.Headers.ContainsKey(HeaderKeys.XAmzDecodedContentLengthHeader)) { request.Headers[HeaderKeys.ContentLengthHeader] = request.Headers[HeaderKeys.XAmzDecodedContentLengthHeader]; } } if (request.Headers.ContainsKey(HeaderKeys.HostHeader)) { request.Headers.Remove(HeaderKeys.HostHeader); } // FinalizeForRedirect() sets the correct endpoint as per the redirected location. base.FinalizeForRedirect(executionContext, redirectedLocation); // Evaluate if this request requires SigV4. The endpoint set by FinalizeForRedirect() // is one of the inputs to decide if SigV4 is required. AmazonS3KmsHandler.EvaluateIfSigV4Required(executionContext.RequestContext.Request); var redirect = new AmazonS3Uri(redirectedLocation); if (AWSConfigsS3.UseSignatureVersion4 || request.UseSigV4 || redirect.Region.GetEndpointForService("s3").SignatureVersionOverride == "4" || redirect.Region.GetEndpointForService("s3").SignatureVersionOverride == null) { // Resign if sigV4 is enabled, the request explicitly requires SigV4 or if the redirected region mandates sigV4. // resign appropriately for the redirected region, re-instating the user's client // config to original state when done request.AuthenticationRegion = redirect.Region.SystemName; Signer.SignRequest(executionContext.RequestContext); } }
protected override void FinalizeForRedirect(IExecutionContext executionContext, string redirectedLocation) { IRequest request = executionContext.get_RequestContext().get_Request(); if (request.get_UseChunkEncoding() && request.get_Headers().ContainsKey("X-Amz-Decoded-Content-Length")) { request.get_Headers()["Content-Length"] = request.get_Headers()["X-Amz-Decoded-Content-Length"]; } if (request.get_Headers().ContainsKey("host")) { request.get_Headers().Remove("host"); } this.FinalizeForRedirect(executionContext, redirectedLocation); AmazonS3KmsHandler.EvaluateIfSigV4Required(executionContext.get_RequestContext().get_Request()); AmazonS3Uri amazonS3Uri = new AmazonS3Uri(redirectedLocation); if (AWSConfigsS3.UseSignatureVersion4 || request.get_UseSigV4() || amazonS3Uri.Region.GetEndpointForService("s3").get_SignatureVersionOverride() == "4" || amazonS3Uri.Region.GetEndpointForService("s3").get_SignatureVersionOverride() == null) { request.set_AuthenticationRegion(amazonS3Uri.Region.get_SystemName()); Signer.SignRequest(executionContext.get_RequestContext()); } }
public bool?RetryForExceptionSync(IExecutionContext executionContext, Exception exception) { AmazonServiceException val = exception as AmazonServiceException; if (val != null) { if (val.get_StatusCode() == HttpStatusCode.OK) { Type type = ((object)executionContext.get_RequestContext().get_OriginalRequest()).GetType(); if (RequestsWith200Error.Contains(type)) { return(true); } } if (val.get_StatusCode() == HttpStatusCode.BadRequest) { if (new Uri(executionContext.get_RequestContext().get_ClientConfig().DetermineServiceURL()).Host.Equals("s3.amazonaws.com") && (((Exception)val).Message.Contains("AWS4-HMAC-SHA256") || ((Exception)val).Message.Contains("AWS KMS managed keys require AWS Signature Version 4"))) { this.get_Logger().InfoFormat("Request {0}: the bucket you are attempting to access should be addressed using a region-specific endpoint. Additional calls will be made to attempt to determine the correct region to be used. For better performance configure your client to use the correct region.", new object[1] { executionContext.get_RequestContext().get_RequestName() }); IRequest request = executionContext.get_RequestContext().get_Request(); AmazonS3Uri amazonS3Uri = new AmazonS3Uri(request.get_Endpoint()); string uriString = string.Format(CultureInfo.InvariantCulture, "https://{0}.{1}", amazonS3Uri.Bucket, "s3-external-1.amazonaws.com"); request.set_Endpoint(new Uri(uriString)); if (((Exception)val).Message.Contains("AWS KMS managed keys require AWS Signature Version 4")) { request.set_UseSigV4(true); request.set_AuthenticationRegion(RegionEndpoint.USEast1.get_SystemName()); executionContext.get_RequestContext().set_IsSigned(false); } return(true); } return(null); } } return(this.RetryForException(executionContext, exception)); }
/// <summary> /// The SDK allows us to deploy a template from a URL, but does not apply parameters from a URL. So we /// must download the parameters file and parse it locally. /// </summary> static void DownloadS3(IVariables variables, ILog log, string templateParameterS3Url) { try { var environment = AwsEnvironmentGeneration.Create(log, variables).GetAwaiter().GetResult(); var s3Uri = new AmazonS3Uri(templateParameterS3Url); using (IAmazonS3 client = ClientHelpers.CreateS3Client(environment)) { var request = new GetObjectRequest { BucketName = s3Uri.Bucket, Key = s3Uri.Key }; var response = client.GetObjectAsync(request).GetAwaiter().GetResult(); response.WriteResponseStreamToFileAsync(ParametersFile, false, new CancellationTokenSource().Token).GetAwaiter().GetResult(); } } catch (UriFormatException ex) { log.Error($"The parameters URL of {templateParameterS3Url} is invalid"); throw; } }
public override void Sign(IRequest request, IClientConfig clientConfig, RequestMetrics metrics, string awsAccessKeyId, string awsSecretAccessKey) { AWS4Signer val = this.SelectSigner(this, _useSigV4, request, clientConfig) as AWS4Signer; if (val != null) { AmazonS3Uri amazonS3Uri; RegionEndpoint alternateEndpoint = default(RegionEndpoint); if (AmazonS3Uri.TryParseAmazonS3Uri(request.get_Endpoint(), out amazonS3Uri) && amazonS3Uri.Bucket != null && BucketRegionDetector.BucketRegionCache.TryGetValue(amazonS3Uri.Bucket, ref alternateEndpoint)) { request.set_AlternateEndpoint(alternateEndpoint); } AWS4SigningResult val2 = val.SignRequest(request, clientConfig, metrics, awsAccessKeyId, awsSecretAccessKey); request.get_Headers()["Authorization"] = val2.get_ForAuthorizationHeader(); if (request.get_UseChunkEncoding()) { request.set_AWS4SignerResult(val2); } } else { SignRequest(request, metrics, awsAccessKeyId, awsSecretAccessKey); } }
public WebRequest Create(Uri uri) { IAmazonS3 client = null; AmazonS3Uri amazonS3Uri = null; // Create client from config AmazonS3Uri.TryParseAmazonS3Uri(uri, out amazonS3Uri); AWSOptions awsOptions = options; if (options == null) { awsOptions = new AWSOptions(); } client = awsOptions.CreateServiceClient <IAmazonS3>(); if (amazonS3Uri == null || amazonS3Uri.IsPathStyle || S3UriParser.IsKnownScheme(uri.Scheme)) { (client.Config as AmazonS3Config).ForcePathStyle = true; } S3WebRequest s3WebRequest = new S3WebRequest(uri, logger, (AmazonS3Client)client, s3BucketsConfiguration); return(s3WebRequest); }
public void S3UriWithPlusForSpaceDecodeTest() { var s3Uri = new AmazonS3Uri("https://testbucket.s3.us-east-2.amazonaws.com/C8624825821_S3+Log", true); Assert.AreEqual(s3Uri.Key, "C8624825821_S3 Log"); }
public void S3EndpointAbsoluterLegacyS3UriTest() { bool isS3Uri = AmazonS3Uri.IsAmazonS3Endpoint("https://s3.amazonaws.com/"); Assert.IsTrue(isS3Uri); }