private Task <TResp> SendRequest <TReq, TResp>(TReq request, CancellationToken token) where TResp : IResponse, new() where TReq : IRequest
{
request.Timestamp = DateTimeOffset.UtcNow;
request.RequestId = Guid.NewGuid();
_logger.LogTrace("Handling {RequestType} with request id {RequestId}", typeof(TReq).Name, request.RequestId);
S3Config config = _options.Value;
Stream? requestStream = _marshaller.MarshalRequest(request, config);
_validator.ValidateAndThrow(request);
StringBuilder sb = StringBuilderPool.Shared.Rent(200);
RequestHelper.AppendScheme(sb, config);
int schemeLength = sb.Length;
RequestHelper.AppendHost(sb, config, request);
request.SetHeader(HttpHeaders.Host, sb.ToString(schemeLength, sb.Length - schemeLength));
request.SetHeader(AmzHeaders.XAmzDate, request.Timestamp, DateTimeFormat.Iso8601DateTime);
if (requestStream != null)
{
foreach (IRequestStreamWrapper wrapper in _requestStreamWrappers)
{
if (wrapper.IsSupported(request))
{
requestStream = wrapper.Wrap(requestStream, request);
}
}
}
if (!request.Headers.TryGetValue(AmzHeaders.XAmzContentSha256, out string contentHash))
{
if (config.PayloadSignatureMode == SignatureMode.Unsigned)
{
contentHash = "UNSIGNED-PAYLOAD";
}
else
{
contentHash = requestStream == null ? "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" : CryptoHelper.Sha256Hash(requestStream, true).HexEncode();
}
request.SetHeader(AmzHeaders.XAmzContentSha256, contentHash);
}
_logger.LogDebug("ContentSha256 is {ContentSha256}", contentHash);
//We add the authorization header here because we need ALL other headers to be present when we do
_authBuilder.BuildAuthorization(request);
RequestHelper.AppendUrl(sb, config, request);
RequestHelper.AppendQueryParameters(sb, request);
string url = sb.ToString();
StringBuilderPool.Shared.Return(sb);
return(HandleResponse <TReq, TResp>(request, url, requestStream, token));
}
public string SignRequest <TReq>(TReq request, TimeSpan expiresIn) where TReq : IRequest
{
request.Timestamp = DateTimeOffset.UtcNow;
request.RequestId = Guid.NewGuid();
_logger.LogTrace("Handling {RequestType} with request id {RequestId}", typeof(TReq).Name, request.RequestId);
S3Config config = _options.Value;
_marshaller.MarshalRequest(request, config);
_validator.ValidateAndThrow(request);
StringBuilder sb = StringBuilderPool.Shared.Rent(200);
RequestHelper.AppendScheme(sb, config);
int schemeLength = sb.Length;
RequestHelper.AppendHost(sb, config, request);
request.SetHeader(HttpHeaders.Host, sb.ToString(schemeLength, sb.Length - schemeLength));
string scope = _scopeBuilder.CreateScope("s3", request.Timestamp);
request.SetQueryParameter(AmzParameters.XAmzAlgorithm, SigningConstants.AlgorithmTag);
request.SetQueryParameter(AmzParameters.XAmzCredential, _options.Value.Credentials.KeyId + '/' + scope);
request.SetQueryParameter(AmzParameters.XAmzDate, request.Timestamp.ToString(DateTimeFormats.Iso8601DateTime, DateTimeFormatInfo.InvariantInfo));
request.SetQueryParameter(AmzParameters.XAmzExpires, expiresIn.TotalSeconds.ToString(NumberFormatInfo.InvariantInfo));
request.SetQueryParameter(AmzParameters.XAmzSignedHeaders, string.Join(";", SigningConstants.FilterHeaders(request.Headers).Select(x => x.Key)));
//Copy all headers to query parameters
foreach (KeyValuePair <string, string> header in request.Headers)
{
if (header.Key == HttpHeaders.Host)
{
continue;
}
request.SetQueryParameter(header.Key, header.Value);
}
_authBuilder.BuildAuthorization(request);
//Clear sensitive material from the request
if (request is IContainSensitiveMaterial sensitive)
{
sensitive.ClearSensitiveMaterial();
}
RequestHelper.AppendUrl(sb, config, request);
RequestHelper.AppendQueryParameters(sb, request);
string url = sb.ToString();
StringBuilderPool.Shared.Return(sb);
return(url);
}
private Task <TResp> SendRequestInternalAsync <TReq, TResp>(TReq request, CancellationToken token) where TResp : IResponse, new() where TReq : IRequest
{
request.Timestamp = DateTimeOffset.UtcNow;
request.RequestId = Guid.NewGuid();
_logger.LogTrace("Handling {RequestType} with request id {RequestId}", typeof(TReq).Name, request.RequestId);
Stream?requestStream = _marshaller.MarshalRequest(_config, request);
_requestValidator.ValidateAndThrow(request);
IEndpointData endpointData = _endpointBuilder.GetEndpoint(request);
request.SetHeader(HttpHeaders.Host, endpointData.Host);
request.SetHeader(AmzHeaders.XAmzDate, request.Timestamp, DateTimeFormat.Iso8601DateTime);
if (requestStream != null)
{
foreach (IRequestStreamWrapper wrapper in _requestStreamWrappers)
{
if (wrapper.IsSupported(request))
{
requestStream = wrapper.Wrap(requestStream, request);
}
}
}
if (!request.Headers.TryGetValue(AmzHeaders.XAmzContentSha256, out string contentHash))
{
if (_config.PayloadSignatureMode == SignatureMode.Unsigned)
{
contentHash = "UNSIGNED-PAYLOAD";
}
else
{
contentHash = requestStream == null ? Constants.EmptySha256 : CryptoHelper.Sha256Hash(requestStream, true).HexEncode();
}
request.SetHeader(AmzHeaders.XAmzContentSha256, contentHash);
}
_logger.LogDebug("ContentSha256 is {ContentSha256}", contentHash);
//We add the authorization header here because we need ALL other headers to be present when we do
_authBuilder.BuildAuthorization(request);
StringBuilder sb = StringBuilderPool.Shared.Rent(200);
sb.Append(endpointData.Endpoint);
RequestHelper.AppendPath(sb, _config, request);
RequestHelper.AppendQueryParameters(sb, request);
string url = StringBuilderPool.Shared.ReturnString(sb);
return(HandleResponse <TReq, TResp>(request, url, requestStream, token));
}
public string SignRequest <TReq>(TReq request, TimeSpan expiresIn) where TReq : IRequest
{
request.Timestamp = DateTimeOffset.UtcNow;
request.RequestId = Guid.NewGuid();
_logger.LogTrace("Handling {RequestType} with request id {RequestId}", typeof(TReq).Name, request.RequestId);
_marshaller.MarshalRequest(_config, request);
IEndpointData endpointData = _endpointBuilder.GetEndpoint(request);
request.SetHeader(HttpHeaders.Host, endpointData.Host);
string scope = _scopeBuilder.CreateScope("s3", request.Timestamp);
request.SetQueryParameter(AmzParameters.XAmzAlgorithm, SigningConstants.AlgorithmTag);
request.SetQueryParameter(AmzParameters.XAmzCredential, _config.Credentials.KeyId + '/' + scope);
request.SetQueryParameter(AmzParameters.XAmzDate, request.Timestamp.ToString(DateTimeFormats.Iso8601DateTime, DateTimeFormatInfo.InvariantInfo));
request.SetQueryParameter(AmzParameters.XAmzExpires, expiresIn.TotalSeconds.ToString(NumberFormatInfo.InvariantInfo));
request.SetQueryParameter(AmzParameters.XAmzSignedHeaders, string.Join(";", HeaderWhitelist.FilterHeaders(request.Headers).Select(x => x.Key)));
//Copy all headers to query parameters
foreach (KeyValuePair <string, string> header in request.Headers)
{
if (header.Key == HttpHeaders.Host)
{
continue;
}
request.SetQueryParameter(header.Key, header.Value);
}
_authBuilder.BuildAuthorization(request);
//Clear sensitive material from the request
if (request is IContainSensitiveMaterial sensitive)
{
sensitive.ClearSensitiveMaterial();
}
StringBuilder sb = StringBuilderPool.Shared.Rent(200);
sb.Append(endpointData.Endpoint);
RequestHelper.AppendPath(sb, _config, request);
RequestHelper.AppendQueryParameters(sb, request);
return(StringBuilderPool.Shared.ReturnString(sb));
}
public async Task <TResp> SendRequestAsync <TReq, TResp>(TReq request, CancellationToken cancellationToken = default) where TResp : IResponse, new() where TReq : IRequest
{
cancellationToken.ThrowIfCancellationRequested();
request.Timestamp = DateTimeOffset.UtcNow;
request.RequestId = Guid.NewGuid();
_logger.LogTrace("Sending {RequestType} with request id {RequestId}", typeof(TReq).Name, request.RequestId);
S3Config config = _options.Value;
Stream requestStream = _marshaller.MarshalRequest(request, config);
_validator.ValidateAndThrow(request);
string bucketName = null;
if (request is IHasBucketName bn)
{
bucketName = bn.BucketName;
}
string objectKey = null;
if (request is IHasObjectKey ok)
{
objectKey = ok.ObjectKey;
}
//Ensure that the object key is encoded
string encodedResource = objectKey != null?UrlHelper.UrlPathEncode(objectKey) : null;
if (config.Endpoint == null || config.NamingMode == NamingMode.PathStyle)
{
if (bucketName != null)
{
objectKey = bucketName + '/' + encodedResource;
}
else
{
objectKey = encodedResource;
}
}
else
{
objectKey = encodedResource;
}
StringBuilder sb = StringBuilderPool.Shared.Rent(100);
Uri endpoint = config.Endpoint;
if (endpoint != null)
{
sb.Append(endpoint.Host);
if (!endpoint.IsDefaultPort)
{
sb.Append(':').Append(endpoint.Port);
}
}
else
{
if (config.NamingMode == NamingMode.VirtualHost)
{
if (bucketName != null)
{
sb.Append(bucketName).Append(".s3.").Append(ValueHelper.EnumToString(config.Region)).Append(".amazonaws.com");
}
else
{
sb.Append("s3.").Append(ValueHelper.EnumToString(config.Region)).Append(".amazonaws.com");
}
}
else
{
sb.Append("s3.").Append(ValueHelper.EnumToString(config.Region)).Append(".amazonaws.com");
}
}
request.SetHeader(HttpHeaders.Host, sb.ToString());
request.SetHeader(AmzHeaders.XAmzDate, request.Timestamp, DateTimeFormat.Iso8601DateTime);
if (requestStream != null)
{
foreach (IRequestStreamWrapper wrapper in _requestStreamWrappers)
{
if (wrapper.IsSupported(request))
{
requestStream = wrapper.Wrap(requestStream, request);
}
}
}
if (!request.Headers.TryGetValue(AmzHeaders.XAmzContentSha256, out string contentHash))
{
if (config.PayloadSignatureMode == SignatureMode.Unsigned)
{
contentHash = "UNSIGNED-PAYLOAD";
}
else
{
contentHash = requestStream == null ? "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" : CryptoHelper.Sha256Hash(requestStream, true).HexEncode();
}
request.SetHeader(AmzHeaders.XAmzContentSha256, contentHash);
}
_logger.LogDebug("ContentSha256 is {ContentSha256}", contentHash);
//We add the authorization header here because we need ALL other headers to be present when we do
request.SetHeader(HttpHeaders.Authorization, _authBuilder.BuildAuthorization(request));
sb.Append('/').Append(objectKey);
//Map all the parameters on to the url
if (request.QueryParameters.Count > 0)
{
sb.Append('?').Append(UrlHelper.CreateQueryString(request.QueryParameters));
}
string scheme = endpoint == null ? config.UseTLS ? "https" : "http" : endpoint.Scheme;
string fullUrl = scheme + "://" + sb;
StringBuilderPool.Shared.Return(sb);
_logger.LogDebug("Building request for {Url}", fullUrl);
(int statusCode, IDictionary <string, string> headers, Stream responseStream) = await _networkDriver.SendRequestAsync(request.Method, fullUrl, request.Headers, requestStream, cancellationToken).ConfigureAwait(false);
//Clear sensitive material from the request
if (request is IContainSensitiveMaterial sensitive)
{
sensitive.ClearSensitiveMaterial();
}
TResp response = new TResp();
response.StatusCode = statusCode;
response.ContentLength = headers.GetHeaderLong(HttpHeaders.ContentLength);
response.ConnectionClosed = "closed".Equals(headers.GetHeader(HttpHeaders.Connection), StringComparison.OrdinalIgnoreCase);
response.Date = headers.GetHeaderDate(HttpHeaders.Date, DateTimeFormat.Rfc1123);
response.Server = headers.GetHeader(HttpHeaders.Server);
response.ResponseId = headers.GetHeader(AmzHeaders.XAmzId2);
response.RequestId = headers.GetHeader(AmzHeaders.XAmzRequestId);
// https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
response.IsSuccess = !(statusCode == 403 || //Forbidden
statusCode == 400 || //BadRequest
statusCode == 500 || //InternalServerError
statusCode == 416 || //RequestedRangeNotSatisfiable
statusCode == 405 || //MethodNotAllowed
statusCode == 411 || //LengthRequired
statusCode == 404 || //NotFound
statusCode == 501 || //NotImplemented
statusCode == 504 || //GatewayTimeout
statusCode == 301 || //MovedPermanently
statusCode == 412 || //PreconditionFailed
statusCode == 307 || //TemporaryRedirect
statusCode == 409 || //Conflict
statusCode == 503); //ServiceUnavailable
//Only marshal successful responses
if (response.IsSuccess)
{
_marshaller.MarshalResponse(config, request, response, headers, responseStream);
}
else
{
MemoryStream ms = new MemoryStream();
responseStream.CopyTo(ms);
if (ms.Length > 0)
{
ms.Seek(0, SeekOrigin.Begin);
using (responseStream)
response.Error = ErrorHandler.Create(ms);
_logger.LogError("Received error: '{Message}'. Details: '{Details}'", response.Error.Message, response.Error.GetExtraData());
}
}
return(response);
}
public async Task <TResp> SendRequestAsync <TReq, TResp>(TReq request, CancellationToken cancellationToken) where TResp : IResponse, new() where TReq : IRequest
{
_logger.LogTrace($"Sending {typeof(TReq)} to bucket '{request.BucketName}' as resource '{request.Resource}'");
Stream requestStream = _marshaller.MarshalRequest(request);
_validator.ValidateAndThrow(request);
//Ensure that the resource is encoded
string encodedResource = UrlHelper.UrlPathEncode(request.Resource);
if (_options.Value.Endpoint == null || _options.Value.NamingType == NamingType.PathStyle)
{
if (!string.IsNullOrEmpty(request.BucketName))
{
request.Resource = request.BucketName + '/' + encodedResource;
}
else
{
request.Resource = encodedResource;
}
}
else
{
request.Resource = encodedResource;
}
StringBuilder sb = new StringBuilder(512);
if (_options.Value.Endpoint != null)
{
sb.Append(_options.Value.Endpoint.Host);
}
else
{
if (_options.Value.NamingType == NamingType.VirtualHost)
{
if (!string.IsNullOrEmpty(request.BucketName))
{
sb.Append(request.BucketName).Append(".s3.").Append(ValueHelper.EnumToString(_options.Value.Region)).Append(".amazonaws.com");
}
else
{
sb.Append("s3.").Append(ValueHelper.EnumToString(_options.Value.Region)).Append(".amazonaws.com");
}
}
else
{
sb.Append("s3.").Append(ValueHelper.EnumToString(_options.Value.Region)).Append(".amazonaws.com");
}
}
request.AddHeader(HttpHeaders.Host, sb.ToString());
request.AddHeader(AmzHeaders.XAmzDate, request.Date, DateTimeFormat.Iso8601DateTime);
if (requestStream != null && _requestStreamWrappers != null)
{
foreach (IRequestStreamWrapper wrapper in _requestStreamWrappers)
{
if (wrapper.IsSupported(request))
{
requestStream = wrapper.Wrap(requestStream, request);
}
}
}
//We check if it was already added here because it might have been due to streaming support
if (!request.Headers.ContainsKey(AmzHeaders.XAmzContentSha256))
{
string contentHash;
if (!_options.Value.EnablePayloadSigning)
{
contentHash = "UNSIGNED-PAYLOAD";
}
else
{
contentHash = requestStream == null ? "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" : CryptoHelper.Sha256Hash(requestStream, true).HexEncode();
}
_logger.LogDebug("ContentSha256 is {ContentSha256}", contentHash);
request.AddHeader(AmzHeaders.XAmzContentSha256, contentHash);
}
//We add the authorization header here because we need ALL other headers to be present when we do
request.AddHeader(HttpHeaders.Authorization, _authBuilder.BuildAuthorization(request));
sb.Append('/').Append(request.Resource);
//Map all the parameters on to the url
if (request.QueryParameters.Count > 0)
{
sb.Append('?').Append(UrlHelper.CreateQueryString(request.QueryParameters));
}
string fullUrl = "https://" + sb;
_logger.LogDebug("Building request for {Url}", fullUrl);
(int statusCode, IDictionary <string, string> headers, Stream responseStream) = await _networkDriver.SendRequestAsync(request.Method, fullUrl, request.Headers, requestStream, cancellationToken).ConfigureAwait(false);
//Clear sensitive material from the request
if (request is IContainSensitiveMaterial sensitive)
{
sensitive.ClearSensitiveMaterial();
}
TResp response = new TResp();
response.StatusCode = statusCode;
response.ContentLength = headers.GetHeaderLong(HttpHeaders.ContentLength);
response.ConnectionClosed = "closed".Equals(headers.GetHeader(HttpHeaders.Connection), StringComparison.OrdinalIgnoreCase);
response.Date = headers.GetHeaderDate(HttpHeaders.Date, DateTimeFormat.Rfc1123);
response.Server = headers.GetHeader(HttpHeaders.Server);
response.ResponseId = headers.GetHeader(AmzHeaders.XAmzId2);
response.RequestId = headers.GetHeader(AmzHeaders.XAmzRequestId);
// https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
response.IsSuccess = !(statusCode == 403 || //Forbidden
statusCode == 400 || //BadRequest
statusCode == 500 || //InternalServerError
statusCode == 416 || //RequestedRangeNotSatisfiable
statusCode == 405 || //MethodNotAllowed
statusCode == 411 || //LengthRequired
statusCode == 404 || //NotFound
statusCode == 501 || //NotImplemented
statusCode == 504 || //GatewayTimeout
statusCode == 301 || //MovedPermanently
statusCode == 412 || //PreconditionFailed
statusCode == 307 || //TemporaryRedirect
statusCode == 409 || //Conflict
statusCode == 503); //ServiceUnavailable
//Don't know why, but ContentLength seems to be 0 sometimes, even though it is in the headers
if (!response.IsSuccess && (response.ContentLength > 0 || responseStream.Length > 0))
{
using (responseStream)
response.Error = ErrorHandler.Create(responseStream);
_logger.LogError("Received error: '{Message}'. Details: '{Details}'", response.Error.Message, response.Error.GetExtraData());
}
//Only marshal successful responses
if (response.IsSuccess)
{
_marshaller.MarshalResponse(request, response, headers, responseStream);
}
return(response);
}