private bool TryDownloadFile(AmazonS3Client Client, string BucketName, string Key, FileReference OutputFile)
{
try
{
GetObjectRequest Request = new GetObjectRequest();
Request.BucketName = BucketName;
Request.Key = Key;
using (GetObjectResponse Response = Client.GetObject(Request))
{
if ((int)Response.HttpStatusCode >= 200 && (int)Response.HttpStatusCode <= 299)
{
Response.WriteResponseStreamToFile(OutputFile.FullName);
return(true);
}
}
return(false);
}
catch (Exception Ex)
{
AmazonS3Exception AmazonEx = Ex as AmazonS3Exception;
if (AmazonEx == null || AmazonEx.StatusCode != HttpStatusCode.NotFound)
{
Log.TraceWarning("Exception while downloading {0}: {1}", Key, Ex);
}
return(false);
}
}
protected override void ExecuteTask()
{
string result = null;
try {
this.ExecuteS3Task();
result = null; // No news is good news
} catch (Exception ex) {
BuildException buildEx = ex as BuildException;
if (buildEx != null)
{
this.SetResultProperty(buildEx.Message);
throw;
}
AmazonS3Exception awsEx = ex as AmazonS3Exception;
result = awsEx != null ? awsEx.ErrorCode : ex.Message;
if (string.IsNullOrEmpty(result))
{
result = "Exception thrown: " + ex.GetType().FullName;
}
}
this.SetResultProperty(result);
if (!string.IsNullOrEmpty(result) && this.FailOnError)
{
throw new BuildException(result);
}
if (!string.IsNullOrEmpty(result))
{
this.Log(Level.Error, result);
}
}
void DeleteBucketPolicyWebResponse(object sender, ResponseEventArgs args)
{
AmazonS3Exception response = args.Response as AmazonS3Exception;
S3Common.client.OnS3Response -= DeleteBucketPolicyWebResponse;
if (null != response)
{
S3ErrorResponse(response);
return;
}
DeleteBucketPolicyResponse policyResponse = args.Response as DeleteBucketPolicyResponse;
if (null != policyResponse)
{
if (!string.IsNullOrEmpty(policyResponse.AmazonId2) &&
(!string.IsNullOrEmpty(policyResponse.RequestId)))
{
this.Dispatcher.BeginInvoke(() =>
{
GetBucketResult = string.Format(CultureInfo.InvariantCulture, "Bucket Policy Deleted Successfully for Bucket: {0}", SelectedBucketName);
});
}
}
else
{
GetBucketResult = string.Format(CultureInfo.InvariantCulture, "Unsuccessful Deletion of Policy for Bucket: {0}", SelectedBucketName);
}
}
public static StorageException ToStorageException(this AmazonS3Exception asex)
{
switch (asex?.ErrorCode)
{
case "InvalidAccessKeyId":
case "Forbidden":
return(new StorageException(StorageErrorCode.InvalidCredentials, asex));
case "InvalidSecurity":
case "AccessDenied":
case "AccountProblem":
case "NotSignedUp":
case "InvalidPayer":
case "RequestTimeTooSkewed":
case "SignatureDoesNotMatch":
return(new StorageException(StorageErrorCode.InvalidAccess, asex));
case "NoSuchBucket":
case "NoSuchKey":
case "NoSuchUpload":
case "NoSuchVersion":
return(new StorageException(StorageErrorCode.NotFound, asex));
case "InvalidBucketName":
case "KeyTooLong":
return(new StorageException(StorageErrorCode.InvalidName, asex));
default:
return(new StorageException(StorageErrorCode.GenericException, asex));
}
}
void GetBucketPolicyWebResponse(object sender, ResponseEventArgs args)
{
AmazonS3Exception result = args.Response as AmazonS3Exception;
S3Common.client.OnS3Response -= GetBucketPolicyWebResponse;
if (null != result)
{
S3ErrorResponse(result);
return;
}
GetBucketPolicyResponse policyResponse = args.Response as GetBucketPolicyResponse;
if (null != policyResponse)
{
if (!string.IsNullOrEmpty(policyResponse.AmazonId2) &&
(!string.IsNullOrEmpty(policyResponse.RequestId)))
{
this.Dispatcher.BeginInvoke(() =>
{
GetBucketResult = policyResponse.Policy;
});
}
}
else
{
this.Dispatcher.BeginInvoke(() =>
{
GetBucketResult = string.Format(CultureInfo.InvariantCulture, "Error Processing GetPolicy Request");
});
}
}
void GetBucketLocationWebResponse(object sender, ResponseEventArgs args)
{
AmazonS3Exception response = args.Response as AmazonS3Exception;
S3Common.client.OnS3Response -= GetBucketLocationWebResponse;
if (null != response)
{
S3ErrorResponse(response);
return;
}
GetBucketLocationResponse locationResponse = args.Response as GetBucketLocationResponse;
if (null != locationResponse)
{
if (!string.IsNullOrEmpty(locationResponse.AmazonId2) &&
(!string.IsNullOrEmpty(locationResponse.RequestId)))
{
this.Dispatcher.BeginInvoke(() =>
{
GetBucketResult = String.Format(CultureInfo.InvariantCulture, "The Location of {0} Bucket: {1}", SelectedBucketName,
locationResponse.Location);
});
}
}
else
{
this.Dispatcher.BeginInvoke(() =>
{
GetBucketResult = string.Format(CultureInfo.InvariantCulture, "Error Processing GetBucketLocation Request");
});
}
}
void DeleteBucketWebResponse(object sender, ResponseEventArgs args)
{
AmazonS3Exception response = args.Response as AmazonS3Exception;
S3Common.client.OnS3Response -= DeleteBucketWebResponse;
if (null != response)
{
S3ErrorResponse(response);
return;
}
DeleteBucketResponse deleteBucketResult = args.Response as DeleteBucketResponse;
if (null != deleteBucketResult)
{
if (!string.IsNullOrEmpty(deleteBucketResult.AmazonId2) &&
(!string.IsNullOrEmpty(deleteBucketResult.RequestId)))
{
this.Dispatcher.BeginInvoke(() =>
{
DeleteBucketText = string.Format(CultureInfo.InvariantCulture, "{0} bucket deleted Successfully", SelectedBucketName);
this.BucketNameSelected = false;
});
}
}
else
{
this.Dispatcher.BeginInvoke(() =>
{
DeleteBucketText = string.Format(CultureInfo.InvariantCulture, "Unsuccessful deletetion of Bucket: {0}", SelectedBucketName);
this.BucketNameSelected = false;
});
}
}
void CreateBucketWebResponse(object sender, ResponseEventArgs args)
{
AmazonS3Exception response = args.Response as AmazonS3Exception;
S3Common.client.OnS3Response -= CreateBucketWebResponse;
if (null != response)
{
S3ErrorResponse(response);
return;
}
PutBucketResponse putBucketResponse = args.Response as PutBucketResponse;
if (null != putBucketResponse)
{
if (!string.IsNullOrEmpty(putBucketResponse.AmazonId2) &&
(!string.IsNullOrEmpty(putBucketResponse.RequestId)))
{
this.Dispatcher.BeginInvoke(() =>
{
GetBucketResult = string.Format(CultureInfo.InvariantCulture, "{0} bucket created Successfully", CreateBucketName);
});
}
}
else
{
this.Dispatcher.BeginInvoke(() =>
{
GetBucketResult = string.Format(CultureInfo.InvariantCulture, "Unsuccessul Creation of Bucket: {0}", CreateBucketName);
});
}
}
public void ChangeSkewBasedOnRequestEndpointAndNotConfigEndpoint()
{
Uri configEndpoint = new Uri("https://s3.amazonaws.com");
Uri requestEndpoint = new Uri("https://bucketname.s3.amazonaws.com");
ReflectionHelpers.Invoke(typeof(CorrectClockSkew), "SetClockCorrectionForEndpoint",
new object[] { configEndpoint.ToString(), TimeSpan.FromHours(-1) });
ReflectionHelpers.Invoke(typeof(CorrectClockSkew), "SetClockCorrectionForEndpoint",
new object[] { requestEndpoint.ToString(), TimeSpan.Zero });
Assert.AreEqual(TimeSpan.Zero, CorrectClockSkew.GetClockCorrectionForEndpoint(requestEndpoint.ToString()));
Tester.Reset();
Tester.Action = (int callCount) =>
{
var timeString = DateTime.UtcNow.AddHours(-1).ToString(AWSSDKUtils.ISO8601BasicDateTimeFormat);
var exception = new AmazonS3Exception("(" + timeString + " - ");
exception.ErrorCode = "RequestTimeTooSkewed";
throw exception;
};
Utils.AssertExceptionExpected(() =>
{
var request = CreateTestContext();
request.RequestContext.Request.Endpoint = requestEndpoint;
RuntimePipeline.InvokeSync(request);
},
typeof(AmazonServiceException));
// RetryPolicy should see that the clock skew for bucketname.s3.amazonaws.com is zero and change it to ~ -1 hour
Assert.IsTrue(CorrectClockSkew.GetClockCorrectionForEndpoint(requestEndpoint.ToString()) < TimeSpan.FromMinutes(-55));
}
private static void TryHandleException(AmazonS3Exception ex)
{
if (IsDoesntExist(ex))
{
throw new StorageException(ErrorCode.NotFound, ex);
}
}
public void SetUp()
{
var file1 = new Mock <IFileInfoWrap>();
file1.Setup(x => x.Name).Returns("File1.jpg");
file1.Setup(x => x.FullName).Returns("c:\\files\\File1.jpg");
var file2 = new Mock <IFileInfoWrap>();
file2.Setup(x => x.Name).Returns("File2.jpg");
file2.Setup(x => x.FullName).Returns("c:\\files\\File2.jpg");
var file3 = new Mock <IFileInfoWrap>();
file3.Setup(x => x.Name).Returns("File3.jpg");
file3.Setup(x => x.FullName).Returns("c:\\files\\morefiles\\File3.jpg");
var file4 = new Mock <IFileInfoWrap>();
file4.Setup(x => x.Name).Returns("File4.jpg");
file4.Setup(x => x.FullName).Returns("c:\\files\\morefiles\\File4.jpg");
var file5 = new Mock <IFileInfoWrap>();
file5.Setup(x => x.Name).Returns("File5.jpg");
file5.Setup(x => x.FullName).Returns("c:\\files\\otherfiles\\File5.jpg");
var moreFilesDirectory = new Mock <IDirectoryInfoWrap>();
moreFilesDirectory.Setup(x => x.GetFiles()).Returns(() => new IFileInfoWrap[] { file3.Object, file4.Object });
moreFilesDirectory.Setup(x => x.GetDirectories()).Returns(() => new IDirectoryInfoWrap[] { });
moreFilesDirectory.Setup(x => x.Exists).Returns(() => true);
moreFilesDirectory.Setup(x => x.Name).Returns(() => "morefiles");
moreFilesDirectory.Setup(x => x.FullName).Returns(() => "c:\\files\\morefiles");
var otherfilesDirectory = new Mock <IDirectoryInfoWrap>();
otherfilesDirectory.Setup(x => x.GetFiles()).Returns(() => new IFileInfoWrap[] { file5.Object });
otherfilesDirectory.Setup(x => x.GetDirectories()).Returns(() => new IDirectoryInfoWrap[] { });
otherfilesDirectory.Setup(x => x.Exists).Returns(() => true);
otherfilesDirectory.Setup(x => x.Name).Returns(() => "otherfiles");
otherfilesDirectory.Setup(x => x.FullName).Returns(() => "c:\\files\\otherfiles");
DirectoryInfo = new Mock <IDirectoryInfoWrap>();
DirectoryInfo.Setup(x => x.GetFiles()).Returns(() => new IFileInfoWrap[] { file1.Object, file2.Object });
DirectoryInfo.Setup(x => x.GetDirectories()).Returns(() => new IDirectoryInfoWrap[] { moreFilesDirectory.Object, otherfilesDirectory.Object });
DirectoryInfo.Setup(x => x.Exists).Returns(() => true);
DirectoryInfo.Setup(x => x.Name).Returns(() => "files");
DirectoryInfo.Setup(x => x.FullName).Returns(() => "c:\\files");
var notFoundException = new AmazonS3Exception("Key not found", HttpStatusCode.NotFound);
AmazonS3 = new Mock <AmazonS3>();
AmazonS3.Setup(x => x.PutObject(It.IsAny <PutObjectRequest>())).Returns(new PutObjectResponse()).Verifiable();
AmazonS3.Setup(x => x.GetObjectMetadata(It.IsAny <GetObjectMetadataRequest>())).Throws(notFoundException);
S3FileSystem = new S3FileSystem(AmazonS3.Object, GetFileStreamFromFile);
}
private static bool TryHandleException(AmazonS3Exception ex)
{
if (IsDoesntExist(ex))
{
throw new StorageException(ErrorCode.NotFound, ex);
}
return(false);
}
private static bool TryHandleException(AmazonS3Exception ex)
{
if (ex.ErrorCode == "NoSuchKey")
{
throw new StorageException(ErrorCode.NotFound, ex);
}
return(false);
}
public void test_that_gateway_throws_imageNotFound_exception_when_S3_throws_key_not_found_error()
{
//arrange
var s3Exception = new AmazonS3Exception("The specified key does not exist.");
var expectedException = new AggregateException("One or more errors occured.", s3Exception);
mockS3Client.Setup(x => x.retrieveImage(It.IsAny <AWSCredentials>(), It.IsAny <string>(), It.IsAny <string>())).Throws(expectedException);
//assert
Assert.Throws <ImageNotFound>(() => classUnderTest.RetrieveImage(It.IsAny <string>()));
}
private StorageException AnAwsRelatedException(AmazonS3Exception ex)
{
if ((ex.ErrorCode != null) &&
((ex.ErrorCode.Equals("InvalidAccessKeyId")) || (ex.ErrorCode.Equals("InvalidSecurity"))))
{
return(new StorageException("The provided AWS credentials are invalid.", ex));
}
return(new StorageException("An AWS service related error occurred while processing your request.", ex));
}
private async Task <(bool success, AmazonS3Exception exception)> TryRetryableFileUploadAsyncCore(string bucketName, string prefix, Action <UploadProgressArgs> uploadCallback, ExponentialBackoff exponentialBackoff, List <Queue <S3FileHashStatus> > queueList, List <Queue <S3FileHashStatus> > retryQueueList)
{
// How many times it retry uploading QueueList.
var retryLimit = 5;
var currentRetry = 0;
AmazonS3Exception exception = null;
foreach (var queue in queueList)
{
try
{
await ConcurretFileUploadAsync(bucketName, queue, prefix, uploadCallback);
Log($"Partial Complete : Upload to S3. ({queue.Count})");
}
catch (AmazonS3Exception ex)
{
exception = ex;
switch (ex.StatusCode)
{
case HttpStatusCode.ServiceUnavailable:
{
// Put error queue list into retry queue list.
retryQueueList.Add(queue);
// re-throw when retry limit exceeded.
if (currentRetry >= retryLimit)
{
Error($"Error : Exceeded retry count limit ({currentRetry}/{retryLimit}). Stop execution.");
throw ex;
}
// Request reejected because "Too many Request"? Wait for Exponential Backoff.
// Sample Error :
// (Status Code : 502) Unhandled Exception: Amazon.S3.AmazonS3Exception: Please reduce your request rate. --->Amazon.Runtime.Internal.HttpErrorResponseException: Exception of type 'Amazon.Runtime.Internal.HttpErrorResponseException' was thrown.
var waitTime = exponentialBackoff.GetNextDelay();
Warn($"Warning : Exception happen during upload, re-queue to last then wait {waitTime.TotalSeconds}sec for next retry. Exception count in Queue List ({currentRetry}/{retryLimit}). {ex.GetType().FullName}, {ex.Message}, {ex.StackTrace}");
// Adjust next retry timing : wait for exponential Backoff
await Task.Delay(waitTime);
// increment retry count
currentRetry++;
continue;
}
default:
throw ex;
}
}
}
return((exception == null), exception);
}
private string GetMessageFromException(AmazonS3Exception e)
{
if (!string.IsNullOrEmpty(e.ErrorCode) &&
(e.ErrorCode == InvalidAccessKeyErrorCode ||
e.ErrorCode == InvalidSecurityErrorCode))
{
return("S3 error: check your access key and secret access key");
}
return(string.Format("S3 error: code [{0}], message [{1}]", e.ErrorCode, e.Message));
}
/// <summary>Format and display an exception</summary>
/// <param name="ex">Exception to display</param>
public void ShowError(AmazonS3Exception ex)
{
if (ex.ErrorCode != null && (ex.ErrorCode.Equals("InvalidAccessKeyId") || ex.ErrorCode.Equals("InvalidSecurity")))
{
Project.Log(Level.Error, "Please check the provided AWS Credentials.");
}
else
{
Project.Log(Level.Error, "An Error, number {0}, occurred with the message '{1}'",
ex.ErrorCode, ex.Message);
}
}
public void GetThrowsDataAccessExceptionIfGetCallThrowsAmazonS3ExceptionTest()
{
var ex = new AmazonS3Exception("test");
_mockS3Client
.Setup(x => x.GetObjectAsync(It.IsAny <GetObjectRequest>(), It.IsAny <CancellationToken>()))
.Throws(ex);
var thrown = Assert.ThrowsAsync <DataAccessException>(() => _store.Get(1)).Result;
Assert.Equal("Error Getting File", thrown.Message);
Assert.Equal(ex, thrown.InnerException);
}
private void HandleNotFound(string bucket, string key, AmazonS3Exception exception)
{
var httpError = exception.InnerException as HttpErrorResponseException;
if (httpError?.Response.StatusCode == HttpStatusCode.NotFound)
{
var message =
$"{key} is missing from the {bucket} bucket in Amazon 23 store. It has either expired or an invalid id has been supplied";
_logger.LogWarning(httpError, message);
throw new BlobNotFoundException(message, exception);
}
}
/// <summary>Format and display an exception</summary>
/// <param name="ex">Exception to display</param>
private void ShowError(AmazonS3Exception ex)
{
if (ex.ErrorCode != null && (ex.ErrorCode.Equals("InvalidAccessKeyId") || ex.ErrorCode.Equals("InvalidSecurity")))
{
Project.Log(Level.Error, "Please check the provided AWS Credentials.");
Project.Log(Level.Error, "If you haven't signed up for Amazon S3, please visit http://aws.amazon.com/s3");
}
else
{
Project.Log(Level.Error, "An Error, number {0}, occurred with the message '{1}'",
ex.ErrorCode, ex.Message);
}
}
/// <summary>
/// Serialize an AmazonS3 exception, including the data dictionary
/// </summary>
/// <param name="ex">The exception</param>
/// <returns>A serialized exception string.</returns>
private static string WrapAmazonS3Exception(AmazonS3Exception ex)
{
StringBuilder sb = new StringBuilder();
if (ex.Data != null)
{
foreach (object key in ex.Data.Keys)
{
sb.AppendFormat(CultureInfo.CurrentCulture, "{0} : {1}\r\n", key.ToString(), ex.Data[key].ToString());
}
}
return(String.Format(CultureInfo.InvariantCulture, "Error moving file on S3: Message:{0}\r\nHelp:\r\n{1}\r\nData:\r\n{2}", ex.Message, ex.HelpLink ?? string.Empty, sb.ToString()));
}
/// <summary>
/// Standardized error handler for S3 exceptions - provides debug output.
/// </summary>
/// <param name="amazonS3Exception">Exception to handle.</param>
public static void AWSerror(AmazonS3Exception amazonS3Exception)
{
if (amazonS3Exception.ErrorCode != null &&
(amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId") ||
amazonS3Exception.ErrorCode.Equals("InvalidSecurity")))
{
Debug.WriteLine("Please check the provided AWS Credentials.");
}
else
{
Debug.WriteLine("An Error, number {0}, occurred when listing buckets with the message '{1}", amazonS3Exception.ErrorCode, amazonS3Exception.Message);
}
}
public bool GetMessageFromException(Exception ex, SdtMessages_Message msg)
{
try
{
AmazonS3Exception s3ex = (AmazonS3Exception)ex;
msg.gxTpr_Id = s3ex.ErrorCode;
return(true);
}
catch (Exception)
{
return(false);
}
}
private async Task MaybeHandleExceptionAsync(AmazonS3Exception exception)
{
switch (exception.StatusCode)
{
case HttpStatusCode.Moved:
await AssertBucketLocationAsync();
break;
case HttpStatusCode.Forbidden:
await AssertBucketPermissionsAsync();
break;
}
}
private Task HandleAmazonS3Exception(HttpContext context, AmazonS3Exception e)
{
SentrySdk.CaptureException(e);
var statusCode = HttpStatusCode.InternalServerError;
context.Response.ContentType = "application/json";
context.Response.StatusCode = (int)statusCode;
var message = Messages.Misc_ThirdParty;
#if DEBUG
message = e.Message;
#endif
var rsp = new Response(false, statusCode, message);
return(ReturnResult(context, rsp));
}
public void GetReturnsObjectDefaultWhenKeyDoesntExistFromS3Test()
{
var ex = new AmazonS3Exception("test ex");
ex.StatusCode = System.Net.HttpStatusCode.NotFound;
_mockS3Client
.Setup(
x => x.GetObjectAsync(
It.Is <GetObjectRequest>(
y => y.BucketName == "mybucket" && y.Key == "keyprefix/1"),
It.IsAny <CancellationToken>()))
.ThrowsAsync(ex);
var result = _store.Get(1).Result;
Assert.Equal(default(TestObject), result);
}
public override AmazonServiceException UnmarshallException(XmlUnmarshallerContext context, Exception innerException, HttpStatusCode statusCode)
{
//IL_0014: Unknown result type (might be due to invalid IL or missing references)
S3ErrorResponse s3ErrorResponse = S3ErrorResponseUnmarshaller.Instance.Unmarshall(context);
AmazonS3Exception ex = new AmazonS3Exception(s3ErrorResponse.get_Message(), innerException, s3ErrorResponse.get_Type(), s3ErrorResponse.get_Code(), s3ErrorResponse.get_RequestId(), statusCode, s3ErrorResponse.Id2, s3ErrorResponse.AmzCfId);
ex.Region = s3ErrorResponse.Region;
if (s3ErrorResponse.ParsingException != null)
{
string responseBody = context.get_ResponseBody();
if (!string.IsNullOrEmpty(responseBody))
{
ex.ResponseBody = responseBody;
}
}
return(ex);
}
void S3ErrorResponse(AmazonS3Exception error)
{
StringBuilder errorBuilder = new StringBuilder();
errorBuilder.AppendLine(string.Format(CultureInfo.InvariantCulture, "ERROR CODE: {0}", error.ErrorCode));
errorBuilder.AppendLine(string.Format(CultureInfo.InvariantCulture, "HOST ID: {0}", error.HostId));
errorBuilder.AppendLine(string.Format(CultureInfo.InvariantCulture, "INNER EXCEPTION: {0}", error.InnerException));
errorBuilder.AppendLine(string.Format(CultureInfo.InvariantCulture, "MESSAGE: {0}", error.Message));
errorBuilder.AppendLine(string.Format(CultureInfo.InvariantCulture, "REQUEST ID: {0}", error.RequestId));
errorBuilder.AppendLine(string.Format(CultureInfo.InvariantCulture, "STATUS CODE: {0}", error.StatusCode));
errorBuilder.AppendLine(string.Format(CultureInfo.InvariantCulture, "REQUEST ADDRESS: {0}", error.S3RequestAddress));
errorBuilder.AppendLine(string.Format(CultureInfo.InvariantCulture, "\nERROR RESPONSE:\n {0}", error.XML));
this.Dispatcher.BeginInvoke(() =>
{
MessageBox.Show(errorBuilder.ToString(), "Error Occured", MessageBoxButton.OK);
});
}
public static string FormatExceptionLogMessage(GetObjectRequest request, AmazonS3Exception ex, StringBuilder builder = null)
{
builder = builder ?? new StringBuilder();
builder.Append($"{ex.GetType()} Exception - bucketName: {request.BucketName}, key: {request.Key}.");
builder.AppendLine();
builder.Append($"\tAmazon ID2 Token: {ex.AmazonId2}");
builder.AppendLine();
builder.Append($"\tStatus Code: {ex.StatusCode}");
builder.AppendLine();
builder.Append($"\tError Code: {ex.ErrorCode}");
builder.AppendLine();
return(ExceptionLogFormatter.FormatExceptionLogMessage(ex, builder));
}
