private async Task CheckBucketExists()
{
if (!_bucketExist)
{
_bucketExist = await _s3Client.DoesS3BucketExistAsync(_bucketName);
while (_bucketExist == false)
{
S3Region s3region = S3Region.FindValue(_config.AmazonRegion);
var putBucketRequest = new PutBucketRequest {
BucketName = _bucketName,
BucketRegion = s3region,
};
try
{
EnsureValidResponse(await _s3Client.PutBucketAsync(putBucketRequest), HttpStatusCode.OK);
}
catch (AmazonS3Exception ex)
{
if (ex.ErrorCode == "BucketAlreadyOwnedByYou")
{
break;
}
throw;
}
_bucketExist = await _s3Client.DoesS3BucketExistAsync(_bucketName);
}
}
}
public override async Task <Context> Execute(Context context)
{
var getCallerResult = await sts.GetCallerIdentityAsync(new GetCallerIdentityRequest());
var accountId = getCallerResult.Account;
var region = Environment.GetEnvironmentVariable("AWS_DEFAULT_REGION") ?? "us-west-2";
var bucketName = $"{typeof(SimpleCalculator).Name}-{region}-{accountId}".ToLower();
if (!s3.DoesS3BucketExistAsync(bucketName).Result)
{
await s3.PutBucketAsync(
new PutBucketRequest
{
BucketName = bucketName,
UseClientRegion = true
});
}
var putResult = await s3.PutObjectAsync(new PutObjectRequest
{
BucketName = bucketName,
Key = "state.json",
ContentType = "application/json",
ServerSideEncryptionMethod = ServerSideEncryptionMethod.AES256,
ContentBody = JsonConvert.SerializeObject(context)
});
return(context);
}
/// <summary>
/// Initializes a new instance of the <see cref="AWSS3Repository" /> class.
/// </summary>
/// <param name="s3client">AWS S3 client</param>
/// <param name="bucketName">Bucket name</param>
public AWSS3Repository(
IAmazonS3 s3client,
string bucketName)
{
this.s3client = s3client;
this.bucketName = bucketName;
var getBucketLocationTask =
s3client.GetBucketLocationAsync(new GetBucketLocationRequest
{
BucketName = this.bucketName
});
var doesS3BucketExistTask = s3client.DoesS3BucketExistAsync(this.bucketName);
doesS3BucketExistTask.Wait();
if (!doesS3BucketExistTask.Result)
{
// Note that CreateBucketRequest does not specify region. So bucket is
// created in the region specified in the client.
s3client.PutBucketAsync(bucketName).Wait();
}
getBucketLocationTask.Wait();
if (getBucketLocationTask.Result != null)
{
bucketLocation = getBucketLocationTask.Result.Location;
}
}
public async Task <GenericResponse> CreateBucket(string bucketName)
{
GenericResponse result = new GenericResponse();
try
{
if (await _client.DoesS3BucketExistAsync(bucketName) == false)
{
var putBucketRequest = new PutBucketRequest()
{
BucketName = bucketName, UseClientRegion = true
};
var w = await _client.PutBucketAsync(putBucketRequest);
}
}
catch (AmazonS3Exception awsEx)
{
var errorMessage = awsEx.ErrorCode;
}
catch (Exception ex)
{
var errorMessage = ex.Message;
}
return(result);
}
public async Task CreateBucket(string bucketName)
{
try
{
Validation.ValidateBucketName(bucketName);
await _requestRetryPolicy.ExecuteAsync(async() =>
{
var exists = await _client.DoesS3BucketExistAsync(bucketName.ToLower());
if (exists)
{
throw new BucketExistsException(bucketName);
}
var response = await _client.PutBucketAsync(bucketName.ToLower());
});
}
catch (AmazonServiceException ex) when(ex is AmazonS3Exception
)
{
throw new EndpointUnreachableException(_endpoint, ex.ToString());
}
catch (S3BaseException)
{
throw;
}
catch (Exception ex)
{
throw new S3BaseException(ex.Message, ex.ToString());
}
}
public async Task InitializeAsync(CancellationToken ct = default)
{
try
{
var amazonS3Config = new AmazonS3Config {
ForcePathStyle = options.ForcePathStyle
};
if (!string.IsNullOrWhiteSpace(options.ServiceUrl))
{
amazonS3Config.ServiceURL = options.ServiceUrl;
}
else
{
amazonS3Config.RegionEndpoint = RegionEndpoint.GetBySystemName(options.RegionName);
}
s3Client = new AmazonS3Client(options.AccessKey, options.SecretKey, amazonS3Config);
transferUtility = new TransferUtility(s3Client);
var exists = await s3Client.DoesS3BucketExistAsync(options.Bucket);
if (!exists)
{
throw new ConfigurationException($"Cannot connect to Amazon S3 bucket '{options.Bucket}'.");
}
}
catch (AmazonS3Exception ex)
{
throw new ConfigurationException($"Cannot connect to Amazon S3 bucket '{options.Bucket}'.", ex);
}
}
public async Task <bool> TryCreateBucket(string bucketName, CancellationToken cancellationToken)
{
try
{
bool bucketExist = await _s3Client.DoesS3BucketExistAsync(bucketName);
if (!bucketExist)
{
var request = new PutBucketRequest {
BucketName = bucketName
};
var response = await _s3Client.PutBucketAsync(request, cancellationToken);
return(response.HttpStatusCode == System.Net.HttpStatusCode.OK);
}
return(bucketExist);
}
catch (AmazonS3Exception amazonS3Exception)
{
_logger.LogError("S3 error occured. Exception:", amazonS3Exception.ToString());
throw;
}
catch (Exception e)
{
_logger.LogError(e.InnerException, e.Message);
throw;
}
}
public override async Task <bool> HasBucket(ILogger log, CancellationToken token)
{
if (_hasBucket == null)
{
_hasBucket = await _client.DoesS3BucketExistAsync(_bucketName);
}
return(_hasBucket == true);
}
public Task <CreatedBucket?> CreateBucketAsync(string bucketName)
{
if (string.IsNullOrWhiteSpace(bucketName))
{
throw new ArgumentException("A bucket name is required", nameof(bucketName));
}
return(createBucketAsync());
async Task <CreatedBucket?> createBucketAsync()
{
try
{
if (await _amazonS3.DoesS3BucketExistAsync(bucketName))
{
_logger.LogWarning($"A bucket having name '{bucketName}' already exists.");
return(null);
}
var response = await _amazonS3.PutBucketAsync(bucketName);
if (response.HttpStatusCode != HttpStatusCode.OK)
{
_logger.LogWarning($"A bucket having name '{bucketName}' could not be created.");
return(null);
}
return(new CreatedBucket
{
Name = bucketName
});
}
catch (AmazonS3Exception e)
{
_logger.LogError(e, e.Message);
throw;
}
}
}
public async Task InitializeAsync()
{
if (!await _client.DoesS3BucketExistAsync(_bucketName))
{
var putBucketRequest = new PutBucketRequest
{
BucketName = _bucketName,
UseClientRegion = true
};
await _client.PutBucketAsync(putBucketRequest);
}
}
/// <summary>
/// Upload a file to an S3, here four files are uploaded in four different ways
/// </summary>
/// <param name="fileKey"></param>
/// <param name="data"></param>
/// <param name="metadata"></param>
/// <returns></returns>
public async Task <SimpleS3Response> Upload(string fileKey, byte[] data, IDictionary <string, string> metadata)
{
// Nothing needs to be done ...
if (!_connected)
{
return(new SimpleS3Response(HttpStatusCode.BadRequest, "Not connected!"));
}
try
{
if (await _client.DoesS3BucketExistAsync(_s3ServiceConfig.BucketName))
{
var fileTransferUtility = new TransferUtility(_client);
var fileTransferUtilityRequest = new TransferUtilityUploadRequest
{
Key = $"{_s3ServiceConfig.Prefix}/{fileKey}",
InputStream = new MemoryStream(data),
BucketName = _s3ServiceConfig.BucketName,
CannedACL = S3CannedACL.NoACL
};
foreach (var(key, value) in metadata)
{
fileTransferUtilityRequest.Metadata.Add(key, value);
}
await fileTransferUtility.UploadAsync(fileTransferUtilityRequest);
return(new SimpleS3Response(HttpStatusCode.OK, "Successfully uploaded to S3"));
}
// Bucket not found
throw new Exception($"Bucket: {_s3ServiceConfig.BucketName} does not exist");
}
// Catch specific amazon errors
catch (AmazonS3Exception e)
{
_logger.LogError(e.AmazonId2, e);
return(new SimpleS3Response(e.StatusCode, e.Message));
}
// Catch other errors
catch (Exception e)
{
_logger.LogError(e.Message, e);
return(new SimpleS3Response(HttpStatusCode.BadRequest, e.Message));
}
}
public override async Task <bool> Validate(ILogger log, CancellationToken token)
{
log.LogInformation($"Verifying {bucketName} exists.");
var isBucketFound = await client.DoesS3BucketExistAsync(bucketName).ConfigureAwait(false);
if (!isBucketFound)
{
log.LogError(
$"Unable to find {bucketName}. Verify that the Amazon account and bucket exists. The bucket " +
"must be created manually before using this feed.");
}
return(isBucketFound);
}
public static async Task DeleteAllBucketsWithPrefix(IAmazonS3 s3Client, string namePrefix)
{
var listBucketsResponse = await s3Client.ListBucketsAsync(new ListBucketsRequest()).ConfigureAwait(false);
await Task.WhenAll(listBucketsResponse.Buckets.Where(x => x.BucketName.StartsWith(namePrefix, StringComparison.OrdinalIgnoreCase))
.Select(x => x.BucketName).Select(async bucketName =>
{
try
{
if (!await s3Client.DoesS3BucketExistAsync(bucketName))
{
return;
}
var response = await s3Client.GetBucketLocationAsync(bucketName);
S3Region region;
switch (response.Location)
{
case "":
{
region = new S3Region("us-east-1");
break;
}
case "EU":
{
region = S3Region.EUW1;
break;
}
default:
region = response.Location;
break;
}
await s3Client.DeleteBucketAsync(new DeleteBucketRequest
{
BucketName = bucketName,
BucketRegion = region
});
}
catch (AmazonS3Exception)
{
Console.WriteLine($"Unable to delete bucket '{bucketName}'");
}
}));
}
async Task <bool> checkIfBucketExists(string bucket)
{
try
{
s3Client = new AmazonS3Client();
return(await s3Client.DoesS3BucketExistAsync(bucket));
}
catch (AmazonS3Exception ex)
{
log.Error(string.Format("S3 Error encountered on server when checking bucket exists, bucket={0}", bucket), ex);
}
catch (Exception ex)
{
log.Error(string.Format("Unknown S3 error on server when checking bucket exists. bucket={0}", bucket), ex);
}
return(false);
}
private async Task <IAmazonS3> CreateBucketIfNoExists(IAmazonS3 client, string bucketName)
{
if (await client.DoesS3BucketExistAsync(bucketName))
{
return(client);
}
var response = await client.PutBucketAsync(new PutBucketRequest
{
BucketName = bucketName,
});
if (!response.HttpStatusCode.IsSuccessStatusCode())
{
throw new ApplicationException($"Error on {nameof(CreateBucketIfNoExists)}");
}
return(client);
}
public static async Task Delete(IAmazonS3 s3, string endpointName, string bucketName)
{
await Console.Out.WriteLineAsync($"Delete bucket with name '{bucketName}' for endpoint '{endpointName}'.");
if (await s3.DoesS3BucketExistAsync(bucketName))
{
var response = await s3.GetBucketLocationAsync(bucketName);
S3Region region;
switch (response.Location)
{
case "":
{
region = new S3Region("us-east-1");
break;
}
case "EU":
{
region = S3Region.EUW1;
break;
}
default:
region = response.Location;
break;
}
var deleteRequest = new DeleteBucketRequest
{
BucketName = bucketName,
BucketRegion = region,
};
await s3.DeleteBucketAsync(deleteRequest).ConfigureAwait(false);
await Console.Out.WriteLineAsync($"Delete bucket with name '{bucketName}' for endpoint '{endpointName}'.");
}
else
{
await Console.Out.WriteLineAsync($"Bucket with name '{bucketName}' does not exist.");
}
}
public static async Task Create(IAmazonS3 s3, string endpointName, string bucketName)
{
await Console.Out.WriteLineAsync($"Creating bucket with name '{bucketName}' for endpoint '{endpointName}'.");
if (!await s3.DoesS3BucketExistAsync(bucketName))
{
await s3.RetryConflictsAsync(async() =>
await s3.PutBucketAsync(new PutBucketRequest {
BucketName = bucketName
}).ConfigureAwait(false),
onRetry : async x => { await Console.Out.WriteLineAsync($"Conflict when creating S3 bucket, retrying after {x}ms."); }).ConfigureAwait(false);
await Console.Out.WriteLineAsync($"Created bucket with name '{bucketName}' for endpoint '{endpointName}'.");
}
else
{
await Console.Out.WriteLineAsync($"Bucket with name '{bucketName}' already exists.");
}
}
static AWS()
{
// Validate AWS credentials
if (Environment.GetEnvironmentVariable("AWS_ACCESS_KEY_ID") == null)
{
throw new ArgumentNullException("The environment variable AWS_ACCESS_KEY_ID must not be null");
}
accessKey = Environment.GetEnvironmentVariable("AWS_ACCESS_KEY_ID");
if (accessKey == "")
{
throw new ArgumentException("The environment variable AWS_ACCESS_KEY_ID must not be empty");
}
if (Environment.GetEnvironmentVariable("AWS_SECRET_ACCESS_KEY") == null)
{
throw new ArgumentNullException("The environment variable AWS_SECRET_ACCESS_KEY must not be null");
}
secretKey = Environment.GetEnvironmentVariable("AWS_SECRET_ACCESS_KEY");
if (secretKey == "")
{
throw new ArgumentException("The environment variable AWS_SECRET_ACCESS_KEY must not be empty");
}
credentials = new BasicAWSCredentials(accessKey, secretKey);
client = new AmazonS3Client(credentials, region);
// Validate the AWS client and bucket
Task <Boolean> t = client.DoesS3BucketExistAsync(bucket);
t.Wait();
bool bucketExists = t.Result;
if (!t.IsCompletedSuccessfully)
{
throw new WebException("DoesS3BucketExistAsync() must run successfully");
}
if (!bucketExists)
{
throw new WebException($"Amazon S3 bucket must exist");
}
}
public async Task InitializeAsync(CancellationToken ct = default)
{
try
{
s3Client = new AmazonS3Client(
accessKey,
secretKey,
bucketRegion);
transferUtility = new TransferUtility(s3Client);
var exists = await s3Client.DoesS3BucketExistAsync(bucketName);
if (!exists)
{
throw new ConfigurationException($"Cannot connect to Amazon S3 bucket '${bucketName}'.");
}
}
catch (AmazonS3Exception ex)
{
throw new ConfigurationException($"Cannot connect to Amazon S3 bucket '${bucketName}'.", ex);
}
}
private async Task <bool> FileExistsAsync(IAmazonS3 client, string containerName, string fileId, CancellationToken cancellationToken)
{
// Make sure Blob Container exists.
if (await client.DoesS3BucketExistAsync(containerName))
{
try
{
var getObjectMetadataResponse = await client.GetObjectMetadataAsync(containerName, fileId, cancellationToken);
return(getObjectMetadataResponse.HttpStatusCode == HttpStatusCode.OK);
}
catch (Exception e)
{
if (e is AmazonS3Exception)
{
return(false);
}
throw;
}
}
return(false);
}
public async Task <bool> DoesS3BucketExists(string bucketName)
{
return(await _s3Client.DoesS3BucketExistAsync(bucketName));
}
public AmazonPictureService(IRepository <Picture> pictureRepository,
ISettingService settingService,
IWebHelper webHelper,
ILogger logger,
IEventPublisher eventPublisher,
MediaSettings mediaSettings,
GrandConfig config)
: base(pictureRepository,
settingService,
webHelper,
logger,
eventPublisher,
mediaSettings)
{
this._config = config;
//Arguments guard
if (string.IsNullOrEmpty(_config.AmazonAwsAccessKeyId))
{
throw new ArgumentNullException("AmazonAwsAccessKeyId");
}
if (string.IsNullOrEmpty(_config.AmazonAwsSecretAccessKey))
{
throw new ArgumentNullException("AmazonAwsSecretAccessKey");
}
if (string.IsNullOrEmpty(_config.AmazonBucketName))
{
throw new ArgumentNullException("AmazonBucketName");
}
//Region guard
RegionEndpoint regionEndpoint = RegionEndpoint.GetBySystemName(_config.AmazonRegion);
if (regionEndpoint.DisplayName == "Unknown")
{
throw new NullReferenceException("specified Region is invalid");
}
//Client guard
_s3Client = new AmazonS3Client(_config.AmazonAwsAccessKeyId, _config.AmazonAwsSecretAccessKey, regionEndpoint);
try
{
EnsureValidResponse(_s3Client.ListBucketsAsync().Result, HttpStatusCode.OK);
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
//Bucket guard
_bucketName = _config.AmazonBucketName;
var bucketExists = _s3Client.DoesS3BucketExistAsync(_bucketName).Result;
while (bucketExists == false)
{
S3Region s3region = S3Region.FindValue(_config.AmazonRegion);
var putBucketRequest = new PutBucketRequest
{
BucketName = _bucketName,
BucketRegion = s3region,
};
try
{
EnsureValidResponse(_s3Client.PutBucketAsync(putBucketRequest).Result, HttpStatusCode.OK);
}
catch (AmazonS3Exception ex)
{
if (ex.ErrorCode == "BucketAlreadyOwnedByYou")
{
break;
}
throw;
}
bucketExists = _s3Client.DoesS3BucketExistAsync(_bucketName).Result;
}
}
public virtual async Task <bool> DoesS3BucketExist(string bucketName)
{
var bucketExistAsync = await _amazonS3.DoesS3BucketExistAsync(bucketName);
return(bucketExistAsync);
}
public async Task <bool> DoesS3BucketExist(string bucketName) => await _s3Client.DoesS3BucketExistAsync(bucketName);
internal bool IsBucketExists(string bucketName)
{
bool rez = _client.DoesS3BucketExistAsync(bucketName).Result;
return(rez);
}
/// <summary>
/// Store file stream to Amazon S3 buckets
/// </summary>
/// <param name="bucketName">The bucket name where uploaded files will be stored</param>
/// <param name="uploadItems">File streams with key and metatadata to uploaded</param>
/// <returns>
/// UploadResponse contains:
/// - StatusCode: an error code causing the method failed to run (OK if no error)
/// - HttpStatusCode: The extra error code decribing the reason of failure when StatusCode is ServiceStatusCode.HttpStatusCode
/// - UploadedItems: The items in uploadItems which were successfully uploaded to S3 buckets
/// - FailedItems: The items in uploadItems which were failed to uploaded,
/// with a UploadItemResponseCode and a possible exception for the details of filure
/// </returns>
public async Task <UploadResponse> UploadAsync(string bucketName, IList <IUploadItem> uploadItems)
{
var response = new UploadResponse();
if (string.IsNullOrEmpty(bucketName))
{
response.StatusCode = ServiceStatusCode.InvalidBucketName;
return(response);
}
if (!await client.DoesS3BucketExistAsync(bucketName))
{
response.StatusCode = ServiceStatusCode.BucketDoesNotExist;
return(response);
}
using (TransferUtility transferUtility = new TransferUtility(client))
{
response.StatusCode = ServiceStatusCode.OK;
response.UploadedItems = new List <IUploadItem>();
response.FailedItems = new List <IUploadItemStatus>();
foreach (var item in uploadItems)
{
if (string.IsNullOrEmpty(item.KeyName))
{
response.FailedItems.Add(new UploadItemStatus(item, UploadItemStatsCode.InvalidKeyName));
continue;
}
try
{
var content = item.GetStream();
TransferUtilityUploadRequest request = new TransferUtilityUploadRequest
{
AutoCloseStream = true,
BucketName = bucketName,
Key = item.KeyName,
InputStream = item.GetStream(),
};
await transferUtility.UploadAsync(request);
// client.PutObjectAsync
response.UploadedItems.Add(item);
}
catch (AmazonS3Exception ex)
{
response.FailedItems.Add(new UploadItemStatus(item, ex));
}
catch (IOException ex)
{
response.FailedItems.Add(new UploadItemStatus(item, ex));
}
catch (Exception ex)
{
response.FailedItems.Add(new UploadItemStatus(item, ex));
}
}
}
return(response);
}
