private void EnsureBucketExists()
{
var exists = _client.DoesS3BucketExistAsync(_bucketName).Result;
if (!exists)
{
_client.EnsureBucketExistsAsync(_bucketName).Wait();
}
}
public AwsS3Storage(IOptions <AwsS3Config> options)
{
var region = RegionEndpoint.EnumerableAllRegions
.ToList()
.SingleOrDefault(x => x.SystemName == options.Value.AwsRegion)
?? throw new InvalidOperationException($"Cannot resolve {nameof(options.Value.AwsRegion)} ({options.Value.AwsRegion}) from valid options {string.Join(", ", RegionEndpoint.EnumerableAllRegions.Select(x => x.SystemName))}");
var config = new AmazonS3Config
{
RegionEndpoint = region,
ForcePathStyle = true
};
if (options.Value.AwsServiceURL != null)
{
config.ServiceURL = options.Value.AwsServiceURL;
}
_s3Client = new AmazonS3Client(
options.Value.AccessKey ?? throw new InvalidOperationException($"Missing configuration {nameof(options.Value.AccessKey)}"),
options.Value.SecretKey ?? throw new InvalidOperationException($"Missing configuration {nameof(options.Value.AccessKey)}"),
config);
_bucketName = options.Value.AwsS3BucketName ?? throw new InvalidOperationException($"Missing configuration {nameof(options.Value.AwsS3BucketName)}");
_s3Client.EnsureBucketExistsAsync(_bucketName);
}
public async Task <InvokeResult> InitAsync(DataStream stream)
{
_stream = stream;
var options = new CredentialProfileOptions
{
AccessKey = stream.AwsAccessKey,
SecretKey = stream.AwsSecretKey
};
var profile = new Amazon.Runtime.CredentialManagement.CredentialProfile($"awsprofile_{stream.Id}", options);
var netSDKFile = new NetSDKCredentialsFile();
netSDKFile.RegisterProfile(profile);
var creds = AWSCredentialsFactory.GetAWSCredentials(profile, netSDKFile);
try
{
_s3Client = new AmazonS3Client(creds, AWSRegionMappings.MapRegion(stream.AwsRegion));
await _s3Client.EnsureBucketExistsAsync(stream.S3BucketName);
}
catch (AmazonS3Exception amazonS3Exception)
{
_logger.AddException("AWSS3Connector_InitAsync", amazonS3Exception);
return(InvokeResult.FromException("AWSS3Connector_InitAsync", amazonS3Exception));
}
return(InvokeResult.Success);
}
/// <summary>
/// Does the actual work of initialization.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
async Task InitInternalAsync(CancellationToken cancellationToken)
{
logger.Verbose("InitInternalAsync()");
await client.EnsureBucketExistsAsync(options.Value.BucketName);
init = false;
}
public async Task <string> Execute()
{
//make sure the queue exists
await _client.EnsureBucketExistsAsync(_settings.Bucket);
using var transferUtility = new TransferUtility(_client);
using var sr = new StreamReader(Filename);
var uploadRequest = new TransferUtilityUploadRequest
{
InputStream = sr.BaseStream,
ContentType = "content/text",
BucketName = _settings.Bucket,
Key = Filename,
TagSet = new List <Tag>
{
new Tag {
Key = "key1", Value = "value1"
},
new Tag {
Key = "key2", Value = "value2"
}
}
};
await transferUtility.UploadAsync(uploadRequest);
return("uploaded");
}
public async Task <string> Execute()
{
//make sure the queue exists
await _client.EnsureBucketExistsAsync(_settings.Bucket);
var result = _client.GetPreSignedURL(new GetPreSignedUrlRequest
{
BucketName = _settings.Bucket,
Key = Filename,
Expires = DateTime.Now.AddMinutes(1),
Protocol = Protocol.HTTP
});
return(result);
}
public async Task WaitForInit(TimeSpan timeout)
{
var cts = new CancellationTokenSource();
cts.CancelAfter(timeout);
var request = new GraphRequest
{
AwsEnvironment = "dev"
};
while (!cts.IsCancellationRequested)
{
try
{
using (var httpCliet = new HttpClient())
{
LogHelper.Log(LogLevel.INFO, "Checking LocalStack health...");
httpCliet.Timeout = TimeSpan.FromSeconds(3);
var content = new StringContent(JsonConvert.SerializeObject(request), Encoding.UTF8, "application/json");
var response = await httpCliet.PostAsync(_settings.DashboardGraphUrl, content, cts.Token);
if (response.IsSuccessStatusCode)
{
await _s3Client.EnsureBucketExistsAsync(_settings.Bucket);
LogHelper.Log(LogLevel.INFO, "LocalStack S3 ready!");
break;
}
}
}
catch (Exception ex)
{
LogHelper.Log(LogLevel.WARN, $"Health check response: {ex.Message}");
await Task.Delay(1000, cts.Token);
}
}
}
public override async Task CreateBucket(ILogger log, CancellationToken token)
{
if (!await HasBucket(log, token))
{
log.LogInformation($"Creating new bucket: {_bucketName}");
await _client.EnsureBucketExistsAsync(_bucketName);
var tries = 0;
var maxTries = 30;
var success = false;
while (tries < maxTries && !success)
{
tries++;
try
{
var policyRequest = new PutBucketPolicyRequest()
{
BucketName = _bucketName,
Policy = GetReadOnlyPolicy(_bucketName).ToString()
};
log.LogInformation($"Adding policy for public read access to bucket: ${_bucketName}");
await _client.PutBucketPolicyAsync(policyRequest);
success = true;
}
catch (AmazonS3Exception ex) when(tries < (maxTries - 1))
{
log.LogWarning($"Failed to update policy for bucket: {ex.Message} Trying again.");
await Task.Delay(TimeSpan.FromSeconds(tries));
}
}
if (tries >= maxTries)
{
throw new InvalidOperationException("Unable to create bucket");
}
// Get and release the lock to ensure that everything will work for the next operation.
// In the E2E tests there are often failures due to the bucket saying it is not available
// even though the above checks passed. To work around this wait until a file can be
// successfully created in the bucket before returning.
tries = 0;
success = false;
// Pass the null logger to avoid noise
using (var feedLock = CreateLock(NullLogger.Instance))
{
while (tries < maxTries && !success)
{
tries++;
try
{
// Attempt to get the lock, since this is a new container it will be available.
// This will fail if the bucket it not yet ready.
if (await feedLock.GetLock(TimeSpan.FromMinutes(1), "Container create lock test", token))
{
feedLock.Release();
success = true;
}
}
catch (AmazonS3Exception ex) when(tries < (maxTries - 1) && ex.StatusCode != HttpStatusCode.BadRequest)
{
// Ignore exceptions until the last exception
await Task.Delay(TimeSpan.FromSeconds(tries));
}
}
}
_hasBucket = true;
}
}
public async Task Execute()
{
await _client.EnsureBucketExistsAsync(_settings.Bucket);
LogHelper.Log(LogLevel.INFO, $"Create '{_settings.Bucket}' bucket - DONE");
}
protected virtual async Task CreateBucketIfNotExists(IAmazonS3 client, string containerName)
{
await client.EnsureBucketExistsAsync(containerName);
}
public async Task <IActionResult> CreateIfNotExists(string bucketName)
{
await _s3Client.EnsureBucketExistsAsync(bucketName);
return(Ok());
}
