private void EnsureBucketExists() { var exists = _client.DoesS3BucketExistAsync(_bucketName).Result; if (!exists) { _client.EnsureBucketExistsAsync(_bucketName).Wait(); } }
public AwsS3Storage(IOptions <AwsS3Config> options) { var region = RegionEndpoint.EnumerableAllRegions .ToList() .SingleOrDefault(x => x.SystemName == options.Value.AwsRegion) ?? throw new InvalidOperationException($"Cannot resolve {nameof(options.Value.AwsRegion)} ({options.Value.AwsRegion}) from valid options {string.Join(", ", RegionEndpoint.EnumerableAllRegions.Select(x => x.SystemName))}"); var config = new AmazonS3Config { RegionEndpoint = region, ForcePathStyle = true }; if (options.Value.AwsServiceURL != null) { config.ServiceURL = options.Value.AwsServiceURL; } _s3Client = new AmazonS3Client( options.Value.AccessKey ?? throw new InvalidOperationException($"Missing configuration {nameof(options.Value.AccessKey)}"), options.Value.SecretKey ?? throw new InvalidOperationException($"Missing configuration {nameof(options.Value.AccessKey)}"), config); _bucketName = options.Value.AwsS3BucketName ?? throw new InvalidOperationException($"Missing configuration {nameof(options.Value.AwsS3BucketName)}"); _s3Client.EnsureBucketExistsAsync(_bucketName); }
public async Task <InvokeResult> InitAsync(DataStream stream) { _stream = stream; var options = new CredentialProfileOptions { AccessKey = stream.AwsAccessKey, SecretKey = stream.AwsSecretKey }; var profile = new Amazon.Runtime.CredentialManagement.CredentialProfile($"awsprofile_{stream.Id}", options); var netSDKFile = new NetSDKCredentialsFile(); netSDKFile.RegisterProfile(profile); var creds = AWSCredentialsFactory.GetAWSCredentials(profile, netSDKFile); try { _s3Client = new AmazonS3Client(creds, AWSRegionMappings.MapRegion(stream.AwsRegion)); await _s3Client.EnsureBucketExistsAsync(stream.S3BucketName); } catch (AmazonS3Exception amazonS3Exception) { _logger.AddException("AWSS3Connector_InitAsync", amazonS3Exception); return(InvokeResult.FromException("AWSS3Connector_InitAsync", amazonS3Exception)); } return(InvokeResult.Success); }
/// <summary> /// Does the actual work of initialization. /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> async Task InitInternalAsync(CancellationToken cancellationToken) { logger.Verbose("InitInternalAsync()"); await client.EnsureBucketExistsAsync(options.Value.BucketName); init = false; }
public async Task <string> Execute() { //make sure the queue exists await _client.EnsureBucketExistsAsync(_settings.Bucket); using var transferUtility = new TransferUtility(_client); using var sr = new StreamReader(Filename); var uploadRequest = new TransferUtilityUploadRequest { InputStream = sr.BaseStream, ContentType = "content/text", BucketName = _settings.Bucket, Key = Filename, TagSet = new List <Tag> { new Tag { Key = "key1", Value = "value1" }, new Tag { Key = "key2", Value = "value2" } } }; await transferUtility.UploadAsync(uploadRequest); return("uploaded"); }
public async Task <string> Execute() { //make sure the queue exists await _client.EnsureBucketExistsAsync(_settings.Bucket); var result = _client.GetPreSignedURL(new GetPreSignedUrlRequest { BucketName = _settings.Bucket, Key = Filename, Expires = DateTime.Now.AddMinutes(1), Protocol = Protocol.HTTP }); return(result); }
public async Task WaitForInit(TimeSpan timeout) { var cts = new CancellationTokenSource(); cts.CancelAfter(timeout); var request = new GraphRequest { AwsEnvironment = "dev" }; while (!cts.IsCancellationRequested) { try { using (var httpCliet = new HttpClient()) { LogHelper.Log(LogLevel.INFO, "Checking LocalStack health..."); httpCliet.Timeout = TimeSpan.FromSeconds(3); var content = new StringContent(JsonConvert.SerializeObject(request), Encoding.UTF8, "application/json"); var response = await httpCliet.PostAsync(_settings.DashboardGraphUrl, content, cts.Token); if (response.IsSuccessStatusCode) { await _s3Client.EnsureBucketExistsAsync(_settings.Bucket); LogHelper.Log(LogLevel.INFO, "LocalStack S3 ready!"); break; } } } catch (Exception ex) { LogHelper.Log(LogLevel.WARN, $"Health check response: {ex.Message}"); await Task.Delay(1000, cts.Token); } } }
public override async Task CreateBucket(ILogger log, CancellationToken token) { if (!await HasBucket(log, token)) { log.LogInformation($"Creating new bucket: {_bucketName}"); await _client.EnsureBucketExistsAsync(_bucketName); var tries = 0; var maxTries = 30; var success = false; while (tries < maxTries && !success) { tries++; try { var policyRequest = new PutBucketPolicyRequest() { BucketName = _bucketName, Policy = GetReadOnlyPolicy(_bucketName).ToString() }; log.LogInformation($"Adding policy for public read access to bucket: ${_bucketName}"); await _client.PutBucketPolicyAsync(policyRequest); success = true; } catch (AmazonS3Exception ex) when(tries < (maxTries - 1)) { log.LogWarning($"Failed to update policy for bucket: {ex.Message} Trying again."); await Task.Delay(TimeSpan.FromSeconds(tries)); } } if (tries >= maxTries) { throw new InvalidOperationException("Unable to create bucket"); } // Get and release the lock to ensure that everything will work for the next operation. // In the E2E tests there are often failures due to the bucket saying it is not available // even though the above checks passed. To work around this wait until a file can be // successfully created in the bucket before returning. tries = 0; success = false; // Pass the null logger to avoid noise using (var feedLock = CreateLock(NullLogger.Instance)) { while (tries < maxTries && !success) { tries++; try { // Attempt to get the lock, since this is a new container it will be available. // This will fail if the bucket it not yet ready. if (await feedLock.GetLock(TimeSpan.FromMinutes(1), "Container create lock test", token)) { feedLock.Release(); success = true; } } catch (AmazonS3Exception ex) when(tries < (maxTries - 1) && ex.StatusCode != HttpStatusCode.BadRequest) { // Ignore exceptions until the last exception await Task.Delay(TimeSpan.FromSeconds(tries)); } } } _hasBucket = true; } }
public async Task Execute() { await _client.EnsureBucketExistsAsync(_settings.Bucket); LogHelper.Log(LogLevel.INFO, $"Create '{_settings.Bucket}' bucket - DONE"); }
protected virtual async Task CreateBucketIfNotExists(IAmazonS3 client, string containerName) { await client.EnsureBucketExistsAsync(containerName); }
public async Task <IActionResult> CreateIfNotExists(string bucketName) { await _s3Client.EnsureBucketExistsAsync(bucketName); return(Ok()); }