public override async Task TryAwake(params object[] arg) { var listResponse = await S3Client.ListObjectsV2Async(new ListObjectsV2Request { BucketName = BucketName }); }
public async Task InternalDeleteFilesByPrefixesAsync(IEnumerable <string> filePrefixes, CancellationToken cancellationToken = default) { var remover = CreateS3BatchRemover(cancellationToken); var producer = new TransformManyBlock <string, string>(prefix => { return(GetFiles()); IEnumerable <string> GetFiles() { var request = new ListObjectsV2Request { BucketName = BucketName, Prefix = AdjustKey(prefix) }; ListObjectsV2Response response; do { // in case of C# 8.0 it might be async operation with async enumerator result response = S3Client.ListObjectsV2Async(request, cancellationToken).GetAwaiter().GetResult(); foreach (var key in response.S3Objects.Select(x => x.Key)) { yield return(key); } request.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated); } }, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Math.Max(Environment.ProcessorCount / 2, 1) }); _ = producer.LinkTo(remover, new DataflowLinkOptions { PropagateCompletion = true }); foreach (var prefix in filePrefixes) { _ = await producer.SendAsync(prefix, cancellationToken).ConfigureAwait(false); } producer.Complete(); await remover.Completion.AggregateExceptions <S3BucketException>( $"Deletion files by prefix from '{BucketName}' bucket failed."); }
public async Task <JsonResult> Get() { var listResponse = await S3Client.ListObjectsV2Async(new ListObjectsV2Request { BucketName = BucketName }); try { Response.ContentType = "text/json"; return(new JsonResult(listResponse.S3Objects, new JsonSerializerSettings { Formatting = Formatting.Indented })); } catch (AmazonS3Exception e) { Response.StatusCode = (int)e.StatusCode; return(new JsonResult(e.Message)); } }
/// <summary> /// The only artifact should be the one passed to the above function. /// </summary> /// <param name="evnt"></param> /// <param name="context"></param> /// <returns></returns> public async Task <string> CopyArtifactExtractedFiles(CodePipelineEvent evnt, ILambdaContext context) { context.Logger.LogLine(Newtonsoft.Json.JsonConvert.SerializeObject(evnt, Newtonsoft.Json.Formatting.Indented)); var artifact = evnt.Job.data.inputArtifacts.First(); var toCopy = JsonConvert.DeserializeObject <ArtifactsToCopy>(evnt.Job.data.actionConfiguration.configuration.UserParameters); var outputFolderFromPriorStep = artifact.location.s3Location.objectKey + "_output/"; var filePrefix = outputFolderFromPriorStep + toCopy.SourcePrefix; var objects = await S3Client.ListObjectsV2Async(new Amazon.S3.Model.ListObjectsV2Request { BucketName = artifact.location.s3Location.bucketName, Prefix = filePrefix, MaxKeys = 100000 }); context.Logger.LogLine("Objects: Keys=" + objects.KeyCount + " NextConinuationToken: " + objects.NextContinuationToken); foreach (var o in objects.S3Objects) { context.Logger.LogLine("File: " + o.Key); await S3Client.CopyObjectAsync(artifact.location.s3Location.bucketName, o.Key, toCopy.DestBucket, o.Key.Replace(filePrefix, "")); } context.Logger.LogLine("Call PutJobSuccessResultAsync, JobID: " + evnt.Job.id); await CodePipelineClient.PutJobSuccessResultAsync(new Amazon.CodePipeline.Model.PutJobSuccessResultRequest { JobId = evnt.Job.id }); context.Logger.LogLine("Completed"); return("Success"); }
protected async Task <bool> InternalFileExistsAsync(string fullFileName, CancellationToken cancellationToken = default) { fullFileName = AdjustKey(fullFileName); var request = new ListObjectsV2Request { BucketName = BucketName, Prefix = fullFileName }; ListObjectsV2Response response; do { response = await S3Client.ListObjectsV2Async(request, cancellationToken).ConfigureAwait(false); if (response.S3Objects.Any(x => string.Equals(x.Key, fullFileName, StringComparison.OrdinalIgnoreCase))) { return(true); } request.ContinuationToken = response.NextContinuationToken; } while (response.IsTruncated); return(false); }