public override async Task TryAwake(params object[] arg)
{
var listResponse = await S3Client.ListObjectsV2Async(new ListObjectsV2Request
{
BucketName = BucketName
});
}
public async Task InternalDeleteFilesByPrefixesAsync(IEnumerable <string> filePrefixes,
CancellationToken cancellationToken = default)
{
var remover = CreateS3BatchRemover(cancellationToken);
var producer = new TransformManyBlock <string, string>(prefix =>
{
return(GetFiles());
IEnumerable <string> GetFiles()
{
var request = new ListObjectsV2Request {
BucketName = BucketName, Prefix = AdjustKey(prefix)
};
ListObjectsV2Response response;
do
{
// in case of C# 8.0 it might be async operation with async enumerator result
response = S3Client.ListObjectsV2Async(request, cancellationToken).GetAwaiter().GetResult();
foreach (var key in response.S3Objects.Select(x => x.Key))
{
yield return(key);
}
request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated);
}
}, new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationToken,
MaxDegreeOfParallelism = Math.Max(Environment.ProcessorCount / 2, 1)
});
_ = producer.LinkTo(remover, new DataflowLinkOptions {
PropagateCompletion = true
});
foreach (var prefix in filePrefixes)
{
_ = await producer.SendAsync(prefix, cancellationToken).ConfigureAwait(false);
}
producer.Complete();
await remover.Completion.AggregateExceptions <S3BucketException>(
$"Deletion files by prefix from '{BucketName}' bucket failed.");
}
public async Task <JsonResult> Get()
{
var listResponse = await S3Client.ListObjectsV2Async(new ListObjectsV2Request
{
BucketName = BucketName
});
try
{
Response.ContentType = "text/json";
return(new JsonResult(listResponse.S3Objects, new JsonSerializerSettings {
Formatting = Formatting.Indented
}));
}
catch (AmazonS3Exception e)
{
Response.StatusCode = (int)e.StatusCode;
return(new JsonResult(e.Message));
}
}
/// <summary>
/// The only artifact should be the one passed to the above function.
/// </summary>
/// <param name="evnt"></param>
/// <param name="context"></param>
/// <returns></returns>
public async Task <string> CopyArtifactExtractedFiles(CodePipelineEvent evnt, ILambdaContext context)
{
context.Logger.LogLine(Newtonsoft.Json.JsonConvert.SerializeObject(evnt, Newtonsoft.Json.Formatting.Indented));
var artifact = evnt.Job.data.inputArtifacts.First();
var toCopy = JsonConvert.DeserializeObject <ArtifactsToCopy>(evnt.Job.data.actionConfiguration.configuration.UserParameters);
var outputFolderFromPriorStep = artifact.location.s3Location.objectKey + "_output/";
var filePrefix = outputFolderFromPriorStep + toCopy.SourcePrefix;
var objects = await S3Client.ListObjectsV2Async(new Amazon.S3.Model.ListObjectsV2Request
{
BucketName = artifact.location.s3Location.bucketName,
Prefix = filePrefix,
MaxKeys = 100000
});
context.Logger.LogLine("Objects: Keys=" + objects.KeyCount + " NextConinuationToken: " + objects.NextContinuationToken);
foreach (var o in objects.S3Objects)
{
context.Logger.LogLine("File: " + o.Key);
await S3Client.CopyObjectAsync(artifact.location.s3Location.bucketName, o.Key, toCopy.DestBucket, o.Key.Replace(filePrefix, ""));
}
context.Logger.LogLine("Call PutJobSuccessResultAsync, JobID: " + evnt.Job.id);
await CodePipelineClient.PutJobSuccessResultAsync(new Amazon.CodePipeline.Model.PutJobSuccessResultRequest
{
JobId = evnt.Job.id
});
context.Logger.LogLine("Completed");
return("Success");
}
protected async Task <bool> InternalFileExistsAsync(string fullFileName, CancellationToken cancellationToken = default)
{
fullFileName = AdjustKey(fullFileName);
var request = new ListObjectsV2Request {
BucketName = BucketName, Prefix = fullFileName
};
ListObjectsV2Response response;
do
{
response = await S3Client.ListObjectsV2Async(request, cancellationToken).ConfigureAwait(false);
if (response.S3Objects.Any(x => string.Equals(x.Key, fullFileName, StringComparison.OrdinalIgnoreCase)))
{
return(true);
}
request.ContinuationToken = response.NextContinuationToken;
} while (response.IsTruncated);
return(false);
}
