public async Task <object> Process(JsonDocument request, ILambdaContext context)
{
var s3 = new AmazonS3Client();
// easier than doing math on the timestamps in logs
var timer = new Stopwatch();
timer.Start();
context.Logger.LogLine($"{timer.Elapsed}: Getting started.");
using (var output = new S3UploadStream(s3, BUCKET, OUTPUT_KEY)) {
using var zip = new ZipArchive(output, ZipArchiveMode.Create);
await foreach (var o in s3.Paginators.ListObjects(new ListObjectsRequest()
{
BucketName = BUCKET, Prefix = PREFIX
}).S3Objects)
{
context.Logger.LogLine($"{timer.Elapsed}: Starting on {o.Key}.");
using var stream = (await s3.GetObjectAsync(BUCKET, o.Key)).ResponseStream;
using var entry = zip.CreateEntry(o.Key.Substring(PREFIX.Length)).Open();
await stream.CopyToAsync(entry);
context.Logger.LogLine($"{timer.Elapsed}: Done with {o.Key}.");
}
}
context.Logger.LogLine($"{timer.Elapsed}: Done.");
timer.Stop();
return(new {
Prefix = PREFIX,
ZipFile = $"s3://{BUCKET}/{OUTPUT_KEY}",
Status = "ok"
});
}
public async Task <object> Process(JsonDocument request, ILambdaContext context)
{
var s3 = new AmazonS3Client();
// easier than doing math on the timestamps in logs
var timer = new Stopwatch();
timer.Start();
context.Logger.LogLine($"{timer.Elapsed}: Getting started.");
using var stream = (await s3.GetObjectAsync(BUCKET, AES_DATA)).ResponseStream;
// setup a decryptor
using var aes = AesManaged.Create();
aes.IV = Convert.FromBase64String("EqYoED0ag4vlPnFkWZMCog==");
aes.Key = Convert.FromBase64String("Sgf9NocncDHSBqMXrMthXbToAQmthMpC6eJ6Hw51Ghg=");
using var idecrypt = aes.CreateDecryptor();
using var cstream = new CryptoStream(stream, idecrypt, CryptoStreamMode.Read);
using (var output = new S3UploadStream(s3, BUCKET, OUTPUT_KEY)) {
await cstream.CopyToAsync(output);
}
context.Logger.LogLine($"{timer.Elapsed}: Done copying.");
timer.Stop();
return(new {
AesFile = $"s3://{BUCKET}/{AES_DATA}",
CsvFile = $"s3://{BUCKET}/{OUTPUT_KEY}",
Status = "ok"
});
}
public async Task <object> Process(JsonDocument request, ILambdaContext context)
{
var s3 = new AmazonS3Client();
// easier than doing math on the timestamps in logs
var timer = new Stopwatch();
timer.Start();
context.Logger.LogLine($"{timer.Elapsed}: Getting started.");
using var stream = (await s3.GetObjectAsync(BUCKET, GZ_DATA)).ResponseStream;
using var decompress = new GZipStream(stream, CompressionMode.Decompress);
using (var output = new S3UploadStream(s3, BUCKET, OUTPUT_KEY)) {
await decompress.CopyToAsync(output);
}
context.Logger.LogLine($"{timer.Elapsed}: Done copying.");
timer.Stop();
return(new {
AesFile = $"s3://{BUCKET}/{GZ_DATA}",
CsvFile = $"s3://{BUCKET}/{OUTPUT_KEY}",
Status = "ok"
});
}
public async Task <object> Process(JsonDocument request, ILambdaContext context)
{
var s3 = new AmazonS3Client();
// easier than doing math on the timestamps in logs
var timer = new Stopwatch();
timer.Start();
context.Logger.LogLine($"{timer.Elapsed}: Getting started.");
using var data = new SeekableS3Stream(s3, BUCKET, PGP_DATA, 5 * 1024 * 1024, 10);
using var key = new SeekableS3Stream(s3, BUCKET, PGP_PRIVATE_KEY, 32 * 1024);
using var pgp = new PGP();
using (var output = new S3UploadStream(s3, BUCKET, OUTPUT_KEY)) {
await pgp.DecryptStreamAsync(data, output, key, PGP_PASSWORD);
}
context.Logger.LogLine($"{timer.Elapsed}: Done copying.");
timer.Stop();
return(new {
PgpFile = $"s3://{BUCKET}/{PGP_DATA}",
CsvFile = $"s3://{BUCKET}/{OUTPUT_KEY}",
Status = "ok"
});
}
public bool SendFile(Stream localFile, string destFullPath, bool closeInputStream)
{
// try to upload file
for (int i = 0; i < Details.RetryCount; i++)
{
// try to open
if (!Open())
{
return(false);
}
try
{
// upload
using (Stream ostream = new S3UploadStream(client, destFullPath, Details.Get("useReducedRedundancy", false), true, Details.Get("makePublic", false), Details.Get("partSize", 20 * 1024 * 1024)))
{
using (var file = localFile)
{
file.CopyTo(ostream, FileServiceConnectionInfo.DefaultWriteBufferSize >> 2);
}
}
_setStatus(true);
return(true);
}
catch (Exception ex)
{
_setStatus(ex);
// disconnect on error
Close();
}
finally
{
if (closeInputStream)
{
localFile.Close();
}
}
}
return(false);
}
