public async Task <object> Process(JsonDocument request, ILambdaContext context) { var s3 = new AmazonS3Client(); // easier than doing math on the timestamps in logs var timer = new Stopwatch(); timer.Start(); context.Logger.LogLine($"{timer.Elapsed}: Getting started."); using (var output = new S3UploadStream(s3, BUCKET, OUTPUT_KEY)) { using var zip = new ZipArchive(output, ZipArchiveMode.Create); await foreach (var o in s3.Paginators.ListObjects(new ListObjectsRequest() { BucketName = BUCKET, Prefix = PREFIX }).S3Objects) { context.Logger.LogLine($"{timer.Elapsed}: Starting on {o.Key}."); using var stream = (await s3.GetObjectAsync(BUCKET, o.Key)).ResponseStream; using var entry = zip.CreateEntry(o.Key.Substring(PREFIX.Length)).Open(); await stream.CopyToAsync(entry); context.Logger.LogLine($"{timer.Elapsed}: Done with {o.Key}."); } } context.Logger.LogLine($"{timer.Elapsed}: Done."); timer.Stop(); return(new { Prefix = PREFIX, ZipFile = $"s3://{BUCKET}/{OUTPUT_KEY}", Status = "ok" }); }
public async Task <object> Process(JsonDocument request, ILambdaContext context) { var s3 = new AmazonS3Client(); // easier than doing math on the timestamps in logs var timer = new Stopwatch(); timer.Start(); context.Logger.LogLine($"{timer.Elapsed}: Getting started."); using var stream = (await s3.GetObjectAsync(BUCKET, AES_DATA)).ResponseStream; // setup a decryptor using var aes = AesManaged.Create(); aes.IV = Convert.FromBase64String("EqYoED0ag4vlPnFkWZMCog=="); aes.Key = Convert.FromBase64String("Sgf9NocncDHSBqMXrMthXbToAQmthMpC6eJ6Hw51Ghg="); using var idecrypt = aes.CreateDecryptor(); using var cstream = new CryptoStream(stream, idecrypt, CryptoStreamMode.Read); using (var output = new S3UploadStream(s3, BUCKET, OUTPUT_KEY)) { await cstream.CopyToAsync(output); } context.Logger.LogLine($"{timer.Elapsed}: Done copying."); timer.Stop(); return(new { AesFile = $"s3://{BUCKET}/{AES_DATA}", CsvFile = $"s3://{BUCKET}/{OUTPUT_KEY}", Status = "ok" }); }
public async Task <object> Process(JsonDocument request, ILambdaContext context) { var s3 = new AmazonS3Client(); // easier than doing math on the timestamps in logs var timer = new Stopwatch(); timer.Start(); context.Logger.LogLine($"{timer.Elapsed}: Getting started."); using var stream = (await s3.GetObjectAsync(BUCKET, GZ_DATA)).ResponseStream; using var decompress = new GZipStream(stream, CompressionMode.Decompress); using (var output = new S3UploadStream(s3, BUCKET, OUTPUT_KEY)) { await decompress.CopyToAsync(output); } context.Logger.LogLine($"{timer.Elapsed}: Done copying."); timer.Stop(); return(new { AesFile = $"s3://{BUCKET}/{GZ_DATA}", CsvFile = $"s3://{BUCKET}/{OUTPUT_KEY}", Status = "ok" }); }
public async Task <object> Process(JsonDocument request, ILambdaContext context) { var s3 = new AmazonS3Client(); // easier than doing math on the timestamps in logs var timer = new Stopwatch(); timer.Start(); context.Logger.LogLine($"{timer.Elapsed}: Getting started."); using var data = new SeekableS3Stream(s3, BUCKET, PGP_DATA, 5 * 1024 * 1024, 10); using var key = new SeekableS3Stream(s3, BUCKET, PGP_PRIVATE_KEY, 32 * 1024); using var pgp = new PGP(); using (var output = new S3UploadStream(s3, BUCKET, OUTPUT_KEY)) { await pgp.DecryptStreamAsync(data, output, key, PGP_PASSWORD); } context.Logger.LogLine($"{timer.Elapsed}: Done copying."); timer.Stop(); return(new { PgpFile = $"s3://{BUCKET}/{PGP_DATA}", CsvFile = $"s3://{BUCKET}/{OUTPUT_KEY}", Status = "ok" }); }
public bool SendFile(Stream localFile, string destFullPath, bool closeInputStream) { // try to upload file for (int i = 0; i < Details.RetryCount; i++) { // try to open if (!Open()) { return(false); } try { // upload using (Stream ostream = new S3UploadStream(client, destFullPath, Details.Get("useReducedRedundancy", false), true, Details.Get("makePublic", false), Details.Get("partSize", 20 * 1024 * 1024))) { using (var file = localFile) { file.CopyTo(ostream, FileServiceConnectionInfo.DefaultWriteBufferSize >> 2); } } _setStatus(true); return(true); } catch (Exception ex) { _setStatus(ex); // disconnect on error Close(); } finally { if (closeInputStream) { localFile.Close(); } } } return(false); }