static async Task Main(string[] args)
{
var s3 = new AmazonS3Client();
using var stream = new Cppl.Utilities.AWS.SeekableS3Stream(s3, BUCKET, KEY, 1 * 1024 * 1024, 4);
using var parquet = new ParquetReader(stream);
var fields = parquet.Schema.GetDataFields();
await Console.Out.WriteLineAsync($"{stream.TotalRead:0,000} read {stream.TotalLoaded:0,000} loaded of {stream.Length:0,000} bytes");
}
static async Task Main(string[] args)
{
var s3 = new AmazonS3Client();
using var stream = new Cppl.Utilities.AWS.SeekableS3Stream(s3, BUCKET, KEY, 1 * 1024 * 1024, 4);
using var iso = new CDReader(stream, true);
using var file = iso.OpenFile(FILENAME, FileMode.Open, FileAccess.Read);
using var reader = new StreamReader(file);
var content = await reader.ReadToEndAsync();
await Console.Out.WriteLineAsync($"{stream.TotalRead:0,000} read {stream.TotalLoaded:0,000} loaded of {stream.Length:0,000} bytes");
}
static async Task Main(string[] args)
{
var s3 = new AmazonS3Client();
using var stream = new Cppl.Utilities.AWS.SeekableS3Stream(s3, BUCKET, KEY, 128 * 1024, 12);
using var iso = new CDReader(stream, true);
using var embedded = iso.OpenFile(ZIPNAME, FileMode.Open, FileAccess.Read);
using var zip = new ZipArchive(embedded);
using var file = zip.GetEntry(FILENAME).Open();
using var reader = new StreamReader(file);
await reader.ReadLineAsync();
await Console.Out.WriteLineAsync($"{stream.TotalRead:0,000} read {stream.TotalLoaded:0,000} loaded of {stream.Length:0,000} bytes");
}
public async Task <object> Get(JsonDocument request, ILambdaContext context)
{
var s3 = new AmazonS3Client();
// easier than doing math on the timestamps in logs
var timer = new Stopwatch();
timer.Start();
context.Logger.LogLine($"{timer.Elapsed}: Getting started.");
using var stream = new Cppl.Utilities.AWS.SeekableS3Stream(s3, BUCKET, KEY, 12 * 1024 * 1024, 5);
using var iso = new CDReader(stream, true);
using var embedded = iso.OpenFile(ZIPNAME, FileMode.Open, FileAccess.Read);
using var zip = new ZipArchive(embedded, ZipArchiveMode.Read);
var entry = zip.GetEntry(FILENAME);
using var file = entry.Open();
using var reader = new StreamReader(file);
// how soon do we get the first line?
var line = await reader.ReadLineAsync();
context.Logger.LogLine($"{timer.Elapsed}: First row received.");
// read all of the remainline lines (it'll take a while...)
ulong rows = 1;
while ((line = await reader.ReadLineAsync()) != null)
{
++rows;
}
context.Logger.LogLine($"{timer.Elapsed}: Done reading rows.");
// the total amount read should be close to the total file size, but the amount loaded may be greated than
// the file size if too few ranges are held in the MRU and end-up being loaded multiple times.
context.Logger.LogLine($"{timer.Elapsed}: {stream.TotalRead:0,000} read {stream.TotalLoaded:0,000} loaded of {stream.Length:0,000} bytes");
timer.Stop();
return(new {
IsoPath = $"s3://{BUCKET}/{KEY}",
stream.TotalRead,
stream.TotalLoaded,
entry.Length,
TotalRows = rows,
Status = "ok"
});
}
static async Task Main(string[] args)
{
var s3 = new AmazonS3Client();
var images = await s3.ListObjectsAsync(BUCKET, PREFIX);
long size = 0L;
long loaded = 0L;
long read = 0L;
foreach (var image in images.S3Objects.Where(o => o.Key.EndsWith(".jpg")).Take(10))
{
size += image.Size;
using var stream = new Cppl.Utilities.AWS.SeekableS3Stream(s3, BUCKET, image.Key, 16 * 1024, 8);
var directories = ImageMetadataReader.ReadMetadata(stream);
read = stream.TotalRead;
loaded += stream.TotalLoaded;
}
// loads less than 1% of the image file content
await Console.Out.WriteLineAsync($"{read:0,000} read {loaded:0,000} loaded of {size:0,000} bytes");
}