public override async Task ConsumeAsync(int index)
{
var filename = $"table-{index}.csv";
var outputPath = Path.Join(Destination, filename);
var converter = new TableEntityConverter();
using (var tw = File.CreateText(outputPath))
using (var csv = new CsvWriter(tw))
{
csv.WriteField("PartitionKey");
csv.WriteField("RowKey");
csv.WriteField("Timestamp");
csv.WriteField("Data");
await csv.NextRecordAsync();
foreach (var row in PubSub.GetConsumingEnumerable())
{
csv.WriteField(row.PartitionKey);
csv.WriteField(row.RowKey);
csv.WriteField(row.Timestamp.UtcDateTime.ToString("O"));
var data = JsonConvert.SerializeObject(
row.Properties,
converter
);
csv.WriteField(data);
await csv.NextRecordAsync();
}
}
}
public void Configure(IApplicationBuilder app)
{
app.Run(async context =>
{
var converter = new TableEntityConverter();
using (var ms = new MemoryStream(8192))
{
do
{
using (var tw = new StreamWriter(ms, Encoding.UTF8, 4096, true))
using (var csv = new CsvWriter(tw))
{
foreach (var row in PubSub.Take(1000))
{
csv.WriteField(row.PartitionKey);
csv.WriteField(row.RowKey);
csv.WriteField(row.Timestamp.UtcDateTime.ToString("O"));
var data = JsonConvert.SerializeObject(
row.Properties.Select(p => (
key: p.Key,
value: p.Value.PropertyAsObject
)),
converter
);
csv.WriteField(data);
await csv.NextRecordAsync();
}
}
await context.Response.Body.WriteAsync(ms.ToArray());
ms.Position = 0;
ms.SetLength(0);
}while (PubSub.IsAddingCompleted == false);
}
await context.Response.Body.FlushAsync();
Completion.SetResult(0);
});
}