void SaveMessage(MessageHandlerClosure closure, CellarWriter writer)
{
try {
var message = closure.Message;
var keySegment = new ArraySegment <byte>(message.Key);
using (var mem = new MemoryStream(message.Value))
using (var zip = new LZ4Stream(mem, CompressionMode.Decompress))
using (var pooled = _recyclableStreamManagerWrapper.GetStream("mv")) {
Streams.Copy(zip, pooled, _buffer);
pooled.Seek(0, SeekOrigin.Begin);
writer.Append(keySegment, pooled, (int)pooled.Length);
}
}
catch (Exception ex) {
Console.WriteLine("Failed to read message {0}", ex.Message);
}
}
public override void Run()
{
var streams = SourceStreams.Split(new[] { ';', ',' }, StringSplitOptions.RemoveEmptyEntries);
var rawSize = 1024 * 1024 * ChunkSize;
var key = Convert.FromBase64String(EncryptionKey);
using (var output = CellarWriter.Create(OutputStream, rawSize, key)) {
foreach (var stream in streams)
{
var fetcher = GetFetcher(stream);
var inputPos = output.GetCheckpoint(stream + "-pos");
using (var cts = new CancellationTokenSource()) {
// launch reading
var task = fetcher.ReadAll(cts.Token, inputPos, int.MaxValue);
while (!cts.IsCancellationRequested)
{
var start = Stopwatch.StartNew();
var result = task.Result;
if (result.ReadRecords <= 0)
{
break;
}
inputPos = result.CurrentPosition;
// launch next task in advance
task = fetcher.ReadAll(cts.Token, inputPos, int.MaxValue);
foreach (var message in result.Messages)
{
SaveMessage(message, output);
}
output.Checkpoint(stream + "-pos", inputPos);
var stats = output.EstimateSize();
var totalSize = streams.Sum(s => output.GetCheckpoint(s + "-pos"));
var compression = totalSize * 1D / stats.DiskSize;
var streamCompletion = 100F * result.CurrentPosition / result.MaxPosition;
start.Stop();
var bytes = result.CurrentPosition - result.StartingPosition;
var mbpersec = 1D * bytes / 1024 / 1024 / start.Elapsed.TotalSeconds;
Console.WriteLine(
"{4}: {0:##0.0}% at {1:###.0}Mb/s. {2:000} chunks with {3:##.0}x compression",
streamCompletion, mbpersec,
stats.ChunkCount,
compression, stream);
}
}
var size = output.EstimateSize();
Console.WriteLine("Total records: {0} and {1} of data", size.Records,
Print.Bytes(size.ByteSize));
output.Checkpoint("stream-pos", inputPos);
output.Checkpoint("raw-records", size.Records);
output.Checkpoint("raw-bytes", size.ByteSize);
output.Checkpoint("timestamp-secs", DateTimeOffset.UtcNow.ToUnixTimeSeconds());
}
Console.WriteLine("Done");
}
}
