public override void Run()
{
if (!Directory.Exists(DatabasePath))
{
Console.WriteLine("Path not found {0}", DatabasePath);
Environment.Exit(1);
}
var outPath = DatabasePath.TrimEnd('/') + ".pack";
if (Directory.Exists(outPath))
{
Directory.Delete(outPath, true);
}
Directory.CreateDirectory(outPath);
using (var db = LmdbEnv.CreateDb(DatabasePath, 0, EnvironmentOpenFlags.ReadOnly)) {
using (var tx = db.Read()) {
var stats = tx.GetUsedSize();
Console.WriteLine("Entry count: {0}. Size: {1}", stats.EntryCount,
Print.Bytes(stats.UsedBytes));
}
db.CopyTo(outPath, true);
}
var inFile = Path.Combine(outPath, "data.mdb");
Console.WriteLine("Packed into {0}", Print.Bytes(new FileInfo(inFile).Length));
var outFile = Path.Combine(outPath, "data.mdb.gzip");
using (var input = File.OpenRead(inFile))
using (var output = File.Create(outFile))
using (var gz = new GZipStream(output, CompressionLevel.Optimal)) {
input.CopyTo(gz);
}
Console.WriteLine("Gzipped into {0}", Print.Bytes(new FileInfo(outFile).Length));
}
public override void Run()
{
var streams = SourceStreams.Split(new[] { ';', ',' }, StringSplitOptions.RemoveEmptyEntries);
var rawSize = 1024 * 1024 * ChunkSize;
var key = Convert.FromBase64String(EncryptionKey);
using (var output = CellarWriter.Create(OutputStream, rawSize, key)) {
foreach (var stream in streams)
{
var fetcher = GetFetcher(stream);
var inputPos = output.GetCheckpoint(stream + "-pos");
using (var cts = new CancellationTokenSource()) {
// launch reading
var task = fetcher.ReadAll(cts.Token, inputPos, int.MaxValue);
while (!cts.IsCancellationRequested)
{
var start = Stopwatch.StartNew();
var result = task.Result;
if (result.ReadRecords <= 0)
{
break;
}
inputPos = result.CurrentPosition;
// launch next task in advance
task = fetcher.ReadAll(cts.Token, inputPos, int.MaxValue);
foreach (var message in result.Messages)
{
SaveMessage(message, output);
}
output.Checkpoint(stream + "-pos", inputPos);
var stats = output.EstimateSize();
var totalSize = streams.Sum(s => output.GetCheckpoint(s + "-pos"));
var compression = totalSize * 1D / stats.DiskSize;
var streamCompletion = 100F * result.CurrentPosition / result.MaxPosition;
start.Stop();
var bytes = result.CurrentPosition - result.StartingPosition;
var mbpersec = 1D * bytes / 1024 / 1024 / start.Elapsed.TotalSeconds;
Console.WriteLine(
"{4}: {0:##0.0}% at {1:###.0}Mb/s. {2:000} chunks with {3:##.0}x compression",
streamCompletion, mbpersec,
stats.ChunkCount,
compression, stream);
}
}
var size = output.EstimateSize();
Console.WriteLine("Total records: {0} and {1} of data", size.Records,
Print.Bytes(size.ByteSize));
output.Checkpoint("stream-pos", inputPos);
output.Checkpoint("raw-records", size.Records);
output.Checkpoint("raw-bytes", size.ByteSize);
output.Checkpoint("timestamp-secs", DateTimeOffset.UtcNow.ToUnixTimeSeconds());
}
Console.WriteLine("Done");
}
}
