public override void Run() { if (!Directory.Exists(DatabasePath)) { Console.WriteLine("Path not found {0}", DatabasePath); Environment.Exit(1); } var outPath = DatabasePath.TrimEnd('/') + ".pack"; if (Directory.Exists(outPath)) { Directory.Delete(outPath, true); } Directory.CreateDirectory(outPath); using (var db = LmdbEnv.CreateDb(DatabasePath, 0, EnvironmentOpenFlags.ReadOnly)) { using (var tx = db.Read()) { var stats = tx.GetUsedSize(); Console.WriteLine("Entry count: {0}. Size: {1}", stats.EntryCount, Print.Bytes(stats.UsedBytes)); } db.CopyTo(outPath, true); } var inFile = Path.Combine(outPath, "data.mdb"); Console.WriteLine("Packed into {0}", Print.Bytes(new FileInfo(inFile).Length)); var outFile = Path.Combine(outPath, "data.mdb.gzip"); using (var input = File.OpenRead(inFile)) using (var output = File.Create(outFile)) using (var gz = new GZipStream(output, CompressionLevel.Optimal)) { input.CopyTo(gz); } Console.WriteLine("Gzipped into {0}", Print.Bytes(new FileInfo(outFile).Length)); }
public override void Run() { var streams = SourceStreams.Split(new[] { ';', ',' }, StringSplitOptions.RemoveEmptyEntries); var rawSize = 1024 * 1024 * ChunkSize; var key = Convert.FromBase64String(EncryptionKey); using (var output = CellarWriter.Create(OutputStream, rawSize, key)) { foreach (var stream in streams) { var fetcher = GetFetcher(stream); var inputPos = output.GetCheckpoint(stream + "-pos"); using (var cts = new CancellationTokenSource()) { // launch reading var task = fetcher.ReadAll(cts.Token, inputPos, int.MaxValue); while (!cts.IsCancellationRequested) { var start = Stopwatch.StartNew(); var result = task.Result; if (result.ReadRecords <= 0) { break; } inputPos = result.CurrentPosition; // launch next task in advance task = fetcher.ReadAll(cts.Token, inputPos, int.MaxValue); foreach (var message in result.Messages) { SaveMessage(message, output); } output.Checkpoint(stream + "-pos", inputPos); var stats = output.EstimateSize(); var totalSize = streams.Sum(s => output.GetCheckpoint(s + "-pos")); var compression = totalSize * 1D / stats.DiskSize; var streamCompletion = 100F * result.CurrentPosition / result.MaxPosition; start.Stop(); var bytes = result.CurrentPosition - result.StartingPosition; var mbpersec = 1D * bytes / 1024 / 1024 / start.Elapsed.TotalSeconds; Console.WriteLine( "{4}: {0:##0.0}% at {1:###.0}Mb/s. {2:000} chunks with {3:##.0}x compression", streamCompletion, mbpersec, stats.ChunkCount, compression, stream); } } var size = output.EstimateSize(); Console.WriteLine("Total records: {0} and {1} of data", size.Records, Print.Bytes(size.ByteSize)); output.Checkpoint("stream-pos", inputPos); output.Checkpoint("raw-records", size.Records); output.Checkpoint("raw-bytes", size.ByteSize); output.Checkpoint("timestamp-secs", DateTimeOffset.UtcNow.ToUnixTimeSeconds()); } Console.WriteLine("Done"); } }