public DecompressOption GetTccDecompressOption(string decompressedFolder) { string src; if (Files != null) { src = String.Join(" ", Files.Select(i => i.FullName)); } else if (Directories != null) { src = String.Join(" ", Directories.Select(i => i.FullName)); } else { throw new MissingMemberException(); } Target = decompressedFolder; var decompressOption = new DecompressOption { SourceDirOrFile = src, DestinationDir = Target, Threads = Environment.ProcessorCount }; return(decompressOption); }
public async Task <OperationSummary> Decompress(DecompressOption option) { IEnumerable <Block> blocks = BlockHelper.PreprareDecompressBlocks(option); var sw = Stopwatch.StartNew(); var po = new ParallelizeOption { FailMode = option.FailFast ? Fail.Fast : Fail.Smart, MaxDegreeOfParallelism = option.Threads }; var operationBlocks = await blocks .AsAsyncStream(_cancellationTokenSource.Token) .CountAsync(out var counter) // Prepare decryption .ParallelizeStreamAsync(async(b, token) => { await _encryptionCommands.PrepareDecryptionKey(b, option, token); return(b); }, po) // Core loop .ParallelizeStreamAsync(async(block, token) => { _logger.LogInformation($"Starting {block.Source}"); CommandResult result = null; try { string cmd = _compressionCommands.DecompressCommand(block, option); result = await cmd.Run(block.OperationFolder, token); _logger.LogInformation($"Finished {block.Source} on {result?.ElapsedMilliseconds} ms"); } catch (Exception e) { _logger.LogError(e, $"Error on {block.Source}"); } return(new OperationBlock(block, result)); }, po) // Cleanup loop .ParallelizeStreamAsync(async(opb, token) => { await _encryptionCommands.CleanupKey(opb.Block, option, opb.CommandResult, Mode.Compress); return(opb); }, po) .ForEachAsync((i, ct) => { _blockListener.OnBlockReport(new BlockReport(i.Item.CommandResult, i.Item.Block, counter.Count)); return(Task.CompletedTask); }) .AsEnumerableAsync(); sw.Stop(); return(new OperationSummary(operationBlocks, option.Threads, sw)); }
public static IEnumerable <Block> PreprareDecompressBlocks(DecompressOption decompressOption) { bool yielded = false; var dstDir = new DirectoryInfo(decompressOption.DestinationDir); bool crypted = decompressOption.PasswordOption.PasswordMode != PasswordMode.None; if (Directory.Exists(decompressOption.SourceDirOrFile)) { var srcDir = new DirectoryInfo(decompressOption.SourceDirOrFile); foreach (FileInfo fi in srcDir.EnumerateFiles("*" + ExtensionFromAlgo(CompressionAlgo.Lz4, crypted))) { yielded = true; yield return(GenerateDecompressBlock(fi, dstDir, CompressionAlgo.Lz4)); } foreach (FileInfo fi in srcDir.EnumerateFiles("*" + ExtensionFromAlgo(CompressionAlgo.Brotli, crypted))) { yielded = true; yield return(GenerateDecompressBlock(fi, dstDir, CompressionAlgo.Brotli)); } foreach (FileInfo fi in srcDir.EnumerateFiles("*" + ExtensionFromAlgo(CompressionAlgo.Zstd, crypted))) { yielded = true; yield return(GenerateDecompressBlock(fi, dstDir, CompressionAlgo.Zstd)); } } else if (File.Exists(decompressOption.SourceDirOrFile)) { var file = new FileInfo(decompressOption.SourceDirOrFile); yielded = true; yield return(GenerateDecompressBlock(file, dstDir, AlgoFromExtension(file.Extension))); } if (yielded && !dstDir.Exists) { dstDir.Create(); } }
public static TccCommand ParseCommandLine(this string[] args) { TccCommand command; var parsed = CommandLine.Parser.Default.ParseArguments <CompressCmdOptions, DecompressOptions, BenchmarkOptions>(args); try { command = parsed.MapResult( (CompressCmdOptions opts) => { var option = new CompressOption { Algo = opts.Algorithm, CompressionRatio = opts.Ratio, DestinationDir = opts.Output, FailFast = opts.FailFast, Verbose = opts.Verbose, SourceDirOrFile = opts.Source.FirstOrDefault(), BlockMode = opts.Individual ? BlockMode.Individual : BlockMode.Explicit, Threads = ExtractThreads(opts) }; ExtractPasswordInfo(opts, option, Mode.Compress); return(new TccCommand { Mode = Mode.Compress, Option = option }); }, (DecompressOptions opts) => { var option = new DecompressOption { DestinationDir = opts.Output, FailFast = opts.FailFast, Verbose = opts.Verbose, SourceDirOrFile = opts.Source.FirstOrDefault(), Threads = ExtractThreads(opts) }; ExtractPasswordInfo(opts, option, Mode.Decompress); return(new TccCommand { Mode = Mode.Decompress, Option = option }); }, (BenchmarkOptions opts) => { bool IsExplicitMode() { return(!args.Any(i => BenchmarkOptions.AutoTestDataOptions.Contains(i))); } var option = new BenchmarkOption { Algorithm = opts.Algorithm, Ratios = opts.Ratios, Encrypt = opts.Encrypt, Source = IsExplicitMode() ? opts.Source : null, Content = opts.Content, NumberOfFiles = opts.NumberOfFiles, FileSize = opts.FileSize, Threads = opts.Threads, OutputCompressed = opts.OutputCompressed, OutputDecompressed = opts.OutputDecompressed, Cleanup = opts.Cleanup }; return(new TccCommand { Mode = Mode.Benchmark, BenchmarkOption = option }); }, errs => new TccCommand { ReturnCode = 1 }); } catch (CommandLineException ae) { Console.Out.WriteLine(ae.Message); return(new TccCommand { ReturnCode = 1 }); } catch (Exception e) { Console.Out.WriteLine(e.ToString()); return(new TccCommand { ReturnCode = 1 }); } return(command); }
public async Task <OperationSummary> RunBenchmark(BenchmarkOption benchmarkOption) { var operationSummaries = new List <OperationSummary>(); var keysFolder = TestFileHelper.NewFolder(); var iterations = await _iterationGenerator.PrepareIteration(benchmarkOption); var threads = benchmarkOption.Threads == 0 ? Environment.ProcessorCount : benchmarkOption.Threads; foreach (var iteration in iterations) { PasswordMode pm = iteration.Encryption ? PasswordMode.PublicKey : PasswordMode.None; var compressedFolder = TestFileHelper.NewFolder(benchmarkOption.OutputCompressed); var outputFolder = TestFileHelper.NewFolder(benchmarkOption.OutputDecompressed); // compress var compressOption = new CompressOption { Algo = iteration.Algo, CompressionRatio = iteration.CompressionRatio, BlockMode = BlockMode.Individual, SourceDirOrFile = iteration.Content.Source, DestinationDir = compressedFolder, Threads = threads, PasswordOption = await _benchmarkOptionHelper.GenerateCompressPasswordOption(pm, keysFolder) }; OperationSummary resultCompress = await _tarCompressCrypt.Compress(compressOption); if (_cancellationTokenSource.IsCancellationRequested) { await Cleanup(); return(null); } operationSummaries.Add(resultCompress); resultCompress.ThrowOnError(); // decompress var decompressOption = new DecompressOption { SourceDirOrFile = compressedFolder, DestinationDir = outputFolder, Threads = threads, PasswordOption = _benchmarkOptionHelper.GenerateDecompressPasswordOption(pm, keysFolder) }; OperationSummary resultDecompress = await _tarCompressCrypt.Decompress(decompressOption); if (_cancellationTokenSource.IsCancellationRequested) { await Cleanup(); return(null); } operationSummaries.Add(resultDecompress); resultDecompress.ThrowOnError(); StringBuilder sb = FormatResultSummary(iteration, resultCompress, resultDecompress); Console.Out.WriteLine(sb.ToString()); async Task Cleanup() { if (benchmarkOption.Cleanup) { await "del /f /s /q * > NUL".Run(compressedFolder, CancellationToken.None); Directory.Delete(compressedFolder, true); await "del /f /s /q * > NUL".Run(outputFolder, CancellationToken.None); Directory.Delete(outputFolder, true); } } await Cleanup(); } return(new OperationSummary(operationSummaries.SelectMany(i => i.OperationBlocks), 0, default)); }