public DecompressOption GetTccDecompressOption(string decompressedFolder)
{
string src;
if (Files != null)
{
src = String.Join(" ", Files.Select(i => i.FullName));
}
else if (Directories != null)
{
src = String.Join(" ", Directories.Select(i => i.FullName));
}
else
{
throw new MissingMemberException();
}
Target = decompressedFolder;
var decompressOption = new DecompressOption
{
SourceDirOrFile = src,
DestinationDir = Target,
Threads = Environment.ProcessorCount
};
return(decompressOption);
}
public async Task <OperationSummary> Decompress(DecompressOption option)
{
IEnumerable <Block> blocks = BlockHelper.PreprareDecompressBlocks(option);
var sw = Stopwatch.StartNew();
var po = new ParallelizeOption
{
FailMode = option.FailFast ? Fail.Fast : Fail.Smart,
MaxDegreeOfParallelism = option.Threads
};
var operationBlocks = await blocks
.AsAsyncStream(_cancellationTokenSource.Token)
.CountAsync(out var counter)
// Prepare decryption
.ParallelizeStreamAsync(async(b, token) =>
{
await _encryptionCommands.PrepareDecryptionKey(b, option, token);
return(b);
}, po)
// Core loop
.ParallelizeStreamAsync(async(block, token) =>
{
_logger.LogInformation($"Starting {block.Source}");
CommandResult result = null;
try
{
string cmd = _compressionCommands.DecompressCommand(block, option);
result = await cmd.Run(block.OperationFolder, token);
_logger.LogInformation($"Finished {block.Source} on {result?.ElapsedMilliseconds} ms");
}
catch (Exception e)
{
_logger.LogError(e, $"Error on {block.Source}");
}
return(new OperationBlock(block, result));
}, po)
// Cleanup loop
.ParallelizeStreamAsync(async(opb, token) =>
{
await _encryptionCommands.CleanupKey(opb.Block, option, opb.CommandResult, Mode.Compress);
return(opb);
}, po)
.ForEachAsync((i, ct) =>
{
_blockListener.OnBlockReport(new BlockReport(i.Item.CommandResult, i.Item.Block, counter.Count));
return(Task.CompletedTask);
})
.AsEnumerableAsync();
sw.Stop();
return(new OperationSummary(operationBlocks, option.Threads, sw));
}
public static IEnumerable <Block> PreprareDecompressBlocks(DecompressOption decompressOption)
{
bool yielded = false;
var dstDir = new DirectoryInfo(decompressOption.DestinationDir);
bool crypted = decompressOption.PasswordOption.PasswordMode != PasswordMode.None;
if (Directory.Exists(decompressOption.SourceDirOrFile))
{
var srcDir = new DirectoryInfo(decompressOption.SourceDirOrFile);
foreach (FileInfo fi in srcDir.EnumerateFiles("*" + ExtensionFromAlgo(CompressionAlgo.Lz4, crypted)))
{
yielded = true;
yield return(GenerateDecompressBlock(fi, dstDir, CompressionAlgo.Lz4));
}
foreach (FileInfo fi in srcDir.EnumerateFiles("*" + ExtensionFromAlgo(CompressionAlgo.Brotli, crypted)))
{
yielded = true;
yield return(GenerateDecompressBlock(fi, dstDir, CompressionAlgo.Brotli));
}
foreach (FileInfo fi in srcDir.EnumerateFiles("*" + ExtensionFromAlgo(CompressionAlgo.Zstd, crypted)))
{
yielded = true;
yield return(GenerateDecompressBlock(fi, dstDir, CompressionAlgo.Zstd));
}
}
else if (File.Exists(decompressOption.SourceDirOrFile))
{
var file = new FileInfo(decompressOption.SourceDirOrFile);
yielded = true;
yield return(GenerateDecompressBlock(file, dstDir, AlgoFromExtension(file.Extension)));
}
if (yielded && !dstDir.Exists)
{
dstDir.Create();
}
}
public static TccCommand ParseCommandLine(this string[] args)
{
TccCommand command;
var parsed = CommandLine.Parser.Default.ParseArguments <CompressCmdOptions, DecompressOptions, BenchmarkOptions>(args);
try
{
command = parsed.MapResult(
(CompressCmdOptions opts) =>
{
var option = new CompressOption
{
Algo = opts.Algorithm,
CompressionRatio = opts.Ratio,
DestinationDir = opts.Output,
FailFast = opts.FailFast,
Verbose = opts.Verbose,
SourceDirOrFile = opts.Source.FirstOrDefault(),
BlockMode = opts.Individual ? BlockMode.Individual : BlockMode.Explicit,
Threads = ExtractThreads(opts)
};
ExtractPasswordInfo(opts, option, Mode.Compress);
return(new TccCommand
{
Mode = Mode.Compress,
Option = option
});
},
(DecompressOptions opts) =>
{
var option = new DecompressOption
{
DestinationDir = opts.Output,
FailFast = opts.FailFast,
Verbose = opts.Verbose,
SourceDirOrFile = opts.Source.FirstOrDefault(),
Threads = ExtractThreads(opts)
};
ExtractPasswordInfo(opts, option, Mode.Decompress);
return(new TccCommand
{
Mode = Mode.Decompress,
Option = option
});
},
(BenchmarkOptions opts) =>
{
bool IsExplicitMode()
{
return(!args.Any(i => BenchmarkOptions.AutoTestDataOptions.Contains(i)));
}
var option = new BenchmarkOption
{
Algorithm = opts.Algorithm,
Ratios = opts.Ratios,
Encrypt = opts.Encrypt,
Source = IsExplicitMode() ? opts.Source : null,
Content = opts.Content,
NumberOfFiles = opts.NumberOfFiles,
FileSize = opts.FileSize,
Threads = opts.Threads,
OutputCompressed = opts.OutputCompressed,
OutputDecompressed = opts.OutputDecompressed,
Cleanup = opts.Cleanup
};
return(new TccCommand
{
Mode = Mode.Benchmark,
BenchmarkOption = option
});
},
errs => new TccCommand {
ReturnCode = 1
});
}
catch (CommandLineException ae)
{
Console.Out.WriteLine(ae.Message);
return(new TccCommand {
ReturnCode = 1
});
}
catch (Exception e)
{
Console.Out.WriteLine(e.ToString());
return(new TccCommand {
ReturnCode = 1
});
}
return(command);
}
public async Task <OperationSummary> RunBenchmark(BenchmarkOption benchmarkOption)
{
var operationSummaries = new List <OperationSummary>();
var keysFolder = TestFileHelper.NewFolder();
var iterations = await _iterationGenerator.PrepareIteration(benchmarkOption);
var threads = benchmarkOption.Threads == 0 ? Environment.ProcessorCount : benchmarkOption.Threads;
foreach (var iteration in iterations)
{
PasswordMode pm = iteration.Encryption ? PasswordMode.PublicKey : PasswordMode.None;
var compressedFolder = TestFileHelper.NewFolder(benchmarkOption.OutputCompressed);
var outputFolder = TestFileHelper.NewFolder(benchmarkOption.OutputDecompressed);
// compress
var compressOption = new CompressOption
{
Algo = iteration.Algo,
CompressionRatio = iteration.CompressionRatio,
BlockMode = BlockMode.Individual,
SourceDirOrFile = iteration.Content.Source,
DestinationDir = compressedFolder,
Threads = threads,
PasswordOption = await _benchmarkOptionHelper.GenerateCompressPasswordOption(pm, keysFolder)
};
OperationSummary resultCompress = await _tarCompressCrypt.Compress(compressOption);
if (_cancellationTokenSource.IsCancellationRequested)
{
await Cleanup();
return(null);
}
operationSummaries.Add(resultCompress);
resultCompress.ThrowOnError();
// decompress
var decompressOption = new DecompressOption
{
SourceDirOrFile = compressedFolder,
DestinationDir = outputFolder,
Threads = threads,
PasswordOption = _benchmarkOptionHelper.GenerateDecompressPasswordOption(pm, keysFolder)
};
OperationSummary resultDecompress = await _tarCompressCrypt.Decompress(decompressOption);
if (_cancellationTokenSource.IsCancellationRequested)
{
await Cleanup();
return(null);
}
operationSummaries.Add(resultDecompress);
resultDecompress.ThrowOnError();
StringBuilder sb = FormatResultSummary(iteration, resultCompress, resultDecompress);
Console.Out.WriteLine(sb.ToString());
async Task Cleanup()
{
if (benchmarkOption.Cleanup)
{
await "del /f /s /q * > NUL".Run(compressedFolder, CancellationToken.None);
Directory.Delete(compressedFolder, true);
await "del /f /s /q * > NUL".Run(outputFolder, CancellationToken.None);
Directory.Delete(outputFolder, true);
}
}
await Cleanup();
}
return(new OperationSummary(operationSummaries.SelectMany(i => i.OperationBlocks), 0, default));
}
