public void UnpackInternal(IAbsoluteFilePath sourceFile, IAbsoluteDirectoryPath outputFolder,
bool overwrite = false, bool fullPath = true, bool checkFileIntegrity = true,
ITProgress progress = null)
{
if (sourceFile == null)
{
throw new ArgumentNullException(nameof(sourceFile));
}
if (outputFolder == null)
{
throw new ArgumentNullException(nameof(outputFolder));
}
var ext = sourceFile.FileExtension;
var options = new ExtractionOptions {
PreserveFileTime = true
};
if (fullPath)
{
options.ExtractFullPath = true;
}
if (overwrite)
{
options.Overwrite = true;
}
using (var archive = GetArchiveWithGzWorkaround(sourceFile, ext))
UnpackArchive(outputFolder, overwrite, archive, options, sourceFile);
}
public virtual string Gzip(IAbsoluteFilePath file, IAbsoluteFilePath dest = null,
bool preserveFileNameAndModificationTime = true, ITProgress status = null) {
Contract.Requires<ArgumentNullException>(file != null);
Contract.Requires<ArgumentException>(file.Exists);
var defDest = (file + ".gz").ToAbsoluteFilePath();
if (dest == null)
dest = defDest;
var cmd = $"-f --best --rsyncable --keep \"{file}\"";
if (!preserveFileNameAndModificationTime)
cmd = "-n " + cmd;
dest.RemoveReadonlyWhenExists();
var startInfo =
new ProcessStartInfoBuilder(Common.Paths.ToolPath.GetChildFileWithName("gzip.exe"), cmd) {
WorkingDirectory = file.ParentDirectoryPath
}.Build();
var srcSize = file.FileInfo.Length;
ProcessExitResultWithOutput ret;
var predictedSize = srcSize*DefaultPredictedCompressionRatio;
using (StatusProcessor.Conditional(defDest, status, (long) predictedSize))
ret = ProcessManager.LaunchAndGrabTool(startInfo, "Gzip pack");
if (Path.GetFullPath(dest.ToString()) != Path.GetFullPath(defDest.ToString()))
FileUtil.Ops.MoveWithRetry(defDest, dest);
return ret.StandardOutput + ret.StandardError;
}
internal StatusProcessor(IAbsoluteFilePath dest, ITProgress status, long predictedSize)
{
_dest = dest;
_status = status;
_predictedSize = predictedSize;
_startTime = DateTime.UtcNow;
_timer = new TimerWithElapsedCancellationOnExceptionOnly(TimeSpan.FromMilliseconds(250),
CalculateProgressAndSpeed);
}
static async Task CopyAsyncInternal(IAbsoluteFilePath source, IAbsoluteFilePath destination,
bool overwrite = true, ITProgress status = null)
{
using (
var sourceStream = File.Open(source.ToString(), FileMode.Open, FileAccess.Read, FileShare.Read))
using (var destinationStream = File.Create(destination.ToString()))
using (new StatusProcessor(destination, status, source.FileInfo.Length))
await sourceStream.CopyToAsync(destinationStream).ConfigureAwait(false);
CopyTimestamps(source, destination);
}
public void UnpackInternal(IAbsoluteFilePath sourceFile, IAbsoluteDirectoryPath outputFolder,
bool overwrite = false, bool fullPath = true, bool checkFileIntegrity = true,
ITProgress progress = null) {
Contract.Requires<ArgumentNullException>(sourceFile != null);
Contract.Requires<ArgumentNullException>(outputFolder != null);
var ext = sourceFile.FileExtension;
var options = new ExtractionOptions {PreserveFileTime = true};
if (fullPath)
options.ExtractFullPath = true;
if (overwrite)
options.Overwrite = true;
using (var archive = GetArchiveWithGzWorkaround(sourceFile, ext))
UnpackArchive(outputFolder, overwrite, archive, options, sourceFile);
}
public void UnpackGzip(IAbsoluteFilePath sourceFile, IAbsoluteFilePath destFile, ITProgress progress = null) {
using (var archive = GZipArchive.Open(sourceFile.ToString())) {
try {
TryUnpackArchive(destFile, progress, archive);
} catch (ZlibException ex) {
if ((ex.Message == "Not a valid GZIP stream.") || (ex.Message == "Bad GZIP header.")
|| (ex.Message == "Unexpected end-of-file reading GZIP header.")
|| (ex.Message == "Unexpected EOF reading GZIP header.")) {
var header = TryReadHeader(sourceFile);
throw new CompressedFileException(
$"The archive appears corrupt: {sourceFile}. Header:\n{header}", ex);
}
throw;
}
}
}
public virtual void Unpack(IAbsoluteFilePath sourceFile, IAbsoluteDirectoryPath outputFolder,
bool overwrite = false, bool fullPath = true, bool force7z = false, bool checkFileIntegrity = true,
ITProgress progress = null) {
Contract.Requires<ArgumentNullException>(sourceFile != null);
Contract.Requires<ArgumentNullException>(outputFolder != null);
var ext = sourceFile.FileExtension;
if (force7z ||
SevenzipArchiveFormats.Any(x => ext.Equals(x, StringComparison.OrdinalIgnoreCase))) {
using (var extracter = new SevenZipExtractor(sourceFile.ToString()))
UnpackArchive(sourceFile, outputFolder, overwrite, checkFileIntegrity, extracter);
} else {
var options = fullPath ? ExtractOptions.ExtractFullPath : ExtractOptions.None;
using (var archive = GetArchiveWithGzWorkaround(sourceFile, ext))
UnpackArchive(outputFolder, overwrite, archive, options);
}
}
private async Task OnlyCopyIfHashMisMatch(IAbsoluteFilePath source, IAbsoluteFilePath destination,
ITProgress status)
{
if (
await
TaskExt.StartLongRunningTask(
() => HashEncryption.MD5FileHash(source) != HashEncryption.MD5FileHash(destination))
.ConfigureAwait(false))
{
await CopyAsyncInternal(source, destination, true, status).ConfigureAwait(false);
return;
}
this.Logger()
.Info("Source and destination files equal. Source: {0}, Destination: {1}", source,
destination);
}
private static void HandleExtractionDestination(FileInfoWithData <State> f, IAbsoluteFilePath c,
ITProgress s)
{
var tmpFile = (c + ".six-tmp").ToAbsoluteFilePath();
if (Common.Flags.Verbose)
{
MainLog.Logger.Info(
$"Decompressing {f.DownloadInfo.LocalFilePath} to {c} (for {string.Join(", ", f.Destinations)})");
}
tmpFile.MakeSureParentPathExists();
Tools.Compression.Gzip.UnpackSingleGzip(f.DownloadInfo.LocalFilePath, tmpFile, s);
if (Tools.HashEncryption.SHA1FileHash(tmpFile) != f.Data.Checksum)
{
throw new ChecksumException("The checksum does not match");
}
tmpFile.Move(c);
}
public Task CopyAsync(IAbsoluteFilePath source, IAbsoluteFilePath destination, bool overwrite = true,
bool checkMd5 = false, ITProgress status = null)
{
if (FileUtil.ComparePathsEqualCase(source.ToString(), destination.ToString()))
{
throw new ArgumentException("Source and destination paths cannot be equal");
}
var destinationExists = destination.Exists;
if (!overwrite && destinationExists)
{
throw new IOException("destination file exists " + destination);
}
if (!checkMd5 || !destinationExists)
{
return(CopyAsyncInternal(source, destination, overwrite, status));
}
return(OnlyCopyIfHashMisMatch(source, destination, status));
}
private static void TryUnpackArchive(IAbsoluteFilePath destFile, ITProgress progress, GZipArchive archive)
{
destFile.RemoveReadonlyWhenExists();
var entry = archive.Entries.First();
if (progress != null)
{
var startTime = DateTime.UtcNow;
archive.CompressedBytesRead += (sender, args) => {
double prog = args.CompressedBytesRead / (float)archive.TotalSize;
if (prog > 1)
{
prog = 1;
}
var totalMilliseconds = (DateTime.UtcNow - startTime).TotalMilliseconds;
progress.Update(
totalMilliseconds > 0 ? (long?)(args.CompressedBytesRead / (totalMilliseconds / 1000.0)) : null,
prog * 100);
};
}
entry.WriteToFile(destFile.ToString());
progress?.Update(null, 100);
}
public virtual string GzipStdOut(IAbsoluteFilePath inputFile, IAbsoluteFilePath outputFile = null,
bool preserveFileNameAndModificationTime = true, ITProgress status = null) {
Contract.Requires<ArgumentException>(inputFile != null);
Contract.Requires<ArgumentException>(inputFile.Exists);
if (outputFile == null)
outputFile = (inputFile + ".gz").ToAbsoluteFilePath();
var cmd = $"-f --best --rsyncable --keep --stdout \"{inputFile}\" > \"{outputFile}\"";
if (!preserveFileNameAndModificationTime)
cmd = "-n " + cmd;
outputFile.RemoveReadonlyWhenExists();
var startInfo =
new ProcessStartInfoBuilder(Common.Paths.ToolPath.GetChildFileWithName("gzip.exe"), cmd) {
WorkingDirectory = Common.Paths.LocalDataPath
}.Build();
var srcSize = inputFile.FileInfo.Length;
ProcessExitResultWithOutput ret;
var predictedSize = srcSize*DefaultPredictedCompressionRatio;
using (StatusProcessor.Conditional(outputFile, status, (long) predictedSize))
ret = ProcessManager.LaunchAndGrabToolCmd(startInfo, "Gzip pack");
return ret.StandardOutput + ret.StandardError;
}
public virtual void Unpack(IAbsoluteFilePath sourceFile, IAbsoluteDirectoryPath outputFolder,
bool overwrite = false, bool fullPath = true, bool checkFileIntegrity = true,
ITProgress progress = null)
=>
_compressionUtil.UnpackInternal(sourceFile, outputFolder, overwrite, fullPath, checkFileIntegrity,
progress);
public static Task CopyAsync(this IAbsoluteFilePath src, IAbsoluteDirectoryPath destination,
bool overwrite = true,
bool checkMd5 = false, ITProgress status = null)
=>
Tools.FileUtil.Ops.CopyAsync(src, destination.GetChildFileWithName(src.FileName), overwrite, checkMd5,
status);
private static Task HandleCopyFileInternal(IAbsoluteFilePath file, IAbsoluteFilePath dst, ITProgress status)
{
if (Common.Flags.Verbose)
{
MainLog.Logger.Info($"Copying {file} to {dst}");
}
dst.MakeSureParentPathExists();
return(file.CopyAsync(dst, status: status));
}
internal StatusProcessor(IAbsoluteFilePath dest, ITProgress status, long predictedSize) {
_dest = dest;
_status = status;
_predictedSize = predictedSize;
_startTime = DateTime.UtcNow;
_timer = new TimerWithElapsedCancellationOnExceptionOnly(TimeSpan.FromMilliseconds(250),
CalculateProgressAndSpeed);
}
public virtual void Unpack(IAbsoluteFilePath sourceFile, IAbsoluteDirectoryPath outputFolder,
bool overwrite = false, bool fullPath = true, bool checkFileIntegrity = true,
ITProgress progress = null)
=>
_compressionUtil.UnpackInternal(sourceFile, outputFolder, overwrite, fullPath, checkFileIntegrity,
progress);
public virtual string Gzip(IAbsoluteFilePath file, IAbsoluteFilePath dest = null,
bool preserveFileNameAndModificationTime = true, ITProgress status = null)
{
if (file == null)
{
throw new ArgumentNullException(nameof(file));
}
if (!(file.Exists))
{
throw new ArgumentException("file.Exists");
}
var defDest = (file + ".gz").ToAbsoluteFilePath();
if (dest == null)
{
dest = defDest;
}
var cmd = $"-f --best --rsyncable --keep \"{file}\"";
if (!preserveFileNameAndModificationTime)
{
cmd = "-n " + cmd;
}
dest.RemoveReadonlyWhenExists();
var startInfo =
new ProcessStartInfoBuilder(Common.Paths.ToolPath.GetChildFileWithName("gzip.exe"), cmd)
{
WorkingDirectory = file.ParentDirectoryPath
}.Build();
var srcSize = file.FileInfo.Length;
ProcessExitResultWithOutput ret;
var predictedSize = srcSize * DefaultPredictedCompressionRatio;
using (StatusProcessor.Conditional(defDest, status, (long)predictedSize))
ret = ProcessManager.LaunchAndGrabTool(startInfo, "Gzip pack");
if (Path.GetFullPath(dest.ToString()) != Path.GetFullPath(defDest.ToString()))
{
FileUtil.Ops.MoveWithRetry(defDest, dest);
}
return(ret.StandardOutput + ret.StandardError);
}
public void UnpackGzip(IAbsoluteFilePath sourceFile, IAbsoluteFilePath destFile, ITProgress progress = null)
{
using (var archive = GZipArchive.Open(sourceFile.ToString())) {
try {
TryUnpackArchive(destFile, progress, archive);
} catch (ZlibException ex) {
if ((ex.Message == "Not a valid GZIP stream.") || (ex.Message == "Bad GZIP header.") ||
(ex.Message == "Unexpected end-of-file reading GZIP header.") ||
(ex.Message == "Unexpected EOF reading GZIP header."))
{
var header = TryReadHeader(sourceFile);
throw new CompressedFileException(
$"The archive appears corrupt: {sourceFile}. Header:\n{header}", ex);
}
throw;
}
}
}
public static void Unpack(this IAbsoluteFilePath src, IAbsoluteDirectoryPath outputFolder,
bool overwrite = false, bool fullPath = true, bool checkFileIntegrity = true, ITProgress progress = null)
=> Tools.Compression.Unpack(src, outputFolder, overwrite, fullPath, checkFileIntegrity, progress);
private static void TryUnpackArchive(IAbsoluteFilePath destFile, ITProgress progress, GZipArchive archive) {
destFile.RemoveReadonlyWhenExists();
var entry = archive.Entries.First();
if (progress != null) {
var startTime = DateTime.UtcNow;
archive.CompressedBytesRead += (sender, args) => {
double prog = args.CompressedBytesRead/(float) archive.TotalSize;
if (prog > 1)
prog = 1;
var totalMilliseconds = (DateTime.UtcNow - startTime).TotalMilliseconds;
progress.Update(
totalMilliseconds > 0 ? (long?) (args.CompressedBytesRead/(totalMilliseconds/1000.0)) : null,
prog*100);
};
}
entry.WriteToFile(destFile.ToString());
progress?.Update(null, 100);
}
public virtual string GzipAuto(IAbsoluteFilePath inputFile, IAbsoluteFilePath outputFile = null,
bool preserveFileNameAndModificationTime = true, ITProgress status = null) {
if (inputFile.ToString().EndsWith(".gz", StringComparison.OrdinalIgnoreCase))
return GzipStdOut(inputFile, outputFile, preserveFileNameAndModificationTime, status);
return Gzip(inputFile, outputFile, preserveFileNameAndModificationTime, status);
}
public StatusWrapper(ITProgress wrapped, Action <double, long?> act)
{
_wrapped = wrapped;
_act = act;
}
public static IDisposable Conditional(IAbsoluteFilePath dest, ITProgress status, long predictedSize)
=> (status == null) || (predictedSize == 0) ? null : new StatusProcessor(dest, status, predictedSize);
public virtual string GzipAuto(IAbsoluteFilePath inputFile, IAbsoluteFilePath outputFile = null,
bool preserveFileNameAndModificationTime = true, ITProgress status = null)
{
if (inputFile.ToString().EndsWith(".gz", StringComparison.OrdinalIgnoreCase))
{
return(GzipStdOut(inputFile, outputFile, preserveFileNameAndModificationTime, status));
}
return(Gzip(inputFile, outputFile, preserveFileNameAndModificationTime, status));
}
public static Task CopyAsync(this IAbsoluteFilePath src, IAbsoluteFilePath destination,
bool overwrite = true,
bool checkMd5 = false, ITProgress status = null)
=> Tools.FileUtil.Ops.CopyAsync(src, destination, overwrite, checkMd5, status);
public void UnpackSingleGzip(IAbsoluteFilePath sourceFile, IAbsoluteFilePath destFile,
ITProgress progress = null)
{
_compressionUtil.UnpackGzip(sourceFile, destFile, progress);
}
public static Task CopyAsync(this IAbsoluteFilePath src, IAbsoluteDirectoryPath destination,
bool overwrite = true,
bool checkMd5 = false, ITProgress status = null)
=>
Tools.FileUtil.Ops.CopyAsync(src, destination.GetChildFileWithName(src.FileName), overwrite, checkMd5,
status);
public static IDisposable Conditional(IAbsoluteFilePath dest, ITProgress status, long predictedSize) => (status == null) || (predictedSize == 0) ? null : new StatusProcessor(dest, status, predictedSize);
public static Task CopyAsync(this IAbsoluteFilePath src, IAbsoluteFilePath destination,
bool overwrite = true,
bool checkMd5 = false, ITProgress status = null)
=> Tools.FileUtil.Ops.CopyAsync(src, destination, overwrite, checkMd5, status);
public virtual string GzipStdOut(IAbsoluteFilePath inputFile, IAbsoluteFilePath outputFile = null,
bool preserveFileNameAndModificationTime = true, ITProgress status = null)
{
if (!(inputFile != null))
{
throw new ArgumentException("inputFile != null");
}
if (!(inputFile.Exists))
{
throw new ArgumentException("inputFile.Exists");
}
if (outputFile == null)
{
outputFile = (inputFile + ".gz").ToAbsoluteFilePath();
}
var cmd = $"-f --best --rsyncable --keep --stdout \"{inputFile}\" > \"{outputFile}\"";
if (!preserveFileNameAndModificationTime)
{
cmd = "-n " + cmd;
}
outputFile.RemoveReadonlyWhenExists();
var startInfo =
new ProcessStartInfoBuilder(Common.Paths.ToolPath.GetChildFileWithName("gzip.exe"), cmd)
{
WorkingDirectory = Common.Paths.LocalDataPath
}.Build();
var srcSize = inputFile.FileInfo.Length;
ProcessExitResultWithOutput ret;
var predictedSize = srcSize * DefaultPredictedCompressionRatio;
using (StatusProcessor.Conditional(outputFile, status, (long)predictedSize))
ret = ProcessManager.LaunchAndGrabToolCmd(startInfo, "Gzip pack");
return(ret.StandardOutput + ret.StandardError);
}
public static void Unpack(this IAbsoluteFilePath src, IAbsoluteDirectoryPath outputFolder,
bool overwrite = false, bool fullPath = true, bool checkFileIntegrity = true, ITProgress progress = null)
=> Tools.Compression.Unpack(src, outputFolder, overwrite, fullPath, checkFileIntegrity, progress);
public void UnpackSingleGzip(IAbsoluteFilePath sourceFile, IAbsoluteFilePath destFile,
ITProgress progress = null) {
using (var archive = GZipArchive.Open(sourceFile.ToString())) {
if (progress != null) {
archive.CompressedBytesRead += (sender, args) => {
double prog = (args.CompressedBytesRead/(float) archive.TotalSize);
if (prog > 1)
prog = 1;
progress.Progress = prog*100;
};
}
destFile.RemoveReadonlyWhenExists();
var entry = archive.Entries.First();
entry.WriteToFile(destFile.ToString());
}
}
public void UnpackSingleGzip(IAbsoluteFilePath sourceFile, IAbsoluteFilePath destFile,
ITProgress progress = null) {
_compressionUtil.UnpackGzip(sourceFile, destFile, progress);
}