async Task TryDownloadAsync(TransferSpec spec, IWebClient webClient, IAbsoluteFilePath tmpFile) {
try {
tmpFile.RemoveReadonlyWhenExists();
if (!string.IsNullOrWhiteSpace(spec.Uri.UserInfo))
webClient.SetAuthInfo(spec.Uri);
using (webClient.HandleCancellationToken(spec))
await webClient.DownloadFileTaskAsync(spec.Uri, tmpFile.ToString()).ConfigureAwait(false);
VerifyIfNeeded(spec, tmpFile);
_fileOps.Move(tmpFile, spec.LocalFile);
} catch (OperationCanceledException e) {
_fileOps.DeleteIfExists(tmpFile.ToString());
throw CreateTimeoutException(spec, e);
} catch (WebException ex) {
_fileOps.DeleteIfExists(tmpFile.ToString());
var cancelledEx = ex.InnerException as OperationCanceledException;
if (cancelledEx != null)
throw CreateTimeoutException(spec, cancelledEx);
if (ex.Status == WebExceptionStatus.RequestCanceled)
throw CreateTimeoutException(spec, ex);
var response = ex.Response as HttpWebResponse;
if (response == null)
throw GenerateDownloadException(spec, ex);
switch (response.StatusCode) {
case HttpStatusCode.NotFound:
throw new RequestFailedException("Received a 404: NotFound response", ex);
case HttpStatusCode.Forbidden:
throw new RequestFailedException("Received a 403: Forbidden response", ex);
case HttpStatusCode.Unauthorized:
throw new RequestFailedException("Received a 401: Unauthorized response", ex);
}
throw GenerateDownloadException(spec, ex);
}
}
public virtual void UnpackFile(IAbsoluteFilePath srcFile, IAbsoluteFilePath dstFile, IStatus status = null) {
var dstPath = dstFile.ParentDirectoryPath;
dstPath.MakeSurePathExists();
dstFile.RemoveReadonlyWhenExists();
Tools.Compression.Unpack(srcFile, dstPath, true, progress: status);
}
public virtual string Gzip(IAbsoluteFilePath file, IAbsoluteFilePath dest = null,
bool preserveFileNameAndModificationTime = false) {
Contract.Requires<ArgumentNullException>(file != null);
Contract.Requires<ArgumentException>(file.Exists);
var defDest = (file + ".gz").ToAbsoluteFilePath();
if (dest == null)
dest = defDest;
var cmd = String.Format("-f --best --rsyncable --keep \"{0}\"", file);
if (!preserveFileNameAndModificationTime)
cmd = "-n " + cmd;
dest.RemoveReadonlyWhenExists();
var startInfo =
new ProcessStartInfoBuilder(Common.Paths.ToolPath.GetChildFileWithName("gzip.exe"), cmd) {
WorkingDirectory = file.ParentDirectoryPath.ToString()
}.Build();
var ret = ProcessManager.LaunchAndGrabTool(startInfo, "Gzip pack");
if (Path.GetFullPath(dest.ToString()) != Path.GetFullPath(defDest.ToString()))
FileUtil.Ops.MoveWithRetry(defDest, dest);
return ret.StandardOutput + ret.StandardError;
}
public virtual string Gzip(IAbsoluteFilePath file, IAbsoluteFilePath dest = null,
bool preserveFileNameAndModificationTime = true, ITProgress status = null) {
Contract.Requires<ArgumentNullException>(file != null);
Contract.Requires<ArgumentException>(file.Exists);
var defDest = (file + ".gz").ToAbsoluteFilePath();
if (dest == null)
dest = defDest;
var cmd = $"-f --best --rsyncable --keep \"{file}\"";
if (!preserveFileNameAndModificationTime)
cmd = "-n " + cmd;
dest.RemoveReadonlyWhenExists();
var startInfo =
new ProcessStartInfoBuilder(Common.Paths.ToolPath.GetChildFileWithName("gzip.exe"), cmd) {
WorkingDirectory = file.ParentDirectoryPath
}.Build();
var srcSize = file.FileInfo.Length;
ProcessExitResultWithOutput ret;
var predictedSize = srcSize*DefaultPredictedCompressionRatio;
using (StatusProcessor.Conditional(defDest, status, (long) predictedSize))
ret = ProcessManager.LaunchAndGrabTool(startInfo, "Gzip pack");
if (Path.GetFullPath(dest.ToString()) != Path.GetFullPath(defDest.ToString()))
FileUtil.Ops.MoveWithRetry(defDest, dest);
return ret.StandardOutput + ret.StandardError;
}
public void Move(IAbsoluteFilePath source, IAbsoluteFilePath destination, bool overwrite = true,
bool checkMd5 = false)
{
if (FileUtil.ComparePathsEqualCase(source.ToString(), destination.ToString()))
{
throw new ArgumentException("Source and destination paths cannot be equal");
}
if (!source.Exists)
{
throw new FileNotFoundException("File doesnt exist: " + source);
}
if (checkMd5 && SumsAreEqual(source, destination))
{
this.Logger()
.Info("Source and destination files equal. Source: {0}, Destination: {1}", source,
destination);
if (!FileUtil.ComparePathsOsCaseSensitive(source, destination))
{
DeleteIfExists(source.ToString());
}
return;
}
source.RemoveReadonlyWhenExists();
if (FileUtil.ComparePathsOsCaseSensitive(source, destination))
{
CaseChangeMove(source, destination);
}
else
{
RealMove(source, destination, overwrite);
}
}
public virtual void UnpackFile(IAbsoluteFilePath srcFile, IAbsoluteFilePath dstFile, IStatus status = null)
{
var dstPath = dstFile.ParentDirectoryPath;
dstPath.MakeSurePathExists();
dstFile.RemoveReadonlyWhenExists();
Tools.Compression.Unpack(srcFile, dstPath, true, progress: status);
}
public virtual void Pack(IAbsoluteFilePath file, IAbsoluteFilePath dest = null,
string archiveFormat = Repository.DefaultArchiveFormat) {
if (dest == null)
dest = (file + archiveFormat).ToAbsoluteFilePath();
dest.ParentDirectoryPath.MakeSurePathExists();
dest.RemoveReadonlyWhenExists();
if (archiveFormat == Repository.DefaultArchiveFormat)
Tools.Compression.Gzip.GzipAuto(file, dest);
else
Tools.Compression.PackSevenZipNative(file, dest);
}
async Task TryDownloadAsync(TransferSpec spec, IWebClient webClient, IAbsoluteFilePath tmpFile)
{
try {
tmpFile.RemoveReadonlyWhenExists();
if (!string.IsNullOrWhiteSpace(spec.Uri.UserInfo))
{
webClient.SetAuthInfo(spec.Uri);
}
using (webClient.HandleCancellationToken(spec))
await webClient.DownloadFileTaskAsync(spec.Uri, tmpFile.ToString()).ConfigureAwait(false);
VerifyIfNeeded(spec, tmpFile);
_fileOps.Move(tmpFile, spec.LocalFile);
} catch (OperationCanceledException e) {
_fileOps.DeleteIfExists(tmpFile.ToString());
throw CreateTimeoutException(spec, e);
} catch (WebException ex) {
_fileOps.DeleteIfExists(tmpFile.ToString());
var cancelledEx = ex.InnerException as OperationCanceledException;
if (cancelledEx != null)
{
throw CreateTimeoutException(spec, cancelledEx);
}
if (ex.Status == WebExceptionStatus.RequestCanceled)
{
throw CreateTimeoutException(spec, ex);
}
var response = ex.Response as HttpWebResponse;
if (response == null)
{
throw GenerateDownloadException(spec, ex);
}
switch (response.StatusCode)
{
case HttpStatusCode.NotFound:
throw new RequestFailedException("Received a 404: NotFound response", ex);
case HttpStatusCode.Forbidden:
throw new RequestFailedException("Received a 403: Forbidden response", ex);
case HttpStatusCode.Unauthorized:
throw new RequestFailedException("Received a 401: Unauthorized response", ex);
}
throw GenerateDownloadException(spec, ex);
}
}
public virtual string Gzip(IAbsoluteFilePath file, IAbsoluteFilePath dest = null,
bool preserveFileNameAndModificationTime = true, ITProgress status = null)
{
if (file == null)
{
throw new ArgumentNullException(nameof(file));
}
if (!(file.Exists))
{
throw new ArgumentException("file.Exists");
}
var defDest = (file + ".gz").ToAbsoluteFilePath();
if (dest == null)
{
dest = defDest;
}
var cmd = $"-f --best --rsyncable --keep \"{file}\"";
if (!preserveFileNameAndModificationTime)
{
cmd = "-n " + cmd;
}
dest.RemoveReadonlyWhenExists();
var startInfo =
new ProcessStartInfoBuilder(Common.Paths.ToolPath.GetChildFileWithName("gzip.exe"), cmd)
{
WorkingDirectory = file.ParentDirectoryPath
}.Build();
var srcSize = file.FileInfo.Length;
ProcessExitResultWithOutput ret;
var predictedSize = srcSize * DefaultPredictedCompressionRatio;
using (StatusProcessor.Conditional(defDest, status, (long)predictedSize))
ret = ProcessManager.LaunchAndGrabTool(startInfo, "Gzip pack");
if (Path.GetFullPath(dest.ToString()) != Path.GetFullPath(defDest.ToString()))
{
FileUtil.Ops.MoveWithRetry(defDest, dest);
}
return(ret.StandardOutput + ret.StandardError);
}
public virtual void Pack(IAbsoluteFilePath file, IAbsoluteFilePath dest = null,
string archiveFormat = Repository.DefaultArchiveFormat)
{
if (dest == null)
{
dest = (file + archiveFormat).ToAbsoluteFilePath();
}
dest.ParentDirectoryPath.MakeSurePathExists();
dest.RemoveReadonlyWhenExists();
if (archiveFormat == Repository.DefaultArchiveFormat)
{
Tools.Compression.Gzip.GzipAuto(file, dest);
}
else
{
throw new NotSupportedException($"{archiveFormat} is not supported, use .gz");
}
//Tools.Compression.PackSevenZipNative(file, dest);
}
public virtual string GzipStdOut(IAbsoluteFilePath inputFile, IAbsoluteFilePath outputFile = null,
bool preserveFileNameAndModificationTime = false) {
Contract.Requires<ArgumentException>(inputFile != null);
Contract.Requires<ArgumentException>(inputFile.Exists);
if (outputFile == null)
outputFile = (inputFile + ".gz").ToAbsoluteFilePath();
var cmd = String.Format("-f --best --rsyncable --keep --stdout \"{0}\" > \"{1}\"",
inputFile, outputFile);
if (!preserveFileNameAndModificationTime)
cmd = "-n " + cmd;
outputFile.RemoveReadonlyWhenExists();
var startInfo =
new ProcessStartInfoBuilder(Common.Paths.ToolPath.GetChildFileWithName("gzip.exe"), cmd) {
WorkingDirectory = Common.Paths.LocalDataPath.ToString()
}.Build();
var ret = ProcessManager.LaunchAndGrabToolCmd(startInfo, "Gzip pack");
return ret.StandardOutput + ret.StandardError;
}
public virtual string GzipStdOut(IAbsoluteFilePath inputFile, IAbsoluteFilePath outputFile = null,
bool preserveFileNameAndModificationTime = true, ITProgress status = null)
{
if (!(inputFile != null))
{
throw new ArgumentException("inputFile != null");
}
if (!(inputFile.Exists))
{
throw new ArgumentException("inputFile.Exists");
}
if (outputFile == null)
{
outputFile = (inputFile + ".gz").ToAbsoluteFilePath();
}
var cmd = $"-f --best --rsyncable --keep --stdout \"{inputFile}\" > \"{outputFile}\"";
if (!preserveFileNameAndModificationTime)
{
cmd = "-n " + cmd;
}
outputFile.RemoveReadonlyWhenExists();
var startInfo =
new ProcessStartInfoBuilder(Common.Paths.ToolPath.GetChildFileWithName("gzip.exe"), cmd)
{
WorkingDirectory = Common.Paths.LocalDataPath
}.Build();
var srcSize = inputFile.FileInfo.Length;
ProcessExitResultWithOutput ret;
var predictedSize = srcSize * DefaultPredictedCompressionRatio;
using (StatusProcessor.Conditional(outputFile, status, (long)predictedSize))
ret = ProcessManager.LaunchAndGrabToolCmd(startInfo, "Gzip pack");
return(ret.StandardOutput + ret.StandardError);
}
async Task TryDownloadAsync(TransferSpec spec, IWebClient webClient, IAbsoluteFilePath tmpFile) {
try {
tmpFile.RemoveReadonlyWhenExists();
if (!string.IsNullOrWhiteSpace(spec.Uri.UserInfo))
webClient.SetAuthInfo(spec.Uri);
using (webClient.HandleCancellationToken(spec))
await webClient.DownloadFileTaskAsync(spec.Uri, tmpFile.ToString()).ConfigureAwait(false);
VerifyIfNeeded(spec, tmpFile);
_fileOps.Move(tmpFile, spec.LocalFile);
} catch (OperationCanceledException e) {
_fileOps.DeleteIfExists(tmpFile.ToString());
throw CreateTimeoutException(spec, e);
} catch (WebException e) {
_fileOps.DeleteIfExists(tmpFile.ToString());
var cancelledEx = e.InnerException as OperationCanceledException;
if (cancelledEx != null)
throw CreateTimeoutException(spec, cancelledEx);
if (e.Status == WebExceptionStatus.RequestCanceled)
throw CreateTimeoutException(spec, e);
GenerateDownloadException(spec, e);
}
}
private static void TryUnpackArchive(IAbsoluteFilePath destFile, ITProgress progress, GZipArchive archive)
{
destFile.RemoveReadonlyWhenExists();
var entry = archive.Entries.First();
if (progress != null)
{
var startTime = DateTime.UtcNow;
archive.CompressedBytesRead += (sender, args) => {
double prog = args.CompressedBytesRead / (float)archive.TotalSize;
if (prog > 1)
{
prog = 1;
}
var totalMilliseconds = (DateTime.UtcNow - startTime).TotalMilliseconds;
progress.Update(
totalMilliseconds > 0 ? (long?)(args.CompressedBytesRead / (totalMilliseconds / 1000.0)) : null,
prog * 100);
};
}
entry.WriteToFile(destFile.ToString());
progress?.Update(null, 100);
}
public virtual string GzipStdOut(IAbsoluteFilePath inputFile, IAbsoluteFilePath outputFile = null,
bool preserveFileNameAndModificationTime = true, ITProgress status = null) {
Contract.Requires<ArgumentException>(inputFile != null);
Contract.Requires<ArgumentException>(inputFile.Exists);
if (outputFile == null)
outputFile = (inputFile + ".gz").ToAbsoluteFilePath();
var cmd = $"-f --best --rsyncable --keep --stdout \"{inputFile}\" > \"{outputFile}\"";
if (!preserveFileNameAndModificationTime)
cmd = "-n " + cmd;
outputFile.RemoveReadonlyWhenExists();
var startInfo =
new ProcessStartInfoBuilder(Common.Paths.ToolPath.GetChildFileWithName("gzip.exe"), cmd) {
WorkingDirectory = Common.Paths.LocalDataPath
}.Build();
var srcSize = inputFile.FileInfo.Length;
ProcessExitResultWithOutput ret;
var predictedSize = srcSize*DefaultPredictedCompressionRatio;
using (StatusProcessor.Conditional(outputFile, status, (long) predictedSize))
ret = ProcessManager.LaunchAndGrabToolCmd(startInfo, "Gzip pack");
return ret.StandardOutput + ret.StandardError;
}
public void UnpackSingleGzip(IAbsoluteFilePath sourceFile, IAbsoluteFilePath destFile,
ITProgress progress = null) {
using (var archive = GZipArchive.Open(sourceFile.ToString())) {
if (progress != null) {
archive.CompressedBytesRead += (sender, args) => {
double prog = (args.CompressedBytesRead/(float) archive.TotalSize);
if (prog > 1)
prog = 1;
progress.Progress = prog*100;
};
}
destFile.RemoveReadonlyWhenExists();
var entry = archive.Entries.First();
entry.WriteToFile(destFile.ToString());
}
}
private static void TryUnpackArchive(IAbsoluteFilePath destFile, ITProgress progress, GZipArchive archive) {
destFile.RemoveReadonlyWhenExists();
var entry = archive.Entries.First();
if (progress != null) {
var startTime = DateTime.UtcNow;
archive.CompressedBytesRead += (sender, args) => {
double prog = args.CompressedBytesRead/(float) archive.TotalSize;
if (prog > 1)
prog = 1;
var totalMilliseconds = (DateTime.UtcNow - startTime).TotalMilliseconds;
progress.Update(
totalMilliseconds > 0 ? (long?) (args.CompressedBytesRead/(totalMilliseconds/1000.0)) : null,
prog*100);
};
}
entry.WriteToFile(destFile.ToString());
progress?.Update(null, 100);
}
