private void CabManagerOnProgress(object sender, ICabProgressionEventArgs e)
{
if (e.EventType == CabEventType.GlobalProgression)
{
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(e.CabPath, e.RelativePathInCab, e.PercentageDone));
}
}
/// <inheritdoc cref="IArchiver.ArchiveFileSet"/>
public int ArchiveFileSet(IEnumerable <IFileToArchive> filesToArchiveIn)
{
if (filesToArchiveIn == null)
{
return(0);
}
var fileToArchive = filesToArchiveIn.ToList();
fileToArchive.ForEach(f => f.Processed = false);
int totalFiles = fileToArchive.Count;
int totalFilesDone = 0;
try {
foreach (var file in fileToArchive)
{
_cancelToken?.ThrowIfCancellationRequested();
if (!File.Exists(file.SourcePath))
{
continue;
}
ConvertFile(file.SourcePath, _encode, Path.Combine(file.ArchivePath ?? "", file.PathInArchive));
totalFilesDone++;
file.Processed = true;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(file.ArchivePath, file.PathInArchive, Math.Round(totalFilesDone / (double)totalFiles * 100, 2)));
}
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to archive. {e.Message}", e);
}
return(totalFilesDone);
}
private void OnProgressionEvent(object sender, ProLibrarySaveEventArgs e)
{
if (sender is ProLibrary library)
{
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(library.FilePath, e.RelativePathInPl, Math.Round(e.TotalBytesDone / (double)e.TotalBytesToProcess * 100, 2)));
}
}
protected void ArchiverOnOnProgress(object sender, ArchiverEventArgs e)
{
if (e.PercentageDone < 0 || e.PercentageDone > 100)
{
throw new Exception($"Wrong value for percentage done : {e.PercentageDone}%.");
}
_hasReceivedGlobalProgression = true;
_cancelSource?.Cancel();
}
/// <inheritdoc cref="IArchiverMove.MoveFileSet"/>
public int MoveFileSet(IEnumerable <IFileInArchiveToMove> filesToMoveIn)
{
if (filesToMoveIn == null)
{
return(0);
}
var filesToMove = filesToMoveIn.ToList();
filesToMove.ForEach(f => f.Processed = false);
int totalFiles = filesToMove.Count;
int totalFilesDone = 0;
foreach (var zipGroupedFiles in filesToMove.GroupBy(f => f.ArchivePath))
{
if (!File.Exists(zipGroupedFiles.Key))
{
continue;
}
var tempPath = Path.Combine(Path.GetDirectoryName(zipGroupedFiles.Key) ?? Path.GetTempPath(), $"~{Path.GetRandomFileName()}");
Utils.CreateDirectoryIfNeeded(tempPath, FileAttributes.Hidden);
try {
using (var zip = ZipFile.Open(zipGroupedFiles.Key, ZipArchiveMode.Update)) {
foreach (var entry in zip.Entries.ToList())
{
_cancelToken?.ThrowIfCancellationRequested();
var fileToMove = zipGroupedFiles.FirstOrDefault(f => entry.FullName.ToCleanRelativePathUnix().EqualsCi(f.PathInArchive.ToCleanRelativePathUnix()));
if (fileToMove != null)
{
try {
var exportPath = Path.Combine(tempPath, "temp");
entry.ExtractToFile(exportPath);
entry.Delete();
zip.CreateEntryFromFile(exportPath, fileToMove.NewRelativePathInArchive, _compressionLevel);
File.Delete(exportPath);
} catch (Exception e) {
throw new ArchiverException($"Failed to move {fileToMove.PathInArchive.PrettyQuote()} to {fileToMove.NewRelativePathInArchive.PrettyQuote()} in {zipGroupedFiles.Key.PrettyQuote()}.", e);
}
totalFilesDone++;
fileToMove.Processed = true;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(zipGroupedFiles.Key, fileToMove.PathInArchive, Math.Round(totalFilesDone / (double)totalFiles * 100, 2)));
}
}
}
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to move files from {zipGroupedFiles.Key.PrettyQuote()}.", e);
} finally {
Directory.Delete(tempPath, true);
}
}
return(totalFilesDone);
}
/// <inheritdoc cref="IArchiverExtract.ExtractFileSet"/>
public int ExtractFileSet(IEnumerable <IFileInArchiveToExtract> filesToExtractIn)
{
if (filesToExtractIn == null)
{
return(0);
}
var filesToExtract = filesToExtractIn.ToList();
filesToExtract.ForEach(f => f.Processed = false);
int totalFiles = filesToExtract.Count;
int totalFilesDone = 0;
foreach (var zipGroupedFiles in filesToExtract.GroupBy(f => f.ArchivePath))
{
if (!File.Exists(zipGroupedFiles.Key))
{
continue;
}
try {
// create all necessary extraction folders
foreach (var extractDirGroupedFiles in zipGroupedFiles.GroupBy(f => Path.GetDirectoryName(f.ExtractionPath)))
{
if (!Directory.Exists(extractDirGroupedFiles.Key))
{
Directory.CreateDirectory(extractDirGroupedFiles.Key);
}
}
using (var zip = ZipFile.OpenRead(zipGroupedFiles.Key)) {
foreach (var entry in zip.Entries)
{
_cancelToken?.ThrowIfCancellationRequested();
var fileToExtract = zipGroupedFiles.FirstOrDefault(f => entry.FullName.ToCleanRelativePathUnix().EqualsCi(f.PathInArchive.ToCleanRelativePathUnix()));
if (fileToExtract != null)
{
try {
entry.ExtractToFile(fileToExtract.ExtractionPath, true);
} catch (Exception e) {
throw new ArchiverException($"Failed to extract {fileToExtract.ExtractionPath.PrettyQuote()} from {zipGroupedFiles.Key.PrettyQuote()} and relative archive path {fileToExtract.PathInArchive}.", e);
}
totalFilesDone++;
fileToExtract.Processed = true;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(zipGroupedFiles.Key, fileToExtract.PathInArchive, Math.Round(totalFilesDone / (double)totalFiles * 100, 2)));
}
}
}
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to extract files from {zipGroupedFiles.Key.PrettyQuote()}.", e);
}
}
return(totalFilesDone);
}
/// <inheritdoc cref="IArchiverDelete.DeleteFileSet"/>
public int DeleteFileSet(IEnumerable <IFileInArchiveToDelete> filesToDeleteIn)
{
if (filesToDeleteIn == null)
{
return(0);
}
var filesToDelete = filesToDeleteIn.ToList();
filesToDelete.ForEach(f => f.Processed = false);
int totalFiles = filesToDelete.Count;
int totalFilesDone = 0;
foreach (var zipGroupedFiles in filesToDelete.GroupBy(f => f.ArchivePath))
{
if (!File.Exists(zipGroupedFiles.Key))
{
continue;
}
try {
using (var zip = ZipFile.Open(zipGroupedFiles.Key, ZipArchiveMode.Update)) {
foreach (var entry in zip.Entries.ToList())
{
_cancelToken?.ThrowIfCancellationRequested();
var fileToDelete = zipGroupedFiles.FirstOrDefault(f => entry.FullName.ToCleanRelativePathUnix().EqualsCi(f.PathInArchive.ToCleanRelativePathUnix()));
if (fileToDelete != null)
{
try {
entry.Delete();
} catch (Exception e) {
throw new ArchiverException($"Failed to delete {fileToDelete.PathInArchive.PrettyQuote()} from {zipGroupedFiles.Key.PrettyQuote()}.", e);
}
totalFilesDone++;
fileToDelete.Processed = true;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(zipGroupedFiles.Key, fileToDelete.PathInArchive, Math.Round(totalFilesDone / (double)totalFiles * 100, 2)));
}
}
}
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to delete files from {zipGroupedFiles.Key.PrettyQuote()}.", e);
}
}
return(totalFilesDone);
}
/// <inheritdoc cref="IArchiver.ArchiveFileSet"/>
public int ArchiveFileSet(IEnumerable <IFileToArchive> filesToArchive)
{
if (filesToArchive == null)
{
return(0);
}
var filesToPack = filesToArchive.ToList();
filesToPack.ForEach(f => f.Processed = false);
int totalFiles = filesToPack.Count;
int totalFilesDone = 0;
foreach (var zipGroupedFiles in filesToPack.GroupBy(f => f.ArchivePath))
{
try {
CreateArchiveFolder(zipGroupedFiles.Key);
var zipMode = File.Exists(zipGroupedFiles.Key) ? ZipArchiveMode.Update : ZipArchiveMode.Create;
using (var zip = ZipFile.Open(zipGroupedFiles.Key, zipMode)) {
foreach (var file in zipGroupedFiles)
{
_cancelToken?.ThrowIfCancellationRequested();
if (!File.Exists(file.SourcePath))
{
continue;
}
try {
zip.CreateEntryFromFile(file.SourcePath, file.PathInArchive.ToCleanRelativePathUnix(), _compressionLevel);
} catch (Exception e) {
throw new ArchiverException($"Failed to pack {file.SourcePath.PrettyQuote()} into {zipGroupedFiles.Key.PrettyQuote()} and relative archive path {file.PathInArchive}.", e);
}
totalFilesDone++;
file.Processed = true;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(zipGroupedFiles.Key, file.PathInArchive, Math.Round(totalFilesDone / (double)totalFiles * 100, 2)));
}
}
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to pack to {zipGroupedFiles.Key.PrettyQuote()}.", e);
}
}
return(totalFilesDone);
}
private int DoForFiles(IEnumerable <IFileArchivedBase> filesIn, ActionType action)
{
if (filesIn == null)
{
return(0);
}
var files = filesIn.ToList();
files.ForEach(f => f.Processed = false);
var totalFiles = files.Count;
var totalFilesDone = 0;
try {
foreach (var file in files)
{
_cancelToken?.ThrowIfCancellationRequested();
string source = (action == ActionType.Pack ? ((IFileToArchive)file).SourcePath : Path.Combine(file.ArchivePath ?? "", file.PathInArchive)).ToCleanPath();
string target = null;
switch (action)
{
case ActionType.Pack:
target = Path.Combine(file.ArchivePath ?? "", ((IFileToArchive)file).PathInArchive).ToCleanPath();
break;
case ActionType.Extract:
target = ((IFileInArchiveToExtract)file).ExtractionPath.ToCleanPath();
break;
case ActionType.Move:
target = Path.Combine(file.ArchivePath ?? "", ((IFileInArchiveToMove)file).NewRelativePathInArchive).ToCleanPath();
break;
}
// ignore non existing files
if (string.IsNullOrEmpty(source) || !File.Exists(source))
{
continue;
}
// create the necessary target folder
string dir;
if (!string.IsNullOrEmpty(target) && !string.IsNullOrEmpty(dir = Path.GetDirectoryName(target)))
{
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
}
try {
switch (action)
{
case ActionType.Pack:
case ActionType.Extract:
if (string.IsNullOrEmpty(target))
{
throw new NullReferenceException("Target should not be null.");
}
if (!source.PathEquals(target))
{
if (File.Exists(target))
{
File.Delete(target);
}
try {
var buffer = new byte[BufferSize];
using (var sourceFileStream = File.OpenRead(source)) {
long fileLength = sourceFileStream.Length;
using (var dest = File.OpenWrite(target)) {
long totalBytes = 0;
int currentBlockSize;
while ((currentBlockSize = sourceFileStream.Read(buffer, 0, buffer.Length)) > 0)
{
totalBytes += currentBlockSize;
dest.Write(buffer, 0, currentBlockSize);
if (totalBytes != fileLength)
{
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(file.ArchivePath, file.PathInArchive, Math.Round((totalFilesDone + (double)totalBytes / fileLength) / totalFiles * 100, 2)));
}
_cancelToken?.ThrowIfCancellationRequested();
}
}
}
} catch (OperationCanceledException) {
// cleanup the potentially unfinished file copy
if (File.Exists(target))
{
File.Delete(target);
}
throw;
}
}
break;
case ActionType.Move:
if (string.IsNullOrEmpty(target))
{
throw new NullReferenceException("Target should not be null.");
}
if (!source.PathEquals(target))
{
if (File.Exists(target))
{
File.Delete(target);
}
File.Move(source, target);
}
break;
case ActionType.Delete:
File.Delete(source);
break;
default:
throw new ArgumentOutOfRangeException(nameof(action), action, null);
}
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to {action.ToString().ToLower()} {source.PrettyQuote()}{(string.IsNullOrEmpty(target) ? "" : $" in {target.PrettyQuote()}")}.", e);
}
totalFilesDone++;
file.Processed = true;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(file.ArchivePath, file.PathInArchive, Math.Round((double)totalFilesDone / totalFiles * 100, 2)));
}
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to {action.ToString().ToLower()} files.", e);
}
return(totalFilesDone);
}
/// <inheritdoc cref="IArchiverBasic.ArchiveFileSet"/>
public int ArchiveFileSet(IEnumerable <IFileToArchive> filesToArchive)
{
var filesToPack = filesToArchive.ToList();
filesToPack.ForEach(f => f.Processed = false);
int totalFiles = filesToPack.Count;
int totalFilesDone = 0;
foreach (var plGroupedFiles in filesToPack.GroupBy(f => f.ArchivePath))
{
string uniqueTempFolder = null;
try {
var archiveFolder = CreateArchiveFolder(plGroupedFiles.Key);
// create a unique temp folder for this .pl
uniqueTempFolder = Path.Combine(archiveFolder, $"{Path.GetFileName(plGroupedFiles.Key)}~{Path.GetRandomFileName()}");
var dirInfo = Directory.CreateDirectory(uniqueTempFolder);
dirInfo.Attributes |= FileAttributes.Hidden;
var subFolders = new Dictionary <string, List <FilesToMove> >();
foreach (var file in plGroupedFiles)
{
var subFolderPath = Path.GetDirectoryName(Path.Combine(uniqueTempFolder, file.PathInArchive));
if (!string.IsNullOrEmpty(subFolderPath))
{
if (!subFolders.ContainsKey(subFolderPath))
{
subFolders.Add(subFolderPath, new List <FilesToMove>());
if (!Directory.Exists(subFolderPath))
{
Directory.CreateDirectory(subFolderPath);
}
}
if (File.Exists(file.SourcePath))
{
subFolders[subFolderPath].Add(new FilesToMove(file.SourcePath, Path.Combine(uniqueTempFolder, file.PathInArchive), file.PathInArchive));
}
}
}
var prolibExe = new ProcessIo(_prolibPath)
{
WorkingDirectory = uniqueTempFolder
};
foreach (var subFolder in subFolders)
{
_cancelToken?.ThrowIfCancellationRequested();
// move files to the temp subfolder
Parallel.ForEach(subFolder.Value, file => {
if (file.Move)
{
File.Move(file.Origin, file.Temp);
}
else
{
File.Copy(file.Origin, file.Temp);
}
});
// for files containing a space, we don't have a choice, call extract for each...
foreach (var file in subFolder.Value.Where(f => f.RelativePath.Contains(" ")))
{
if (!prolibExe.TryExecute(new ProcessArgs().Append(plGroupedFiles.Key, "-create", "-nowarn", "-add", file.RelativePath)))
{
throw new ArchiverException($"Failed to pack {file.Origin.PrettyQuote()} into {plGroupedFiles.Key.PrettyQuote()} and relative archive path {file.RelativePath}.", new ArchiverException(prolibExe.BatchOutput.ToString()));
}
}
var remainingFiles = subFolder.Value.Where(f => !f.RelativePath.Contains(" ")).ToList();
if (remainingFiles.Count > 0)
{
// for the other files, we can use the -pf parameter
var pfContent = new StringBuilder();
pfContent.AppendLine("-create -nowarn -add");
foreach (var file in remainingFiles)
{
pfContent.AppendLine(file.RelativePath);
}
var pfPath = Path.Combine(uniqueTempFolder, $"{Path.GetFileName(plGroupedFiles.Key)}~{Path.GetRandomFileName()}.pf");
File.WriteAllText(pfPath, pfContent.ToString(), _encoding);
if (!prolibExe.TryExecute(new ProcessArgs().Append(plGroupedFiles.Key, "-pf", pfPath)))
{
throw new ArchiverException($"Failed to pack to {plGroupedFiles.Key.PrettyQuote()}.", new Exception(prolibExe.BatchOutput.ToString()));
}
if (File.Exists(pfPath))
{
File.Delete(pfPath);
}
}
// move files from the temp subfolder
foreach (var file in subFolder.Value)
{
try {
if (file.Move)
{
File.Move(file.Temp, file.Origin);
}
else if (!File.Exists(file.Temp))
{
throw new ArchiverException($"Failed to move back the temporary file {file.Origin} from {file.Temp}.");
}
} catch (Exception e) {
throw new ArchiverException($"Failed to move back the temporary file {file.Origin} from {file.Temp}.", e);
}
totalFilesDone++;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(plGroupedFiles.Key, file.RelativePath, Math.Round(totalFilesDone / (double)totalFiles * 100, 2)));
}
}
// compress .pl
prolibExe.TryExecute(new ProcessArgs().Append(plGroupedFiles.Key, "-compress", "-nowarn"));
foreach (var file in plGroupedFiles)
{
file.Processed = true;
}
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to pack to {plGroupedFiles.Key.PrettyQuote()}.", e);
} finally {
// delete temp folder
if (Directory.Exists(uniqueTempFolder))
{
Directory.Delete(uniqueTempFolder, true);
}
}
}
return(totalFilesDone);
}
private int DoAction(IEnumerable <IFileArchivedBase> filesIn, Action action)
{
if (filesIn == null)
{
return(0);
}
var files = filesIn.ToList();
// total size to handle
long totalSizeDone = 0;
long totalSize = 0;
try {
switch (action)
{
case Action.Upload:
foreach (var file in files.OfType <IFileToArchive>())
{
if (File.Exists(file.SourcePath))
{
totalSize += new FileInfo(file.SourcePath).Length;
}
}
break;
case Action.Download:
foreach (var file in files.OfType <IFileInArchiveToExtract>())
{
var response = HttpRequest.GetFileSize(WebUtility.UrlEncode(file.PathInArchive.ToCleanRelativePathUnix()), out long size);
if (response.Success)
{
totalSize += size;
}
}
break;
}
} catch (Exception e) {
throw new ArchiverException($"Failed to assess the total file size to handle during {action.ToString().ToLower()}.", e);
}
bool totalSizeNotFound = totalSize == 0;
int nbFilesProcessed = 0;
foreach (var serverGroupedFiles in files.GroupBy(f => f.ArchivePath))
{
try {
HttpRequest.UseBaseUrl(serverGroupedFiles.Key);
foreach (var file in serverGroupedFiles)
{
bool requestOk;
HttpResponse response;
var fileRelativePath = WebUtility.UrlEncode(file.PathInArchive.ToCleanRelativePathUnix());
switch (action)
{
case Action.Upload:
if (!File.Exists(((IFileToArchive)file).SourcePath))
{
// skip to next file
continue;
}
response = HttpRequest.PutFile(fileRelativePath, ((IFileToArchive)file).SourcePath, progress => {
totalSizeDone += progress.NumberOfBytesDoneSinceLastProgress;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(serverGroupedFiles.Key, fileRelativePath, Math.Round(totalSizeDone / (double)totalSize * 100, 2)));
});
requestOk = response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.Created;
break;
case Action.Download:
response = HttpRequest.DownloadFile(fileRelativePath, ((IFileInArchiveToExtract)file).ExtractionPath, progress => {
if (totalSizeNotFound && progress.NumberOfBytesDoneSinceLastProgress == progress.NumberOfBytesDoneTotal)
{
totalSize += progress.NumberOfBytesTotal;
}
totalSizeDone += progress.NumberOfBytesDoneSinceLastProgress;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(serverGroupedFiles.Key, fileRelativePath, Math.Round(totalSizeDone / (double)totalSize * 100, 2)));
});
requestOk = response.StatusCode == HttpStatusCode.OK;
if (response.StatusCode == HttpStatusCode.NotFound || response.Exception is WebException we && we.Status == WebExceptionStatus.NameResolutionFailure)
{
// skip to next file
continue;
}
break;
case Action.Delete:
response = HttpRequest.DeleteFile(fileRelativePath);
requestOk = response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.NoContent;
if (response.StatusCode == HttpStatusCode.NotFound || response.Exception is WebException we1 && we1.Status == WebExceptionStatus.NameResolutionFailure)
{
// skip to next file
continue;
}
break;
default:
throw new ArgumentOutOfRangeException(nameof(action), action, null);
}
if (!requestOk)
{
if (response.Exception != null)
{
throw response.Exception;
}
throw new ArchiverException($"The server returned {response.StatusCode} : {response.StatusDescription} for {fileRelativePath}.");
}
nbFilesProcessed++;
file.Processed = true;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(serverGroupedFiles.Key, fileRelativePath, Math.Round(nbFilesProcessed / (double)files.Count * 100, 2)));
}
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to {action.ToString().ToLower()} at {serverGroupedFiles.Key}.", e);
}
}
return(nbFilesProcessed);
}
/// <inheritdoc cref="IArchiver.ExtractFileSet"/>
public int ExtractFileSet(IEnumerable <IFileInArchiveToExtract> filesToExtractIn)
{
var filesToExtract = filesToExtractIn.ToList();
filesToExtract.ForEach(f => f.Processed = false);
int totalFiles = filesToExtract.Count;
int totalFilesDone = 0;
var prolibExe = new ProcessIo(_prolibPath);
foreach (var plGroupedFiles in filesToExtract.GroupBy(f => f.ArchivePath))
{
if (!File.Exists(plGroupedFiles.Key))
{
continue;
}
// process only files that actually exist
var archiveFileList = ListFiles(plGroupedFiles.Key).Select(f => f.PathInArchive).ToHashSet();
var plGroupedFilesFiltered = plGroupedFiles.Where(f => archiveFileList.Contains(f.PathInArchive)).ToList();
try {
foreach (var extractDirGroupedFiles in plGroupedFilesFiltered.GroupBy(f => Path.GetDirectoryName(f.ExtractionPath)))
{
prolibExe.WorkingDirectory = extractDirGroupedFiles.Key;
Directory.CreateDirectory(extractDirGroupedFiles.Key);
// for files containing a space, we don't have a choice, call extract for each...
foreach (var file in extractDirGroupedFiles.Where(deploy => deploy.PathInArchive.Contains(" ")))
{
_cancelToken?.ThrowIfCancellationRequested();
if (File.Exists(file.ExtractionPath))
{
File.Delete(file.ExtractionPath);
}
if (!prolibExe.TryExecute(new ProcessArgs().Append(plGroupedFiles.Key, "-nowarn", "-yank", file.PathInArchive)))
{
throw new ArchiverException($"Failed to extract {file.PathInArchive.PrettyQuote()} from {plGroupedFiles.Key.PrettyQuote()}.", new Exception(prolibExe.BatchOutput.ToString()));
}
totalFilesDone++;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(plGroupedFiles.Key, file.PathInArchive, Math.Round(totalFilesDone / (double)totalFiles * 100, 2)));
}
_cancelToken?.ThrowIfCancellationRequested();
var remainingFiles = extractDirGroupedFiles.Where(deploy => !deploy.PathInArchive.Contains(" ")).ToList();
if (remainingFiles.Count > 0)
{
// for the other files, we can use the -pf parameter
var pfContent = new StringBuilder();
pfContent.AppendLine("-nowarn");
pfContent.AppendLine("-yank");
foreach (var file in remainingFiles)
{
pfContent.AppendLine(file.PathInArchive);
if (File.Exists(file.ExtractionPath))
{
File.Delete(file.ExtractionPath);
}
}
var pfPath = Path.Combine(extractDirGroupedFiles.Key, $"{Path.GetFileName(plGroupedFiles.Key)}~{Path.GetRandomFileName()}.pf");
File.WriteAllText(pfPath, pfContent.ToString(), _encoding);
if (!prolibExe.TryExecute(new ProcessArgs().Append(plGroupedFiles.Key, "-pf", pfPath)))
{
throw new ArchiverException($"Failed to extract from {plGroupedFiles.Key.PrettyQuote()}.", new Exception(prolibExe.BatchOutput.ToString()));
}
foreach (var file in remainingFiles)
{
totalFilesDone++;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(plGroupedFiles.Key, file.PathInArchive, Math.Round(totalFilesDone / (double)totalFiles * 100, 2)));
}
if (File.Exists(pfPath))
{
File.Delete(pfPath);
}
}
}
foreach (var file in plGroupedFiles)
{
file.Processed = true;
}
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to process {plGroupedFiles.Key.PrettyQuote()}.", e);
}
}
return(totalFilesDone);
}
/// <inheritdoc cref="IArchiver.MoveFileSet"/>
public int MoveFileSet(IEnumerable <IFileInArchiveToMove> filesToMoveIn)
{
var filesToMove = filesToMoveIn.ToList();
filesToMove.ForEach(f => f.Processed = false);
int totalFiles = filesToMove.Count;
int totalFilesDone = 0;
foreach (var plGroupedFiles in filesToMove.GroupBy(f => f.ArchivePath))
{
string uniqueTempFolder = null;
try {
// process only files that actually exist
var archiveFileList = ListFiles(plGroupedFiles.Key).Select(f => f.PathInArchive).ToHashSet();
var plGroupedFilesFiltered = plGroupedFiles.Where(f => archiveFileList.Contains(f.PathInArchive)).ToList();
if (!plGroupedFilesFiltered.Any())
{
continue;
}
var archiveFolder = CreateArchiveFolder(plGroupedFiles.Key);
// create a unique temp folder for this .pl
uniqueTempFolder = Path.Combine(archiveFolder, $"{Path.GetFileName(plGroupedFiles.Key)}~{Path.GetRandomFileName()}");
var dirInfo = Directory.CreateDirectory(uniqueTempFolder);
dirInfo.Attributes |= FileAttributes.Hidden;
var subFolders = new Dictionary <string, List <FilesToMove> >();
foreach (var file in plGroupedFilesFiltered)
{
var subFolderPath = Path.GetDirectoryName(Path.Combine(uniqueTempFolder, file.NewRelativePathInArchive));
if (!string.IsNullOrEmpty(subFolderPath))
{
if (!subFolders.ContainsKey(subFolderPath))
{
subFolders.Add(subFolderPath, new List <FilesToMove>());
if (!Directory.Exists(subFolderPath))
{
Directory.CreateDirectory(subFolderPath);
}
}
subFolders[subFolderPath].Add(new FilesToMove(file.PathInArchive, Path.Combine(uniqueTempFolder, file.NewRelativePathInArchive), file.NewRelativePathInArchive));
}
}
var prolibExe = new ProcessIo(_prolibPath);
foreach (var subFolder in subFolders)
{
_cancelToken?.ThrowIfCancellationRequested();
foreach (var file in subFolder.Value)
{
prolibExe.WorkingDirectory = Path.GetDirectoryName(file.Temp);
if (!prolibExe.TryExecute(new ProcessArgs().Append(plGroupedFiles.Key, "-nowarn", "yank", file.Origin)))
{
throw new ArchiverException($"Failed to extract {file.Origin.PrettyQuote()} from {plGroupedFiles.Key.PrettyQuote()}.", new Exception(prolibExe.BatchOutput.ToString()));
}
if (!prolibExe.TryExecute(new ProcessArgs().Append(plGroupedFiles.Key, "-nowarn", "-delete", file.Origin)))
{
throw new ArchiverException($"Failed to delete {file.Origin.PrettyQuote()} in {plGroupedFiles.Key.PrettyQuote()}.", new ArchiverException(prolibExe.BatchOutput.ToString()));
}
File.Move(Path.Combine(prolibExe.WorkingDirectory, Path.GetFileName(file.Origin)), file.Temp);
prolibExe.WorkingDirectory = uniqueTempFolder;
prolibExe.TryExecute(new ProcessArgs().Append(plGroupedFiles.Key, "-nowarn", "-delete", file.RelativePath));
if (!prolibExe.TryExecute(new ProcessArgs().Append(plGroupedFiles.Key, "-create", "-nowarn", "-add", file.RelativePath)))
{
throw new ArchiverException($"Failed to pack {file.Origin.PrettyQuote()} into {plGroupedFiles.Key.PrettyQuote()} and relative archive path {file.RelativePath}.", new ArchiverException(prolibExe.BatchOutput.ToString()));
}
totalFilesDone++;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(plGroupedFiles.Key, file.RelativePath, Math.Round(totalFilesDone / (double)totalFiles * 100, 2)));
}
}
// compress .pl
prolibExe.TryExecute(new ProcessArgs().Append(plGroupedFiles.Key, "-compress", "-nowarn"));
// delete temp folder
Directory.Delete(uniqueTempFolder, true);
foreach (var file in plGroupedFiles)
{
file.Processed = true;
}
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to pack to {plGroupedFiles.Key.PrettyQuote()}.", e);
} finally {
// delete temp folder
if (Directory.Exists(uniqueTempFolder))
{
Directory.Delete(uniqueTempFolder, true);
}
}
}
return(totalFilesDone);
}
/// <inheritdoc cref="IArchiver.ArchiveFileSet"/>
public int ArchiveFileSet(IEnumerable <IFileToArchive> filesToArchive)
{
if (filesToArchive == null)
{
return(0);
}
var filesToPack = filesToArchive.ToList();
filesToPack.ForEach(f => f.Processed = false);
int totalFiles = filesToPack.Count;
int totalFilesDone = 0;
foreach (var ftpGroupedFiles in filesToPack.GroupBy(f => f.ArchivePath))
{
try {
if (!ftpGroupedFiles.Key.ParseFtpAddress(out var uri, out var userName, out var passWord, out var host, out var port, out var relativePath))
{
throw new ArchiverException($"The ftp uri is invalid, the typical format is ftp://user:pass@server:port/path. Input uri was : {uri.PrettyQuote()}.");
}
_cancelToken?.ThrowIfCancellationRequested();
var ftp = FtpsClient.Instance.Get(uri);
ConnectOrReconnectFtp(ftp, userName, passWord, host, port);
foreach (var file in ftpGroupedFiles)
{
file.Processed = false;
if (!File.Exists(file.SourcePath))
{
continue;
}
_cancelToken?.ThrowIfCancellationRequested();
try {
var pathInArchive = Path.Combine(relativePath ?? "", file.PathInArchive);
var filesDone = totalFilesDone;
void TransferCallback(FtpsClient sender, ETransferActions action, string local, string remote, ulong done, ulong?total, ref bool cancel)
{
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(ftpGroupedFiles.Key, file.PathInArchive, Math.Round((filesDone + (double)done / (total ?? 0)) / totalFiles * 100, 2)));
}
try {
ftp.PutFile(file.SourcePath, pathInArchive, TransferCallback);
} catch (Exception) {
// try to create the directory and then push the file again
ftp.MakeDir(Path.GetDirectoryName(pathInArchive) ?? "", true);
ftp.SetCurrentDirectory("/");
ftp.PutFile(file.SourcePath, pathInArchive, TransferCallback);
}
totalFilesDone++;
file.Processed = true;
} catch (Exception e) {
throw new ArchiverException($"Failed to send {file.SourcePath.PrettyQuote()} to {file.ArchivePath.PrettyQuote()} and distant path {file.PathInArchive.PrettyQuote()}.", e);
}
}
FtpsClient.Instance.DisconnectFtp();
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to send files to {ftpGroupedFiles.Key.PrettyQuote()}.", e);
}
}
return(totalFilesDone);
}
/// <inheritdoc />
public int MoveFileSet(IEnumerable <IFileInArchiveToMove> filesToMoveIn)
{
if (filesToMoveIn == null)
{
return(0);
}
var filesToMove = filesToMoveIn.ToList();
filesToMove.ForEach(f => f.Processed = false);
int totalFiles = filesToMove.Count;
int totalFilesDone = 0;
foreach (var ftpGroupedFiles in filesToMove.GroupBy(f => f.ArchivePath))
{
try {
if (!ftpGroupedFiles.Key.ParseFtpAddress(out var uri, out var userName, out var passWord, out var host, out var port, out var relativePath))
{
throw new ArchiverException($"The ftp uri is invalid, the typical format is ftp://user:pass@server:port/path. Input uri was : {uri.PrettyQuote()}.");
}
_cancelToken?.ThrowIfCancellationRequested();
var ftp = FtpsClient.Instance.Get(uri);
ConnectOrReconnectFtp(ftp, userName, passWord, host, port);
foreach (var file in ftpGroupedFiles)
{
_cancelToken?.ThrowIfCancellationRequested();
try {
var pathInArchive = Path.Combine(relativePath ?? "", file.PathInArchive);
var newPathInArchive = Path.Combine(relativePath ?? "", file.NewRelativePathInArchive);
try {
ftp.RenameFile(pathInArchive, newPathInArchive);
} catch (FtpCommandException e) {
if (e.ErrorCode == 550)
{
// path does not exist
continue;
}
if (e.ErrorCode == 553)
{
// target already exists
ftp.DeleteFile(newPathInArchive);
ftp.RenameFile(pathInArchive, newPathInArchive);
}
else
{
throw;
}
}
totalFilesDone++;
file.Processed = true;
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(ftpGroupedFiles.Key, file.PathInArchive, Math.Round(totalFilesDone / (double)totalFiles * 100, 2)));
} catch (Exception e) {
throw new ArchiverException($"Failed to move {file.PathInArchive.PrettyQuote()} to {file.NewRelativePathInArchive.PrettyQuote()} in {file.ArchivePath.PrettyQuote()}.", e);
}
}
FtpsClient.Instance.DisconnectFtp();
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to move files from {ftpGroupedFiles.Key.PrettyQuote()}.", e);
}
}
return(totalFilesDone);
}
/// <inheritdoc cref="IArchiverExtract.ExtractFileSet"/>
public int ExtractFileSet(IEnumerable <IFileInArchiveToExtract> filesToExtractIn)
{
if (filesToExtractIn == null)
{
return(0);
}
var filesToExtract = filesToExtractIn.ToList();
filesToExtract.ForEach(f => f.Processed = false);
int totalFiles = filesToExtract.Count;
int totalFilesDone = 0;
foreach (var ftpGroupedFiles in filesToExtract.GroupBy(f => f.ArchivePath))
{
try {
// create all necessary extraction folders
foreach (var extractDirGroupedFiles in ftpGroupedFiles.GroupBy(f => Path.GetDirectoryName(f.ExtractionPath)))
{
if (!Directory.Exists(extractDirGroupedFiles.Key))
{
Directory.CreateDirectory(extractDirGroupedFiles.Key);
}
}
if (!ftpGroupedFiles.Key.ParseFtpAddress(out var uri, out var userName, out var passWord, out var host, out var port, out var relativePath))
{
throw new ArchiverException($"The ftp uri is invalid, the typical format is ftp://user:pass@server:port/path. Input uri was : {uri.PrettyQuote()}.");
}
_cancelToken?.ThrowIfCancellationRequested();
var ftp = FtpsClient.Instance.Get(uri);
ConnectOrReconnectFtp(ftp, userName, passWord, host, port);
foreach (var file in ftpGroupedFiles)
{
_cancelToken?.ThrowIfCancellationRequested();
try {
try {
var filesDone = totalFilesDone;
void TransferCallback(FtpsClient sender, ETransferActions action, string local, string remote, ulong done, ulong?total, ref bool cancel)
{
OnProgress?.Invoke(this, ArchiverEventArgs.NewProgress(ftpGroupedFiles.Key, file.PathInArchive, Math.Round((filesDone + (double)done / (total ?? 0)) / totalFiles * 100, 2)));
}
ftp.GetFile(Path.Combine(relativePath ?? "", file.PathInArchive), file.ExtractionPath, TransferCallback);
} catch (FtpCommandException e) {
if (e.ErrorCode == 550)
{
// path does not exist
continue;
}
throw;
}
totalFilesDone++;
file.Processed = true;
} catch (Exception e) {
throw new ArchiverException($"Failed to get {file.ExtractionPath.PrettyQuote()} from {file.ArchivePath.PrettyQuote()} and distant path {file.PathInArchive.PrettyQuote()}.", e);
}
}
FtpsClient.Instance.DisconnectFtp();
} catch (OperationCanceledException) {
throw;
} catch (Exception e) {
throw new ArchiverException($"Failed to get files from {ftpGroupedFiles.Key.PrettyQuote()}.", e);
}
}
return(totalFilesDone);
}
