private TempFile coreDoGetPiping(FileEntryItem item, Interface.IEncryption useDecrypter, out long retDownloadSize, out string retHashcode)
{
// With piping allowed, we will parallelize the operation with buffered pipes to maximize throughput:
// Separated: Download (only for streaming) - Hashing - Decryption
// The idea is to use DirectStreamLink's that are inserted in the stream stack, creating a fork to run
// the crypto operations on.
retDownloadSize = -1;
retHashcode = null;
bool enableStreaming = (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers);
System.Threading.Tasks.Task<string> taskHasher = null;
DirectStreamLink linkForkHasher = null;
System.Threading.Tasks.Task taskDecrypter = null;
DirectStreamLink linkForkDecryptor = null;
// keep potential temp files and their streams for cleanup (cannot use using here).
TempFile retTarget = null, dlTarget = null, decryptTarget = null;
System.IO.Stream dlToStream = null, decryptToStream = null;
try
{
System.IO.Stream nextTierWriter = null; // target of our stacked streams
if (!enableStreaming) // we will always need dlTarget if not streaming...
dlTarget = new TempFile();
else if (enableStreaming && useDecrypter == null)
{
dlTarget = new TempFile();
dlToStream = System.IO.File.OpenWrite(dlTarget);
nextTierWriter = dlToStream; // actually write through to file.
}
// setup decryption: fork off a StreamLink from stack, and setup decryptor task
if (useDecrypter != null)
{
linkForkDecryptor = new DirectStreamLink(1 << 16, false, false, nextTierWriter);
nextTierWriter = linkForkDecryptor.WriterStream;
linkForkDecryptor.SetKnownLength(item.Size, false); // Set length to allow AES-decryption (not streamable yet)
decryptTarget = new TempFile();
decryptToStream = System.IO.File.OpenWrite(decryptTarget);
taskDecrypter = new System.Threading.Tasks.Task(() =>
{
using (var input = linkForkDecryptor.ReaderStream)
using (var output = decryptToStream)
lock (m_encryptionLock) { useDecrypter.Decrypt(input, output); }
}
);
}
// setup hashing: fork off a StreamLink from stack, then task computes hash
linkForkHasher = new DirectStreamLink(1 << 16, false, false, nextTierWriter);
nextTierWriter = linkForkHasher.WriterStream;
taskHasher = new System.Threading.Tasks.Task<string>(() =>
{
using (var input = linkForkHasher.ReaderStream)
return CalculateFileHash(input);
}
);
// OK, forks with tasks are set up, so let's do the download which is performed in main thread.
bool hadException = false;
try
{
if (enableStreaming)
{
using (var ss = new ShaderStream(nextTierWriter, false))
{
using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
{
taskHasher.Start(); // We do not start tasks earlier to be sure the input always gets closed.
if (taskDecrypter != null) taskDecrypter.Start();
((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs);
}
retDownloadSize = ss.TotalBytesWritten;
}
}
else
{
m_backend.Get(item.RemoteFilename, dlTarget);
retDownloadSize = new System.IO.FileInfo(dlTarget).Length;
using (dlToStream = System.IO.File.OpenRead(dlTarget))
{
taskHasher.Start(); // We do not start tasks earlier to be sure the input always gets closed.
if (taskDecrypter != null) taskDecrypter.Start();
new DirectStreamLink.DataPump(dlToStream, nextTierWriter).Run();
}
}
}
catch (Exception)
{ hadException = true; throw; }
finally
{
// This nested try-catch-finally blocks will make sure we do not miss any exceptions ans all started tasks
// are properly ended and tidied up. For what is thrown: If exceptions in main thread occured (download) it is thrown,
// then hasher task is checked and last decryption. This resembles old logic.
try { retHashcode = taskHasher.Result; }
catch (AggregateException ex) { if (!hadException) { hadException = true; throw ex.InnerExceptions[0]; } }
finally
{
if (taskDecrypter != null)
{
try { taskDecrypter.Wait(); }
catch (AggregateException ex)
{
if (!hadException)
{
hadException = true;
if (ex.InnerExceptions[0] is System.Security.Cryptography.CryptographicException)
throw ex.InnerExceptions[0];
else
throw new System.Security.Cryptography.CryptographicException(ex.InnerExceptions[0].Message, ex.InnerExceptions[0]);
}
}
}
}
}
if (useDecrypter != null) // return decrypted temp file
{ retTarget = decryptTarget; decryptTarget = null; }
else // return downloaded file
{ retTarget = dlTarget; dlTarget = null; }
}
finally
{
// Be tidy: manually do some cleanup to temp files, as we could not use using's.
// Unclosed streams should only occur if we failed even before tasks were started.
if (dlToStream != null) dlToStream.Dispose();
if (dlTarget != null) dlTarget.Dispose();
if (decryptToStream != null) decryptToStream.Dispose();
if (decryptTarget != null) decryptTarget.Dispose();
}
return retTarget;
}
private TempFile coreDoGetSequential(FileEntryItem item, Interface.IEncryption useDecrypter, out long retDownloadSize, out string retHashcode)
{
retHashcode = null;
retDownloadSize = -1;
TempFile retTarget, dlTarget = null, decryptTarget = null;
try
{
dlTarget = new Library.Utility.TempFile();
if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
{
Func<string> getFileHash;
// extended to use stacked streams
using (var fs = System.IO.File.OpenWrite(dlTarget))
using (var hs = GetFileHasherStream(fs, System.Security.Cryptography.CryptoStreamMode.Write, out getFileHash))
using (var ss = new ShaderStream(hs, true))
{
using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
{ ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs); }
ss.Flush();
retDownloadSize = ss.TotalBytesWritten;
retHashcode = getFileHash();
}
}
else
{
m_backend.Get(item.RemoteFilename, dlTarget);
retDownloadSize = new System.IO.FileInfo(dlTarget).Length;
retHashcode = CalculateFileHash(dlTarget);
}
// Decryption is not placed in the stream stack because there seemed to be an effort
// to throw a CryptographicException on fail. If in main stack, we cannot differentiate
// in which part of the stack the source of an exception resides.
if (useDecrypter != null)
{
decryptTarget = new Library.Utility.TempFile();
lock (m_encryptionLock)
{
try { useDecrypter.Decrypt(dlTarget, decryptTarget); }
// If we fail here, make sure that we throw a crypto exception
catch (System.Security.Cryptography.CryptographicException) { throw; }
catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); }
}
retTarget = decryptTarget;
decryptTarget = null;
}
else
{
retTarget = dlTarget;
dlTarget = null;
}
}
finally
{
if (dlTarget != null) dlTarget.Dispose();
if (decryptTarget != null) decryptTarget.Dispose();
}
return retTarget;
}
public static string RestoreControlFiles(string source, string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(source, options))
return i.RestoreControlFiles(target);
}
public static List<KeyValuePair<BackupEntryBase, Exception>> VerifyBackup(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.VerifyBackupChain();
}
public static string[] ListSourceFolders(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.ListSourceFolders();
}
/// <summary>
/// Returns a list of full backup manifests, with a list of volumes and an incremental chain
/// </summary>
/// <param name="target">The backend to read the data from</param>
/// <param name="options">A list of options</param>
/// <returns>A list of full backup manifests</returns>
public static List<ManifestEntry> ParseFileList(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.GetBackupSets();
}
public static List<KeyValuePair<RSync.RSyncDir.PatchFileType, string>> ListActualSignatureFiles(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.ListActualSignatureFiles();
}
public static string Backup(string[] source, string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.Backup(source);
}
public static string DeleteOlderThan(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.DeleteOlderThan();
}
public static List<KeyValuePair<string, DateTime>> FindLastFileVersion(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.FindLastFileVersion();
}
public static string DeleteAllButNFull(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.DeleteAllButNFull();
}
public static void CreateFolder(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
i.CreateFolder();
}
public static string Cleanup(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.Cleanup();
}
public void ExecuteTask(IDuplicityTask task)
{
Dictionary<string, string> options = new Dictionary<string,string>();
//Set the log level to be that of the GUI
options["log-level"] = Duplicati.Library.Logging.Log.LogLevel.ToString();
string destination = task.GetConfiguration(options);
string results = "";
string parsedMessage = "";
m_isAborted = false;
try
{
//TODO: Its a bit dirty to set the options after creating the instance
using (Interface i = new Interface(destination, options))
{
lock (m_lock)
{
m_stopReason = System.Windows.Forms.CloseReason.None;
m_currentBackupControlInterface = i;
}
SetupControlInterface();
i.OperationProgress += new OperationProgressEvent(Duplicati_OperationProgress);
switch (task.TaskType)
{
case DuplicityTaskType.FullBackup:
case DuplicityTaskType.IncrementalBackup:
{
//Activate auto-cleanup
options["auto-cleanup"] = "";
options["force"] = "";
if (task.Schedule.Task.KeepFull > 0)
m_extraOperations++;
if (!string.IsNullOrEmpty(task.Schedule.Task.KeepTime))
m_extraOperations++;
Library.Utility.TempFolder tf = null;
try
{
if (ProgressEvent != null)
ProgressEvent(DuplicatiOperation.Backup, RunnerState.Started, task.Schedule.Name, "", 0, -1);
if (task.Task.IncludeSetup)
{
//Make a copy of the current database
tf = new Duplicati.Library.Utility.TempFolder();
string filename = System.IO.Path.Combine(tf, System.IO.Path.GetFileName(Program.DatabasePath));
System.IO.File.Copy(Program.DatabasePath, filename, true);
using (System.Data.IDbConnection con = (System.Data.IDbConnection)Activator.CreateInstance(SQLiteLoader.SQLiteConnectionType))
{
con.ConnectionString = "Data Source=" + filename;
//Open the database, handle any encryption issues automatically
Program.OpenDatabase(con);
using (System.Data.IDbCommand cmd = con.CreateCommand())
{
//Remove all log data to minimize the size of the database
cmd.CommandText = "DELETE FROM CommandQueue;";
cmd.ExecuteNonQuery();
cmd.CommandText = "DELETE FROM Log;";
cmd.ExecuteNonQuery();
cmd.CommandText = "DELETE FROM LogBlob;";
cmd.ExecuteNonQuery();
//Free up unused space
cmd.CommandText = "VACUUM;";
cmd.ExecuteNonQuery();
}
}
options["signature-control-files"] = filename;
}
options["full-if-sourcefolder-changed"] = "";
List<KeyValuePair<bool, string>> filters = new List<KeyValuePair<bool, string>>();
string[] sourceFolders = DynamicSetupHelper.GetSourceFolders(task.Task, new ApplicationSettings(task.Task.DataParent), filters);
if (options.ContainsKey("filter"))
filters.AddRange(Library.Utility.FilenameFilter.DecodeFilter(options["filter"]));
options["filter"] = Library.Utility.FilenameFilter.EncodeAsFilter(filters);
//At this point we register the backup as being in progress
((FullOrIncrementalTask)task).WriteBackupInProgress(Strings.DuplicatiRunner.ShutdownWhileBackupInprogress);
results = i.Backup(sourceFolders);
}
finally
{
if (tf != null)
tf.Dispose();
if (ProgressEvent != null)
ProgressEvent(DuplicatiOperation.Backup, RunnerState.Stopped, task.Schedule.Name, "", 100, -1);
}
break;
}
case DuplicityTaskType.ListBackups:
List<string> res = new List<string>();
foreach (ManifestEntry be in i.GetBackupSets())
{
res.Add(be.Time.ToString());
foreach (ManifestEntry bei in be.Incrementals)
res.Add(bei.Time.ToString());
}
(task as ListBackupsTask).Backups = res.ToArray();
break;
case DuplicityTaskType.ListBackupEntries:
(task as ListBackupEntriesTask).Backups = i.GetBackupSets();
break;
case DuplicityTaskType.ListFiles:
(task as ListFilesTask).Files = i.ListCurrentFiles();
break;
case DuplicityTaskType.ListSourceFolders:
(task as ListSourceFoldersTask).Files = new List<string>(i.ListSourceFolders() ?? new string[0]);
break;
case DuplicityTaskType.ListActualFiles:
(task as ListActualFilesTask).Files = i.ListActualSignatureFiles();
break;
case DuplicityTaskType.RemoveAllButNFull:
results = i.DeleteAllButNFull();
break;
case DuplicityTaskType.RemoveOlderThan:
results = i.DeleteOlderThan();
break;
case DuplicityTaskType.Restore:
options["file-to-restore"] = ((RestoreTask)task).SourceFiles;
if (options.ContainsKey("filter"))
options.Remove("filter");
try
{
if (ProgressEvent != null)
ProgressEvent(DuplicatiOperation.Restore, RunnerState.Started, task.Schedule.Name, "", 0, -1);
results = i.Restore(task.LocalPath.Split(System.IO.Path.PathSeparator));
}
finally
{
if (ProgressEvent != null)
ProgressEvent(DuplicatiOperation.Restore, RunnerState.Stopped, task.Schedule.Name, "", 100, -1);
}
break;
case DuplicityTaskType.RestoreSetup:
i.RestoreControlFiles(task.LocalPath);
break;
default:
return;
}
}
}
catch (Exception ex)
{
while (ex is System.Reflection.TargetInvocationException && ex.InnerException != null)
ex = ex.InnerException;
if (ex is System.Threading.ThreadAbortException)
{
m_isAborted = true;
System.Threading.Thread.ResetAbort();
}
else if (ex is Library.Main.LiveControl.ExecutionStoppedException)
m_isAborted = true;
if (m_isAborted && m_stopReason != System.Windows.Forms.CloseReason.None)
{
//If the user has stopped the backup for some reason, write a nicer message
switch (m_stopReason)
{
case System.Windows.Forms.CloseReason.ApplicationExitCall:
parsedMessage = Strings.DuplicatiRunner.ApplicationExitLogMesssage;
break;
case System.Windows.Forms.CloseReason.TaskManagerClosing:
parsedMessage = Strings.DuplicatiRunner.TaskManagerCloseMessage;
break;
case System.Windows.Forms.CloseReason.UserClosing:
parsedMessage = Strings.DuplicatiRunner.UserClosingMessage;
break;
case System.Windows.Forms.CloseReason.WindowsShutDown:
parsedMessage = Strings.DuplicatiRunner.WindowsShutdownMessage;
break;
default:
parsedMessage = string.Format(Strings.DuplicatiRunner.OtherAbortMessage, m_stopReason);
break;
}
if (task.Schedule != null)
{
//If the application is going down, the backup should resume on next launch
switch (m_stopReason)
{
case System.Windows.Forms.CloseReason.ApplicationExitCall:
case System.Windows.Forms.CloseReason.TaskManagerClosing:
case System.Windows.Forms.CloseReason.WindowsShutDown:
task.Schedule.ScheduledRunFailed();
break;
}
}
}
else
parsedMessage = string.Format(Strings.DuplicatiRunner.ErrorMessage, ex.Message);
results = "Error: " + ex.ToString(); //Don't localize
while (ex.InnerException != null)
{
ex = ex.InnerException;
results += Environment.NewLine + "InnerError: " + ex.ToString(); //Don't localize
}
}
finally
{
lock (m_lock)
m_currentBackupControlInterface = null;
}
try
{
if (!m_isAborted && (task.TaskType == DuplicityTaskType.FullBackup || task.TaskType == DuplicityTaskType.IncrementalBackup))
{
if (task.Schedule.Task.KeepFull > 0)
{
m_lastPGProgress = 100;
m_lastPGmessage = Strings.DuplicatiRunner.CleaningUpMessage;
m_lastPGSubmessage = "";
m_lastPGSubprogress = -1;
ReinvokeLastProgressEvent();
m_extraOperations--;
RemoveAllButNFullTask tmpTask = new RemoveAllButNFullTask(task.Schedule, (int)task.Schedule.Task.KeepFull);
ExecuteTask(tmpTask);
results += Environment.NewLine + Strings.DuplicatiRunner.CleanupLogdataHeader + Environment.NewLine + tmpTask.Result;
}
if (!string.IsNullOrEmpty(task.Schedule.Task.KeepTime))
{
m_lastPGProgress = 100;
m_lastPGmessage = Strings.DuplicatiRunner.CleaningUpMessage;
m_lastPGSubmessage = "";
m_lastPGSubprogress = -1;
ReinvokeLastProgressEvent();
m_extraOperations--;
RemoveOlderThanTask tmpTask = new RemoveOlderThanTask(task.Schedule, task.Schedule.Task.KeepTime);
ExecuteTask(tmpTask);
results += Environment.NewLine + Strings.DuplicatiRunner.CleanupLogdataHeader + Environment.NewLine + tmpTask.Result;
}
if (task.Schedule.Task.KeepFull > 0 || !string.IsNullOrEmpty(task.Schedule.Task.KeepTime))
ReinvokeLastProgressEvent();
if (ProgressEvent != null)
ProgressEvent(DuplicatiOperation.Backup, RunnerState.Stopped, task.Schedule.Name, "", 100, -1);
}
}
catch (Exception ex)
{
results += Environment.NewLine + string.Format(Strings.DuplicatiRunner.CleanupError, ex.Message);
}
task.IsAborted = m_isAborted;
task.Result = results;
task.RaiseTaskCompleted(results, parsedMessage);
if (ResultEvent != null && task is FullBackupTask || task is IncrementalBackupTask)
{
Log[] logs = Program.DataConnection.GetObjects<Log>("TaskID = ? AND SubAction LIKE ? ORDER BY EndTime DESC", task.Task.ID, "Primary");
if (logs != null && logs.Length > 0)
{
Datamodel.Log l = logs[0];
RunnerResult r = RunnerResult.Error;
if (l.ParsedStatus == DuplicatiOutputParser.ErrorStatus)
r = RunnerResult.Error;
else if (l.ParsedStatus == DuplicatiOutputParser.OKStatus || l.ParsedStatus == DuplicatiOutputParser.NoChangedFiles)
r = RunnerResult.OK;
else if (l.ParsedStatus == DuplicatiOutputParser.PartialStatus)
r = RunnerResult.Partial;
else if (l.ParsedStatus == DuplicatiOutputParser.WarningStatus)
r = RunnerResult.Warning;
ResultEvent(r, parsedMessage, results);
}
}
if (task.Schedule != null && !m_isAborted)
task.Schedule.ScheduledRunCompleted(); //Register as completed if not aborted
}
public static IList<string> ListCurrentFiles(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.ListCurrentFiles();
}
void Duplicati_OperationProgress(Interface caller, DuplicatiOperation operation, DuplicatiOperationMode specificmode, int progress, int subprogress, string message, string submessage)
{
m_lastPGOperation = operation;
m_lastPGMode = specificmode;
m_lastPGProgress = progress;
m_lastPGSubprogress = subprogress;
m_lastPGmessage = message;
m_lastPGSubmessage = submessage;
//If there are extra operations, reserve some space for it by reducing the displayed progress
if (m_extraOperations > 0 && progress > 0)
progress = (int)((m_lastPGProgress / 100.0) * (100 - (m_extraOperations * PERCENT_PR_EXTRA_OPERATION)));
if (ProgressEvent != null)
try { ProgressEvent(operation, RunnerState.Running, message, submessage, progress, subprogress); }
catch { }
}
/// <summary>
/// Event handler for the OperationProgres, used to store the last status message
/// </summary>
private void Interface_OperationProgress(Interface caller, DuplicatiOperation operation, DuplicatiOperationMode specificoperation, int progress, int subprogress, string message, string submessage)
{
m_lastProgressMessage = message;
}
