private TempFile coreDoGetPiping(FileEntryItem item, Interface.IEncryption useDecrypter, out long retDownloadSize, out string retHashcode) { // With piping allowed, we will parallelize the operation with buffered pipes to maximize throughput: // Separated: Download (only for streaming) - Hashing - Decryption // The idea is to use DirectStreamLink's that are inserted in the stream stack, creating a fork to run // the crypto operations on. retDownloadSize = -1; retHashcode = null; bool enableStreaming = (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers); System.Threading.Tasks.Task<string> taskHasher = null; DirectStreamLink linkForkHasher = null; System.Threading.Tasks.Task taskDecrypter = null; DirectStreamLink linkForkDecryptor = null; // keep potential temp files and their streams for cleanup (cannot use using here). TempFile retTarget = null, dlTarget = null, decryptTarget = null; System.IO.Stream dlToStream = null, decryptToStream = null; try { System.IO.Stream nextTierWriter = null; // target of our stacked streams if (!enableStreaming) // we will always need dlTarget if not streaming... dlTarget = new TempFile(); else if (enableStreaming && useDecrypter == null) { dlTarget = new TempFile(); dlToStream = System.IO.File.OpenWrite(dlTarget); nextTierWriter = dlToStream; // actually write through to file. } // setup decryption: fork off a StreamLink from stack, and setup decryptor task if (useDecrypter != null) { linkForkDecryptor = new DirectStreamLink(1 << 16, false, false, nextTierWriter); nextTierWriter = linkForkDecryptor.WriterStream; linkForkDecryptor.SetKnownLength(item.Size, false); // Set length to allow AES-decryption (not streamable yet) decryptTarget = new TempFile(); decryptToStream = System.IO.File.OpenWrite(decryptTarget); taskDecrypter = new System.Threading.Tasks.Task(() => { using (var input = linkForkDecryptor.ReaderStream) using (var output = decryptToStream) lock (m_encryptionLock) { useDecrypter.Decrypt(input, output); } } ); } // setup hashing: fork off a StreamLink from stack, then task computes hash linkForkHasher = new DirectStreamLink(1 << 16, false, false, nextTierWriter); nextTierWriter = linkForkHasher.WriterStream; taskHasher = new System.Threading.Tasks.Task<string>(() => { using (var input = linkForkHasher.ReaderStream) return CalculateFileHash(input); } ); // OK, forks with tasks are set up, so let's do the download which is performed in main thread. bool hadException = false; try { if (enableStreaming) { using (var ss = new ShaderStream(nextTierWriter, false)) { using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond)) using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress)) { taskHasher.Start(); // We do not start tasks earlier to be sure the input always gets closed. if (taskDecrypter != null) taskDecrypter.Start(); ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs); } retDownloadSize = ss.TotalBytesWritten; } } else { m_backend.Get(item.RemoteFilename, dlTarget); retDownloadSize = new System.IO.FileInfo(dlTarget).Length; using (dlToStream = System.IO.File.OpenRead(dlTarget)) { taskHasher.Start(); // We do not start tasks earlier to be sure the input always gets closed. if (taskDecrypter != null) taskDecrypter.Start(); new DirectStreamLink.DataPump(dlToStream, nextTierWriter).Run(); } } } catch (Exception) { hadException = true; throw; } finally { // This nested try-catch-finally blocks will make sure we do not miss any exceptions ans all started tasks // are properly ended and tidied up. For what is thrown: If exceptions in main thread occured (download) it is thrown, // then hasher task is checked and last decryption. This resembles old logic. try { retHashcode = taskHasher.Result; } catch (AggregateException ex) { if (!hadException) { hadException = true; throw ex.InnerExceptions[0]; } } finally { if (taskDecrypter != null) { try { taskDecrypter.Wait(); } catch (AggregateException ex) { if (!hadException) { hadException = true; if (ex.InnerExceptions[0] is System.Security.Cryptography.CryptographicException) throw ex.InnerExceptions[0]; else throw new System.Security.Cryptography.CryptographicException(ex.InnerExceptions[0].Message, ex.InnerExceptions[0]); } } } } } if (useDecrypter != null) // return decrypted temp file { retTarget = decryptTarget; decryptTarget = null; } else // return downloaded file { retTarget = dlTarget; dlTarget = null; } } finally { // Be tidy: manually do some cleanup to temp files, as we could not use using's. // Unclosed streams should only occur if we failed even before tasks were started. if (dlToStream != null) dlToStream.Dispose(); if (dlTarget != null) dlTarget.Dispose(); if (decryptToStream != null) decryptToStream.Dispose(); if (decryptTarget != null) decryptTarget.Dispose(); } return retTarget; }
private TempFile coreDoGetSequential(FileEntryItem item, Interface.IEncryption useDecrypter, out long retDownloadSize, out string retHashcode) { retHashcode = null; retDownloadSize = -1; TempFile retTarget, dlTarget = null, decryptTarget = null; try { dlTarget = new Library.Utility.TempFile(); if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers) { Func<string> getFileHash; // extended to use stacked streams using (var fs = System.IO.File.OpenWrite(dlTarget)) using (var hs = GetFileHasherStream(fs, System.Security.Cryptography.CryptoStreamMode.Write, out getFileHash)) using (var ss = new ShaderStream(hs, true)) { using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond)) using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress)) { ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs); } ss.Flush(); retDownloadSize = ss.TotalBytesWritten; retHashcode = getFileHash(); } } else { m_backend.Get(item.RemoteFilename, dlTarget); retDownloadSize = new System.IO.FileInfo(dlTarget).Length; retHashcode = CalculateFileHash(dlTarget); } // Decryption is not placed in the stream stack because there seemed to be an effort // to throw a CryptographicException on fail. If in main stack, we cannot differentiate // in which part of the stack the source of an exception resides. if (useDecrypter != null) { decryptTarget = new Library.Utility.TempFile(); lock (m_encryptionLock) { try { useDecrypter.Decrypt(dlTarget, decryptTarget); } // If we fail here, make sure that we throw a crypto exception catch (System.Security.Cryptography.CryptographicException) { throw; } catch (Exception ex) { throw new System.Security.Cryptography.CryptographicException(ex.Message, ex); } } retTarget = decryptTarget; decryptTarget = null; } else { retTarget = dlTarget; dlTarget = null; } } finally { if (dlTarget != null) dlTarget.Dispose(); if (decryptTarget != null) decryptTarget.Dispose(); } return retTarget; }
public static string RestoreControlFiles(string source, string target, Dictionary<string, string> options) { using (Interface i = new Interface(source, options)) return i.RestoreControlFiles(target); }
public static List<KeyValuePair<BackupEntryBase, Exception>> VerifyBackup(string target, Dictionary<string, string> options) { using (Interface i = new Interface(target, options)) return i.VerifyBackupChain(); }
public static string[] ListSourceFolders(string target, Dictionary<string, string> options) { using (Interface i = new Interface(target, options)) return i.ListSourceFolders(); }
/// <summary> /// Returns a list of full backup manifests, with a list of volumes and an incremental chain /// </summary> /// <param name="target">The backend to read the data from</param> /// <param name="options">A list of options</param> /// <returns>A list of full backup manifests</returns> public static List<ManifestEntry> ParseFileList(string target, Dictionary<string, string> options) { using (Interface i = new Interface(target, options)) return i.GetBackupSets(); }
public static List<KeyValuePair<RSync.RSyncDir.PatchFileType, string>> ListActualSignatureFiles(string target, Dictionary<string, string> options) { using (Interface i = new Interface(target, options)) return i.ListActualSignatureFiles(); }
public static string Backup(string[] source, string target, Dictionary<string, string> options) { using (Interface i = new Interface(target, options)) return i.Backup(source); }
public static string DeleteOlderThan(string target, Dictionary<string, string> options) { using (Interface i = new Interface(target, options)) return i.DeleteOlderThan(); }
public static List<KeyValuePair<string, DateTime>> FindLastFileVersion(string target, Dictionary<string, string> options) { using (Interface i = new Interface(target, options)) return i.FindLastFileVersion(); }
public static string DeleteAllButNFull(string target, Dictionary<string, string> options) { using (Interface i = new Interface(target, options)) return i.DeleteAllButNFull(); }
public static void CreateFolder(string target, Dictionary<string, string> options) { using (Interface i = new Interface(target, options)) i.CreateFolder(); }
public static string Cleanup(string target, Dictionary<string, string> options) { using (Interface i = new Interface(target, options)) return i.Cleanup(); }
public void ExecuteTask(IDuplicityTask task) { Dictionary<string, string> options = new Dictionary<string,string>(); //Set the log level to be that of the GUI options["log-level"] = Duplicati.Library.Logging.Log.LogLevel.ToString(); string destination = task.GetConfiguration(options); string results = ""; string parsedMessage = ""; m_isAborted = false; try { //TODO: Its a bit dirty to set the options after creating the instance using (Interface i = new Interface(destination, options)) { lock (m_lock) { m_stopReason = System.Windows.Forms.CloseReason.None; m_currentBackupControlInterface = i; } SetupControlInterface(); i.OperationProgress += new OperationProgressEvent(Duplicati_OperationProgress); switch (task.TaskType) { case DuplicityTaskType.FullBackup: case DuplicityTaskType.IncrementalBackup: { //Activate auto-cleanup options["auto-cleanup"] = ""; options["force"] = ""; if (task.Schedule.Task.KeepFull > 0) m_extraOperations++; if (!string.IsNullOrEmpty(task.Schedule.Task.KeepTime)) m_extraOperations++; Library.Utility.TempFolder tf = null; try { if (ProgressEvent != null) ProgressEvent(DuplicatiOperation.Backup, RunnerState.Started, task.Schedule.Name, "", 0, -1); if (task.Task.IncludeSetup) { //Make a copy of the current database tf = new Duplicati.Library.Utility.TempFolder(); string filename = System.IO.Path.Combine(tf, System.IO.Path.GetFileName(Program.DatabasePath)); System.IO.File.Copy(Program.DatabasePath, filename, true); using (System.Data.IDbConnection con = (System.Data.IDbConnection)Activator.CreateInstance(SQLiteLoader.SQLiteConnectionType)) { con.ConnectionString = "Data Source=" + filename; //Open the database, handle any encryption issues automatically Program.OpenDatabase(con); using (System.Data.IDbCommand cmd = con.CreateCommand()) { //Remove all log data to minimize the size of the database cmd.CommandText = "DELETE FROM CommandQueue;"; cmd.ExecuteNonQuery(); cmd.CommandText = "DELETE FROM Log;"; cmd.ExecuteNonQuery(); cmd.CommandText = "DELETE FROM LogBlob;"; cmd.ExecuteNonQuery(); //Free up unused space cmd.CommandText = "VACUUM;"; cmd.ExecuteNonQuery(); } } options["signature-control-files"] = filename; } options["full-if-sourcefolder-changed"] = ""; List<KeyValuePair<bool, string>> filters = new List<KeyValuePair<bool, string>>(); string[] sourceFolders = DynamicSetupHelper.GetSourceFolders(task.Task, new ApplicationSettings(task.Task.DataParent), filters); if (options.ContainsKey("filter")) filters.AddRange(Library.Utility.FilenameFilter.DecodeFilter(options["filter"])); options["filter"] = Library.Utility.FilenameFilter.EncodeAsFilter(filters); //At this point we register the backup as being in progress ((FullOrIncrementalTask)task).WriteBackupInProgress(Strings.DuplicatiRunner.ShutdownWhileBackupInprogress); results = i.Backup(sourceFolders); } finally { if (tf != null) tf.Dispose(); if (ProgressEvent != null) ProgressEvent(DuplicatiOperation.Backup, RunnerState.Stopped, task.Schedule.Name, "", 100, -1); } break; } case DuplicityTaskType.ListBackups: List<string> res = new List<string>(); foreach (ManifestEntry be in i.GetBackupSets()) { res.Add(be.Time.ToString()); foreach (ManifestEntry bei in be.Incrementals) res.Add(bei.Time.ToString()); } (task as ListBackupsTask).Backups = res.ToArray(); break; case DuplicityTaskType.ListBackupEntries: (task as ListBackupEntriesTask).Backups = i.GetBackupSets(); break; case DuplicityTaskType.ListFiles: (task as ListFilesTask).Files = i.ListCurrentFiles(); break; case DuplicityTaskType.ListSourceFolders: (task as ListSourceFoldersTask).Files = new List<string>(i.ListSourceFolders() ?? new string[0]); break; case DuplicityTaskType.ListActualFiles: (task as ListActualFilesTask).Files = i.ListActualSignatureFiles(); break; case DuplicityTaskType.RemoveAllButNFull: results = i.DeleteAllButNFull(); break; case DuplicityTaskType.RemoveOlderThan: results = i.DeleteOlderThan(); break; case DuplicityTaskType.Restore: options["file-to-restore"] = ((RestoreTask)task).SourceFiles; if (options.ContainsKey("filter")) options.Remove("filter"); try { if (ProgressEvent != null) ProgressEvent(DuplicatiOperation.Restore, RunnerState.Started, task.Schedule.Name, "", 0, -1); results = i.Restore(task.LocalPath.Split(System.IO.Path.PathSeparator)); } finally { if (ProgressEvent != null) ProgressEvent(DuplicatiOperation.Restore, RunnerState.Stopped, task.Schedule.Name, "", 100, -1); } break; case DuplicityTaskType.RestoreSetup: i.RestoreControlFiles(task.LocalPath); break; default: return; } } } catch (Exception ex) { while (ex is System.Reflection.TargetInvocationException && ex.InnerException != null) ex = ex.InnerException; if (ex is System.Threading.ThreadAbortException) { m_isAborted = true; System.Threading.Thread.ResetAbort(); } else if (ex is Library.Main.LiveControl.ExecutionStoppedException) m_isAborted = true; if (m_isAborted && m_stopReason != System.Windows.Forms.CloseReason.None) { //If the user has stopped the backup for some reason, write a nicer message switch (m_stopReason) { case System.Windows.Forms.CloseReason.ApplicationExitCall: parsedMessage = Strings.DuplicatiRunner.ApplicationExitLogMesssage; break; case System.Windows.Forms.CloseReason.TaskManagerClosing: parsedMessage = Strings.DuplicatiRunner.TaskManagerCloseMessage; break; case System.Windows.Forms.CloseReason.UserClosing: parsedMessage = Strings.DuplicatiRunner.UserClosingMessage; break; case System.Windows.Forms.CloseReason.WindowsShutDown: parsedMessage = Strings.DuplicatiRunner.WindowsShutdownMessage; break; default: parsedMessage = string.Format(Strings.DuplicatiRunner.OtherAbortMessage, m_stopReason); break; } if (task.Schedule != null) { //If the application is going down, the backup should resume on next launch switch (m_stopReason) { case System.Windows.Forms.CloseReason.ApplicationExitCall: case System.Windows.Forms.CloseReason.TaskManagerClosing: case System.Windows.Forms.CloseReason.WindowsShutDown: task.Schedule.ScheduledRunFailed(); break; } } } else parsedMessage = string.Format(Strings.DuplicatiRunner.ErrorMessage, ex.Message); results = "Error: " + ex.ToString(); //Don't localize while (ex.InnerException != null) { ex = ex.InnerException; results += Environment.NewLine + "InnerError: " + ex.ToString(); //Don't localize } } finally { lock (m_lock) m_currentBackupControlInterface = null; } try { if (!m_isAborted && (task.TaskType == DuplicityTaskType.FullBackup || task.TaskType == DuplicityTaskType.IncrementalBackup)) { if (task.Schedule.Task.KeepFull > 0) { m_lastPGProgress = 100; m_lastPGmessage = Strings.DuplicatiRunner.CleaningUpMessage; m_lastPGSubmessage = ""; m_lastPGSubprogress = -1; ReinvokeLastProgressEvent(); m_extraOperations--; RemoveAllButNFullTask tmpTask = new RemoveAllButNFullTask(task.Schedule, (int)task.Schedule.Task.KeepFull); ExecuteTask(tmpTask); results += Environment.NewLine + Strings.DuplicatiRunner.CleanupLogdataHeader + Environment.NewLine + tmpTask.Result; } if (!string.IsNullOrEmpty(task.Schedule.Task.KeepTime)) { m_lastPGProgress = 100; m_lastPGmessage = Strings.DuplicatiRunner.CleaningUpMessage; m_lastPGSubmessage = ""; m_lastPGSubprogress = -1; ReinvokeLastProgressEvent(); m_extraOperations--; RemoveOlderThanTask tmpTask = new RemoveOlderThanTask(task.Schedule, task.Schedule.Task.KeepTime); ExecuteTask(tmpTask); results += Environment.NewLine + Strings.DuplicatiRunner.CleanupLogdataHeader + Environment.NewLine + tmpTask.Result; } if (task.Schedule.Task.KeepFull > 0 || !string.IsNullOrEmpty(task.Schedule.Task.KeepTime)) ReinvokeLastProgressEvent(); if (ProgressEvent != null) ProgressEvent(DuplicatiOperation.Backup, RunnerState.Stopped, task.Schedule.Name, "", 100, -1); } } catch (Exception ex) { results += Environment.NewLine + string.Format(Strings.DuplicatiRunner.CleanupError, ex.Message); } task.IsAborted = m_isAborted; task.Result = results; task.RaiseTaskCompleted(results, parsedMessage); if (ResultEvent != null && task is FullBackupTask || task is IncrementalBackupTask) { Log[] logs = Program.DataConnection.GetObjects<Log>("TaskID = ? AND SubAction LIKE ? ORDER BY EndTime DESC", task.Task.ID, "Primary"); if (logs != null && logs.Length > 0) { Datamodel.Log l = logs[0]; RunnerResult r = RunnerResult.Error; if (l.ParsedStatus == DuplicatiOutputParser.ErrorStatus) r = RunnerResult.Error; else if (l.ParsedStatus == DuplicatiOutputParser.OKStatus || l.ParsedStatus == DuplicatiOutputParser.NoChangedFiles) r = RunnerResult.OK; else if (l.ParsedStatus == DuplicatiOutputParser.PartialStatus) r = RunnerResult.Partial; else if (l.ParsedStatus == DuplicatiOutputParser.WarningStatus) r = RunnerResult.Warning; ResultEvent(r, parsedMessage, results); } } if (task.Schedule != null && !m_isAborted) task.Schedule.ScheduledRunCompleted(); //Register as completed if not aborted }
public static IList<string> ListCurrentFiles(string target, Dictionary<string, string> options) { using (Interface i = new Interface(target, options)) return i.ListCurrentFiles(); }
void Duplicati_OperationProgress(Interface caller, DuplicatiOperation operation, DuplicatiOperationMode specificmode, int progress, int subprogress, string message, string submessage) { m_lastPGOperation = operation; m_lastPGMode = specificmode; m_lastPGProgress = progress; m_lastPGSubprogress = subprogress; m_lastPGmessage = message; m_lastPGSubmessage = submessage; //If there are extra operations, reserve some space for it by reducing the displayed progress if (m_extraOperations > 0 && progress > 0) progress = (int)((m_lastPGProgress / 100.0) * (100 - (m_extraOperations * PERCENT_PR_EXTRA_OPERATION))); if (ProgressEvent != null) try { ProgressEvent(operation, RunnerState.Running, message, submessage, progress, subprogress); } catch { } }
/// <summary> /// Event handler for the OperationProgres, used to store the last status message /// </summary> private void Interface_OperationProgress(Interface caller, DuplicatiOperation operation, DuplicatiOperationMode specificoperation, int progress, int subprogress, string message, string submessage) { m_lastProgressMessage = message; }