public void Run(string[] sources, Library.Utility.IFilter filter) { var storeSymlinks = m_options.SymlinkPolicy == Duplicati.Library.Main.Options.SymlinkStrategy.Store; var sourcefilter = new Library.Utility.FilterExpression(sources, true); using (var snapshot = BackupHandler.GetSnapshot(sources, m_options)) { foreach (var path in new BackupHandler.FilterHandler(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy).EnumerateFilesAndFolders()) { var fa = FileAttributes.Normal; try { fa = snapshot.GetAttributes(path); } catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); } if (storeSymlinks && snapshot.IsSymlink(path, fa)) { Logging.Log.WriteVerboseMessage(LOGTAG, "StoreSymlink", "Storing symlink: {0}", path); } else if ((fa & FileAttributes.Directory) == FileAttributes.Directory) { Logging.Log.WriteVerboseMessage(LOGTAG, "AddDirectory", "Including folder {0}", path); } else { m_result.FileCount++; var size = -1L; try { size = snapshot.GetFileSize(path); m_result.FileSize += size; } catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "SizeReadFailed", "Failed to read length of file {0}: {1}", path, ex.Message); } if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan) { Logging.Log.WriteVerboseMessage(LOGTAG, "IncludeFile", "Including file: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); } else { Logging.Log.WriteVerboseMessage(LOGTAG, "ExcludeLargeFile", "Excluding file due to size: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); } } } } }
public void Run(string[] sources, Library.Utility.IFilter filter) { var storeSymlinks = m_options.SymlinkPolicy == Duplicati.Library.Main.Options.SymlinkStrategy.Store; var sourcefilter = new Library.Utility.FilterExpression(sources, true); using (var snapshot = BackupHandler.GetSnapshot(sources, m_options, m_result)) { foreach (var path in new BackupHandler.FilterHandler(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_result).EnumerateFilesAndFolders()) { var fa = FileAttributes.Normal; try { fa = snapshot.GetAttributes(path); } catch { } if (storeSymlinks && ((fa & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint)) { m_result.AddVerboseMessage("Storing symlink: {0}", path); } else if ((fa & FileAttributes.Directory) == FileAttributes.Directory) { m_result.AddVerboseMessage("Including folder: {0}", path); } else { m_result.FileCount++; var size = -1L; try { size = snapshot.GetFileSize(path); m_result.FileSize += size; } catch { } if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan) { m_result.AddVerboseMessage("Including file: {0} ({1})", path, size <= 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); } else { m_result.AddVerboseMessage("Excluding file due to size: {0} ({1})", path, size <= 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); } } } } }
public Duplicati.Library.Interface.IBackupResults Backup(string[] inputsources, IFilter filter = null) { Library.UsageReporter.Reporter.Report("USE_BACKEND", new Library.Utility.Uri(m_backend).Scheme); Library.UsageReporter.Reporter.Report("USE_COMPRESSION", m_options.CompressionModule); Library.UsageReporter.Reporter.Report("USE_ENCRYPTION", m_options.EncryptionModule); return(RunAction(new BackupResults(), ref inputsources, ref filter, (result) => { if (inputsources == null || inputsources.Length == 0) { throw new Duplicati.Library.Interface.UserInformationException(Strings.Controller.NoSourceFoldersError); } var sources = new List <string>(inputsources); //Make sure they all have the same format and exist for (int i = 0; i < sources.Count; i++) { try { sources[i] = System.IO.Path.GetFullPath(sources[i]); } catch (Exception ex) { throw new Duplicati.Library.Interface.UserInformationException(Strings.Controller.InvalidPathError(sources[i], ex.Message), ex); } var fi = new System.IO.FileInfo(sources[i]); var di = new System.IO.DirectoryInfo(sources[i]); if (!(fi.Exists || di.Exists) && !m_options.AllowMissingSource) { throw new System.IO.IOException(Strings.Controller.SourceIsMissingError(sources[i])); } if (!fi.Exists) { sources[i] = Library.Utility.Utility.AppendDirSeparator(sources[i]); } } //Sanity check for duplicate files/folders var pathDuplicates = sources.GroupBy(x => x, Library.Utility.Utility.ClientFilenameStringComparer) .Where(g => g.Count() > 1).Select(y => y.Key).ToList(); foreach (var pathDuplicate in pathDuplicates) { result.AddVerboseMessage(string.Format("Removing duplicate source: {0}", pathDuplicate)); } sources = sources.Distinct(Library.Utility.Utility.ClientFilenameStringComparer).OrderBy(a => a).ToList(); //Sanity check for multiple inclusions of the same folder for (int i = 0; i < sources.Count; i++) { for (int j = 0; j < sources.Count; j++) { if (i != j && sources[i].StartsWith(sources[j], Library.Utility.Utility.ClientFilenameStringComparision)) { if (filter != null) { bool includes; bool excludes; FilterExpression.AnalyzeFilters(filter, out includes, out excludes); // If there are no excludes, there is no need to keep the folder as a filter if (excludes) { result.AddVerboseMessage("Removing source \"{0}\" because it is a subfolder of \"{1}\", and using it as an include filter", sources[i], sources[j]); filter = Library.Utility.JoinedFilterExpression.Join(new FilterExpression(sources[i]), filter); } else { result.AddVerboseMessage("Removing source \"{0}\" because it is a subfolder or subfile of \"{1}\"", sources[i], sources[j]); } } else { result.AddVerboseMessage("Removing source \"{0}\" because it is a subfolder or subfile of \"{1}\"", sources[i], sources[j]); } sources.RemoveAt(i); i--; break; } } } using (var h = new Operation.BackupHandler(m_backend, m_options, result)) h.Run(sources.ToArray(), filter); Library.UsageReporter.Reporter.Report("BACKUP_FILECOUNT", result.ExaminedFiles); Library.UsageReporter.Reporter.Report("BACKUP_FILESIZE", result.SizeOfExaminedFiles); Library.UsageReporter.Reporter.Report("BACKUP_DURATION", (long)result.Duration.TotalSeconds); })); }
public Duplicati.Library.Interface.IBackupResults Backup(string[] inputsources, IFilter filter = null) { return(RunAction(new BackupResults(), ref inputsources, (result) => { if (inputsources == null || inputsources.Length == 0) { throw new Exception(Strings.Controller.NoSourceFoldersError); } var sources = new List <string>(inputsources); //Make sure they all have the same format and exist for (int i = 0; i < sources.Count; i++) { try { sources[i] = System.IO.Path.GetFullPath(sources[i]); } catch (Exception ex) { throw new ArgumentException(Strings.Controller.InvalidPathError(sources[i], ex.Message), ex); } var fi = new System.IO.FileInfo(sources[i]); var di = new System.IO.DirectoryInfo(sources[i]); if (!(fi.Exists || di.Exists) && !m_options.AllowMissingSource) { throw new System.IO.IOException(Strings.Controller.SourceIsMissingError(sources[i])); } if (!fi.Exists) { sources[i] = Library.Utility.Utility.AppendDirSeparator(sources[i]); } } //Sanity check for duplicate folders and multiple inclusions of the same folder for (int i = 0; i < sources.Count - 1; i++) { for (int j = i + 1; j < sources.Count; j++) { if (sources[i].Equals(sources[j], Library.Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase)) { result.AddVerboseMessage("Removing duplicate source: {0}", sources[j]); sources.RemoveAt(j); j--; } else if (sources[i].StartsWith(sources[j], Library.Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase)) { result.AddVerboseMessage("Removing source \"{0}\" because it is a subfolder of \"{1}\"", sources[i], sources[j]); filter = Library.Utility.JoinedFilterExpression.Join(new FilterExpression(sources[i]), filter); sources.RemoveAt(i); i--; break; } } } using (var h = new Operation.BackupHandler(m_backend, m_options, result)) h.Run(sources.ToArray(), filter); })); }
public void Run(string[] sources, Library.Utility.IFilter filter) { var storeSymlinks = m_options.SymlinkPolicy == Duplicati.Library.Main.Options.SymlinkStrategy.Store; var sourcefilter = new Library.Utility.FilterExpression(sources, true); using (var snapshot = BackupHandler.GetSnapshot(sources, m_options)) using (new IsolatedChannelScope()) { var source = Operation.Backup.FileEnumerationProcess.Run(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_options.ExcludeEmptyFolders, m_options.IgnoreFilenames, null, m_result.TaskReader); var sink = CoCoL.AutomationExtensions.RunTask( new { source = Operation.Backup.Channels.SourcePaths.ForRead }, async self => { while (true) { var path = await self.source.ReadAsync(); var fa = FileAttributes.Normal; try { fa = snapshot.GetAttributes(path); } catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); } // Analyze symlinks var isSymlink = snapshot.IsSymlink(path, fa); string symlinkTarget = null; if (isSymlink) { try { symlinkTarget = snapshot.GetSymlinkTarget(path); } catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", path); } } if (isSymlink && m_options.SymlinkPolicy == Options.SymlinkStrategy.Store && !string.IsNullOrWhiteSpace(symlinkTarget)) { // Skip stored symlinks continue; } // Go for the symlink target, as we know we follow symlinks if (!string.IsNullOrWhiteSpace(symlinkTarget)) { path = symlinkTarget; fa = FileAttributes.Normal; try { fa = snapshot.GetAttributes(path); } catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); } } // Proceed with non-folders if (!((fa & FileAttributes.Directory) == FileAttributes.Directory)) { m_result.FileCount++; var size = -1L; try { size = snapshot.GetFileSize(path); m_result.FileSize += size; } catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "SizeReadFailed", "Failed to read length of file {0}: {1}", path, ex.Message); } if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan) { Logging.Log.WriteVerboseMessage(LOGTAG, "IncludeFile", "Including file: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); } else { Logging.Log.WriteVerboseMessage(LOGTAG, "ExcludeLargeFile", "Excluding file due to size: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); } } } } ); System.Threading.Tasks.Task.WhenAll(source, sink).WaitForTaskOrThrow(); } }