public void Run(string[] sources, Library.Utility.IFilter filter)
{
var storeSymlinks = m_options.SymlinkPolicy == Duplicati.Library.Main.Options.SymlinkStrategy.Store;
var sourcefilter = new Library.Utility.FilterExpression(sources, true);
using (var snapshot = BackupHandler.GetSnapshot(sources, m_options))
{
foreach (var path in new BackupHandler.FilterHandler(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy).EnumerateFilesAndFolders())
{
var fa = FileAttributes.Normal;
try { fa = snapshot.GetAttributes(path); }
catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); }
if (storeSymlinks && snapshot.IsSymlink(path, fa))
{
Logging.Log.WriteVerboseMessage(LOGTAG, "StoreSymlink", "Storing symlink: {0}", path);
}
else if ((fa & FileAttributes.Directory) == FileAttributes.Directory)
{
Logging.Log.WriteVerboseMessage(LOGTAG, "AddDirectory", "Including folder {0}", path);
}
else
{
m_result.FileCount++;
var size = -1L;
try
{
size = snapshot.GetFileSize(path);
m_result.FileSize += size;
}
catch (Exception ex)
{
Logging.Log.WriteVerboseMessage(LOGTAG, "SizeReadFailed", "Failed to read length of file {0}: {1}", path, ex.Message);
}
if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan)
{
Logging.Log.WriteVerboseMessage(LOGTAG, "IncludeFile", "Including file: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
}
else
{
Logging.Log.WriteVerboseMessage(LOGTAG, "ExcludeLargeFile", "Excluding file due to size: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
}
}
}
}
}
public void Run(string[] sources, Library.Utility.IFilter filter)
{
var storeSymlinks = m_options.SymlinkPolicy == Duplicati.Library.Main.Options.SymlinkStrategy.Store;
var sourcefilter = new Library.Utility.FilterExpression(sources, true);
using (var snapshot = BackupHandler.GetSnapshot(sources, m_options, m_result))
{
foreach (var path in new BackupHandler.FilterHandler(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_result).EnumerateFilesAndFolders())
{
var fa = FileAttributes.Normal;
try { fa = snapshot.GetAttributes(path); }
catch { }
if (storeSymlinks && ((fa & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint))
{
m_result.AddVerboseMessage("Storing symlink: {0}", path);
}
else if ((fa & FileAttributes.Directory) == FileAttributes.Directory)
{
m_result.AddVerboseMessage("Including folder: {0}", path);
}
else
{
m_result.FileCount++;
var size = -1L;
try
{
size = snapshot.GetFileSize(path);
m_result.FileSize += size;
}
catch
{
}
if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan)
{
m_result.AddVerboseMessage("Including file: {0} ({1})", path, size <= 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
}
else
{
m_result.AddVerboseMessage("Excluding file due to size: {0} ({1})", path, size <= 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
}
}
}
}
}
public Duplicati.Library.Interface.IBackupResults Backup(string[] inputsources, IFilter filter = null)
{
Library.UsageReporter.Reporter.Report("USE_BACKEND", new Library.Utility.Uri(m_backend).Scheme);
Library.UsageReporter.Reporter.Report("USE_COMPRESSION", m_options.CompressionModule);
Library.UsageReporter.Reporter.Report("USE_ENCRYPTION", m_options.EncryptionModule);
return(RunAction(new BackupResults(), ref inputsources, ref filter, (result) => {
if (inputsources == null || inputsources.Length == 0)
{
throw new Duplicati.Library.Interface.UserInformationException(Strings.Controller.NoSourceFoldersError);
}
var sources = new List <string>(inputsources);
//Make sure they all have the same format and exist
for (int i = 0; i < sources.Count; i++)
{
try
{
sources[i] = System.IO.Path.GetFullPath(sources[i]);
}
catch (Exception ex)
{
throw new Duplicati.Library.Interface.UserInformationException(Strings.Controller.InvalidPathError(sources[i], ex.Message), ex);
}
var fi = new System.IO.FileInfo(sources[i]);
var di = new System.IO.DirectoryInfo(sources[i]);
if (!(fi.Exists || di.Exists) && !m_options.AllowMissingSource)
{
throw new System.IO.IOException(Strings.Controller.SourceIsMissingError(sources[i]));
}
if (!fi.Exists)
{
sources[i] = Library.Utility.Utility.AppendDirSeparator(sources[i]);
}
}
//Sanity check for duplicate files/folders
var pathDuplicates = sources.GroupBy(x => x, Library.Utility.Utility.ClientFilenameStringComparer)
.Where(g => g.Count() > 1).Select(y => y.Key).ToList();
foreach (var pathDuplicate in pathDuplicates)
{
result.AddVerboseMessage(string.Format("Removing duplicate source: {0}", pathDuplicate));
}
sources = sources.Distinct(Library.Utility.Utility.ClientFilenameStringComparer).OrderBy(a => a).ToList();
//Sanity check for multiple inclusions of the same folder
for (int i = 0; i < sources.Count; i++)
{
for (int j = 0; j < sources.Count; j++)
{
if (i != j && sources[i].StartsWith(sources[j], Library.Utility.Utility.ClientFilenameStringComparision))
{
if (filter != null)
{
bool includes;
bool excludes;
FilterExpression.AnalyzeFilters(filter, out includes, out excludes);
// If there are no excludes, there is no need to keep the folder as a filter
if (excludes)
{
result.AddVerboseMessage("Removing source \"{0}\" because it is a subfolder of \"{1}\", and using it as an include filter", sources[i], sources[j]);
filter = Library.Utility.JoinedFilterExpression.Join(new FilterExpression(sources[i]), filter);
}
else
{
result.AddVerboseMessage("Removing source \"{0}\" because it is a subfolder or subfile of \"{1}\"", sources[i], sources[j]);
}
}
else
{
result.AddVerboseMessage("Removing source \"{0}\" because it is a subfolder or subfile of \"{1}\"", sources[i], sources[j]);
}
sources.RemoveAt(i);
i--;
break;
}
}
}
using (var h = new Operation.BackupHandler(m_backend, m_options, result))
h.Run(sources.ToArray(), filter);
Library.UsageReporter.Reporter.Report("BACKUP_FILECOUNT", result.ExaminedFiles);
Library.UsageReporter.Reporter.Report("BACKUP_FILESIZE", result.SizeOfExaminedFiles);
Library.UsageReporter.Reporter.Report("BACKUP_DURATION", (long)result.Duration.TotalSeconds);
}));
}
public Duplicati.Library.Interface.IBackupResults Backup(string[] inputsources, IFilter filter = null)
{
return(RunAction(new BackupResults(), ref inputsources, (result) => {
if (inputsources == null || inputsources.Length == 0)
{
throw new Exception(Strings.Controller.NoSourceFoldersError);
}
var sources = new List <string>(inputsources);
//Make sure they all have the same format and exist
for (int i = 0; i < sources.Count; i++)
{
try
{
sources[i] = System.IO.Path.GetFullPath(sources[i]);
}
catch (Exception ex)
{
throw new ArgumentException(Strings.Controller.InvalidPathError(sources[i], ex.Message), ex);
}
var fi = new System.IO.FileInfo(sources[i]);
var di = new System.IO.DirectoryInfo(sources[i]);
if (!(fi.Exists || di.Exists) && !m_options.AllowMissingSource)
{
throw new System.IO.IOException(Strings.Controller.SourceIsMissingError(sources[i]));
}
if (!fi.Exists)
{
sources[i] = Library.Utility.Utility.AppendDirSeparator(sources[i]);
}
}
//Sanity check for duplicate folders and multiple inclusions of the same folder
for (int i = 0; i < sources.Count - 1; i++)
{
for (int j = i + 1; j < sources.Count; j++)
{
if (sources[i].Equals(sources[j], Library.Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase))
{
result.AddVerboseMessage("Removing duplicate source: {0}", sources[j]);
sources.RemoveAt(j);
j--;
}
else if (sources[i].StartsWith(sources[j], Library.Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase))
{
result.AddVerboseMessage("Removing source \"{0}\" because it is a subfolder of \"{1}\"", sources[i], sources[j]);
filter = Library.Utility.JoinedFilterExpression.Join(new FilterExpression(sources[i]), filter);
sources.RemoveAt(i);
i--;
break;
}
}
}
using (var h = new Operation.BackupHandler(m_backend, m_options, result))
h.Run(sources.ToArray(), filter);
}));
}
public void Run(string[] sources, Library.Utility.IFilter filter)
{
var storeSymlinks = m_options.SymlinkPolicy == Duplicati.Library.Main.Options.SymlinkStrategy.Store;
var sourcefilter = new Library.Utility.FilterExpression(sources, true);
using (var snapshot = BackupHandler.GetSnapshot(sources, m_options))
using (new IsolatedChannelScope())
{
var source = Operation.Backup.FileEnumerationProcess.Run(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_options.ExcludeEmptyFolders, m_options.IgnoreFilenames, null, m_result.TaskReader);
var sink = CoCoL.AutomationExtensions.RunTask(
new { source = Operation.Backup.Channels.SourcePaths.ForRead },
async self => {
while (true)
{
var path = await self.source.ReadAsync();
var fa = FileAttributes.Normal;
try { fa = snapshot.GetAttributes(path); }
catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); }
// Analyze symlinks
var isSymlink = snapshot.IsSymlink(path, fa);
string symlinkTarget = null;
if (isSymlink)
{
try { symlinkTarget = snapshot.GetSymlinkTarget(path); }
catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "SymlinkTargetReadFailure", ex, "Failed to read symlink target for path: {0}", path); }
}
if (isSymlink && m_options.SymlinkPolicy == Options.SymlinkStrategy.Store && !string.IsNullOrWhiteSpace(symlinkTarget))
{
// Skip stored symlinks
continue;
}
// Go for the symlink target, as we know we follow symlinks
if (!string.IsNullOrWhiteSpace(symlinkTarget))
{
path = symlinkTarget;
fa = FileAttributes.Normal;
try { fa = snapshot.GetAttributes(path); }
catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); }
}
// Proceed with non-folders
if (!((fa & FileAttributes.Directory) == FileAttributes.Directory))
{
m_result.FileCount++;
var size = -1L;
try
{
size = snapshot.GetFileSize(path);
m_result.FileSize += size;
}
catch (Exception ex)
{
Logging.Log.WriteVerboseMessage(LOGTAG, "SizeReadFailed", "Failed to read length of file {0}: {1}", path, ex.Message);
}
if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan)
{
Logging.Log.WriteVerboseMessage(LOGTAG, "IncludeFile", "Including file: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
}
else
{
Logging.Log.WriteVerboseMessage(LOGTAG, "ExcludeLargeFile", "Excluding file due to size: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
}
}
}
}
);
System.Threading.Tasks.Task.WhenAll(source, sink).WaitForTaskOrThrow();
}
}
