public BackupHandler(string backendurl, Options options, BackupResults results)
{
EMPTY_METADATA = Utility.WrapMetadata(new Dictionary<string, string>(), options);
m_options = options;
m_result = results;
m_backendurl = backendurl;
m_attributeFilter = m_options.FileAttributeFilter;
m_symlinkPolicy = m_options.SymlinkPolicy;
m_blocksize = m_options.Blocksize;
m_blockbuffer = new byte[m_options.Blocksize * Math.Max(1, m_options.FileReadBufferSize / m_options.Blocksize)];
m_blocklistbuffer = new byte[m_options.Blocksize];
m_blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
m_filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);
if (m_blockhasher == null)
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm));
if (m_filehasher == null)
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.FileHashAlgorithm));
if (!m_blockhasher.CanReuseTransform)
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm));
if (!m_filehasher.CanReuseTransform)
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.FileHashAlgorithm));
if (options.AllowPassphraseChange)
throw new Exception(Strings.Foresthash.PassphraseChangeUnsupported);
}
public BackupHandler(string backendurl, Options options, BackupResults results)
{
EMPTY_METADATA = Utility.WrapMetadata(new Dictionary<string, string>(), options);
m_options = options;
m_result = results;
m_backendurl = backendurl;
m_attributeFilter = m_options.FileAttributeFilter;
m_symlinkPolicy = m_options.SymlinkPolicy;
if (options.AllowPassphraseChange)
throw new Exception(Strings.Foresthash.PassphraseChangeUnsupported);
}
public FilterHandler(Snapshots.ISnapshotService snapshot, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter filter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, ILogWriter logWriter)
{
m_snapshot = snapshot;
m_attributeFilter = attributeFilter;
m_sourcefilter = sourcefilter;
m_emitfilter = filter;
m_symlinkPolicy = symlinkPolicy;
m_hardlinkPolicy = hardlinkPolicy;
m_logWriter = logWriter;
m_hardlinkmap = new Dictionary<string, string>();
m_mixinqueue = new Queue<string>();
bool includes;
bool excludes;
Library.Utility.FilterExpression.AnalyzeFilters(filter, out includes, out excludes);
if (includes && !excludes)
{
m_enumeratefilter = Library.Utility.FilterExpression.Combine(filter, new Duplicati.Library.Utility.FilterExpression("*" + System.IO.Path.DirectorySeparatorChar, true));
}
else
m_enumeratefilter = m_emitfilter;
}
public static Task Run(IEnumerable <string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, string[] changedfilelist, ITaskReader taskreader)
{
return(AutomationExtensions.RunTask(
new
{
Output = Backup.Channels.SourcePaths.ForWrite
},
async self =>
{
var hardlinkmap = new Dictionary <string, string>();
var mixinqueue = new Queue <string>();
Duplicati.Library.Utility.IFilter enumeratefilter = emitfilter;
bool includes;
bool excludes;
Library.Utility.FilterExpression.AnalyzeFilters(emitfilter, out includes, out excludes);
if (includes && !excludes)
{
enumeratefilter = Library.Utility.FilterExpression.Combine(emitfilter, new Duplicati.Library.Utility.FilterExpression("*" + System.IO.Path.DirectorySeparatorChar, true));
}
// Simplify checking for an empty list
if (ignorenames != null && ignorenames.Length == 0)
{
ignorenames = null;
}
// If we have a specific list, use that instead of enumerating the filesystem
IEnumerable <string> worklist;
if (changedfilelist != null && changedfilelist.Length > 0)
{
worklist = changedfilelist.Where(x =>
{
var fa = FileAttributes.Normal;
try
{
fa = snapshot.GetAttributes(x);
}
catch
{
}
return AttributeFilter(x, fa, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue);
});
}
else
{
Library.Utility.Utility.EnumerationFilterDelegate attributeFilter = (root, path, attr) =>
AttributeFilter(path, attr, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue);
if (journalService != null)
{
// filter sources using USN journal, to obtain a sub-set of files / folders that may have been modified
sources = journalService.GetModifiedSources(attributeFilter);
}
worklist = snapshot.EnumerateFilesAndFolders(sources, attributeFilter, (rootpath, errorpath, ex) =>
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", errorpath);
});
}
var source = ExpandWorkList(worklist, mixinqueue, emitfilter, enumeratefilter);
if (excludeemptyfolders)
{
source = ExcludeEmptyFolders(source);
}
// Process each path, and dequeue the mixins with symlinks as we go
foreach (var s in source)
{
if (!await taskreader.ProgressAsync)
{
return;
}
await self.Output.WriteAsync(s);
}
}));
}
/// <summary>
/// Plugin filter for enumerating a list of files.
/// </summary>
/// <returns>True if the path should be returned, false otherwise.</returns>
/// <param name="path">The current path.</param>
/// <param name="attributes">The file or folder attributes.</param>
private static bool AttributeFilter(string path, FileAttributes attributes, Snapshots.ISnapshotService snapshot, Library.Utility.IFilter sourcefilter, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary <string, string> hardlinkmap, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue <string> mixinqueue)
{
// Step 1, exclude block devices
try
{
if (snapshot.IsBlockDevice(path))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlockDevice", "Excluding block device: {0}", path);
return(false);
}
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path);
return(false);
}
// Check if we explicitly include this entry
Duplicati.Library.Utility.IFilter sourcematch;
bool sourcematches;
if (sourcefilter.Matches(path, out sourcematches, out sourcematch) && sourcematches)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingSourcePath", "Including source path: {0}", path);
return(true);
}
// If we have a hardlink strategy, obey it
if (hardlinkPolicy != Options.HardlinkStrategy.All)
{
try
{
var id = snapshot.HardlinkTargetID(path);
if (id != null)
{
if (hardlinkPolicy == Options.HardlinkStrategy.None)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingHardlinkByPolicy", "Excluding hardlink: {0} ({1})", path, id);
return(false);
}
else if (hardlinkPolicy == Options.HardlinkStrategy.First)
{
string prevPath;
if (hardlinkmap.TryGetValue(id, out prevPath))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", path, id, prevPath);
return(false);
}
else
{
hardlinkmap.Add(id, path);
}
}
}
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path);
return(false);
}
}
if (ignorenames != null && (attributes & FileAttributes.Directory) != 0)
{
try
{
foreach (var n in ignorenames)
{
var ignorepath = SnapshotUtility.SystemIO.PathCombine(path, n);
if (snapshot.FileExists(ignorepath))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathDueToIgnoreFile", "Excluding path because ignore file was found: {0}", ignorepath);
return(false);
}
}
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path);
}
}
// If we exclude files based on attributes, filter that
if ((fileAttributes & attributes) != 0)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", path);
return(false);
}
// Then check if the filename is not explicitly excluded by a filter
Library.Utility.IFilter match;
var filtermatch = false;
if (!Library.Utility.FilterExpression.Matches(enumeratefilter, path, out match))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromFilter", "Excluding path due to filter: {0} => {1}", path, match == null ? "null" : match.ToString());
return(false);
}
else if (match != null)
{
filtermatch = true;
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPathFromFilter", "Including path due to filter: {0} => {1}", path, match.ToString());
}
// If the file is a symlink, apply special handling
var isSymlink = snapshot.IsSymlink(path, attributes);
string symlinkTarget = null;
if (isSymlink)
{
try { symlinkTarget = snapshot.GetSymlinkTarget(path); }
catch (Exception ex) { Logging.Log.WriteExplicitMessage(FILTER_LOGTAG, "SymlinkTargetReadError", ex, "Failed to read symlink target for path: {0}", path); }
}
if (isSymlink)
{
if (!string.IsNullOrWhiteSpace(symlinkTarget))
{
if (symlinkPolicy == Options.SymlinkStrategy.Ignore)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludeSymlink", "Excluding symlink: {0}", path);
return(false);
}
if (symlinkPolicy == Options.SymlinkStrategy.Store)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "StoreSymlink", "Storing symlink: {0}", path);
// We return false because we do not want to recurse into the path,
// but we add the symlink to the mixin so we process the symlink itself
mixinqueue.Enqueue(path);
return(false);
}
}
else
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", path);
}
}
if (!filtermatch)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPath", "Including path as no filters matched: {0}", path);
}
// All the way through, yes!
return(true);
}
public FilterHandler(Snapshots.ISnapshotService snapshot, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter filter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, ILogWriter logWriter)
{
m_snapshot = snapshot;
m_attributeFilter = attributeFilter;
m_sourcefilter = sourcefilter;
m_filter = filter;
m_symlinkPolicy = symlinkPolicy;
m_hardlinkPolicy = hardlinkPolicy;
m_logWriter = logWriter;
m_hardlinkmap = new Dictionary<string, string>();
}
public PathCollector(Snapshots.ISnapshotService snapshot, Options.SymlinkStrategy symlinkHandling, FileAttributes attributeMask, Utility.FilenameFilter filter, CommunicationStatistics stat)
{
m_symlinkHandling = symlinkHandling;
m_filter = filter;
m_attributeMask = attributeMask;
m_snapshot = snapshot;
m_stat = stat;
}
public void SymLinkPolicy(Options.SymlinkStrategy symlinkPolicy)
{
// Create symlink target directory
const string targetDirName = "target";
var targetDir = systemIO.PathCombine(this.DATAFOLDER, targetDirName);
systemIO.DirectoryCreate(targetDir);
// Create files in symlink target directory
var fileNames = new[] { "a.txt", "b.txt", "c.txt" };
foreach (var file in fileNames)
{
var targetFile = systemIO.PathCombine(targetDir, file);
TestUtils.WriteFile(targetFile, Encoding.Default.GetBytes(file));
}
// Create actual symlink directory linking to the target directory
const string symlinkDirName = "symlink";
var symlinkDir = systemIO.PathCombine(this.DATAFOLDER, symlinkDirName);
try
{
systemIO.CreateSymlink(symlinkDir, targetDir, asDir: true);
}
catch (Exception e)
{
// If client cannot create symlinks, mark test as ignored
Assert.Ignore($"Client could not create a symbolic link. Error reported: {e.Message}");
}
// Backup all files with given symlink policy
Dictionary <string, string> restoreOptions = new Dictionary <string, string>(this.TestOptions)
{
["restore-path"] = this.RESTOREFOLDER
};
Dictionary <string, string> backupOptions = new Dictionary <string, string>(this.TestOptions)
{
["symlink-policy"] = symlinkPolicy.ToString()
};
using (Controller c = new Controller("file://" + this.TARGETFOLDER, backupOptions, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Restore all files
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IRestoreResults restoreResults = c.Restore(null);
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
// Verify that symlink policy was followed
var restoreSymlinkDir = systemIO.PathCombine(this.RESTOREFOLDER, symlinkDirName);
switch (symlinkPolicy)
{
case Options.SymlinkStrategy.Store:
// Restore should contain an actual symlink to the original target
Assert.That(systemIO.IsSymlink(restoreSymlinkDir), Is.True);
var restoredSymlinkFullPath = systemIO.PathGetFullPath(systemIO.GetSymlinkTarget(restoreSymlinkDir));
var symlinkTargetFullPath = systemIO.PathGetFullPath(targetDir);
Assert.That(restoredSymlinkFullPath, Is.EqualTo(symlinkTargetFullPath));
break;
case Options.SymlinkStrategy.Follow:
// Restore should contain a regular directory with copies of the files in the symlink target
Assert.That(systemIO.IsSymlink(restoreSymlinkDir), Is.False);
TestUtils.AssertDirectoryTreesAreEquivalent(targetDir, restoreSymlinkDir, true, "Restore");
break;
case Options.SymlinkStrategy.Ignore:
// Restore should not contain the symlink or directory at all
Assert.That(systemIO.DirectoryExists(restoreSymlinkDir), Is.False);
Assert.That(systemIO.FileExists(restoreSymlinkDir), Is.False);
break;
default:
Assert.Fail($"Unexpected symlink policy");
break;
}
}
}