public FilterHandler(Snapshots.ISnapshotService snapshot, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter filter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, ILogWriter logWriter)
{
m_snapshot = snapshot;
m_attributeFilter = attributeFilter;
m_sourcefilter = sourcefilter;
m_emitfilter = filter;
m_symlinkPolicy = symlinkPolicy;
m_hardlinkPolicy = hardlinkPolicy;
m_logWriter = logWriter;
m_hardlinkmap = new Dictionary<string, string>();
m_mixinqueue = new Queue<string>();
bool includes;
bool excludes;
Library.Utility.FilterExpression.AnalyzeFilters(filter, out includes, out excludes);
if (includes && !excludes)
{
m_enumeratefilter = Library.Utility.FilterExpression.Combine(filter, new Duplicati.Library.Utility.FilterExpression("*" + System.IO.Path.DirectorySeparatorChar, true));
}
else
m_enumeratefilter = m_emitfilter;
}
public FilterHandler(Snapshots.ISnapshotService snapshot, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter filter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, ILogWriter logWriter)
{
m_snapshot = snapshot;
m_attributeFilter = attributeFilter;
m_sourcefilter = sourcefilter;
m_filter = filter;
m_symlinkPolicy = symlinkPolicy;
m_hardlinkPolicy = hardlinkPolicy;
m_logWriter = logWriter;
m_hardlinkmap = new Dictionary<string, string>();
}
private static Duplicati.Library.Utility.IFilter ApplyFilter(Duplicati.Server.Serialization.Interface.IBackup backup, DuplicatiOperation mode, Duplicati.Library.Utility.IFilter filter)
{
var f2 = backup.Filters;
if (f2 != null && f2.Length > 0)
{
var nf =
(from n in f2
let exp = Environment.ExpandEnvironmentVariables(n.Expression)
orderby n.Order
select(Duplicati.Library.Utility.IFilter)(new Duplicati.Library.Utility.FilterExpression(exp, n.Include)))
.Aggregate((a, b) => Duplicati.Library.Utility.FilterExpression.Combine(a, b));
return(Duplicati.Library.Utility.FilterExpression.Combine(filter, nf));
}
else
{
return(filter);
}
}
public static Task Run(Snapshots.ISnapshotService snapshot, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, string[] changedfilelist, Common.ITaskReader taskreader)
{
return(AutomationExtensions.RunTask(
new
{
Output = Backup.Channels.SourcePaths.ForWrite
},
async self =>
{
var hardlinkmap = new Dictionary <string, string>();
var mixinqueue = new Queue <string>();
Duplicati.Library.Utility.IFilter enumeratefilter = emitfilter;
bool includes;
bool excludes;
Library.Utility.FilterExpression.AnalyzeFilters(emitfilter, out includes, out excludes);
if (includes && !excludes)
{
enumeratefilter = Library.Utility.FilterExpression.Combine(emitfilter, new Duplicati.Library.Utility.FilterExpression("*" + System.IO.Path.DirectorySeparatorChar, true));
}
// Simplify checking for an empty list
if (ignorenames != null && ignorenames.Length == 0)
{
ignorenames = null;
}
// If we have a specific list, use that instead of enumerating the filesystem
IEnumerable <string> worklist;
if (changedfilelist != null && changedfilelist.Length > 0)
{
worklist = changedfilelist.Where(x =>
{
var fa = FileAttributes.Normal;
try
{
fa = snapshot.GetAttributes(x);
}
catch
{
}
return AttributeFilterAsync(null, x, fa, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, attributeFilter, enumeratefilter, ignorenames, mixinqueue).WaitForTask().Result;
});
}
else
{
worklist = snapshot.EnumerateFilesAndFolders((root, path, attr) =>
{
return AttributeFilterAsync(root, path, attr, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, attributeFilter, enumeratefilter, ignorenames, mixinqueue).WaitForTask().Result;
}, (rootpath, path, ex) =>
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", path);
});
}
var source = ExpandWorkList(worklist, mixinqueue, emitfilter, enumeratefilter);
if (excludeemptyfolders)
{
source = ExcludeEmptyFolders(source);
}
// Process each path, and dequeue the mixins with symlinks as we go
foreach (var s in source)
{
if (!await taskreader.ProgressAsync)
{
return;
}
await self.Output.WriteAsync(s);
}
}));
}
/// <summary>
/// Plugin filter for enumerating a list of files.
/// </summary>
/// <returns>True if the path should be returned, false otherwise.</returns>
/// <param name="rootpath">The root path that initiated this enumeration.</param>
/// <param name="path">The current path.</param>
/// <param name="attributes">The file or folder attributes.</param>
private static async Task <bool> AttributeFilterAsync(string rootpath, string path, FileAttributes attributes, Snapshots.ISnapshotService snapshot, Library.Utility.IFilter sourcefilter, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary <string, string> hardlinkmap, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue <string> mixinqueue)
{
// Step 1, exclude block devices
try
{
if (snapshot.IsBlockDevice(path))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlockDevice", "Excluding block device: {0}", path);
return(false);
}
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path);
return(false);
}
// Check if we explicitly include this entry
Duplicati.Library.Utility.IFilter sourcematch;
bool sourcematches;
if (sourcefilter.Matches(path, out sourcematches, out sourcematch) && sourcematches)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingSourcePath", "Including source path: {0}", path);
return(true);
}
// If we have a hardlink strategy, obey it
if (hardlinkPolicy != Options.HardlinkStrategy.All)
{
try
{
var id = snapshot.HardlinkTargetID(path);
if (id != null)
{
if (hardlinkPolicy == Options.HardlinkStrategy.None)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingHardlinkByPolicy", "Excluding hardlink: {0} ({1})", path, id);
return(false);
}
else if (hardlinkPolicy == Options.HardlinkStrategy.First)
{
string prevPath;
if (hardlinkmap.TryGetValue(id, out prevPath))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", path, id, prevPath);
return(false);
}
else
{
hardlinkmap.Add(id, path);
}
}
}
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path);
return(false);
}
}
if (ignorenames != null && (attributes & FileAttributes.Directory) != 0)
{
try
{
foreach (var n in ignorenames)
{
var ignorepath = SnapshotUtility.SystemIO.PathCombine(path, n);
if (snapshot.FileExists(ignorepath))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathDueToIgnoreFile", "Excluding path because ignore file was found: {0}", ignorepath);
return(false);
}
}
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path);
}
}
// If we exclude files based on attributes, filter that
if ((attributeFilter & attributes) != 0)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", path);
return(false);
}
// Then check if the filename is not explicitly excluded by a filter
Library.Utility.IFilter match;
var filtermatch = false;
if (!Library.Utility.FilterExpression.Matches(enumeratefilter, path, out match))
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromFilter", "Excluding path due to filter: {0} => {1}", path, match == null ? "null" : match.ToString());
return(false);
}
else if (match != null)
{
filtermatch = true;
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPathFromFilter", "Including path due to filter: {0} => {1}", path, match.ToString());
}
// If the file is a symlink, apply special handling
var isSymlink = snapshot.IsSymlink(path, attributes);
string symlinkTarget = null;
if (isSymlink)
{
try { symlinkTarget = snapshot.GetSymlinkTarget(path); }
catch (Exception ex) { Logging.Log.WriteExplicitMessage(FILTER_LOGTAG, "SymlinkTargetReadError", ex, "Failed to read symlink target for path: {0}", path); }
}
if (isSymlink)
{
if (!string.IsNullOrWhiteSpace(symlinkTarget))
{
if (symlinkPolicy == Options.SymlinkStrategy.Ignore)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludeSymlink", "Excluding symlink: {0}", path);
return(false);
}
if (symlinkPolicy == Options.SymlinkStrategy.Store)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "StoreSymlink", "Storing symlink: {0}", path);
// We return false because we do not want to recurse into the path,
// but we add the symlink to the mixin so we process the symlink itself
mixinqueue.Enqueue(path);
return(false);
}
}
else
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", path);
}
}
if (!filtermatch)
{
Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPath", "Including path as no filters matched: {0}", path);
}
// All the way through, yes!
return(true);
}
public static IEnumerable <Duplicati.Library.Main.Volumes.IFileEntry> EnumerateFilesInDList(string file, Duplicati.Library.Utility.IFilter filter, Dictionary <string, string> options)
{
var p = Library.Main.Volumes.VolumeBase.ParseFilename(Path.GetFileName(file));
using (var fs = new System.IO.FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read))
using (var cm = Library.DynamicLoader.CompressionLoader.GetModule(p.CompressionModule, fs, Library.Interface.ArchiveMode.Read, options))
using (var filesetreader = new Library.Main.Volumes.FilesetVolumeReader(cm, new Duplicati.Library.Main.Options(options)))
foreach (var f in filesetreader.Files)
{
if (f.Type != Duplicati.Library.Main.FilelistEntryType.File)
{
continue;
}
bool result;
bool match = filter.Matches(f.Path, out result, out _);
if (!match || result)
{
yield return(f);
}
}
}
public static IEnumerable <Duplicati.Library.Main.Volumes.IFileEntry> EnumerateFilesInDList(string file, Duplicati.Library.Utility.IFilter filter, Dictionary <string, string> options)
{
var p = Library.Main.Volumes.VolumeBase.ParseFilename(file);
using (var cm = Library.DynamicLoader.CompressionLoader.GetModule(p.CompressionModule, file, options))
using (var filesetreader = new Library.Main.Volumes.FilesetVolumeReader(cm, new Duplicati.Library.Main.Options(options)))
foreach (var f in filesetreader.Files)
{
if (f.Type != Duplicati.Library.Main.FilelistEntryType.File)
{
continue;
}
bool result;
Library.Utility.IFilter evfilter;
bool match = filter.Matches(f.Path, out result, out evfilter);
if (!match || (match && result))
{
yield return(f);
}
}
}
