Exemple #1
0
        public BackupHandler(string backendurl, Options options, BackupResults results)
        {
            EMPTY_METADATA = Utility.WrapMetadata(new Dictionary<string, string>(), options);

            m_options = options;
            m_result = results;
            m_backendurl = backendurl;

            m_attributeFilter = m_options.FileAttributeFilter;
            m_symlinkPolicy = m_options.SymlinkPolicy;
            m_blocksize = m_options.Blocksize;

            m_blockbuffer = new byte[m_options.Blocksize * Math.Max(1, m_options.FileReadBufferSize / m_options.Blocksize)];
            m_blocklistbuffer = new byte[m_options.Blocksize];

            m_blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
            m_filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);

            if (m_blockhasher == null)
                throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm));
            if (m_filehasher == null)
                throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.FileHashAlgorithm));

            if (!m_blockhasher.CanReuseTransform)
                throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm));
            if (!m_filehasher.CanReuseTransform)
                throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.FileHashAlgorithm));

            if (options.AllowPassphraseChange)
                throw new Exception(Strings.Foresthash.PassphraseChangeUnsupported);
        }
Exemple #2
0
        public BackupHandler(string backendurl, Options options, BackupResults results)
        {
            EMPTY_METADATA = Utility.WrapMetadata(new Dictionary<string, string>(), options);

            m_options = options;
            m_result = results;
            m_backendurl = backendurl;

            m_attributeFilter = m_options.FileAttributeFilter;
            m_symlinkPolicy = m_options.SymlinkPolicy;

            if (options.AllowPassphraseChange)
                throw new Exception(Strings.Foresthash.PassphraseChangeUnsupported);
        }
Exemple #3
0
            public FilterHandler(Snapshots.ISnapshotService snapshot, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter filter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, ILogWriter logWriter)
            {
                m_snapshot = snapshot;
                m_attributeFilter = attributeFilter;
                m_sourcefilter = sourcefilter;
                m_emitfilter = filter;
                m_symlinkPolicy = symlinkPolicy;
                m_hardlinkPolicy = hardlinkPolicy;
                m_logWriter = logWriter;
                m_hardlinkmap = new Dictionary<string, string>();
                m_mixinqueue = new Queue<string>();

                bool includes;
                bool excludes;
                Library.Utility.FilterExpression.AnalyzeFilters(filter, out includes, out excludes);
                if (includes && !excludes)
                {
                    m_enumeratefilter = Library.Utility.FilterExpression.Combine(filter, new Duplicati.Library.Utility.FilterExpression("*" + System.IO.Path.DirectorySeparatorChar, true));
                }
                else
                    m_enumeratefilter = m_emitfilter;

            }
Exemple #4
0
        public static Task Run(IEnumerable <string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, string[] changedfilelist, ITaskReader taskreader)
        {
            return(AutomationExtensions.RunTask(
                       new
            {
                Output = Backup.Channels.SourcePaths.ForWrite
            },

                       async self =>
            {
                var hardlinkmap = new Dictionary <string, string>();
                var mixinqueue = new Queue <string>();
                Duplicati.Library.Utility.IFilter enumeratefilter = emitfilter;

                bool includes;
                bool excludes;
                Library.Utility.FilterExpression.AnalyzeFilters(emitfilter, out includes, out excludes);
                if (includes && !excludes)
                {
                    enumeratefilter = Library.Utility.FilterExpression.Combine(emitfilter, new Duplicati.Library.Utility.FilterExpression("*" + System.IO.Path.DirectorySeparatorChar, true));
                }

                // Simplify checking for an empty list
                if (ignorenames != null && ignorenames.Length == 0)
                {
                    ignorenames = null;
                }

                // If we have a specific list, use that instead of enumerating the filesystem
                IEnumerable <string> worklist;
                if (changedfilelist != null && changedfilelist.Length > 0)
                {
                    worklist = changedfilelist.Where(x =>
                    {
                        var fa = FileAttributes.Normal;
                        try
                        {
                            fa = snapshot.GetAttributes(x);
                        }
                        catch
                        {
                        }

                        return AttributeFilter(x, fa, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue);
                    });
                }
                else
                {
                    Library.Utility.Utility.EnumerationFilterDelegate attributeFilter = (root, path, attr) =>
                                                                                        AttributeFilter(path, attr, snapshot, sourcefilter, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributes, enumeratefilter, ignorenames, mixinqueue);

                    if (journalService != null)
                    {
                        // filter sources using USN journal, to obtain a sub-set of files / folders that may have been modified
                        sources = journalService.GetModifiedSources(attributeFilter);
                    }

                    worklist = snapshot.EnumerateFilesAndFolders(sources, attributeFilter, (rootpath, errorpath, ex) =>
                    {
                        Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", errorpath);
                    });
                }

                var source = ExpandWorkList(worklist, mixinqueue, emitfilter, enumeratefilter);
                if (excludeemptyfolders)
                {
                    source = ExcludeEmptyFolders(source);
                }

                // Process each path, and dequeue the mixins with symlinks as we go
                foreach (var s in source)
                {
                    if (!await taskreader.ProgressAsync)
                    {
                        return;
                    }

                    await self.Output.WriteAsync(s);
                }
            }));
        }
Exemple #5
0
        /// <summary>
        /// Plugin filter for enumerating a list of files.
        /// </summary>
        /// <returns>True if the path should be returned, false otherwise.</returns>
        /// <param name="path">The current path.</param>
        /// <param name="attributes">The file or folder attributes.</param>
        private static bool AttributeFilter(string path, FileAttributes attributes, Snapshots.ISnapshotService snapshot, Library.Utility.IFilter sourcefilter, Options.HardlinkStrategy hardlinkPolicy, Options.SymlinkStrategy symlinkPolicy, Dictionary <string, string> hardlinkmap, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter enumeratefilter, string[] ignorenames, Queue <string> mixinqueue)
        {
            // Step 1, exclude block devices
            try
            {
                if (snapshot.IsBlockDevice(path))
                {
                    Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlockDevice", "Excluding block device: {0}", path);
                    return(false);
                }
            }
            catch (Exception ex)
            {
                Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path);
                return(false);
            }

            // Check if we explicitly include this entry
            Duplicati.Library.Utility.IFilter sourcematch;
            bool sourcematches;

            if (sourcefilter.Matches(path, out sourcematches, out sourcematch) && sourcematches)
            {
                Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingSourcePath", "Including source path: {0}", path);
                return(true);
            }

            // If we have a hardlink strategy, obey it
            if (hardlinkPolicy != Options.HardlinkStrategy.All)
            {
                try
                {
                    var id = snapshot.HardlinkTargetID(path);
                    if (id != null)
                    {
                        if (hardlinkPolicy == Options.HardlinkStrategy.None)
                        {
                            Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingHardlinkByPolicy", "Excluding hardlink: {0} ({1})", path, id);
                            return(false);
                        }
                        else if (hardlinkPolicy == Options.HardlinkStrategy.First)
                        {
                            string prevPath;
                            if (hardlinkmap.TryGetValue(id, out prevPath))
                            {
                                Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", path, id, prevPath);
                                return(false);
                            }
                            else
                            {
                                hardlinkmap.Add(id, path);
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path);
                    return(false);
                }
            }

            if (ignorenames != null && (attributes & FileAttributes.Directory) != 0)
            {
                try
                {
                    foreach (var n in ignorenames)
                    {
                        var ignorepath = SnapshotUtility.SystemIO.PathCombine(path, n);
                        if (snapshot.FileExists(ignorepath))
                        {
                            Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathDueToIgnoreFile", "Excluding path because ignore file was found: {0}", ignorepath);
                            return(false);
                        }
                    }
                }
                catch (Exception ex)
                {
                    Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path);
                }
            }

            // If we exclude files based on attributes, filter that
            if ((fileAttributes & attributes) != 0)
            {
                Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", path);
                return(false);
            }

            // Then check if the filename is not explicitly excluded by a filter
            Library.Utility.IFilter match;
            var filtermatch = false;

            if (!Library.Utility.FilterExpression.Matches(enumeratefilter, path, out match))
            {
                Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromFilter", "Excluding path due to filter: {0} => {1}", path, match == null ? "null" : match.ToString());
                return(false);
            }
            else if (match != null)
            {
                filtermatch = true;
                Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPathFromFilter", "Including path due to filter: {0} => {1}", path, match.ToString());
            }

            // If the file is a symlink, apply special handling
            var    isSymlink     = snapshot.IsSymlink(path, attributes);
            string symlinkTarget = null;

            if (isSymlink)
            {
                try { symlinkTarget = snapshot.GetSymlinkTarget(path); }
                catch (Exception ex) { Logging.Log.WriteExplicitMessage(FILTER_LOGTAG, "SymlinkTargetReadError", ex, "Failed to read symlink target for path: {0}", path); }
            }

            if (isSymlink)
            {
                if (!string.IsNullOrWhiteSpace(symlinkTarget))
                {
                    if (symlinkPolicy == Options.SymlinkStrategy.Ignore)
                    {
                        Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludeSymlink", "Excluding symlink: {0}", path);
                        return(false);
                    }

                    if (symlinkPolicy == Options.SymlinkStrategy.Store)
                    {
                        Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "StoreSymlink", "Storing symlink: {0}", path);

                        // We return false because we do not want to recurse into the path,
                        // but we add the symlink to the mixin so we process the symlink itself
                        mixinqueue.Enqueue(path);
                        return(false);
                    }
                }
                else
                {
                    Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "FollowingEmptySymlink", "Treating empty symlink as regular path {0}", path);
                }
            }

            if (!filtermatch)
            {
                Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPath", "Including path as no filters matched: {0}", path);
            }

            // All the way through, yes!
            return(true);
        }
Exemple #6
0
 public FilterHandler(Snapshots.ISnapshotService snapshot, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter filter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, ILogWriter logWriter)
 {
     m_snapshot = snapshot;
     m_attributeFilter = attributeFilter;
     m_sourcefilter = sourcefilter;
     m_filter = filter;
     m_symlinkPolicy = symlinkPolicy;
     m_hardlinkPolicy = hardlinkPolicy;
     m_logWriter = logWriter;
     m_hardlinkmap = new Dictionary<string, string>();
 }
Exemple #7
0
 public PathCollector(Snapshots.ISnapshotService snapshot, Options.SymlinkStrategy symlinkHandling, FileAttributes attributeMask, Utility.FilenameFilter filter, CommunicationStatistics stat)
 {
     m_symlinkHandling = symlinkHandling;
     m_filter = filter;
     m_attributeMask = attributeMask;
     m_snapshot = snapshot;
     m_stat = stat;
 }
Exemple #8
0
        public void SymLinkPolicy(Options.SymlinkStrategy symlinkPolicy)
        {
            // Create symlink target directory
            const string targetDirName = "target";
            var          targetDir     = systemIO.PathCombine(this.DATAFOLDER, targetDirName);

            systemIO.DirectoryCreate(targetDir);
            // Create files in symlink target directory
            var fileNames = new[] { "a.txt", "b.txt", "c.txt" };

            foreach (var file in fileNames)
            {
                var targetFile = systemIO.PathCombine(targetDir, file);
                TestUtils.WriteFile(targetFile, Encoding.Default.GetBytes(file));
            }

            // Create actual symlink directory linking to the target directory
            const string symlinkDirName = "symlink";
            var          symlinkDir     = systemIO.PathCombine(this.DATAFOLDER, symlinkDirName);

            try
            {
                systemIO.CreateSymlink(symlinkDir, targetDir, asDir: true);
            }
            catch (Exception e)
            {
                // If client cannot create symlinks, mark test as ignored
                Assert.Ignore($"Client could not create a symbolic link.  Error reported: {e.Message}");
            }

            // Backup all files with given symlink policy
            Dictionary <string, string> restoreOptions = new Dictionary <string, string>(this.TestOptions)
            {
                ["restore-path"] = this.RESTOREFOLDER
            };
            Dictionary <string, string> backupOptions = new Dictionary <string, string>(this.TestOptions)
            {
                ["symlink-policy"] = symlinkPolicy.ToString()
            };

            using (Controller c = new Controller("file://" + this.TARGETFOLDER, backupOptions, null))
            {
                IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
                Assert.AreEqual(0, backupResults.Errors.Count());
                Assert.AreEqual(0, backupResults.Warnings.Count());
            }
            // Restore all files
            using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
            {
                IRestoreResults restoreResults = c.Restore(null);
                Assert.AreEqual(0, restoreResults.Errors.Count());
                Assert.AreEqual(0, restoreResults.Warnings.Count());

                // Verify that symlink policy was followed
                var restoreSymlinkDir = systemIO.PathCombine(this.RESTOREFOLDER, symlinkDirName);
                switch (symlinkPolicy)
                {
                case Options.SymlinkStrategy.Store:
                    // Restore should contain an actual symlink to the original target
                    Assert.That(systemIO.IsSymlink(restoreSymlinkDir), Is.True);
                    var restoredSymlinkFullPath = systemIO.PathGetFullPath(systemIO.GetSymlinkTarget(restoreSymlinkDir));
                    var symlinkTargetFullPath   = systemIO.PathGetFullPath(targetDir);
                    Assert.That(restoredSymlinkFullPath, Is.EqualTo(symlinkTargetFullPath));
                    break;

                case Options.SymlinkStrategy.Follow:
                    // Restore should contain a regular directory with copies of the files in the symlink target
                    Assert.That(systemIO.IsSymlink(restoreSymlinkDir), Is.False);
                    TestUtils.AssertDirectoryTreesAreEquivalent(targetDir, restoreSymlinkDir, true, "Restore");
                    break;

                case Options.SymlinkStrategy.Ignore:
                    // Restore should not contain the symlink or directory at all
                    Assert.That(systemIO.DirectoryExists(restoreSymlinkDir), Is.False);
                    Assert.That(systemIO.FileExists(restoreSymlinkDir), Is.False);
                    break;

                default:
                    Assert.Fail($"Unexpected symlink policy");
                    break;
                }
            }
        }