Esempio n. 1
0
        /// <summary>
        /// Handles a single file.
        /// </summary>
        /// <param name="item">The item.</param>
        /// <param name="configuration">The configuration.</param>
        /// <param name="stats">The statistics.</param>
        /// <param name="seenItems">The seen items.</param>
        private static void _HandleFile(
            FileInfo item,
            Configuration configuration,
            RuntimeStats stats,
            ConcurrentDictionary <long, ConcurrentDictionary <string, FileEntry> > seenItems)
        {
            stats.IncrementFiles();
            var length = item.Length;

            stats.IncrementBytes(length);

            if (length < configuration.MinimumFileSizeInBytes || length > configuration.MaximumFileSizeInBytes)
            {
                return;
            }

            var knownWithThisLength = seenItems.GetOrAdd(length, _ => new ConcurrentDictionary <string, FileEntry>());

            // preventing othre threads from processing files with the same size
            // avoiding a race condition where all known links to a file are removed at once, thus loosing data completely
            lock (knownWithThisLength)
                try {
                    _HandleFileWithGivenSize(item, configuration, stats, knownWithThisLength);
                } catch (Exception e) {
                    Console.WriteLine($"[Error] Could not process file {item.FullName}: {e.Message}");
                }
        }
Esempio n. 2
0
        /// <summary>
        /// Handles files which are already hardlinks.
        /// </summary>
        /// <param name="item">The item.</param>
        /// <param name="configuration">The configuration.</param>
        /// <param name="stats">The statistics.</param>
        /// <param name="knownWithThisLength">Length of the known with this.</param>
        private static void _HandleExistingHardLink(
            FileInfo item,
            Configuration configuration,
            RuntimeStats stats,
            ConcurrentDictionary <string, FileEntry> knownWithThisLength
            )
        {
            if (configuration.DeleteHardLinkedFiles)
            {
                _RemoveFileEntry(item, knownWithThisLength);
                _DeleteLink(item);
                stats.HardLinkStats.IncreaseDeleted();
                Console.WriteLine($"[Info] Deleted Hardlink {item.FullName}");
                return;
            }

            if (configuration.RemoveHardLinks)
            {
                _RemoveFileEntry(item, knownWithThisLength);
                try {
                    _ReplaceFileLinkWithFileContent(item);
                    stats.HardLinkStats.IncreaseRemoved();
                    Console.WriteLine($"[Info] Removed Hardlink {item.FullName}");
                } catch (Exception e) {
                    Console.WriteLine($"[Error] Could not remove Hardlink {item.FullName}: {e.Message}");
                }

                return;
            }

            if (configuration.SetReadOnlyAttributeOnExistingHardLinks && ((item.Attributes & FileAttributes.ReadOnly) != FileAttributes.ReadOnly))
            {
                Console.WriteLine($"[Info] Setting read-only attribute on Hardlink {item.FullName}");
                item.Attributes |= FileAttributes.ReadOnly;
                return;
            }

            Console.WriteLine($"[Info] {item.FullName} is alrady a Hardlink");
        }
Esempio n. 3
0
        /// <summary>
        /// Handles a specific file along with a group of files already seen with the same size.
        /// </summary>
        /// <param name="item">The item.</param>
        /// <param name="configuration">The configuration.</param>
        /// <param name="stats">The statistics.</param>
        /// <param name="knownWithThisLength">Length of the known with this.</param>
        private static void _HandleFileWithGivenSize(
            FileInfo item,
            Configuration configuration,
            RuntimeStats stats,
            ConcurrentDictionary <string, FileEntry> knownWithThisLength)
        {
            var myKey    = _GenerateKey(item);
            var checksum = knownWithThisLength.GetOrAdd(myKey, new FileEntry(item));

            var isHardLink = false;
            IEnumerable <FileInfo> hardlinks;

            try {
                hardlinks = item.GetHardLinkTargets();
            } catch (Exception e) {
                _RemoveFileEntry(item, knownWithThisLength);
                Console.WriteLine($"[Error] Could not enumerate HardLinks {item.FullName}: {e.Message}");
                return;
            }

            foreach (var target in hardlinks)
            {
                isHardLink = true;
                knownWithThisLength.TryAdd(_GenerateKey(target), new FileEntry(target));
            }

            if (isHardLink)
            {
                stats.HardLinkStats.IncreaseSeen();
                if (configuration.ShowInfoOnly)
                {
                    return;
                }

                _HandleExistingHardLink(item, configuration, stats, knownWithThisLength);
                return;
            }

            string symlink;

            try {
                symlink = item.GetSymbolicLinkTarget();
            } catch (Exception e) {
                _RemoveFileEntry(item, knownWithThisLength);
                Console.WriteLine($"[Error] Could not enumerate SymLink {item.FullName}: {e.Message}");
                return;
            }

            if (symlink != null)
            {
                knownWithThisLength.TryAdd(symlink, new FileEntry(new FileInfo(symlink)));
                stats.SymbolicLinkStats.IncreaseSeen();
                if (configuration.ShowInfoOnly)
                {
                    return;
                }

                _HandleExistingSymbolicLink(item, configuration, stats, knownWithThisLength);
                return;
            }

            if (configuration.ShowInfoOnly)
            {
                return;
            }

            // find matching file in seen list and try to hard or symlink
            var sameFiles =
                knownWithThisLength
                .Where(kvp => kvp.Key != myKey)
                .Where(kvp => kvp.Value.Equals(checksum))
                .Select(kvp => kvp.Key)
            ;

            foreach (var sameFile in sameFiles)
            {
                var temporaryFile = _CreateTemporaryFileInSameDirectory(item);
                temporaryFile.Delete();

                var isSymlink = false;

                try {
                    LinkExtensions.CreateHardLinkFrom((FileInfo)temporaryFile, (string)sameFile);
                } catch (Exception e1) {
                    if (configuration.AlsoTrySymbolicLinks)
                    {
                        isSymlink = true;
                        try {
                            LinkExtensions.CreateSymbolicLinkFrom((FileInfo)temporaryFile, (string)sameFile);
                        } catch (Exception e2) {
                            Console.WriteLine(
                                $"[Warning] Could not Symlink {item.FullName}({FilesizeFormatter.FormatUnit(item.Length, true)}) --> {sameFile}: {e2.Message}");
                            continue;
                        }
                    }
                    else
                    {
                        Console.WriteLine(
                            $"[Warning] Could not Hardlink {item.FullName}({FilesizeFormatter.FormatUnit(item.Length, true)}) --> {sameFile}: {e1.Message}");
                        continue;
                    }
                }

                var isAlreadyDeleted = false;

                try {
                    item.Attributes &= ~FileAttributes.ReadOnly;
                    item.Delete();
                    isAlreadyDeleted = true;
                    File.Move(temporaryFile.FullName, item.FullName);
                } catch {
                    if (isAlreadyDeleted)
                    {
                        // undo file deletion
                        temporaryFile.CopyTo(item.FullName, true);
                    }
                    else
                    {
                        // undo temp file creation
                        temporaryFile.Delete();
                    }

                    throw;
                }

                if (isSymlink)
                {
                    stats.SymbolicLinkStats.IncreaseCreated();
                    if (configuration.SetReadOnlyAttributeOnNewSymbolicLinks)
                    {
                        item.Attributes |= FileAttributes.ReadOnly;
                    }
                }
                else
                {
                    stats.HardLinkStats.IncreaseCreated();
                    if (configuration.SetReadOnlyAttributeOnNewHardLinks)
                    {
                        item.Attributes |= FileAttributes.ReadOnly;
                    }
                }

                Console.WriteLine($"[Info] Created {(isSymlink ? "Symlink" : "Hardlink")} for {item.FullName}({FilesizeFormatter.FormatUnit(item.Length, true)}) --> {sameFile}");
                return;
            }
        }
Esempio n. 4
0
        /// <summary>
        /// A worker thread, pulling items from the stack, handling files and directories, etc.
        /// </summary>
        /// <param name="stack">The stack.</param>
        /// <param name="configuration">The configuration.</param>
        /// <param name="seenItems">The seen items.</param>
        /// <param name="waiter">The waiter.</param>
        /// <param name="state">The state.</param>
        private static void _ThreadWorker(ConcurrentStack <DirectoryInfo> stack, Configuration configuration, RuntimeStats stats, ConcurrentDictionary <long, ConcurrentDictionary <string, FileEntry> > seenItems, AutoResetEvent waiter, int[] state)
        {
            while (true)
            {
                if (!stack.TryPop(out var current))
                {
                    // when stack is empty, signal we're lazy and if all other threads are also, end thread
                    if (Interlocked.Decrement(ref state[0]) == 0)
                    {
                        // signal another thread to continue exiting
                        waiter.Set();
                        Console.WriteLine($"Ending Thread #{Thread.CurrentThread.ManagedThreadId}");
                        return;
                    }

                    waiter.WaitOne();
                    Interlocked.Increment(ref state[0]);
                    continue;
                }

                // push directories and wake up any sleeping threads
                foreach (var directory in current.EnumerateDirectories())
                {
                    stack.Push(directory);
                    stats.IncrementFolders();

                    // notify other threads which may be waiting for work
                    waiter.Set();
                }

                foreach (var item in current.EnumerateFiles())
                {
                    _HandleFile(item, configuration, stats, seenItems);
                }
            }
        }
Esempio n. 5
0
        /// <summary>
        /// Processes the given folders with the given configuration.
        /// </summary>
        /// <param name="directories">The directories.</param>
        /// <param name="configuration">The configuration.</param>
        /// <param name="stats">The statistics.</param>
        public static void ProcessFolders(IList <DirectoryInfo> directories, Configuration configuration, RuntimeStats stats)
        {
            var seenFiles = new ConcurrentDictionary <long, ConcurrentDictionary <string, FileEntry> >();
            var stack     = new ConcurrentStack <DirectoryInfo>();

            stack.PushRange(directories);
            stats.IncrementFolders(directories.Count);
            var threads = new Thread[Math.Max(1, configuration.MaximumCrawlerThreads)];

            using (var autoresetEvent = new AutoResetEvent(false)) {
                var runningWorkers = new[] { threads.Length };

                for (var i = 0; i < threads.Length; ++i)
                {
                    threads[i] = new Thread(_ThreadWorker);
                    threads[i].Start(Tuple.Create(stack, configuration, stats, seenFiles, autoresetEvent, runningWorkers));
                }

                foreach (var thread in threads)
                {
                    thread.Join();
                }
            }
        }