private IEnumerable <IndexedEntry> Index( DirectoryPath path, HashSet <DirectoryPath> excludedFolders, HashSet <string> includedExtensions, List <Regex> excludedPatterns) { var stack = new Stack <DirectoryPath>(); stack.Push(path); while (stack.Count > 0) { var current = stack.Pop(); // Folder excluded? if (excludedFolders.Contains(current)) { _log.Debug($"Folder '{current.FullPath}' has been excluded."); continue; } // Folder name not allowed? if (TryMatchPattern(current, excludedPatterns, out var pattern)) { _log.Debug($"Folder '{current.FullPath}' matched pattern '{pattern}' that has been excluded."); continue; } var directory = _fileSystem.GetDirectorySafe(current); if (directory != null) { // Index the directory. yield return(DocumentIndexSourceEntry.Directory(directory.Path, directory.Path.GetDirectoryName(), directory.Path.FullPath, directory.Path.ToUri("shell", new Dictionary <string, string> { { "isDirectory", "true" } }))); // Get all files in the directory. var files = _fileSystem.GetFilesSafe(directory.Path, "*.*", SearchScope.Current); if (files == null) { continue; } // Index all files in the directory. foreach (var file in files) { if (file.Path.GetFilename().FullPath.StartsWith(".")) { // TODO: Temporary fix due to Spectre.System lib. continue; } if ((file.Attributes & FileAttributes.System) == FileAttributes.System) { continue; } if ((file.Attributes & FileAttributes.Hidden) == FileAttributes.Hidden) { continue; } if (!includedExtensions.Contains(file.Path.GetExtension().Name)) { continue; } yield return(DocumentIndexSourceEntry.File(file.Path, file.Path.GetFilename().FullPath, file.Path.FullPath, file.Path.ToUri("shell"))); } // Push immediate child directories to the stack. directory.GetDirectories("*", SearchScope.Current) .ForEach(child => stack.Push(child.Path)); } } }
public Task <bool> Run(CancellationToken token) { return(Task.Run(() => { while (true) { var indexingWatch = new Stopwatch(); indexingWatch.Start(); var result = LoadResults(token); if (token.IsCancellationRequested) { break; } _log.Debug("Updating index..."); var indexUpdateWatch = new Stopwatch(); indexUpdateWatch.Start(); var trie = new Trie <IndexedEntry>(); foreach (var file in result) { // Index individual words as well as combinations. Index(trie, file.Title, file); Index(trie, file.Description, file); // Also index the whole words without tokenization. trie.Insert(file.Title, file); // Is this a file path? if (file is IHasPath entryWithPath) { if (entryWithPath.Path is FilePath filePath) { // Index individual words as well as combinations. Index(trie, filePath.GetFilenameWithoutExtension().FullPath, file); Index(trie, filePath.GetFilename().RemoveExtension().FullPath, file); // Also index the whole words without tokenization. trie.Insert(filePath.GetFilenameWithoutExtension().FullPath, file); trie.Insert(filePath.GetFilename().FullPath, file); } } } indexUpdateWatch.Stop(); _log.Debug($"Building trie took {indexUpdateWatch.ElapsedMilliseconds}ms"); _log.Debug("Writing index..."); Interlocked.Exchange(ref _trie, trie); _log.Verbose($"Nodes: {_trie.NodeCount}"); _log.Verbose($"Items: {_trie.ItemCount}"); indexingWatch.Stop(); _log.Debug($"Indexing done. Took {indexingWatch.ElapsedMilliseconds}ms"); // Wait for a while. var index = WaitHandle.WaitAny(new[] { token.WaitHandle, _trigger }, (int)TimeSpan.FromMinutes(5).TotalMilliseconds); if (index == 0) { _log.Information("We were instructed to stop (2)."); break; } // Triggered update? if (index == 1) { _log.Information("A re-index was triggered."); _trigger.Reset(); } } return true; }, token)); }
public async Task <bool> Run(CancellationToken token) { while (true) { // Ask all sources for files. var result = new HashSet <IndexedEntry>(); foreach (var source in _sources) { if (token.WaitHandle.WaitOne((int)TimeSpan.FromMinutes(0).TotalMilliseconds)) { _log.Information("We were instructed to stop (1). Aborting indexing..."); break; } _log.Debug($"Running '{source.Name}' indexer..."); foreach (var file in source.Index()) { result.Add(file); } } _log.Debug("Updating index..."); var trie = new Trie <IndexedEntry>(); foreach (var file in result) { // Index individual words as well as combinations. Index(trie, file.Title, file); Index(trie, file.Description, file); // Also index the whole words without tokenization. trie.Insert(file.Title, file); // Is this a file path? if (file is IHasPath entryWithPath) { if (entryWithPath.Path is FilePath filePath) { // Index individual words as well as combinations. Index(trie, filePath.GetFilenameWithoutExtension().FullPath, file); Index(trie, filePath.GetFilename().RemoveExtension().FullPath, file); // Also index the whole words without tokenization. trie.Insert(filePath.GetFilenameWithoutExtension().FullPath, file); trie.Insert(filePath.GetFilename().FullPath, file); } } } _log.Debug("Writing index..."); using (await _lock.WriterLockAsync(token)) { _trie = trie; } _log.Verbose($"Nodes: {_trie.NodeCount}"); _log.Verbose($"Items: {_trie.ItemCount}"); // Wait for a minute. _log.Debug("Indexing done."); if (token.WaitHandle.WaitOne((int)TimeSpan.FromMinutes(5).TotalMilliseconds)) { _log.Information("We were instructed to stop (2)."); break; } } return(true); }