private IEnumerable <IndexedEntry> Index(
DirectoryPath path,
HashSet <DirectoryPath> excludedFolders,
HashSet <string> includedExtensions,
List <Regex> excludedPatterns)
{
var stack = new Stack <DirectoryPath>();
stack.Push(path);
while (stack.Count > 0)
{
var current = stack.Pop();
// Folder excluded?
if (excludedFolders.Contains(current))
{
_log.Debug($"Folder '{current.FullPath}' has been excluded.");
continue;
}
// Folder name not allowed?
if (TryMatchPattern(current, excludedPatterns, out var pattern))
{
_log.Debug($"Folder '{current.FullPath}' matched pattern '{pattern}' that has been excluded.");
continue;
}
var directory = _fileSystem.GetDirectorySafe(current);
if (directory != null)
{
// Index the directory.
yield return(DocumentIndexSourceEntry.Directory(directory.Path,
directory.Path.GetDirectoryName(), directory.Path.FullPath,
directory.Path.ToUri("shell", new Dictionary <string, string>
{
{ "isDirectory", "true" }
})));
// Get all files in the directory.
var files = _fileSystem.GetFilesSafe(directory.Path, "*.*", SearchScope.Current);
if (files == null)
{
continue;
}
// Index all files in the directory.
foreach (var file in files)
{
if (file.Path.GetFilename().FullPath.StartsWith("."))
{
// TODO: Temporary fix due to Spectre.System lib.
continue;
}
if ((file.Attributes & FileAttributes.System) == FileAttributes.System)
{
continue;
}
if ((file.Attributes & FileAttributes.Hidden) == FileAttributes.Hidden)
{
continue;
}
if (!includedExtensions.Contains(file.Path.GetExtension().Name))
{
continue;
}
yield return(DocumentIndexSourceEntry.File(file.Path,
file.Path.GetFilename().FullPath, file.Path.FullPath,
file.Path.ToUri("shell")));
}
// Push immediate child directories to the stack.
directory.GetDirectories("*", SearchScope.Current)
.ForEach(child => stack.Push(child.Path));
}
}
}
public Task <bool> Run(CancellationToken token)
{
return(Task.Run(() =>
{
while (true)
{
var indexingWatch = new Stopwatch();
indexingWatch.Start();
var result = LoadResults(token);
if (token.IsCancellationRequested)
{
break;
}
_log.Debug("Updating index...");
var indexUpdateWatch = new Stopwatch();
indexUpdateWatch.Start();
var trie = new Trie <IndexedEntry>();
foreach (var file in result)
{
// Index individual words as well as combinations.
Index(trie, file.Title, file);
Index(trie, file.Description, file);
// Also index the whole words without tokenization.
trie.Insert(file.Title, file);
// Is this a file path?
if (file is IHasPath entryWithPath)
{
if (entryWithPath.Path is FilePath filePath)
{
// Index individual words as well as combinations.
Index(trie, filePath.GetFilenameWithoutExtension().FullPath, file);
Index(trie, filePath.GetFilename().RemoveExtension().FullPath, file);
// Also index the whole words without tokenization.
trie.Insert(filePath.GetFilenameWithoutExtension().FullPath, file);
trie.Insert(filePath.GetFilename().FullPath, file);
}
}
}
indexUpdateWatch.Stop();
_log.Debug($"Building trie took {indexUpdateWatch.ElapsedMilliseconds}ms");
_log.Debug("Writing index...");
Interlocked.Exchange(ref _trie, trie);
_log.Verbose($"Nodes: {_trie.NodeCount}");
_log.Verbose($"Items: {_trie.ItemCount}");
indexingWatch.Stop();
_log.Debug($"Indexing done. Took {indexingWatch.ElapsedMilliseconds}ms");
// Wait for a while.
var index = WaitHandle.WaitAny(new[] { token.WaitHandle, _trigger }, (int)TimeSpan.FromMinutes(5).TotalMilliseconds);
if (index == 0)
{
_log.Information("We were instructed to stop (2).");
break;
}
// Triggered update?
if (index == 1)
{
_log.Information("A re-index was triggered.");
_trigger.Reset();
}
}
return true;
}, token));
}
public async Task <bool> Run(CancellationToken token)
{
while (true)
{
// Ask all sources for files.
var result = new HashSet <IndexedEntry>();
foreach (var source in _sources)
{
if (token.WaitHandle.WaitOne((int)TimeSpan.FromMinutes(0).TotalMilliseconds))
{
_log.Information("We were instructed to stop (1). Aborting indexing...");
break;
}
_log.Debug($"Running '{source.Name}' indexer...");
foreach (var file in source.Index())
{
result.Add(file);
}
}
_log.Debug("Updating index...");
var trie = new Trie <IndexedEntry>();
foreach (var file in result)
{
// Index individual words as well as combinations.
Index(trie, file.Title, file);
Index(trie, file.Description, file);
// Also index the whole words without tokenization.
trie.Insert(file.Title, file);
// Is this a file path?
if (file is IHasPath entryWithPath)
{
if (entryWithPath.Path is FilePath filePath)
{
// Index individual words as well as combinations.
Index(trie, filePath.GetFilenameWithoutExtension().FullPath, file);
Index(trie, filePath.GetFilename().RemoveExtension().FullPath, file);
// Also index the whole words without tokenization.
trie.Insert(filePath.GetFilenameWithoutExtension().FullPath, file);
trie.Insert(filePath.GetFilename().FullPath, file);
}
}
}
_log.Debug("Writing index...");
using (await _lock.WriterLockAsync(token))
{
_trie = trie;
}
_log.Verbose($"Nodes: {_trie.NodeCount}");
_log.Verbose($"Items: {_trie.ItemCount}");
// Wait for a minute.
_log.Debug("Indexing done.");
if (token.WaitHandle.WaitOne((int)TimeSpan.FromMinutes(5).TotalMilliseconds))
{
_log.Information("We were instructed to stop (2).");
break;
}
}
return(true);
}