public static WordListLoaderResult LoadFile(
string path,
int minWordLength = -1,
WordListLoadOptions options = WordListLoadOptions.None)
{
LoadState state = LoadState.Create(options);
LoadFile(path, minWordLength, state);
return(new WordListLoaderResult(
new WordList(state.Words, state.Sequences),
((options & WordListLoadOptions.IgnoreCase) == 0)
? new WordList(state.CaseSensitiveWords, state.CaseSensitiveSequences)
: WordList.CaseSensitive,
FixList.Create(state.Fixes)));
}
public static WordListLoaderResult Load(
IEnumerable <string> paths,
int minWordLength = -1,
WordListLoadOptions options = WordListLoadOptions.None,
CancellationToken cancellationToken = default)
{
LoadState state = LoadState.Create(options);
foreach (string filePath in GetFiles(paths))
{
cancellationToken.ThrowIfCancellationRequested();
LoadFile(filePath, minWordLength, state);
}
Dictionary <string, HashSet <string> > fixes = state.Fixes;
foreach (string word in state.Words)
{
fixes.Remove(word);
}
if (state.CaseSensitiveWords != null)
{
foreach (string word in state.CaseSensitiveWords)
{
fixes.Remove(word);
}
}
foreach (HashSet <string> values in fixes.Values.ToList())
{
foreach (string value in values)
{
fixes.Remove(value);
}
}
return(new WordListLoaderResult(
new WordList(state.Words, state.Sequences),
new WordList(state.CaseSensitiveWords, state.CaseSensitiveSequences),
FixList.Create(fixes)));
}
internal static WordListLoaderResult LoadParallel(
IEnumerable <string> paths,
int minWordLength = -1,
WordListLoadOptions options = WordListLoadOptions.None,
CancellationToken cancellationToken = default)
{
var states = new ConcurrentBag <LoadState>();
var parallelOptions = new ParallelOptions()
{
CancellationToken = cancellationToken
};
Parallel.ForEach(
GetFiles(paths),
parallelOptions,
path =>
{
LoadState state = LoadState.Create(options);
LoadFile(path, minWordLength, state);
states.Add(state);
});
bool isCaseSensitive = (options & WordListLoadOptions.IgnoreCase) == 0;
var words = new List <string>(states.Sum(f => f.Words.Count));
List <string> caseSensitiveWords = null;
List <WordSequence> caseSensitiveSequences = null;
if (isCaseSensitive)
{
caseSensitiveWords = new List <string>(states.Sum(f => f.CaseSensitiveWords !.Count));
caseSensitiveSequences = states.SelectMany(f => f.CaseSensitiveSequences).ToList();
}
Dictionary <string, HashSet <string> > fixes = states.SelectMany(f => f.Fixes).ToDictionary(f => f.Key, f => f.Value);
List <WordSequence> sequences = states.SelectMany(f => f.Sequences).ToList();
foreach (LoadState state in states)
{
foreach (string word in state.Words)
{
fixes.Remove(word);
words.Add(word);
}
if (isCaseSensitive)
{
foreach (string word in state.CaseSensitiveWords !)
{
fixes.Remove(word);
caseSensitiveWords !.Add(word);
}
}
}
foreach (HashSet <string> values in fixes.Values)
{
foreach (string value in values)
{
fixes.Remove(value);
}
}
return(new WordListLoaderResult(
new WordList(words, sequences),
new WordList(caseSensitiveWords, caseSensitiveSequences),
FixList.Create(fixes)));
}