private IEnumerable <Tuple <WordPosition, string> > GetWords(string file)
{
using (Stream stream = File.Open(file, FileMode.Open))
{
var word = new StringBuilder();
while (true)
{
long position = stream.Position;
int data = (char)stream.ReadByte();
{
if (data > byte.MaxValue)
{
break;
}
var ch = (Char)data;
if (char.IsLetter(ch))
{
word.Append(ch);
}
else
{
if (word.Length != 0)
{
var wordPosition = new WordPosition(position, file);
yield return(new Tuple <WordPosition, string>(wordPosition, word.ToString().ToLower()));
word.Clear();
m_WordCount++;
}
}
}
UpdateProgress(position);
}
}
}
private void LoadFile(string fileName)
{
Tuple <WordPosition, string>[] words = GetWords(fileName).ToArray();
foreach (var word in words)
{
string text = word.Item2;
WordPosition wordPosition = word.Item1;
m_PatriciaTrie.Add(text, wordPosition);
}
}