public void Tokenize(ParseDictionary dictionary)
{
var space = new[] { ' ' };
var allSeparators = dictionary.Separators.Union(space).ToArray();
var startToken = 0;
for (var i = 0; i < Text.Length; i++)
{
if (allSeparators.Contains(Text[i]))
{
Tokens.Add(startToken + 1, Text.Substring(startToken, i - startToken));
if (Text[i] != ' ')
{
Tokens.Add(i + 1, Text.Substring(i, 1));
}
startToken = i + 1;
}
}
if (startToken < Text.Length)
{
Tokens.Add(startToken + 1, Text.Substring(startToken));
}
}
public void Populate(TextBuffer textInput, ParseDictionary dictionary)
{
var count = 0;
var index = 1;
foreach (var token in textInput.Tokens)
{
var dictionaryAddress = 0;
var searchTerm = token.Value;
if (searchTerm.Length > 6)
{
searchTerm = searchTerm.Remove(6);
}
if (dictionary.Words.ContainsKey(searchTerm))
{
dictionaryAddress = dictionary.Words[searchTerm].Address;
}
var(msb, lsb) = Bits.BreakWord(dictionaryAddress);
memory.Bytes.Span[index] = msb;
memory.Bytes.Span[index + 1] = lsb;
memory.Bytes.Span[index + 2] = (byte)searchTerm.Length;
memory.Bytes.Span[index + 3] = (byte)token.Key;
count += 1;
index += 4;
if (index >= MaxLength)
{
break;
}
}
memory.Bytes.Span[0] = (byte)count;
}
