public void Tokenize(ParseDictionary dictionary) { var space = new[] { ' ' }; var allSeparators = dictionary.Separators.Union(space).ToArray(); var startToken = 0; for (var i = 0; i < Text.Length; i++) { if (allSeparators.Contains(Text[i])) { Tokens.Add(startToken + 1, Text.Substring(startToken, i - startToken)); if (Text[i] != ' ') { Tokens.Add(i + 1, Text.Substring(i, 1)); } startToken = i + 1; } } if (startToken < Text.Length) { Tokens.Add(startToken + 1, Text.Substring(startToken)); } }
public void Populate(TextBuffer textInput, ParseDictionary dictionary) { var count = 0; var index = 1; foreach (var token in textInput.Tokens) { var dictionaryAddress = 0; var searchTerm = token.Value; if (searchTerm.Length > 6) { searchTerm = searchTerm.Remove(6); } if (dictionary.Words.ContainsKey(searchTerm)) { dictionaryAddress = dictionary.Words[searchTerm].Address; } var(msb, lsb) = Bits.BreakWord(dictionaryAddress); memory.Bytes.Span[index] = msb; memory.Bytes.Span[index + 1] = lsb; memory.Bytes.Span[index + 2] = (byte)searchTerm.Length; memory.Bytes.Span[index + 3] = (byte)token.Key; count += 1; index += 4; if (index >= MaxLength) { break; } } memory.Bytes.Span[0] = (byte)count; }