/// <summary> /// Lexicalizes a word type /// </summary> private void LexicalizeWord() { if (IsKeyword(tokenizer.GetTextValue())) // Is probably keyword { //Check for symbols directly after keyword if (IsSymbol(tokenizer.PeekCharacter())) { //It is not a keyword, followed directly by symbol, so maybe a path, etc. LexicalizeIdentifier(); } else { // We are dealing with an keyword TokenStream.Add(new Token(tokenizer.GetTextValue(), TokenType.KEYWORD, tokenizer.GetScannedLines())); CurrentToken = tokenizer.NextToken(); } } else if (IsIdentifier(tokenizer.GetTextValue())) { LexicalizeIdentifier(); return; } else { throw new StreamTokenizerException("Invalid token: " + CurrentToken, tokenizer.GetScannedLines()); } }