/// <summary> /// Performs a lex to return the next <see cref="MergableLexerResult"/> /// from a <see cref="ITextBufferReader"/> and seeks past it if there is a match. /// </summary> /// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param> /// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current state.</param> /// <returns>A <see cref="MergableLexerResult"/> indicating the lexer result.</returns> public override MergableLexerResult GetNextToken(ITextBufferReader reader, ILexicalState lexicalState) { // Initialize int tokenId = ParentTokenId.Invalid; // Get the next character char ch = reader.Read(); switch (lexicalState.Id) { case ParentLexicalStateId.Default: { // If the character is a letter or digit... if ((Char.IsLetter(ch) || (ch == '_'))) { // Parse the identifier tokenId = this.ParseIdentifier(reader, ch); } else if (Char.IsWhiteSpace(ch)) { // Consume sequential whitespace while (Char.IsWhiteSpace(reader.Peek())) { reader.Read(); } tokenId = ParentTokenId.Whitespace; } else { // Invalid tokenId = ParentTokenId.Invalid; } break; } } if (tokenId != ParentTokenId.Invalid) { return(new MergableLexerResult(MatchType.ExactMatch, new LexicalStateTokenData(lexicalState, tokenId))); } else { reader.ReadReverse(); return(MergableLexerResult.NoMatch); } }
/// <summary> /// Creates an <see cref="IToken"/> that represents an invalid range of text. /// </summary> /// <param name="startOffset">The start offset of the <see cref="IToken"/>.</param> /// <param name="length">The length of the <see cref="IToken"/>.</param> /// <param name="lexicalState">The <see cref="ILexicalState"/> that contains the token.</param> /// <returns>An <see cref="IToken"/> that represents an invalid range of text.</returns> public override IToken CreateInvalidToken(int startOffset, int length, ILexicalState lexicalState) { return new LuatToken(startOffset, length, LexicalParseFlags.None, null, new LexicalStateAndIDTokenLexicalParseData(lexicalState, LuatTokenId.Invalid)); }
/// <summary> /// Creates an <see cref="IToken"/> that represents the end of a document. /// </summary> /// <param name="startOffset">The start offset of the <see cref="IToken"/>.</param> /// <param name="lexicalState">The <see cref="ILexicalState"/> that contains the token.</param> /// <returns>An <see cref="IToken"/> that represents the end of a document.</returns> public override IToken CreateDocumentEndToken(int startOffset, ILexicalState lexicalState) { return new LuatToken(startOffset, 0, LexicalParseFlags.None, null, new LexicalStateAndIDTokenLexicalParseData(lexicalState, LuatTokenId.DocumentEnd)); }
public override MergableLexerResult GetNextToken(ITextBufferReader reader, ILexicalState lexicalState) { // Initialize int tokenId = ExecutionTargetSelectorTokenId.Invalid; // Get the next character char ch = reader.Read(); if ((Char.IsLetter(ch) || (ch == '_'))) { // Parse the identifier tokenId = this.ParseIdentifier(reader, ch); } //else if (Char.IsWhiteSpace(ch)) //{ // while (Char.IsWhiteSpace(reader.Peek())) // reader.Read(); // tokenId = ExecutionTargetSelectorTokenId.Whitespace; //} //else if (Char.IsNumber(ch)) //{ // tokenId = this.ParseNumber(reader, ch); //} else if (ch == '.') { tokenId = ExecutionTargetSelectorTokenId.Dot; } else { tokenId = ExecutionTargetSelectorTokenId.Invalid; } //{ // tokenId = ExecutionTargetSelectorTokenId.Invalid; // switch (ch) // { // case ',': // tokenId = ExecutionTargetSelectorTokenId.Comma; // break; // case '(': // tokenId = ExecutionTargetSelectorTokenId.OpenParenthesis; // break; // case ')': // tokenId = ExecutionTargetSelectorTokenId.CloseParenthesis; // break; // case ';': // tokenId = ExecutionTargetSelectorTokenId.SemiColon; // break; // case '\n': // // Line terminator // tokenId = ExecutionTargetSelectorTokenId.Whitespace; // break; // case '{': // tokenId = ExecutionTargetSelectorTokenId.OpenCurlyBrace; // break; // case '}': // tokenId = ExecutionTargetSelectorTokenId.CloseCurlyBrace; // break; // case '/': // tokenId = ExecutionTargetSelectorTokenId.Division; // switch (reader.Peek()) // { // case '/': // // Parse a single-line comment // tokenId = this.ParseSingleLineComment(reader); // break; // case '*': // // Parse a multi-line comment // tokenId = this.ParseMultiLineComment(reader); // break; // } // break; // case '=': // if (reader.Peek() == '=') // { // reader.Read(); // tokenId = ExecutionTargetSelectorTokenId.Equality; // } // else // tokenId = ExecutionTargetSelectorTokenId.Assignment; // break; // case '!': // if (reader.Peek() == '=') // { // reader.Read(); // tokenId = ExecutionTargetSelectorTokenId.Inequality; // } // break; // case '+': // tokenId = ExecutionTargetSelectorTokenId.Addition; // break; // case '-': // tokenId = ExecutionTargetSelectorTokenId.Subtraction; // break; // case '*': // tokenId = ExecutionTargetSelectorTokenId.Multiplication; // break; // default: // if ((ch >= '0') && (ch <= '9')) // { // // Parse the number // tokenId = this.ParseNumber(reader, ch); // } // break; // } //} if (tokenId != ExecutionTargetSelectorTokenId.Invalid) { return(new MergableLexerResult(MatchType.ExactMatch, new LexicalStateTokenData(lexicalState, tokenId))); } else { reader.ReadReverse(); return(MergableLexerResult.NoMatch); } }
/// <summary> /// Creates an <see cref="IToken"/> that represents an invalid range of text. /// </summary> /// <param name="startOffset">The start offset of the <see cref="IToken"/>.</param> /// <param name="length">The length of the <see cref="IToken"/>.</param> /// <param name="lexicalState">The <see cref="ILexicalState"/> that contains the token.</param> /// <returns>An <see cref="IToken"/> that represents an invalid range of text.</returns> public override IToken CreateInvalidToken(int startOffset, int length, ILexicalState lexicalState) { return(new LuatToken(startOffset, length, LexicalParseFlags.None, null, new LexicalStateAndIDTokenLexicalParseData(lexicalState, LuatTokenId.Invalid))); }
/// <summary> /// Creates an <see cref="IToken"/> that represents the end of a document. /// </summary> /// <param name="startOffset">The start offset of the <see cref="IToken"/>.</param> /// <param name="lexicalState">The <see cref="ILexicalState"/> that contains the token.</param> /// <returns>An <see cref="IToken"/> that represents the end of a document.</returns> public override IToken CreateDocumentEndToken(int startOffset, ILexicalState lexicalState) { return(new LuatToken(startOffset, 0, LexicalParseFlags.None, null, new LexicalStateAndIDTokenLexicalParseData(lexicalState, LuatTokenId.DocumentEnd))); }
/// <summary> /// Performs a lexical parse to return the next <see cref="ITokenLexicalParseData"/> /// from a <see cref="ITextBufferReader"/> and seeks past it if there is a match. /// </summary> /// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param> /// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current context.</param> /// <param name="lexicalParseData">Returns the next <see cref="ITokenLexicalParseData"/> from a <see cref="ITextBufferReader"/>.</param> /// <returns>A <see cref="MatchType"/> indicating the type of match that was made.</returns> public MatchType GetNextTokenLexicalParseData(ITextBufferReader reader, ILexicalState lexicalState, ref ITokenLexicalParseData lexicalParseData) { // Initialize int tokenID = LuatTokenId.Invalid; if ( reader.IsAtEnd ) { lexicalParseData = new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)LuatTokenId.DocumentEnd); return MatchType.ExactMatch; } // Get the next character char ch = reader.Read(); // If the character is a letter or digit... if ((Char.IsLetter(ch) || (ch == '_'))) { // Parse the identifier tokenID = this.ParseIdentifier(reader, ch); } else if ((ch != '\n') && (Char.IsWhiteSpace(ch))) { while ((reader.Peek() != '\n') && (Char.IsWhiteSpace(reader.Peek()))) reader.Read(); tokenID = LuatTokenId.Whitespace; } else { tokenID = LuatTokenId.Invalid; switch (ch) { case ',': tokenID = LuatTokenId.Comma; break; case '(': tokenID = LuatTokenId.OpenParenthesis; break; case ')': tokenID = LuatTokenId.CloseParenthesis; break; case ';': tokenID = LuatTokenId.SemiColon; break; case ':': tokenID = LuatTokenId.Colon; break; case '\n': case '\r': // Line terminator tokenID = LuatTokenId.LineTerminator; break; case '{': tokenID = LuatTokenId.OpenCurlyBrace; break; case '}': tokenID = LuatTokenId.CloseCurlyBrace; break; case '\"': tokenID = this.ParseString( reader, '\"' ); break; case '\'': tokenID = this.ParseString( reader, '\'' ); break; case '-': if ( reader.Peek(1) != '-' ) { tokenID = LuatTokenId.Subtraction; break; } reader.Read(); if ( reader.Peek(1) != '[' || reader.Peek(2) != '[' ) { tokenID = this.ParseSingleLineComment(reader); } else { reader.Read(); reader.Read(); tokenID = this.ParseMultiLineComment( reader ); } break; case '<': if (reader.Peek() == '=') { reader.Read(); tokenID = LuatTokenId.LessThanEqual; } else { tokenID = LuatTokenId.LessThan; } break; case '>': if (reader.Peek() == '=') { reader.Read(); tokenID = LuatTokenId.GreaterThanEqual; } else { tokenID = LuatTokenId.GreaterThan; } break; case '~': if (reader.Peek() == '=') { reader.Read(); tokenID = LuatTokenId.Inequality; } break; case '=': if (reader.Peek() == '=') { reader.Read(); tokenID = LuatTokenId.Equality; } else { tokenID = LuatTokenId.Assignment; } break; case '!': if (reader.Peek() == '=') { reader.Read(); tokenID = LuatTokenId.Inequality; } break; case '+': tokenID = LuatTokenId.Addition; break; case '/': tokenID = LuatTokenId.Division; break; case '*': tokenID = LuatTokenId.Multiplication; break; case '^': tokenID = LuatTokenId.Hat; break; case '#': tokenID = LuatTokenId.Hash; break; case '%': tokenID = LuatTokenId.Modulus; break; case '.': tokenID = LuatTokenId.Dot; if (reader.Peek() == '.') { reader.Read(); tokenID = LuatTokenId.DoubleDot; } if (reader.Peek() == '.') { reader.Read(); tokenID = LuatTokenId.TripleDot; } break; case '[': tokenID = LuatTokenId.OpenSquareBracket; break; case ']': tokenID = LuatTokenId.CloseSquareBracket; break; default: if ((ch >= '0') && (ch <= '9')) { // Parse the number tokenID = this.ParseNumber(reader, ch); } break; } } if (tokenID != LuatTokenId.Invalid) { lexicalParseData = new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)tokenID); return MatchType.ExactMatch; } else { reader.ReadReverse(); return MatchType.NoMatch; } }
///////////////////////////////////////////////////////////////////////////////////////////////////// // PUBLIC PROCEDURES ///////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// Returns a single-character <see cref="ITokenLexicalParseData"/> representing the lexical parse data for the /// default token in the <see cref="ILexicalState"/> and seeks forward one position in the <see cref="ITextBufferReader"/> /// </summary> /// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param> /// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current context.</param> /// <returns>The <see cref="ITokenLexicalParseData"/> for default text in the <see cref="ILexicalState"/>.</returns> public ITokenLexicalParseData GetLexicalStateDefaultTokenLexicalParseData(ITextBufferReader reader, ILexicalState lexicalState) { reader.Read(); return new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)lexicalState.DefaultTokenID); }
/// <summary> /// Performs a lexical parse to return the next <see cref="ITokenLexicalParseData"/> /// from a <see cref="ITextBufferReader"/> and seeks past it if there is a match. /// </summary> /// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param> /// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current context.</param> /// <param name="lexicalParseData">Returns the next <see cref="ITokenLexicalParseData"/> from a <see cref="ITextBufferReader"/>.</param> /// <returns>A <see cref="MatchType"/> indicating the type of match that was made.</returns> public MatchType GetNextTokenLexicalParseData(ITextBufferReader reader, ILexicalState lexicalState, ref ITokenLexicalParseData lexicalParseData) { // Initialize int tokenID = LuatTokenId.Invalid; if (reader.IsAtEnd) { lexicalParseData = new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)LuatTokenId.DocumentEnd); return(MatchType.ExactMatch); } // Get the next character char ch = reader.Read(); // If the character is a letter or digit... if ((Char.IsLetter(ch) || (ch == '_'))) { // Parse the identifier tokenID = this.ParseIdentifier(reader, ch); } else if ((ch != '\n') && (Char.IsWhiteSpace(ch))) { while ((reader.Peek() != '\n') && (Char.IsWhiteSpace(reader.Peek()))) { reader.Read(); } tokenID = LuatTokenId.Whitespace; } else { tokenID = LuatTokenId.Invalid; switch (ch) { case ',': tokenID = LuatTokenId.Comma; break; case '(': tokenID = LuatTokenId.OpenParenthesis; break; case ')': tokenID = LuatTokenId.CloseParenthesis; break; case ';': tokenID = LuatTokenId.SemiColon; break; case ':': tokenID = LuatTokenId.Colon; break; case '\n': case '\r': // Line terminator tokenID = LuatTokenId.LineTerminator; break; case '{': tokenID = LuatTokenId.OpenCurlyBrace; break; case '}': tokenID = LuatTokenId.CloseCurlyBrace; break; case '\"': tokenID = this.ParseString(reader, '\"'); break; case '\'': tokenID = this.ParseString(reader, '\''); break; case '-': if (reader.Peek(1) != '-') { tokenID = LuatTokenId.Subtraction; break; } reader.Read(); if (reader.Peek(1) != '[' || reader.Peek(2) != '[') { tokenID = this.ParseSingleLineComment(reader); } else { reader.Read(); reader.Read(); tokenID = this.ParseMultiLineComment(reader); } break; case '<': if (reader.Peek() == '=') { reader.Read(); tokenID = LuatTokenId.LessThanEqual; } else { tokenID = LuatTokenId.LessThan; } break; case '>': if (reader.Peek() == '=') { reader.Read(); tokenID = LuatTokenId.GreaterThanEqual; } else { tokenID = LuatTokenId.GreaterThan; } break; case '~': if (reader.Peek() == '=') { reader.Read(); tokenID = LuatTokenId.Inequality; } break; case '=': if (reader.Peek() == '=') { reader.Read(); tokenID = LuatTokenId.Equality; } else { tokenID = LuatTokenId.Assignment; } break; case '!': if (reader.Peek() == '=') { reader.Read(); tokenID = LuatTokenId.Inequality; } break; case '+': tokenID = LuatTokenId.Addition; break; case '/': tokenID = LuatTokenId.Division; break; case '*': tokenID = LuatTokenId.Multiplication; break; case '^': tokenID = LuatTokenId.Hat; break; case '#': tokenID = LuatTokenId.Hash; break; case '%': tokenID = LuatTokenId.Modulus; break; case '.': tokenID = LuatTokenId.Dot; if (reader.Peek() == '.') { reader.Read(); tokenID = LuatTokenId.DoubleDot; } if (reader.Peek() == '.') { reader.Read(); tokenID = LuatTokenId.TripleDot; } break; case '[': tokenID = LuatTokenId.OpenSquareBracket; break; case ']': tokenID = LuatTokenId.CloseSquareBracket; break; default: if ((ch >= '0') && (ch <= '9')) { // Parse the number tokenID = this.ParseNumber(reader, ch); } break; } } if (tokenID != LuatTokenId.Invalid) { lexicalParseData = new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)tokenID); return(MatchType.ExactMatch); } else { reader.ReadReverse(); return(MatchType.NoMatch); } }
///////////////////////////////////////////////////////////////////////////////////////////////////// // PUBLIC PROCEDURES ///////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// Returns a single-character <see cref="ITokenLexicalParseData"/> representing the lexical parse data for the /// default token in the <see cref="ILexicalState"/> and seeks forward one position in the <see cref="ITextBufferReader"/> /// </summary> /// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param> /// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current context.</param> /// <returns>The <see cref="ITokenLexicalParseData"/> for default text in the <see cref="ILexicalState"/>.</returns> public ITokenLexicalParseData GetLexicalStateDefaultTokenLexicalParseData(ITextBufferReader reader, ILexicalState lexicalState) { reader.Read(); return(new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)lexicalState.DefaultTokenID)); }
/// <summary> /// Performs a lex to return the next <see cref="MergableLexerResult"/> /// from a <see cref="ITextBufferReader"/> and seeks past it if there is a match. /// </summary> /// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param> /// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current state.</param> /// <returns>A <see cref="MergableLexerResult"/> indicating the lexer result.</returns> public override MergableLexerResult GetNextToken(ITextBufferReader reader, ILexicalState lexicalState) { // Initialize int tokenId = SimpleTokenId.Invalid; // Get the next character char ch = reader.Read(); // If the character is a letter or digit... if ((Char.IsLetter(ch) || (ch == '_'))) { // Parse the identifier tokenId = this.ParseIdentifier(reader, ch); } else if ((ch != '\n') && (Char.IsWhiteSpace(ch))) { while ((reader.Peek() != '\n') && (Char.IsWhiteSpace(reader.Peek()))) { reader.Read(); } tokenId = SimpleTokenId.Whitespace; } else { tokenId = SimpleTokenId.Invalid; switch (ch) { case ',': tokenId = SimpleTokenId.Comma; break; case '(': tokenId = SimpleTokenId.OpenParenthesis; break; case ')': tokenId = SimpleTokenId.CloseParenthesis; break; case ';': tokenId = SimpleTokenId.SemiColon; break; case '\n': // Line terminator tokenId = SimpleTokenId.Whitespace; break; case '{': tokenId = SimpleTokenId.OpenCurlyBrace; break; case '}': tokenId = SimpleTokenId.CloseCurlyBrace; break; case '/': tokenId = SimpleTokenId.Division; switch (reader.Peek()) { case '/': // Parse a single-line comment tokenId = this.ParseSingleLineComment(reader); break; case '*': // Parse a multi-line comment tokenId = this.ParseMultiLineComment(reader); break; } break; case '=': if (reader.Peek() == '=') { reader.Read(); tokenId = SimpleTokenId.Equality; } else { tokenId = SimpleTokenId.Assignment; } break; case '!': if (reader.Peek() == '=') { reader.Read(); tokenId = SimpleTokenId.Inequality; } break; case '+': tokenId = SimpleTokenId.Addition; break; case '-': tokenId = SimpleTokenId.Subtraction; break; case '*': tokenId = SimpleTokenId.Multiplication; break; default: if ((ch >= '0') && (ch <= '9')) { // Parse the number tokenId = this.ParseNumber(reader, ch); } break; } } if (tokenId != SimpleTokenId.Invalid) { return(new MergableLexerResult(MatchType.ExactMatch, new LexicalStateTokenData(lexicalState, tokenId))); } else { reader.ReadReverse(); return(MergableLexerResult.NoMatch); } }