/// <summary>
/// Performs a lex to return the next <see cref="MergableLexerResult"/>
/// from a <see cref="ITextBufferReader"/> and seeks past it if there is a match.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current state.</param>
/// <returns>A <see cref="MergableLexerResult"/> indicating the lexer result.</returns>
public override MergableLexerResult GetNextToken(ITextBufferReader reader, ILexicalState lexicalState)
{
// Initialize
int tokenId = ParentTokenId.Invalid;
// Get the next character
char ch = reader.Read();
switch (lexicalState.Id)
{
case ParentLexicalStateId.Default: {
// If the character is a letter or digit...
if ((Char.IsLetter(ch) || (ch == '_')))
{
// Parse the identifier
tokenId = this.ParseIdentifier(reader, ch);
}
else if (Char.IsWhiteSpace(ch))
{
// Consume sequential whitespace
while (Char.IsWhiteSpace(reader.Peek()))
{
reader.Read();
}
tokenId = ParentTokenId.Whitespace;
}
else
{
// Invalid
tokenId = ParentTokenId.Invalid;
}
break;
}
}
if (tokenId != ParentTokenId.Invalid)
{
return(new MergableLexerResult(MatchType.ExactMatch, new LexicalStateTokenData(lexicalState, tokenId)));
}
else
{
reader.ReadReverse();
return(MergableLexerResult.NoMatch);
}
}
/// <summary>
/// Creates an <see cref="IToken"/> that represents an invalid range of text.
/// </summary>
/// <param name="startOffset">The start offset of the <see cref="IToken"/>.</param>
/// <param name="length">The length of the <see cref="IToken"/>.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that contains the token.</param>
/// <returns>An <see cref="IToken"/> that represents an invalid range of text.</returns>
public override IToken CreateInvalidToken(int startOffset, int length, ILexicalState lexicalState)
{
return new LuatToken(startOffset, length, LexicalParseFlags.None, null, new LexicalStateAndIDTokenLexicalParseData(lexicalState, LuatTokenId.Invalid));
}
/// <summary>
/// Creates an <see cref="IToken"/> that represents the end of a document.
/// </summary>
/// <param name="startOffset">The start offset of the <see cref="IToken"/>.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that contains the token.</param>
/// <returns>An <see cref="IToken"/> that represents the end of a document.</returns>
public override IToken CreateDocumentEndToken(int startOffset, ILexicalState lexicalState)
{
return new LuatToken(startOffset, 0, LexicalParseFlags.None, null, new LexicalStateAndIDTokenLexicalParseData(lexicalState, LuatTokenId.DocumentEnd));
}
public override MergableLexerResult GetNextToken(ITextBufferReader reader, ILexicalState lexicalState)
{
// Initialize
int tokenId = ExecutionTargetSelectorTokenId.Invalid;
// Get the next character
char ch = reader.Read();
if ((Char.IsLetter(ch) || (ch == '_')))
{
// Parse the identifier
tokenId = this.ParseIdentifier(reader, ch);
}
//else if (Char.IsWhiteSpace(ch))
//{
// while (Char.IsWhiteSpace(reader.Peek()))
// reader.Read();
// tokenId = ExecutionTargetSelectorTokenId.Whitespace;
//}
//else if (Char.IsNumber(ch))
//{
// tokenId = this.ParseNumber(reader, ch);
//}
else if (ch == '.')
{
tokenId = ExecutionTargetSelectorTokenId.Dot;
}
else
{
tokenId = ExecutionTargetSelectorTokenId.Invalid;
}
//{
// tokenId = ExecutionTargetSelectorTokenId.Invalid;
// switch (ch)
// {
// case ',':
// tokenId = ExecutionTargetSelectorTokenId.Comma;
// break;
// case '(':
// tokenId = ExecutionTargetSelectorTokenId.OpenParenthesis;
// break;
// case ')':
// tokenId = ExecutionTargetSelectorTokenId.CloseParenthesis;
// break;
// case ';':
// tokenId = ExecutionTargetSelectorTokenId.SemiColon;
// break;
// case '\n':
// // Line terminator
// tokenId = ExecutionTargetSelectorTokenId.Whitespace;
// break;
// case '{':
// tokenId = ExecutionTargetSelectorTokenId.OpenCurlyBrace;
// break;
// case '}':
// tokenId = ExecutionTargetSelectorTokenId.CloseCurlyBrace;
// break;
// case '/':
// tokenId = ExecutionTargetSelectorTokenId.Division;
// switch (reader.Peek())
// {
// case '/':
// // Parse a single-line comment
// tokenId = this.ParseSingleLineComment(reader);
// break;
// case '*':
// // Parse a multi-line comment
// tokenId = this.ParseMultiLineComment(reader);
// break;
// }
// break;
// case '=':
// if (reader.Peek() == '=')
// {
// reader.Read();
// tokenId = ExecutionTargetSelectorTokenId.Equality;
// }
// else
// tokenId = ExecutionTargetSelectorTokenId.Assignment;
// break;
// case '!':
// if (reader.Peek() == '=')
// {
// reader.Read();
// tokenId = ExecutionTargetSelectorTokenId.Inequality;
// }
// break;
// case '+':
// tokenId = ExecutionTargetSelectorTokenId.Addition;
// break;
// case '-':
// tokenId = ExecutionTargetSelectorTokenId.Subtraction;
// break;
// case '*':
// tokenId = ExecutionTargetSelectorTokenId.Multiplication;
// break;
// default:
// if ((ch >= '0') && (ch <= '9'))
// {
// // Parse the number
// tokenId = this.ParseNumber(reader, ch);
// }
// break;
// }
//}
if (tokenId != ExecutionTargetSelectorTokenId.Invalid)
{
return(new MergableLexerResult(MatchType.ExactMatch, new LexicalStateTokenData(lexicalState, tokenId)));
}
else
{
reader.ReadReverse();
return(MergableLexerResult.NoMatch);
}
}
/// <summary>
/// Creates an <see cref="IToken"/> that represents an invalid range of text.
/// </summary>
/// <param name="startOffset">The start offset of the <see cref="IToken"/>.</param>
/// <param name="length">The length of the <see cref="IToken"/>.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that contains the token.</param>
/// <returns>An <see cref="IToken"/> that represents an invalid range of text.</returns>
public override IToken CreateInvalidToken(int startOffset, int length, ILexicalState lexicalState)
{
return(new LuatToken(startOffset, length, LexicalParseFlags.None, null, new LexicalStateAndIDTokenLexicalParseData(lexicalState, LuatTokenId.Invalid)));
}
/// <summary>
/// Creates an <see cref="IToken"/> that represents the end of a document.
/// </summary>
/// <param name="startOffset">The start offset of the <see cref="IToken"/>.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that contains the token.</param>
/// <returns>An <see cref="IToken"/> that represents the end of a document.</returns>
public override IToken CreateDocumentEndToken(int startOffset, ILexicalState lexicalState)
{
return(new LuatToken(startOffset, 0, LexicalParseFlags.None, null, new LexicalStateAndIDTokenLexicalParseData(lexicalState, LuatTokenId.DocumentEnd)));
}
/// <summary>
/// Performs a lexical parse to return the next <see cref="ITokenLexicalParseData"/>
/// from a <see cref="ITextBufferReader"/> and seeks past it if there is a match.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current context.</param>
/// <param name="lexicalParseData">Returns the next <see cref="ITokenLexicalParseData"/> from a <see cref="ITextBufferReader"/>.</param>
/// <returns>A <see cref="MatchType"/> indicating the type of match that was made.</returns>
public MatchType GetNextTokenLexicalParseData(ITextBufferReader reader, ILexicalState lexicalState, ref ITokenLexicalParseData lexicalParseData) {
// Initialize
int tokenID = LuatTokenId.Invalid;
if ( reader.IsAtEnd )
{
lexicalParseData = new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)LuatTokenId.DocumentEnd);
return MatchType.ExactMatch;
}
// Get the next character
char ch = reader.Read();
// If the character is a letter or digit...
if ((Char.IsLetter(ch) || (ch == '_'))) {
// Parse the identifier
tokenID = this.ParseIdentifier(reader, ch);
}
else if ((ch != '\n') && (Char.IsWhiteSpace(ch))) {
while ((reader.Peek() != '\n') && (Char.IsWhiteSpace(reader.Peek())))
reader.Read();
tokenID = LuatTokenId.Whitespace;
}
else {
tokenID = LuatTokenId.Invalid;
switch (ch) {
case ',':
tokenID = LuatTokenId.Comma;
break;
case '(':
tokenID = LuatTokenId.OpenParenthesis;
break;
case ')':
tokenID = LuatTokenId.CloseParenthesis;
break;
case ';':
tokenID = LuatTokenId.SemiColon;
break;
case ':':
tokenID = LuatTokenId.Colon;
break;
case '\n':
case '\r':
// Line terminator
tokenID = LuatTokenId.LineTerminator;
break;
case '{':
tokenID = LuatTokenId.OpenCurlyBrace;
break;
case '}':
tokenID = LuatTokenId.CloseCurlyBrace;
break;
case '\"':
tokenID = this.ParseString( reader, '\"' );
break;
case '\'':
tokenID = this.ParseString( reader, '\'' );
break;
case '-':
if ( reader.Peek(1) != '-' )
{
tokenID = LuatTokenId.Subtraction;
break;
}
reader.Read();
if ( reader.Peek(1) != '[' ||
reader.Peek(2) != '[' )
{
tokenID = this.ParseSingleLineComment(reader);
}
else
{
reader.Read();
reader.Read();
tokenID = this.ParseMultiLineComment( reader );
}
break;
case '<':
if (reader.Peek() == '=') {
reader.Read();
tokenID = LuatTokenId.LessThanEqual;
}
else
{
tokenID = LuatTokenId.LessThan;
}
break;
case '>':
if (reader.Peek() == '=') {
reader.Read();
tokenID = LuatTokenId.GreaterThanEqual;
}
else
{
tokenID = LuatTokenId.GreaterThan;
}
break;
case '~':
if (reader.Peek() == '=') {
reader.Read();
tokenID = LuatTokenId.Inequality;
}
break;
case '=':
if (reader.Peek() == '=') {
reader.Read();
tokenID = LuatTokenId.Equality;
}
else
{
tokenID = LuatTokenId.Assignment;
}
break;
case '!':
if (reader.Peek() == '=') {
reader.Read();
tokenID = LuatTokenId.Inequality;
}
break;
case '+':
tokenID = LuatTokenId.Addition;
break;
case '/':
tokenID = LuatTokenId.Division;
break;
case '*':
tokenID = LuatTokenId.Multiplication;
break;
case '^':
tokenID = LuatTokenId.Hat;
break;
case '#':
tokenID = LuatTokenId.Hash;
break;
case '%':
tokenID = LuatTokenId.Modulus;
break;
case '.':
tokenID = LuatTokenId.Dot;
if (reader.Peek() == '.')
{
reader.Read();
tokenID = LuatTokenId.DoubleDot;
}
if (reader.Peek() == '.')
{
reader.Read();
tokenID = LuatTokenId.TripleDot;
}
break;
case '[':
tokenID = LuatTokenId.OpenSquareBracket;
break;
case ']':
tokenID = LuatTokenId.CloseSquareBracket;
break;
default:
if ((ch >= '0') && (ch <= '9')) {
// Parse the number
tokenID = this.ParseNumber(reader, ch);
}
break;
}
}
if (tokenID != LuatTokenId.Invalid) {
lexicalParseData = new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)tokenID);
return MatchType.ExactMatch;
}
else {
reader.ReadReverse();
return MatchType.NoMatch;
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// PUBLIC PROCEDURES
/////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
/// Returns a single-character <see cref="ITokenLexicalParseData"/> representing the lexical parse data for the
/// default token in the <see cref="ILexicalState"/> and seeks forward one position in the <see cref="ITextBufferReader"/>
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current context.</param>
/// <returns>The <see cref="ITokenLexicalParseData"/> for default text in the <see cref="ILexicalState"/>.</returns>
public ITokenLexicalParseData GetLexicalStateDefaultTokenLexicalParseData(ITextBufferReader reader, ILexicalState lexicalState) {
reader.Read();
return new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)lexicalState.DefaultTokenID);
}
/// <summary>
/// Performs a lexical parse to return the next <see cref="ITokenLexicalParseData"/>
/// from a <see cref="ITextBufferReader"/> and seeks past it if there is a match.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current context.</param>
/// <param name="lexicalParseData">Returns the next <see cref="ITokenLexicalParseData"/> from a <see cref="ITextBufferReader"/>.</param>
/// <returns>A <see cref="MatchType"/> indicating the type of match that was made.</returns>
public MatchType GetNextTokenLexicalParseData(ITextBufferReader reader, ILexicalState lexicalState, ref ITokenLexicalParseData lexicalParseData)
{
// Initialize
int tokenID = LuatTokenId.Invalid;
if (reader.IsAtEnd)
{
lexicalParseData = new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)LuatTokenId.DocumentEnd);
return(MatchType.ExactMatch);
}
// Get the next character
char ch = reader.Read();
// If the character is a letter or digit...
if ((Char.IsLetter(ch) || (ch == '_')))
{
// Parse the identifier
tokenID = this.ParseIdentifier(reader, ch);
}
else if ((ch != '\n') && (Char.IsWhiteSpace(ch)))
{
while ((reader.Peek() != '\n') && (Char.IsWhiteSpace(reader.Peek())))
{
reader.Read();
}
tokenID = LuatTokenId.Whitespace;
}
else
{
tokenID = LuatTokenId.Invalid;
switch (ch)
{
case ',':
tokenID = LuatTokenId.Comma;
break;
case '(':
tokenID = LuatTokenId.OpenParenthesis;
break;
case ')':
tokenID = LuatTokenId.CloseParenthesis;
break;
case ';':
tokenID = LuatTokenId.SemiColon;
break;
case ':':
tokenID = LuatTokenId.Colon;
break;
case '\n':
case '\r':
// Line terminator
tokenID = LuatTokenId.LineTerminator;
break;
case '{':
tokenID = LuatTokenId.OpenCurlyBrace;
break;
case '}':
tokenID = LuatTokenId.CloseCurlyBrace;
break;
case '\"':
tokenID = this.ParseString(reader, '\"');
break;
case '\'':
tokenID = this.ParseString(reader, '\'');
break;
case '-':
if (reader.Peek(1) != '-')
{
tokenID = LuatTokenId.Subtraction;
break;
}
reader.Read();
if (reader.Peek(1) != '[' ||
reader.Peek(2) != '[')
{
tokenID = this.ParseSingleLineComment(reader);
}
else
{
reader.Read();
reader.Read();
tokenID = this.ParseMultiLineComment(reader);
}
break;
case '<':
if (reader.Peek() == '=')
{
reader.Read();
tokenID = LuatTokenId.LessThanEqual;
}
else
{
tokenID = LuatTokenId.LessThan;
}
break;
case '>':
if (reader.Peek() == '=')
{
reader.Read();
tokenID = LuatTokenId.GreaterThanEqual;
}
else
{
tokenID = LuatTokenId.GreaterThan;
}
break;
case '~':
if (reader.Peek() == '=')
{
reader.Read();
tokenID = LuatTokenId.Inequality;
}
break;
case '=':
if (reader.Peek() == '=')
{
reader.Read();
tokenID = LuatTokenId.Equality;
}
else
{
tokenID = LuatTokenId.Assignment;
}
break;
case '!':
if (reader.Peek() == '=')
{
reader.Read();
tokenID = LuatTokenId.Inequality;
}
break;
case '+':
tokenID = LuatTokenId.Addition;
break;
case '/':
tokenID = LuatTokenId.Division;
break;
case '*':
tokenID = LuatTokenId.Multiplication;
break;
case '^':
tokenID = LuatTokenId.Hat;
break;
case '#':
tokenID = LuatTokenId.Hash;
break;
case '%':
tokenID = LuatTokenId.Modulus;
break;
case '.':
tokenID = LuatTokenId.Dot;
if (reader.Peek() == '.')
{
reader.Read();
tokenID = LuatTokenId.DoubleDot;
}
if (reader.Peek() == '.')
{
reader.Read();
tokenID = LuatTokenId.TripleDot;
}
break;
case '[':
tokenID = LuatTokenId.OpenSquareBracket;
break;
case ']':
tokenID = LuatTokenId.CloseSquareBracket;
break;
default:
if ((ch >= '0') && (ch <= '9'))
{
// Parse the number
tokenID = this.ParseNumber(reader, ch);
}
break;
}
}
if (tokenID != LuatTokenId.Invalid)
{
lexicalParseData = new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)tokenID);
return(MatchType.ExactMatch);
}
else
{
reader.ReadReverse();
return(MatchType.NoMatch);
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// PUBLIC PROCEDURES
/////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
/// Returns a single-character <see cref="ITokenLexicalParseData"/> representing the lexical parse data for the
/// default token in the <see cref="ILexicalState"/> and seeks forward one position in the <see cref="ITextBufferReader"/>
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current context.</param>
/// <returns>The <see cref="ITokenLexicalParseData"/> for default text in the <see cref="ILexicalState"/>.</returns>
public ITokenLexicalParseData GetLexicalStateDefaultTokenLexicalParseData(ITextBufferReader reader, ILexicalState lexicalState)
{
reader.Read();
return(new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)lexicalState.DefaultTokenID));
}
/// <summary>
/// Performs a lex to return the next <see cref="MergableLexerResult"/>
/// from a <see cref="ITextBufferReader"/> and seeks past it if there is a match.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current state.</param>
/// <returns>A <see cref="MergableLexerResult"/> indicating the lexer result.</returns>
public override MergableLexerResult GetNextToken(ITextBufferReader reader, ILexicalState lexicalState)
{
// Initialize
int tokenId = SimpleTokenId.Invalid;
// Get the next character
char ch = reader.Read();
// If the character is a letter or digit...
if ((Char.IsLetter(ch) || (ch == '_')))
{
// Parse the identifier
tokenId = this.ParseIdentifier(reader, ch);
}
else if ((ch != '\n') && (Char.IsWhiteSpace(ch)))
{
while ((reader.Peek() != '\n') && (Char.IsWhiteSpace(reader.Peek())))
{
reader.Read();
}
tokenId = SimpleTokenId.Whitespace;
}
else
{
tokenId = SimpleTokenId.Invalid;
switch (ch)
{
case ',':
tokenId = SimpleTokenId.Comma;
break;
case '(':
tokenId = SimpleTokenId.OpenParenthesis;
break;
case ')':
tokenId = SimpleTokenId.CloseParenthesis;
break;
case ';':
tokenId = SimpleTokenId.SemiColon;
break;
case '\n':
// Line terminator
tokenId = SimpleTokenId.Whitespace;
break;
case '{':
tokenId = SimpleTokenId.OpenCurlyBrace;
break;
case '}':
tokenId = SimpleTokenId.CloseCurlyBrace;
break;
case '/':
tokenId = SimpleTokenId.Division;
switch (reader.Peek())
{
case '/':
// Parse a single-line comment
tokenId = this.ParseSingleLineComment(reader);
break;
case '*':
// Parse a multi-line comment
tokenId = this.ParseMultiLineComment(reader);
break;
}
break;
case '=':
if (reader.Peek() == '=')
{
reader.Read();
tokenId = SimpleTokenId.Equality;
}
else
{
tokenId = SimpleTokenId.Assignment;
}
break;
case '!':
if (reader.Peek() == '=')
{
reader.Read();
tokenId = SimpleTokenId.Inequality;
}
break;
case '+':
tokenId = SimpleTokenId.Addition;
break;
case '-':
tokenId = SimpleTokenId.Subtraction;
break;
case '*':
tokenId = SimpleTokenId.Multiplication;
break;
default:
if ((ch >= '0') && (ch <= '9'))
{
// Parse the number
tokenId = this.ParseNumber(reader, ch);
}
break;
}
}
if (tokenId != SimpleTokenId.Invalid)
{
return(new MergableLexerResult(MatchType.ExactMatch, new LexicalStateTokenData(lexicalState, tokenId)));
}
else
{
reader.ReadReverse();
return(MergableLexerResult.NoMatch);
}
}
