/// <summary>
/// Parses a multiple line comment.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseMultiLineComment(ITextBufferReader reader)
{
reader.Read();
while (reader.Offset < reader.Length)
{
if (reader.Peek() == '*')
{
if (reader.Offset + 1 < reader.Length)
{
if (reader.Peek(2) == '/')
{
reader.Read();
reader.Read();
break;
}
}
else
{
reader.Read();
break;
}
}
reader.Read();
}
return(SimpleTokenId.MultiLineCommentText);
}
/// <summary>
/// Parses an identifier.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="ch">The first character of the identifier.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseIdentifier(ITextBufferReader reader, char ch)
{
// Get the entire word
int startOffset = reader.Offset - 1;
while (!reader.IsAtEnd)
{
char ch2 = reader.Read();
// NOTE: This could be improved by supporting \u escape sequences
if ((!char.IsLetterOrDigit(ch2)) && (ch2 != '_'))
{
reader.ReadReverse();
break;
}
}
// Determine if the word is a keyword
if (Char.IsLetter(ch))
{
int value;
String subString = reader.GetSubstring(startOffset, reader.Offset - startOffset);
if (!caseSensitive)
{
subString = subString.ToLowerInvariant();
}
return(keywords.TryGetValue(subString, out value) ? value : SimpleTokenId.Identifier);
}
else
{
return(SimpleTokenId.Identifier);
}
}
/// <summary>
/// Parses a number.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="ch">The first character of the number.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseNumber(ITextBufferReader reader, char ch)
{
while (Char.IsNumber(reader.Peek()))
{
reader.Read();
}
return(SimpleTokenId.Number);
}
/// <summary>
/// Parses a single line comment.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseSingleLineComment(ITextBufferReader reader)
{
while ((!reader.IsAtEnd) && (reader.Peek() != '\n'))
{
reader.Read();
}
return(LuatTokenId.SingleLineComment);
}
protected virtual int ParseIdentifier(ITextBufferReader reader, char ch)
{
// Get the entire word
int startOffset = reader.Offset - 1;
while (!reader.IsAtEnd && (char.IsLetterOrDigit(reader.Peek()) || reader.Peek() == '_'))
{
reader.Read();
}
return(ExecutionTargetSelectorTokenId.Identifier);
}
/// <summary>
/// Parses a string.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseString(ITextBufferReader reader, char quote)
{
Char c0 = '\0';
Char c1 = reader.Read();
while (!(reader.IsAtEnd || (c1 == quote && c0 != '\\')))
{
c0 = c1;
c1 = reader.Read();
}
return(LuatTokenId.String);
}
/// <summary>
/// Represents the method that will handle token matching callbacks.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalScope">The <see cref="ILexicalScope"/> that specifies the lexical scope to check.</param>
/// <returns>A <see cref="MergableLexerResult"/> indicating the lexer result.</returns>
private MergableLexerResult IsChildCodeBlockTransitionStateScopeStart(ITextBufferReader reader, ILexicalScope lexicalScope)
{
if (reader.Peek() == '<')
{
reader.Read();
if (reader.Peek() == '%')
{
reader.Read();
return(new MergableLexerResult(MatchType.ExactMatch, new LexicalScopeTokenData(lexicalScope, ParentTokenId.ChildCodeBlockStart)));
}
reader.ReadReverse();
}
return(MergableLexerResult.NoMatch);
}
/// <summary>
/// Parses a multiple line comment.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseMultiLineComment(ITextBufferReader reader)
{
while (reader.Offset + 2 < reader.Length)
{
if ((reader.Peek(1) == ']' &&
reader.Peek(2) == ']'))
{
reader.Read();
reader.Read();
break;
}
reader.Read();
}
return(LuatTokenId.MultiLineComment);
}
/// <summary>
/// Parses an identifier.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="ch">The first character of the identifier.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseIdentifier(ITextBufferReader reader, char ch)
{
// Get the entire word
int startOffset = reader.Offset - 1;
while (!reader.IsAtEnd)
{
// Accept namespaced identifiers
if (reader.Peek(1) == ':' &&
reader.Peek(2) == ':' &&
char.IsLetterOrDigit(reader.Peek(3)))
{
reader.Read();
reader.Read();
reader.Read();
continue;
}
char ch2 = reader.Read();
// NOTE: This could be improved by supporting \u escape sequences
if ((!char.IsLetterOrDigit(ch2)) && (ch2 != '_'))
{
reader.ReadReverse();
break;
}
}
// Determine if the word is a keyword
if (Char.IsLetter(ch))
{
object value = keywords[reader.GetSubstring(startOffset, reader.Offset - startOffset)];
if (value != null)
{
return((int)value);
}
else
{
return(LuatTokenId.Identifier);
}
}
else
{
return(LuatTokenId.Identifier);
}
}
protected virtual int ParseNumber(ITextBufferReader reader, char ch)
{
while (Char.IsNumber(reader.Peek()))
{
reader.Read();
}
if (reader.Peek() == '.')
{
reader.Read(); // Skip the dot
while (Char.IsNumber(reader.Peek()))
{
reader.Read();
}
return(ExecutionTargetSelectorTokenId.Decimal);
}
return(ExecutionTargetSelectorTokenId.Integer);
}
/// <summary>
/// Performs a lex to return the next <see cref="MergableLexerResult"/>
/// from a <see cref="ITextBufferReader"/> and seeks past it if there is a match.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current state.</param>
/// <returns>A <see cref="MergableLexerResult"/> indicating the lexer result.</returns>
public override MergableLexerResult GetNextToken(ITextBufferReader reader, ILexicalState lexicalState)
{
// Initialize
int tokenId = ParentTokenId.Invalid;
// Get the next character
char ch = reader.Read();
switch (lexicalState.Id)
{
case ParentLexicalStateId.Default: {
// If the character is a letter or digit...
if ((Char.IsLetter(ch) || (ch == '_')))
{
// Parse the identifier
tokenId = this.ParseIdentifier(reader, ch);
}
else if (Char.IsWhiteSpace(ch))
{
// Consume sequential whitespace
while (Char.IsWhiteSpace(reader.Peek()))
{
reader.Read();
}
tokenId = ParentTokenId.Whitespace;
}
else
{
// Invalid
tokenId = ParentTokenId.Invalid;
}
break;
}
}
if (tokenId != ParentTokenId.Invalid)
{
return(new MergableLexerResult(MatchType.ExactMatch, new LexicalStateTokenData(lexicalState, tokenId)));
}
else
{
reader.ReadReverse();
return(MergableLexerResult.NoMatch);
}
}
/// <summary>
/// Parses a number.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="ch">The first character of the number.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseNumber(ITextBufferReader reader, char ch)
{
Char c = reader.Peek();
bool bParsedDot = false;
while (Char.IsNumber(c) || c == '.')
{
if (c == '.')
{
// Only handle a single '.'
if (bParsedDot)
{
return(LuatTokenId.Number);
}
bParsedDot = true;
}
reader.Read();
c = reader.Peek();
}
return(LuatTokenId.Number);
}
/// <summary>
/// Parses an identifier.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="ch">The first character of the identifier.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseIdentifier(ITextBufferReader reader, char ch)
{
// Get the entire word
int startOffset = reader.Offset - 1;
while (!reader.IsAtEnd)
{
char ch2 = reader.Read();
if ((!char.IsLetterOrDigit(ch2)) && (ch2 != '_'))
{
reader.ReadReverse();
break;
}
}
// This language only has one keyword named "date"
if (reader.GetSubstring(startOffset, reader.Offset - startOffset) == "date")
{
return(ParentTokenId.DateKeyword);
}
// Word is an identifier
return(ParentTokenId.Identifier);
}
/// <summary>
/// Performs a lexical parse to return the next <see cref="ITokenLexicalParseData"/>
/// from a <see cref="ITextBufferReader"/> and seeks past it if there is a match.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current context.</param>
/// <param name="lexicalParseData">Returns the next <see cref="ITokenLexicalParseData"/> from a <see cref="ITextBufferReader"/>.</param>
/// <returns>A <see cref="MatchType"/> indicating the type of match that was made.</returns>
public MatchType GetNextTokenLexicalParseData(ITextBufferReader reader, ILexicalState lexicalState, ref ITokenLexicalParseData lexicalParseData) {
// Initialize
int tokenID = LuatTokenId.Invalid;
if ( reader.IsAtEnd )
{
lexicalParseData = new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)LuatTokenId.DocumentEnd);
return MatchType.ExactMatch;
}
// Get the next character
char ch = reader.Read();
// If the character is a letter or digit...
if ((Char.IsLetter(ch) || (ch == '_'))) {
// Parse the identifier
tokenID = this.ParseIdentifier(reader, ch);
}
else if ((ch != '\n') && (Char.IsWhiteSpace(ch))) {
while ((reader.Peek() != '\n') && (Char.IsWhiteSpace(reader.Peek())))
reader.Read();
tokenID = LuatTokenId.Whitespace;
}
else {
tokenID = LuatTokenId.Invalid;
switch (ch) {
case ',':
tokenID = LuatTokenId.Comma;
break;
case '(':
tokenID = LuatTokenId.OpenParenthesis;
break;
case ')':
tokenID = LuatTokenId.CloseParenthesis;
break;
case ';':
tokenID = LuatTokenId.SemiColon;
break;
case ':':
tokenID = LuatTokenId.Colon;
break;
case '\n':
case '\r':
// Line terminator
tokenID = LuatTokenId.LineTerminator;
break;
case '{':
tokenID = LuatTokenId.OpenCurlyBrace;
break;
case '}':
tokenID = LuatTokenId.CloseCurlyBrace;
break;
case '\"':
tokenID = this.ParseString( reader, '\"' );
break;
case '\'':
tokenID = this.ParseString( reader, '\'' );
break;
case '-':
if ( reader.Peek(1) != '-' )
{
tokenID = LuatTokenId.Subtraction;
break;
}
reader.Read();
if ( reader.Peek(1) != '[' ||
reader.Peek(2) != '[' )
{
tokenID = this.ParseSingleLineComment(reader);
}
else
{
reader.Read();
reader.Read();
tokenID = this.ParseMultiLineComment( reader );
}
break;
case '<':
if (reader.Peek() == '=') {
reader.Read();
tokenID = LuatTokenId.LessThanEqual;
}
else
{
tokenID = LuatTokenId.LessThan;
}
break;
case '>':
if (reader.Peek() == '=') {
reader.Read();
tokenID = LuatTokenId.GreaterThanEqual;
}
else
{
tokenID = LuatTokenId.GreaterThan;
}
break;
case '~':
if (reader.Peek() == '=') {
reader.Read();
tokenID = LuatTokenId.Inequality;
}
break;
case '=':
if (reader.Peek() == '=') {
reader.Read();
tokenID = LuatTokenId.Equality;
}
else
{
tokenID = LuatTokenId.Assignment;
}
break;
case '!':
if (reader.Peek() == '=') {
reader.Read();
tokenID = LuatTokenId.Inequality;
}
break;
case '+':
tokenID = LuatTokenId.Addition;
break;
case '/':
tokenID = LuatTokenId.Division;
break;
case '*':
tokenID = LuatTokenId.Multiplication;
break;
case '^':
tokenID = LuatTokenId.Hat;
break;
case '#':
tokenID = LuatTokenId.Hash;
break;
case '%':
tokenID = LuatTokenId.Modulus;
break;
case '.':
tokenID = LuatTokenId.Dot;
if (reader.Peek() == '.')
{
reader.Read();
tokenID = LuatTokenId.DoubleDot;
}
if (reader.Peek() == '.')
{
reader.Read();
tokenID = LuatTokenId.TripleDot;
}
break;
case '[':
tokenID = LuatTokenId.OpenSquareBracket;
break;
case ']':
tokenID = LuatTokenId.CloseSquareBracket;
break;
default:
if ((ch >= '0') && (ch <= '9')) {
// Parse the number
tokenID = this.ParseNumber(reader, ch);
}
break;
}
}
if (tokenID != LuatTokenId.Invalid) {
lexicalParseData = new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)tokenID);
return MatchType.ExactMatch;
}
else {
reader.ReadReverse();
return MatchType.NoMatch;
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// PUBLIC PROCEDURES
/////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
/// Returns a single-character <see cref="ITokenLexicalParseData"/> representing the lexical parse data for the
/// default token in the <see cref="ILexicalState"/> and seeks forward one position in the <see cref="ITextBufferReader"/>
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current context.</param>
/// <returns>The <see cref="ITokenLexicalParseData"/> for default text in the <see cref="ILexicalState"/>.</returns>
public ITokenLexicalParseData GetLexicalStateDefaultTokenLexicalParseData(ITextBufferReader reader, ILexicalState lexicalState) {
reader.Read();
return new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)lexicalState.DefaultTokenID);
}
/// <summary>
/// Parses a string.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseString(ITextBufferReader reader, char quote ) {
Char c0 = '\0';
Char c1 = reader.Read();
while ( !( reader.IsAtEnd || ( c1 == quote && c0 != '\\' ) ) )
{
c0 = c1;
c1 = reader.Read();
}
return LuatTokenId.String;
}
/// <summary>
/// Parses a single line comment.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseSingleLineComment(ITextBufferReader reader) {
while ((!reader.IsAtEnd) && (reader.Peek() != '\n'))
reader.Read();
return LuatTokenId.SingleLineComment;
}
/// <summary>
/// Parses a number.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="ch">The first character of the number.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseNumber(ITextBufferReader reader, char ch) {
Char c = reader.Peek();
bool bParsedDot = false;
while (Char.IsNumber(c) || c == '.')
{
if (c == '.')
{
// Only handle a single '.'
if (bParsedDot)
{
return LuatTokenId.Number;
}
bParsedDot = true;
}
reader.Read();
c = reader.Peek();
}
return LuatTokenId.Number;
}
/// <summary>
/// Parses a multiple line comment.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseMultiLineComment(ITextBufferReader reader) {
while (reader.Offset + 2 < reader.Length) {
if ( ( reader.Peek(1) == ']' &&
reader.Peek(2) == ']' ) )
{
reader.Read();
reader.Read();
break;
}
reader.Read();
}
return LuatTokenId.MultiLineComment;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// PUBLIC PROCEDURES
/////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
/// Creates an <see cref="ITokenReader"/> that is used by the parser to read through tokens.
/// </summary>
/// <param name="reader">The <see cref="ITextBufferReader"/> that provides access to the text buffer.</param>
/// <returns>An <see cref="ITokenReader"/> that is used by the parser to read through tokens.</returns>
public override ITokenReader CreateTokenReader(ITextBufferReader reader)
{
return(new SimpleTokenReader(reader, new Step3c.SimpleLexer(true)));
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// PUBLIC PROCEDURES
/////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
/// Performs a parsing operation using the parameters specified in the supplied <see cref="IParseRequest"/>
/// and returns the resulting parse data.
/// </summary>
/// <param name="request">The <see cref="IParseRequest"/> that contains data about the requested parsing operation.</param>
/// <returns>An <see cref="IParseData"/> that is the result of the parsing operation.</returns>
/// <remarks>
/// A <see cref="IParseRequestDispatcher"/> typically calls this method when a queued parse request is ready to be processed.
/// </remarks>
public override IParseData Parse(IParseRequest request)
{
if (request == null)
{
throw new ArgumentNullException("request");
}
// Create parse data
ParentParseData parseData = new ParentParseData();
parseData.Snapshot = request.Snapshot;
// Initialize generated text
StringBuilder generatedText = new StringBuilder();
generatedText.Append("using System;\n");
generatedText.Append("using System.Collections.Generic;\n\n");
generatedText.Append("using System.Linq;\n\n");
generatedText.Append("[EditorBrowsable(EditorBrowsableState.Never)]\n");
generatedText.Append("class __Generated {\n");
generatedText.Append("\t[EditorBrowsable(EditorBrowsableState.Never)]\n");
generatedText.Append("\tvoid __WriteOutput() {\n");
ITextSnapshotReader sourceReader = request.Snapshot.GetReader(0);
int lastDelimiterOffset = 0;
bool lastDelimiterWasStart = false;
while (!sourceReader.IsAtSnapshotEnd)
{
IToken token = sourceReader.ReadToken();
if (token != null)
{
switch (token.Id)
{
case ParentTokenId.ChildCodeBlockStart:
case ParentTokenId.ChildOutputBlockStart:
if (token.StartOffset - lastDelimiterOffset > 0)
{
// Append generated text
string text = sourceReader.Snapshot.GetSubstring(new TextRange(lastDelimiterOffset, token.StartOffset), LineTerminator.Newline);
generatedText.Append("\t\tResponse.Write(@\"");
generatedText.Append(text.Replace("\"", "\"\""));
generatedText.Append("\");\n");
}
// Store the last delimiter offset
lastDelimiterOffset = token.EndOffset;
lastDelimiterWasStart = true;
break;
case ParentTokenId.ChildCodeBlockEnd:
if ((lastDelimiterWasStart) && (token.StartOffset - lastDelimiterOffset > 0))
{
// Get the text between the delimiters
string text = sourceReader.Snapshot.GetSubstring(new TextRange(lastDelimiterOffset, token.StartOffset), LineTerminator.Newline);
generatedText.Append("\t\t");
// Add a mapping
parseData.TextRangeMappings.Add(Tuple.Create(new TextRange(lastDelimiterOffset, token.StartOffset), TextRange.FromSpan(generatedText.Length, text.Length)));
// Append the text directly
generatedText.Append(text);
generatedText.Append("\n");
}
// Store the last delimiter offset
lastDelimiterOffset = token.EndOffset;
lastDelimiterWasStart = false;
break;
case ParentTokenId.ChildOutputBlockEnd:
if ((lastDelimiterWasStart) && (token.StartOffset - lastDelimiterOffset > 0))
{
// Get the text between the delimiters and append a Response.Write
string text = sourceReader.Snapshot.GetSubstring(new TextRange(lastDelimiterOffset, token.StartOffset), LineTerminator.Newline);
generatedText.Append("\t\tResponse.Write(");
// Add a mapping
parseData.TextRangeMappings.Add(Tuple.Create(new TextRange(lastDelimiterOffset, token.StartOffset), TextRange.FromSpan(generatedText.Length, text.Length)));
// Append the text directly
generatedText.Append(text);
generatedText.Append(");\n");
}
// Store the last delimiter offset
lastDelimiterOffset = token.EndOffset;
lastDelimiterWasStart = false;
break;
}
}
}
if (lastDelimiterOffset < sourceReader.Snapshot.Length)
{
// Append generated text
string text = sourceReader.Snapshot.GetSubstring(new TextRange(lastDelimiterOffset, sourceReader.Snapshot.Length), LineTerminator.Newline);
generatedText.Append("\t\tResponse.Write(@\"");
generatedText.Append(text.Replace("\"", "\"\""));
generatedText.Append("\");\n");
}
// Store the generated text
generatedText.Append("\t}\n");
generatedText.Append("}\n");
// Get parse data for the translated code
CodeDocument generatedDocument = new CodeDocument();
generatedDocument.Language = childLanguage;
generatedDocument.SetText(generatedText.ToString());
// Get a reader
ITextBufferReader generatedReader = generatedDocument.CurrentSnapshot.GetReader(0).BufferReader;
// Create a request
ParseRequest generatedRequest = new ParseRequest(Guid.NewGuid().ToString(), generatedReader, childParser, generatedDocument);
generatedRequest.Snapshot = generatedDocument.CurrentSnapshot;
// Parse
generatedDocument.ParseData = childParser.Parse(generatedRequest);
parseData.GeneratedParseData = generatedDocument.ParseData as ILLParseData;
return(parseData);
}
/// <summary>
/// Performs a lex to return the next <see cref="MergableLexerResult"/>
/// from a <see cref="ITextBufferReader"/> and seeks past it if there is a match.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current state.</param>
/// <returns>A <see cref="MergableLexerResult"/> indicating the lexer result.</returns>
public override MergableLexerResult GetNextToken(ITextBufferReader reader, ILexicalState lexicalState)
{
// Initialize
int tokenId = SimpleTokenId.Invalid;
// Get the next character
char ch = reader.Read();
// If the character is a letter or digit...
if ((Char.IsLetter(ch) || (ch == '_')))
{
// Parse the identifier
tokenId = this.ParseIdentifier(reader, ch);
}
else if ((ch != '\n') && (Char.IsWhiteSpace(ch)))
{
while ((reader.Peek() != '\n') && (Char.IsWhiteSpace(reader.Peek())))
{
reader.Read();
}
tokenId = SimpleTokenId.Whitespace;
}
else
{
tokenId = SimpleTokenId.Invalid;
switch (ch)
{
case ',':
tokenId = SimpleTokenId.Comma;
break;
case '(':
tokenId = SimpleTokenId.OpenParenthesis;
break;
case ')':
tokenId = SimpleTokenId.CloseParenthesis;
break;
case ';':
tokenId = SimpleTokenId.SemiColon;
break;
case '\n':
// Line terminator
tokenId = SimpleTokenId.Whitespace;
break;
case '{':
tokenId = SimpleTokenId.OpenCurlyBrace;
break;
case '}':
tokenId = SimpleTokenId.CloseCurlyBrace;
break;
case '/':
tokenId = SimpleTokenId.Division;
switch (reader.Peek())
{
case '/':
// Parse a single-line comment
tokenId = this.ParseSingleLineComment(reader);
break;
case '*':
// Parse a multi-line comment
tokenId = this.ParseMultiLineComment(reader);
break;
}
break;
case '=':
if (reader.Peek() == '=')
{
reader.Read();
tokenId = SimpleTokenId.Equality;
}
else
{
tokenId = SimpleTokenId.Assignment;
}
break;
case '!':
if (reader.Peek() == '=')
{
reader.Read();
tokenId = SimpleTokenId.Inequality;
}
break;
case '+':
tokenId = SimpleTokenId.Addition;
break;
case '-':
tokenId = SimpleTokenId.Subtraction;
break;
case '*':
tokenId = SimpleTokenId.Multiplication;
break;
default:
if ((ch >= '0') && (ch <= '9'))
{
// Parse the number
tokenId = this.ParseNumber(reader, ch);
}
break;
}
}
if (tokenId != SimpleTokenId.Invalid)
{
return(new MergableLexerResult(MatchType.ExactMatch, new LexicalStateTokenData(lexicalState, tokenId)));
}
else
{
reader.ReadReverse();
return(MergableLexerResult.NoMatch);
}
}
public override MergableLexerResult GetNextToken(ITextBufferReader reader, ILexicalState lexicalState)
{
// Initialize
int tokenId = ExecutionTargetSelectorTokenId.Invalid;
// Get the next character
char ch = reader.Read();
if ((Char.IsLetter(ch) || (ch == '_')))
{
// Parse the identifier
tokenId = this.ParseIdentifier(reader, ch);
}
//else if (Char.IsWhiteSpace(ch))
//{
// while (Char.IsWhiteSpace(reader.Peek()))
// reader.Read();
// tokenId = ExecutionTargetSelectorTokenId.Whitespace;
//}
//else if (Char.IsNumber(ch))
//{
// tokenId = this.ParseNumber(reader, ch);
//}
else if (ch == '.')
{
tokenId = ExecutionTargetSelectorTokenId.Dot;
}
else
{
tokenId = ExecutionTargetSelectorTokenId.Invalid;
}
//{
// tokenId = ExecutionTargetSelectorTokenId.Invalid;
// switch (ch)
// {
// case ',':
// tokenId = ExecutionTargetSelectorTokenId.Comma;
// break;
// case '(':
// tokenId = ExecutionTargetSelectorTokenId.OpenParenthesis;
// break;
// case ')':
// tokenId = ExecutionTargetSelectorTokenId.CloseParenthesis;
// break;
// case ';':
// tokenId = ExecutionTargetSelectorTokenId.SemiColon;
// break;
// case '\n':
// // Line terminator
// tokenId = ExecutionTargetSelectorTokenId.Whitespace;
// break;
// case '{':
// tokenId = ExecutionTargetSelectorTokenId.OpenCurlyBrace;
// break;
// case '}':
// tokenId = ExecutionTargetSelectorTokenId.CloseCurlyBrace;
// break;
// case '/':
// tokenId = ExecutionTargetSelectorTokenId.Division;
// switch (reader.Peek())
// {
// case '/':
// // Parse a single-line comment
// tokenId = this.ParseSingleLineComment(reader);
// break;
// case '*':
// // Parse a multi-line comment
// tokenId = this.ParseMultiLineComment(reader);
// break;
// }
// break;
// case '=':
// if (reader.Peek() == '=')
// {
// reader.Read();
// tokenId = ExecutionTargetSelectorTokenId.Equality;
// }
// else
// tokenId = ExecutionTargetSelectorTokenId.Assignment;
// break;
// case '!':
// if (reader.Peek() == '=')
// {
// reader.Read();
// tokenId = ExecutionTargetSelectorTokenId.Inequality;
// }
// break;
// case '+':
// tokenId = ExecutionTargetSelectorTokenId.Addition;
// break;
// case '-':
// tokenId = ExecutionTargetSelectorTokenId.Subtraction;
// break;
// case '*':
// tokenId = ExecutionTargetSelectorTokenId.Multiplication;
// break;
// default:
// if ((ch >= '0') && (ch <= '9'))
// {
// // Parse the number
// tokenId = this.ParseNumber(reader, ch);
// }
// break;
// }
//}
if (tokenId != ExecutionTargetSelectorTokenId.Invalid)
{
return(new MergableLexerResult(MatchType.ExactMatch, new LexicalStateTokenData(lexicalState, tokenId)));
}
else
{
reader.ReadReverse();
return(MergableLexerResult.NoMatch);
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// PUBLIC PROCEDURES
/////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
/// Returns a single-character <see cref="ITokenLexicalParseData"/> representing the lexical parse data for the
/// default token in the <see cref="ILexicalState"/> and seeks forward one position in the <see cref="ITextBufferReader"/>
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current context.</param>
/// <returns>The <see cref="ITokenLexicalParseData"/> for default text in the <see cref="ILexicalState"/>.</returns>
public ITokenLexicalParseData GetLexicalStateDefaultTokenLexicalParseData(ITextBufferReader reader, ILexicalState lexicalState)
{
reader.Read();
return(new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)lexicalState.DefaultTokenID));
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// OBJECT
/////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
/// Initializes a new instance of the <c>SimpleTokenReader</c> class.
/// </summary>
/// <param name="reader">The <see cref="ITextBufferReader"/> to use for consuming text.</param>
/// <param name="rootLexer">The root <see cref="IMergableLexer"/>.</param>
public SimpleTokenReader(ITextBufferReader reader, IMergableLexer rootLexer) : base(reader, rootLexer)
{
}
/// <summary>
/// Performs a lexical parse to return the next <see cref="ITokenLexicalParseData"/>
/// from a <see cref="ITextBufferReader"/> and seeks past it if there is a match.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="lexicalState">The <see cref="ILexicalState"/> that specifies the current context.</param>
/// <param name="lexicalParseData">Returns the next <see cref="ITokenLexicalParseData"/> from a <see cref="ITextBufferReader"/>.</param>
/// <returns>A <see cref="MatchType"/> indicating the type of match that was made.</returns>
public MatchType GetNextTokenLexicalParseData(ITextBufferReader reader, ILexicalState lexicalState, ref ITokenLexicalParseData lexicalParseData)
{
// Initialize
int tokenID = LuatTokenId.Invalid;
if (reader.IsAtEnd)
{
lexicalParseData = new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)LuatTokenId.DocumentEnd);
return(MatchType.ExactMatch);
}
// Get the next character
char ch = reader.Read();
// If the character is a letter or digit...
if ((Char.IsLetter(ch) || (ch == '_')))
{
// Parse the identifier
tokenID = this.ParseIdentifier(reader, ch);
}
else if ((ch != '\n') && (Char.IsWhiteSpace(ch)))
{
while ((reader.Peek() != '\n') && (Char.IsWhiteSpace(reader.Peek())))
{
reader.Read();
}
tokenID = LuatTokenId.Whitespace;
}
else
{
tokenID = LuatTokenId.Invalid;
switch (ch)
{
case ',':
tokenID = LuatTokenId.Comma;
break;
case '(':
tokenID = LuatTokenId.OpenParenthesis;
break;
case ')':
tokenID = LuatTokenId.CloseParenthesis;
break;
case ';':
tokenID = LuatTokenId.SemiColon;
break;
case ':':
tokenID = LuatTokenId.Colon;
break;
case '\n':
case '\r':
// Line terminator
tokenID = LuatTokenId.LineTerminator;
break;
case '{':
tokenID = LuatTokenId.OpenCurlyBrace;
break;
case '}':
tokenID = LuatTokenId.CloseCurlyBrace;
break;
case '\"':
tokenID = this.ParseString(reader, '\"');
break;
case '\'':
tokenID = this.ParseString(reader, '\'');
break;
case '-':
if (reader.Peek(1) != '-')
{
tokenID = LuatTokenId.Subtraction;
break;
}
reader.Read();
if (reader.Peek(1) != '[' ||
reader.Peek(2) != '[')
{
tokenID = this.ParseSingleLineComment(reader);
}
else
{
reader.Read();
reader.Read();
tokenID = this.ParseMultiLineComment(reader);
}
break;
case '<':
if (reader.Peek() == '=')
{
reader.Read();
tokenID = LuatTokenId.LessThanEqual;
}
else
{
tokenID = LuatTokenId.LessThan;
}
break;
case '>':
if (reader.Peek() == '=')
{
reader.Read();
tokenID = LuatTokenId.GreaterThanEqual;
}
else
{
tokenID = LuatTokenId.GreaterThan;
}
break;
case '~':
if (reader.Peek() == '=')
{
reader.Read();
tokenID = LuatTokenId.Inequality;
}
break;
case '=':
if (reader.Peek() == '=')
{
reader.Read();
tokenID = LuatTokenId.Equality;
}
else
{
tokenID = LuatTokenId.Assignment;
}
break;
case '!':
if (reader.Peek() == '=')
{
reader.Read();
tokenID = LuatTokenId.Inequality;
}
break;
case '+':
tokenID = LuatTokenId.Addition;
break;
case '/':
tokenID = LuatTokenId.Division;
break;
case '*':
tokenID = LuatTokenId.Multiplication;
break;
case '^':
tokenID = LuatTokenId.Hat;
break;
case '#':
tokenID = LuatTokenId.Hash;
break;
case '%':
tokenID = LuatTokenId.Modulus;
break;
case '.':
tokenID = LuatTokenId.Dot;
if (reader.Peek() == '.')
{
reader.Read();
tokenID = LuatTokenId.DoubleDot;
}
if (reader.Peek() == '.')
{
reader.Read();
tokenID = LuatTokenId.TripleDot;
}
break;
case '[':
tokenID = LuatTokenId.OpenSquareBracket;
break;
case ']':
tokenID = LuatTokenId.CloseSquareBracket;
break;
default:
if ((ch >= '0') && (ch <= '9'))
{
// Parse the number
tokenID = this.ParseNumber(reader, ch);
}
break;
}
}
if (tokenID != LuatTokenId.Invalid)
{
lexicalParseData = new LexicalStateAndIDTokenLexicalParseData(lexicalState, (byte)tokenID);
return(MatchType.ExactMatch);
}
else
{
reader.ReadReverse();
return(MatchType.NoMatch);
}
}
public JsonTokenReader(ITextBufferReader reader, IMergableLexer rootLexer) : base(reader, rootLexer)
{
}
/// <summary>
/// Parses an identifier.
/// </summary>
/// <param name="reader">An <see cref="ITextBufferReader"/> that is reading a text source.</param>
/// <param name="ch">The first character of the identifier.</param>
/// <returns>The ID of the token that was matched.</returns>
protected virtual int ParseIdentifier(ITextBufferReader reader, char ch) {
// Get the entire word
int startOffset = reader.Offset - 1;
while (!reader.IsAtEnd) {
// Accept namespaced identifiers
if ( reader.Peek( 1 ) == ':' &&
reader.Peek( 2 ) == ':' &&
char.IsLetterOrDigit( reader.Peek( 3 ) ) )
{
reader.Read();
reader.Read();
reader.Read();
continue;
}
char ch2 = reader.Read();
// NOTE: This could be improved by supporting \u escape sequences
if ((!char.IsLetterOrDigit(ch2)) && (ch2 != '_')) {
reader.ReadReverse();
break;
}
}
// Determine if the word is a keyword
if (Char.IsLetter(ch)) {
object value = keywords[reader.GetSubstring(startOffset, reader.Offset - startOffset)];
if (value != null)
return (int)value;
else
return LuatTokenId.Identifier;
}
else
return LuatTokenId.Identifier;
}
