Token ExtractWord()
{
StringBuilder sb;
Token retval;
sb = new StringBuilder();
this.ReadAppendWord(sb);
string word = sb.ToString();
HappyTokenKind maybeKeyword;
if (_keywords.TryGetValue(word, out maybeKeyword))
retval = new Token(_reader.GetHappySourceLocation(), maybeKeyword, word);
else
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.Identifier, word);
return retval;
}
Token ExtractStatementToken()
{
Token retval;
this.EatWhite();
if (_reader.Eof)
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.EndOfInput, Resources.MiscMessages.EndOfInput);
else
{
if (Char.IsLetter(_reader.Peek()))
retval = ExtractWord();
else if (Char.IsNumber(_reader.Peek(0)) || (_reader.Peek(0) == '-' && Char.IsNumber(_reader.Peek(1))))
retval = ExtractNumber();
else
retval = ExtractOtherStatementToken();
}
return retval;
}
//public void FailIfEof()
//{
// if (_reader.Eof)
// {
// foreach (Token t in _readAhead)
// if (t.HappyTokenKind != HappyTokenKind.EndOfInput)
// return;
// SourceSpan span = new SourceSpan(_enteredLastStatementGroupAt, _reader.CurrentLocation);
// if (_state == LexerState.LexingStatement)
// _errorCollector.UnexpectedEndOfInputWhileParsingSection(_reader.GetHappySourceLocation());
// else
// _errorCollector.UnexpectedEndOfInputInComment(_reader.GetHappySourceLocation());
// throw new AbortParseException(_reader.GetHappySourceLocation());
// }
//}
Token ExtractToken()
{
Token retval;
_reader.ResetStartLocation();
if (_reader.Eof)
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.EndOfInput, Resources.MiscMessages.EndOfInput);
else
{
switch (this.Mode)
{
case LexerMode.LexingVerbatimText:
retval = ExtractVerbatimText();
break;
case LexerMode.LexingVerbatimOutputExpression:
case LexerMode.LexingStatement:
//read until an end of statement is encountered
retval = ExtractStatementToken();
break;
default:
throw new UnhandledCaseSourceException(_reader.GetHappySourceLocation());
}
}
DebugAssert.IsFalse(_reader.Peek() == 0xFFFF && !_reader.Eof, "_reader.Peek() returned 0xFFFF yet _reader.Eof is false");
DebugAssert.IsNotNull(retval, "retval cannot be null");
return retval;
}
Token ExtractNumber()
{
StringBuilder sb;
Token retval = null;
HappySourceLocation startedAt = _reader.GetHappySourceLocation();
sb = new StringBuilder();
if(_reader.Peek(0) == '0' && _reader.Peek(1) == 'x')
{
_reader.Read();
_reader.Read();
char peeked;
do
{
sb.Append(_reader.Read());
peeked = _reader.Peek();
} while (Char.IsNumber(peeked) || (peeked >= 'a' && peeked <= 'z' || peeked >='A' && peeked <='Z'));
if(sb.Length <= 8)
retval = new Token(startedAt, HappyTokenKind.LiteralHexInt32, sb.ToString());
else if(sb.Length <= 16)
retval = new Token(startedAt, HappyTokenKind.LiteralHexInt64, sb.ToString());
else
_errorCollector.TooManyCharactersInHexLiteral(startedAt);
}
else
{
do
{
sb.Append(_reader.Read());
}
while(Char.IsNumber(_reader.Peek(0)) || _reader.Peek(0) == '.');
string aNumber = sb.ToString();
if(aNumber.IndexOf('.') < 0)
retval = new Token(startedAt, HappyTokenKind.LiteralDecimalInt32, aNumber);
else
retval = new Token(startedAt, HappyTokenKind.LiteralDouble, aNumber);
}
return retval;
}
Token ExtractOtherStatementToken()
{
Token retval;
//lexer state changes occur in the first few cases below.
switch (_reader.Peek())
{
case '<':
_reader.Read();
switch (_reader.Peek(0))
{
case '=':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorLessThanOrEqual, "<=");
break;
case '|':
_reader.Read();
if(this.Mode == LexerMode.LexingVerbatimOutputExpression)
_errorCollector.CannotNestTemplatesWithinTemplateOutputExpression(_reader.GetHappySourceLocation());
this.PushState(LexerMode.LexingVerbatimText);
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.BeginTemplate, "<|");
break;
default:
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorLessThan, "<");
break;
}
break;
case '%':
_reader.Read();
if (_reader.Peek(0) != '|')
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorMod, "%");
else
{
_reader.Read();
if(_stateStack.Count <= 1 || this.Mode == LexerMode.LexingVerbatimOutputExpression)
_errorCollector.EndTemplateStatementBlockNotAllowedHere(_reader.GetHappySourceLocation());
this.PopState();
//I don't believe this case can ever happen...
//if(this.Mode != LexerMode.LexingVerbatimText)
// _errorCollector.EndTemplateStatementBlockNotAllowedHere(_reader.GetHappySourceLocation());
DebugAssert.AreEqual(this.Mode, LexerMode.LexingVerbatimText, "Unexpected case. Debug me.");
retval = this.ExtractToken();
}
break;
case '|':
_reader.Read();
switch (_reader.Peek(0))
{
case '>':
if (this.Mode == LexerMode.LexingVerbatimOutputExpression)
_errorCollector.MismatchedVerbatimOutputExpressionDelimiter(_reader.GetHappySourceLocation());
//Do not PopState here as it is already popped in ExtractVerbatimText()
//this.PopState();
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.EndTemplate, "|>");
break;
case '%':
_reader.Read();
_errorCollector.BeginTemplateStatementBlockNotAllowedHere(_reader.GetHappySourceLocation());
throw new AbortParseException(_reader.GetHappySourceLocation());
case '|':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorLogicalOr, "||");
break;
default:
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorBitwiseOr, "|");
break;
}
break;
case '$':
//First time this executes is the next extraction after the switch from LexingVerbatimText.
//The state should just have been changed to LexingVerbatimOutputExpression
//If not, we're in a state (LexingStatement, currently) where $ is not allowed.
if(this.Mode != LexerMode.LexingVerbatimOutputExpression)
_errorCollector.VerbatimOutputExpressionDelimiterNotAllowedHere(_reader.GetHappySourceLocation());
_reader.Read();
this.VerbatimOuputExpressionDelimiterCount++;
if (this.VerbatimOuputExpressionDelimiterCount >= 2)
{
this.PopState();
DebugAssert.AreEqual(LexerMode.LexingVerbatimText, this.Mode, "Previous lexer mode was not LexingVerbatimText. That shoulnd't have happened.");
}
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.VerbatimOutputExpressionDelimiter, "$");
break;
case ';':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.EndOfStatement, ";");
break;
case '(':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorOpenParen, "(");
break;
case ')':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorCloseParen, ")");
break;
case '!':
_reader.Read();
if (_reader.Peek(0) == '=')
{
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorNotEqual, "!=");
_reader.Read();
}
else
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.UnaryOperatorNot, "!");
break;
case '&':
_reader.Read();
if(_reader.Peek() == '&')
{
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorLogicalAnd, "&&");
}
else
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorBitwiseAnd, "&");
break;
case '^':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorXor, "^");
break;
case ',':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.Comma, ",");
break;
case '.':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorDot, ".");
break;
case '[':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorOpenBracket, "[");
break;
case ']':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorCloseBracket, "]");
break;
case '{':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OpenBrace, "{");
break;
case '}':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.CloseBrace, "}");
break;
case '>':
_reader.Read();
if (_reader.Peek(0) == '=')
{
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorGreaterThanOrEqual, ">=");
}
else
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorGreaterThan, ">");
break;
case '=':
_reader.Read();
if (_reader.Peek(0) == '=')
{
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorEqual, "==");
}
else
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorAssign, "=");
break;
case '+':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorAdd, "+");
break;
case '-':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorSubtract, "-");
break;
case '/':
_reader.Read();
//if (_reader.Peek(0) == '*')
// do
// _reader.Read();
// while(_reader.Peek(0) != '*' && _reader.Peek(1) != '/');
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorDivide, "/");
break;
case '*':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.OperatorMultiply, "*");
break;
case '~':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.Output, "~");
break;
case ':':
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.Colon, ":");
break;
case '\'':
_reader.Read();
this.FailIfEof(_reader.GetHappySourceLocation());
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.LiteralChar, _reader.Read().ToString());
this.FailIfEof(_reader.GetHappySourceLocation());
if (_reader.Read() != '\'')
_errorCollector.LiteralCharsMustBeOneCharLong(_reader.GetHappySourceLocation());
break;
case '"':
retval = ExtractLiteralString();
break;
default:
char invalidChar = _reader.Read();
_errorCollector.InvalidCharacter(_reader.GetHappySourceLocation(), invalidChar);
retval = new Token(_reader.GetHappySourceLocation(), HappyTokenKind.InvalidCharacter, Resources.MiscMessages.InvalidCharacter);
break;
}
return retval;
}
Token ExtractLiteralString()
{
StringBuilder sb;
Token retval;
sb = new StringBuilder();
//Eat opening quote
_reader.Read();
while (_reader.Peek() != '"')
{
this.FailIfEof(_reader.GetHappySourceLocation());
if (_reader.Peek() == '\\')
{
_reader.Read();
char escapedChar = _reader.Read();
switch (escapedChar)
{
case 't':
sb.Append('\t');
break;
case 'r':
sb.Append('\r');
break;
case 'n':
sb.Append('\n');
break;
case '"':
sb.Append('"');
break;
case '\\':
sb.Append('\\');
break;
default:
_errorCollector.InvalidEscapeSequence(_reader.GetHappySourceLocation(), escapedChar);
break;
}
}
else
sb.Append(_reader.Read());
if (_reader.Peek() == '\n')
_errorCollector.NewLineInStringLiteral(_reader.GetHappySourceLocation());
}
//Eat closing quote
_reader.Read();
retval = new Token(_reader.GetHappySourceLocation(),
HappyTokenKind.LiteralString, sb.ToString());
return retval;
}
private ExpressionNodeBase PromoteToExpression(Token firstToken)
{
ExpressionNodeBase retval;
int intValue;
switch(firstToken.HappyTokenKind)
{
case HappyTokenKind.Identifier:
HappySourceLocation startsAt, endsAt;
if (this.EatNextTokenIf(HappyTokenKind.OperatorOpenParen, out startsAt))
{
List<ExpressionNodeBase> arguments = new List<ExpressionNodeBase>();
bool parseMore = !this.EatNextTokenIf(HappyTokenKind.OperatorCloseParen, out endsAt);
while (parseMore)
{
arguments.Add(this.ParseExpression(ExpressionContext.ArgumentList));
var commaOrCloseParen = this.Expect(HappyTokenKind.Comma, HappyTokenKind.OperatorCloseParen, MiscMessages.CommaOrCloseParen);
endsAt = commaOrCloseParen.Location;
parseMore = commaOrCloseParen.HappyTokenKind != HappyTokenKind.OperatorCloseParen;
}
retval = new FunctionCallExpression(startsAt, endsAt, firstToken.ToIdentifier(), arguments.ToArray());
}
else
retval = new IdentifierExpression(firstToken.ToIdentifier());
break;
case HappyTokenKind.LiteralBool:
switch(firstToken.Text)
{
case "true":
retval = new LiteralExpression(firstToken.Location, true);
break;
case "false":
retval = new LiteralExpression(firstToken.Location, false);
break;
default:
throw new UnhandledCaseSourceException(firstToken.Location);
}
break;
case HappyTokenKind.LiteralDecimalInt32:
if(!Int32.TryParse(firstToken.Text, out intValue))
throw new InternalSourceException(firstToken.Location, "Failed to parse an Int32 from \"{0}\" _tokenKind is LiteralInt32?!?!", firstToken.Text);
retval = new LiteralExpression(firstToken.Location, intValue);
break;
case HappyTokenKind.LiteralHexInt32:
if(!Int32.TryParse(firstToken.Text, NumberStyles.HexNumber, CultureInfo.InvariantCulture.NumberFormat, out intValue))
throw new InternalSourceException(firstToken.Location, "Failed to parse an Int32 from \"{0}\" _tokenKind is LiteralHexInt32?!?!", firstToken.Text);
retval = new LiteralExpression(firstToken.Location, intValue);
break;
case HappyTokenKind.LiteralHexInt64:
long longValue;
if (!Int64.TryParse(firstToken.Text, NumberStyles.HexNumber, CultureInfo.InvariantCulture.NumberFormat, out longValue))
throw new InternalSourceException(firstToken.Location, "Failed to parse an Int32 from \"{0}\" _tokenKind is LiteralHexInt64?!?!", firstToken.Text);
retval = new LiteralExpression(firstToken.Location, longValue);
break;
case HappyTokenKind.LiteralDouble:
double doubleValue;
try
{
doubleValue = Double.Parse(firstToken.Text);
} catch(Exception e)
{
throw new InternalSourceException(e, firstToken.Location,
"Failed to parse a Double from \"{0}\" but _tokenKind is LiteralDouble?!?!", firstToken.Text);
}
retval = new LiteralExpression(firstToken.Location, doubleValue);
break;
case HappyTokenKind.LiteralString:
retval = new LiteralExpression(firstToken.Location, firstToken.Text);
break;
case HappyTokenKind.LiteralNull:
retval = new NullExpression(firstToken.Location);
break;
case HappyTokenKind.BeginTemplate:
retval = this.ParseAnonymousTemplateExpression(firstToken.Location);
break;
default:
_errorCollector.UnexpectedToken(firstToken);
throw new AbortParseException(firstToken.Location);
}
return retval;
}
public void Expected(string expected, Token actual)
{
this.Add(actual.Location, ErrorCode.Expected_Expected_Actual, expected, actual.Text);
}
public void DuplicateInputValue(Token value)
{
this.Add(value.Location, ErrorCode.DuplicateInputValue_Value, value.Text);
}
public void UnexpectedToken(Token t)
{
this.Add(t.Location, ErrorCode.UnexpectedToken_TokenText, t.Text);
}
