private StateResult Text()
{
var prev = '\0';
while (!EndOfFile &&
!(ParserHelpers.IsWhitespace(CurrentCharacter) || ParserHelpers.IsNewLine(CurrentCharacter)) &&
!AtToken())
{
prev = CurrentCharacter;
TakeCurrent();
}
if (CurrentCharacter == '@')
{
var next = Peek();
if ((ParserHelpers.IsLetter(prev) || ParserHelpers.IsDecimalDigit(prev)) &&
(ParserHelpers.IsLetter(next) || ParserHelpers.IsDecimalDigit(next)))
{
TakeCurrent(); // Take the "@"
return(Stay()); // Stay in the Text state
}
}
// Output the Text token and return to the Data state to tokenize the next character (if there is one)
return(Transition(HtmlTokenizerState.Data, EndToken(SyntaxKind.Text)));
}
private HtmlSymbol Whitespace()
{
while (ParserHelpers.IsWhitespace(CurrentCharacter))
{
TakeCurrent();
}
return(EndSymbol(HtmlSymbolType.WhiteSpace));
}
private SyntaxToken Whitespace()
{
while (ParserHelpers.IsWhitespace(CurrentCharacter))
{
TakeCurrent();
}
return(EndToken(SyntaxKind.Whitespace));
}
// http://dev.w3.org/html5/spec/Overview.html#data-state
private StateResult Data()
{
if (ParserHelpers.IsWhitespace(CurrentCharacter))
{
return(Stay(Whitespace()));
}
else if (ParserHelpers.IsNewLine(CurrentCharacter))
{
return(Stay(Newline()));
}
else if (CurrentCharacter == '@')
{
TakeCurrent();
if (CurrentCharacter == '*')
{
return(Transition(
EndSymbol(HtmlSymbolType.RazorCommentTransition),
AfterRazorCommentTransition
));
}
else if (CurrentCharacter == '@')
{
// Could be escaped comment transition
return(Transition(
EndSymbol(HtmlSymbolType.Transition),
() =>
{
TakeCurrent();
return Transition(EndSymbol(HtmlSymbolType.Transition), Data);
}
));
}
return(Stay(EndSymbol(HtmlSymbolType.Transition)));
}
else if (AtSymbol())
{
return(Stay(Symbol()));
}
else
{
return(Transition(Text));
}
}
private bool AtDateLiteral()
{
if (CurrentCharacter != '#')
{
return(false);
}
int start = Source.Position;
try
{
MoveNext();
while (ParserHelpers.IsWhitespace(CurrentCharacter))
{
MoveNext();
}
return(Char.IsDigit(CurrentCharacter));
}
finally
{
Source.Position = start;
}
}
// http://dev.w3.org/html5/spec/Overview.html#data-state
private StateResult Data()
{
if (ParserHelpers.IsWhitespace(CurrentCharacter))
{
return(Stay(Whitespace()));
}
else if (ParserHelpers.IsNewLine(CurrentCharacter))
{
return(Stay(Newline()));
}
else if (CurrentCharacter == '@')
{
TakeCurrent();
if (CurrentCharacter == '*')
{
return(Transition(
HtmlTokenizerState.AfterRazorCommentTransition,
EndToken(SyntaxKind.RazorCommentTransition)));
}
else if (CurrentCharacter == '@')
{
// Could be escaped comment transition
return(Transition(
HtmlTokenizerState.EscapedRazorCommentTransition,
EndToken(SyntaxKind.Transition)));
}
return(Stay(EndToken(SyntaxKind.Transition)));
}
else if (AtToken())
{
return(Stay(Token()));
}
else
{
return(Transition(HtmlTokenizerState.Text));
}
}
protected Func <char, bool> CharOrWhiteSpace(char character)
{
return(c => c == character || ParserHelpers.IsWhitespace(c) || ParserHelpers.IsNewLine(c));
}
private StateResult Data()
{
// We are accepting more characters and whitespace/newlines then the VB Spec defines, to simplify things
// Since the code must still be compiled by a VB compiler, this will not cause adverse effects.
if (ParserHelpers.IsNewLine(CurrentCharacter))
{
// VB Spec §2.1.1
bool checkTwoCharNewline = CurrentCharacter == '\r';
TakeCurrent();
if (checkTwoCharNewline && CurrentCharacter == '\n')
{
TakeCurrent();
}
return(Stay(EndSymbol(VBSymbolType.NewLine)));
}
else if (ParserHelpers.IsWhitespace(CurrentCharacter))
{
// CSharp Spec §2.1.3
TakeUntil(c => !ParserHelpers.IsWhitespace(c));
return(Stay(EndSymbol(VBSymbolType.WhiteSpace)));
}
else if (VBHelpers.IsSingleQuote(CurrentCharacter))
{
TakeCurrent();
return(CommentBody());
}
else if (IsIdentifierStart())
{
return(Identifier());
}
else if (Char.IsDigit(CurrentCharacter))
{
return(DecimalLiteral());
}
else if (CurrentCharacter == '&')
{
char next = Char.ToLower(Peek(), CultureInfo.InvariantCulture);
if (next == 'h')
{
return(HexLiteral());
}
else if (next == 'o')
{
return(OctLiteral());
}
}
else if (CurrentCharacter == '.' && Char.IsDigit(Peek()))
{
return(FloatingPointLiteralEnd());
}
else if (VBHelpers.IsDoubleQuote(CurrentCharacter))
{
TakeCurrent();
return(Transition(QuotedLiteral));
}
else if (AtDateLiteral())
{
return(DateLiteral());
}
else if (CurrentCharacter == '@')
{
TakeCurrent();
if (CurrentCharacter == '*')
{
return(Transition(EndSymbol(VBSymbolType.RazorCommentTransition), AfterRazorCommentTransition));
}
else if (CurrentCharacter == '@')
{
// Could be escaped comment transition
return(Transition(EndSymbol(VBSymbolType.Transition), () =>
{
TakeCurrent();
return Transition(EndSymbol(VBSymbolType.Transition), Data);
}));
}
else
{
return(Stay(EndSymbol(VBSymbolType.Transition)));
}
}
return(Stay(EndSymbol(Operator())));
}
private StateResult Data()
{
if (ParserHelpers.IsNewLine(CurrentCharacter))
{
// CSharp Spec §2.3.1
var checkTwoCharNewline = CurrentCharacter == '\r';
TakeCurrent();
if (checkTwoCharNewline && CurrentCharacter == '\n')
{
TakeCurrent();
}
return(Stay(EndSymbol(CSharpSymbolType.NewLine)));
}
else if (ParserHelpers.IsWhitespace(CurrentCharacter))
{
// CSharp Spec §2.3.3
TakeUntil(c => !ParserHelpers.IsWhitespace(c));
return(Stay(EndSymbol(CSharpSymbolType.WhiteSpace)));
}
else if (CSharpHelpers.IsIdentifierStart(CurrentCharacter))
{
return(Identifier());
}
else if (Char.IsDigit(CurrentCharacter))
{
return(NumericLiteral());
}
switch (CurrentCharacter)
{
case '@':
return(AtSymbol());
case '\'':
TakeCurrent();
return(Transition(() => QuotedLiteral('\'', CSharpSymbolType.CharacterLiteral)));
case '"':
TakeCurrent();
return(Transition(() => QuotedLiteral('"', CSharpSymbolType.StringLiteral)));
case '.':
if (Char.IsDigit(Peek()))
{
return(RealLiteral());
}
return(Stay(Single(CSharpSymbolType.Dot)));
case '/':
TakeCurrent();
if (CurrentCharacter == '/')
{
TakeCurrent();
return(SingleLineComment());
}
else if (CurrentCharacter == '*')
{
TakeCurrent();
return(Transition(BlockComment));
}
else if (CurrentCharacter == '=')
{
TakeCurrent();
return(Stay(EndSymbol(CSharpSymbolType.DivideAssign)));
}
else
{
return(Stay(EndSymbol(CSharpSymbolType.Slash)));
}
default:
return(Stay(EndSymbol(Operator())));
}
}