private StateResult Text() { var prev = '\0'; while (!EndOfFile && !(ParserHelpers.IsWhitespace(CurrentCharacter) || ParserHelpers.IsNewLine(CurrentCharacter)) && !AtToken()) { prev = CurrentCharacter; TakeCurrent(); } if (CurrentCharacter == '@') { var next = Peek(); if ((ParserHelpers.IsLetter(prev) || ParserHelpers.IsDecimalDigit(prev)) && (ParserHelpers.IsLetter(next) || ParserHelpers.IsDecimalDigit(next))) { TakeCurrent(); // Take the "@" return(Stay()); // Stay in the Text state } } // Output the Text token and return to the Data state to tokenize the next character (if there is one) return(Transition(HtmlTokenizerState.Data, EndToken(SyntaxKind.Text))); }
private HtmlSymbol Whitespace() { while (ParserHelpers.IsWhitespace(CurrentCharacter)) { TakeCurrent(); } return(EndSymbol(HtmlSymbolType.WhiteSpace)); }
private SyntaxToken Whitespace() { while (ParserHelpers.IsWhitespace(CurrentCharacter)) { TakeCurrent(); } return(EndToken(SyntaxKind.Whitespace)); }
// http://dev.w3.org/html5/spec/Overview.html#data-state private StateResult Data() { if (ParserHelpers.IsWhitespace(CurrentCharacter)) { return(Stay(Whitespace())); } else if (ParserHelpers.IsNewLine(CurrentCharacter)) { return(Stay(Newline())); } else if (CurrentCharacter == '@') { TakeCurrent(); if (CurrentCharacter == '*') { return(Transition( EndSymbol(HtmlSymbolType.RazorCommentTransition), AfterRazorCommentTransition )); } else if (CurrentCharacter == '@') { // Could be escaped comment transition return(Transition( EndSymbol(HtmlSymbolType.Transition), () => { TakeCurrent(); return Transition(EndSymbol(HtmlSymbolType.Transition), Data); } )); } return(Stay(EndSymbol(HtmlSymbolType.Transition))); } else if (AtSymbol()) { return(Stay(Symbol())); } else { return(Transition(Text)); } }
private bool AtDateLiteral() { if (CurrentCharacter != '#') { return(false); } int start = Source.Position; try { MoveNext(); while (ParserHelpers.IsWhitespace(CurrentCharacter)) { MoveNext(); } return(Char.IsDigit(CurrentCharacter)); } finally { Source.Position = start; } }
// http://dev.w3.org/html5/spec/Overview.html#data-state private StateResult Data() { if (ParserHelpers.IsWhitespace(CurrentCharacter)) { return(Stay(Whitespace())); } else if (ParserHelpers.IsNewLine(CurrentCharacter)) { return(Stay(Newline())); } else if (CurrentCharacter == '@') { TakeCurrent(); if (CurrentCharacter == '*') { return(Transition( HtmlTokenizerState.AfterRazorCommentTransition, EndToken(SyntaxKind.RazorCommentTransition))); } else if (CurrentCharacter == '@') { // Could be escaped comment transition return(Transition( HtmlTokenizerState.EscapedRazorCommentTransition, EndToken(SyntaxKind.Transition))); } return(Stay(EndToken(SyntaxKind.Transition))); } else if (AtToken()) { return(Stay(Token())); } else { return(Transition(HtmlTokenizerState.Text)); } }
protected Func <char, bool> CharOrWhiteSpace(char character) { return(c => c == character || ParserHelpers.IsWhitespace(c) || ParserHelpers.IsNewLine(c)); }
private StateResult Data() { // We are accepting more characters and whitespace/newlines then the VB Spec defines, to simplify things // Since the code must still be compiled by a VB compiler, this will not cause adverse effects. if (ParserHelpers.IsNewLine(CurrentCharacter)) { // VB Spec §2.1.1 bool checkTwoCharNewline = CurrentCharacter == '\r'; TakeCurrent(); if (checkTwoCharNewline && CurrentCharacter == '\n') { TakeCurrent(); } return(Stay(EndSymbol(VBSymbolType.NewLine))); } else if (ParserHelpers.IsWhitespace(CurrentCharacter)) { // CSharp Spec §2.1.3 TakeUntil(c => !ParserHelpers.IsWhitespace(c)); return(Stay(EndSymbol(VBSymbolType.WhiteSpace))); } else if (VBHelpers.IsSingleQuote(CurrentCharacter)) { TakeCurrent(); return(CommentBody()); } else if (IsIdentifierStart()) { return(Identifier()); } else if (Char.IsDigit(CurrentCharacter)) { return(DecimalLiteral()); } else if (CurrentCharacter == '&') { char next = Char.ToLower(Peek(), CultureInfo.InvariantCulture); if (next == 'h') { return(HexLiteral()); } else if (next == 'o') { return(OctLiteral()); } } else if (CurrentCharacter == '.' && Char.IsDigit(Peek())) { return(FloatingPointLiteralEnd()); } else if (VBHelpers.IsDoubleQuote(CurrentCharacter)) { TakeCurrent(); return(Transition(QuotedLiteral)); } else if (AtDateLiteral()) { return(DateLiteral()); } else if (CurrentCharacter == '@') { TakeCurrent(); if (CurrentCharacter == '*') { return(Transition(EndSymbol(VBSymbolType.RazorCommentTransition), AfterRazorCommentTransition)); } else if (CurrentCharacter == '@') { // Could be escaped comment transition return(Transition(EndSymbol(VBSymbolType.Transition), () => { TakeCurrent(); return Transition(EndSymbol(VBSymbolType.Transition), Data); })); } else { return(Stay(EndSymbol(VBSymbolType.Transition))); } } return(Stay(EndSymbol(Operator()))); }
private StateResult Data() { if (ParserHelpers.IsNewLine(CurrentCharacter)) { // CSharp Spec §2.3.1 var checkTwoCharNewline = CurrentCharacter == '\r'; TakeCurrent(); if (checkTwoCharNewline && CurrentCharacter == '\n') { TakeCurrent(); } return(Stay(EndSymbol(CSharpSymbolType.NewLine))); } else if (ParserHelpers.IsWhitespace(CurrentCharacter)) { // CSharp Spec §2.3.3 TakeUntil(c => !ParserHelpers.IsWhitespace(c)); return(Stay(EndSymbol(CSharpSymbolType.WhiteSpace))); } else if (CSharpHelpers.IsIdentifierStart(CurrentCharacter)) { return(Identifier()); } else if (Char.IsDigit(CurrentCharacter)) { return(NumericLiteral()); } switch (CurrentCharacter) { case '@': return(AtSymbol()); case '\'': TakeCurrent(); return(Transition(() => QuotedLiteral('\'', CSharpSymbolType.CharacterLiteral))); case '"': TakeCurrent(); return(Transition(() => QuotedLiteral('"', CSharpSymbolType.StringLiteral))); case '.': if (Char.IsDigit(Peek())) { return(RealLiteral()); } return(Stay(Single(CSharpSymbolType.Dot))); case '/': TakeCurrent(); if (CurrentCharacter == '/') { TakeCurrent(); return(SingleLineComment()); } else if (CurrentCharacter == '*') { TakeCurrent(); return(Transition(BlockComment)); } else if (CurrentCharacter == '=') { TakeCurrent(); return(Stay(EndSymbol(CSharpSymbolType.DivideAssign))); } else { return(Stay(EndSymbol(CSharpSymbolType.Slash))); } default: return(Stay(EndSymbol(Operator()))); } }