private StateResult QuotedLiteral(char quote, Func <char, bool> isEndQuotedLiteral, SyntaxKind literalType)
{
TakeUntil(isEndQuotedLiteral);
if (CurrentCharacter == '\\')
{
TakeCurrent(); // Take the '\'
// If the next char is the same quote that started this
if (CurrentCharacter == quote || CurrentCharacter == '\\')
{
TakeCurrent(); // Take it so that we don't prematurely end the literal.
}
return(Stay());
}
else if (EndOfFile || ParserHelpers.IsNewLine(CurrentCharacter))
{
CurrentErrors.Add(
RazorDiagnosticFactory.CreateParsing_UnterminatedStringLiteral(
new SourceSpan(CurrentStart, contentLength: 1 /* " */)));
}
else
{
TakeCurrent(); // No-op if at EOF
}
return(Transition(CSharpTokenizerState.Data, EndToken(literalType)));
}
// CSharp Spec §2.4.2
private StateResult Identifier()
{
Debug.Assert(IsIdentifierStart(CurrentCharacter));
TakeCurrent();
TakeUntil(c => !IsIdentifierPart(c));
SyntaxToken token = null;
if (HaveContent)
{
CSharpKeyword keyword;
var type = SyntaxKind.Identifier;
var tokenContent = Buffer.ToString();
if (_keywords.TryGetValue(tokenContent, out keyword))
{
type = SyntaxKind.Keyword;
}
token = SyntaxFactory.Token(type, tokenContent);
Buffer.Clear();
CurrentErrors.Clear();
}
return(Stay(token));
}
// CSharp Spec §2.4.2
private StateResult Identifier()
{
Debug.Assert(IsIdentifierStart(CurrentCharacter));
TakeCurrent();
TakeUntil(c => !IsIdentifierPart(c));
CSharpSymbol symbol = null;
if (HaveContent)
{
CSharpKeyword keyword;
var type = CSharpSymbolType.Identifier;
var symbolContent = Buffer.ToString();
if (_keywords.TryGetValue(symbolContent, out keyword))
{
type = CSharpSymbolType.Keyword;
}
symbol = new CSharpSymbol(symbolContent, type)
{
Keyword = type == CSharpSymbolType.Keyword ? (CSharpKeyword?)keyword : null,
};
Buffer.Clear();
CurrentErrors.Clear();
}
return(Stay(symbol));
}
private StateResult QuotedLiteral(char quote, CSharpSymbolType literalType)
{
TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c));
if (CurrentCharacter == '\\')
{
TakeCurrent(); // Take the '\'
// If the next char is the same quote that started this
if (CurrentCharacter == quote || CurrentCharacter == '\\')
{
TakeCurrent(); // Take it so that we don't prematurely end the literal.
}
return(Stay());
}
else if (EndOfFile || ParserHelpers.IsNewLine(CurrentCharacter))
{
CurrentErrors.Add(
new RazorError(
RazorResources.ParseError_Unterminated_String_Literal,
CurrentStart,
length: 1 /* " */));
}
else
{
TakeCurrent(); // No-op if at EOF
}
return(Transition(EndSymbol(literalType), Data));
}
/// <summary>
/// Tokenizes a quoted literal.
/// </summary>
/// <param name="quote">The quote character.</param>
/// <returns>The state result.</returns>
private StateResult QuotedLiteral(char quote)
{
TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c));
if (CurrentCharacter == '\\')
{
TakeCurrent();
if (CurrentCharacter == quote || CurrentCharacter == '\\')
{
TakeCurrent();
}
return(Stay());
}
if (EndOfFile || ParserHelpers.IsNewLine(CurrentCharacter))
{
CurrentErrors.Add(new Error("Untermined string literal", CurrentStart));
}
else
{
TakeCurrent();
}
return(Stay(EndSymbol(HandlebarsSymbolType.StringLiteral)));
}
protected TSymbol EndSymbol(SourceLocation start, TSymbolType type)
{
TSymbol sym = null;
if (HaveContent)
{
sym = CreateSymbol(start, Buffer.ToString(), type, CurrentErrors.ToArray());
}
StartSymbol();
return(sym);
}
// CSharp Spec §2.3.2
private StateResult BlockComment()
{
TakeUntil(c => c == '*');
if (EndOfFile)
{
CurrentErrors.Add(new RazorError(RazorResources.ParseError_BlockComment_Not_Terminated, CurrentStart));
return(Transition(EndSymbol(CSharpSymbolType.Comment), Data));
}
if (CurrentCharacter == '*')
{
TakeCurrent();
if (CurrentCharacter == '/')
{
TakeCurrent();
return(Transition(EndSymbol(CSharpSymbolType.Comment), Data));
}
}
return(Stay());
}
private StateResult VerbatimStringLiteral()
{
TakeUntil(c => c == '"');
if (CurrentCharacter == '"')
{
TakeCurrent();
if (CurrentCharacter == '"')
{
TakeCurrent();
// Stay in the literal, this is an escaped "
return(Stay());
}
}
else if (EndOfFile)
{
CurrentErrors.Add(new RazorError(RazorResources.ParseError_Unterminated_String_Literal, CurrentStart));
}
return(Transition(EndSymbol(CSharpSymbolType.StringLiteral), Data));
}
private StateResult QuotedLiteral(char quote, CSharpSymbolType literalType)
{
TakeUntil(c => c == '\\' || c == quote || ParserHelpers.IsNewLine(c));
if (CurrentCharacter == '\\')
{
TakeCurrent(); // Take the '\'
TakeCurrent(); // Take the next char as well (multi-char escapes don't matter)
return(Stay());
}
else if (EndOfFile || ParserHelpers.IsNewLine(CurrentCharacter))
{
CurrentErrors.Add(new RazorError(RazorResources.ParseError_Unterminated_String_Literal, CurrentStart));
}
else
{
TakeCurrent(); // No-op if at EOF
}
return(Transition(EndSymbol(literalType), Data));
}
private StateResult VerbatimStringLiteral()
{
TakeUntil(c => c == '"');
if (CurrentCharacter == '"')
{
TakeCurrent();
if (CurrentCharacter == '"')
{
TakeCurrent();
// Stay in the literal, this is an escaped "
return(Stay());
}
}
else if (EndOfFile)
{
CurrentErrors.Add(
RazorDiagnosticFactory.CreateParsing_UnterminatedStringLiteral(
new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));
}
return(Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.StringLiteral)));
}
/// <summary>
/// Attempts to begin a tag by matching the opening braces.
/// </summary>
/// <returns>The state result.</returns>
private StateResult BeginTag()
{
var type = HandlebarsSymbolType.OpenTag;
if (CurrentCharacter != '{')
{
CurrentErrors.Add(new Error("Expected '{'", CurrentLocation));
// We can't process this any more, so stop.
return(Transition(Stop));
}
TakeCurrent();
if (CurrentCharacter != '{')
{
CurrentErrors.Add(new Error("Expected '{'", CurrentLocation));
// We can't process this any more, so stop.
return(Transition(Stop));
}
TakeCurrent();
if (CurrentCharacter == '{')
{
// We're at a raw tag '{{{'
TakeCurrent();
// Change the symbol type.
type = HandlebarsSymbolType.RawOpenTag;
}
else if (CurrentCharacter == '#')
{
// We're at the start of a block tag.
return(Transition(EndSymbol(type), () => BeginTagContent(false)));
}
// Transition to the start of tag content.
return(Transition(EndSymbol(type), () => BeginTagContent(type == HandlebarsSymbolType.RawOpenTag)));
}
/// <summary>
/// Attempts to end a tag by matching the closing braces.
/// </summary>
/// <param name="raw">True if we are expected to end a raw tag '}}}'</param>
/// <returns>The state result.</returns>
private StateResult EndTag(bool raw)
{
if (CurrentCharacter != '}')
{
CurrentErrors.Add(new Error("Expected '}'", CurrentLocation));
// We can't process this any more, so stop.
return(Transition(Stop));
}
TakeCurrent();
if (CurrentCharacter != '}')
{
CurrentErrors.Add(new Error("Expected '}'", CurrentLocation));
// We can't process this any more, so stop.
return(Transition(Stop));
}
TakeCurrent();
if (CurrentCharacter != '}' && raw)
{
CurrentErrors.Add(new Error("Expected '}'", CurrentLocation));
// We can't process this any more, so stop.
return(Transition(Stop));
}
if (CurrentCharacter == '}' && raw)
{
TakeCurrent();
// We're done processing this '}}}' sequence, so let's finish here and return to the Data state.
return(Transition(EndSymbol(HandlebarsSymbolType.RawCloseTag), Data));
}
// We're done processing this '}}' sequence, so let's finish here and return to the Data state.
return(Transition(EndSymbol(HandlebarsSymbolType.CloseTag), Data));
}
// CSharp Spec §2.3.2
private StateResult BlockComment()
{
TakeUntil(c => c == '*');
if (EndOfFile)
{
CurrentErrors.Add(
RazorDiagnosticFactory.CreateParsing_BlockCommentNotTerminated(
new SourceSpan(CurrentStart, contentLength: 1 /* end of file */)));
return(Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.CSharpComment)));
}
if (CurrentCharacter == '*')
{
TakeCurrent();
if (CurrentCharacter == '/')
{
TakeCurrent();
return(Transition(CSharpTokenizerState.Data, EndToken(SyntaxKind.CSharpComment)));
}
}
return(Stay());
}
protected TSymbol EndSymbol(TSymbolType type)
{
TSymbol symbol = null;
if (HaveContent)
{
// Perf: Don't allocate a new errors array unless necessary.
var errors = CurrentErrors.Count == 0 ? RazorDiagnostic.EmptyArray : new RazorDiagnostic[CurrentErrors.Count];
for (var i = 0; i < CurrentErrors.Count; i++)
{
errors[i] = CurrentErrors[i];
}
var symbolContent = GetSymbolContent(type);
Debug.Assert(string.Equals(symbolContent, Buffer.ToString(), StringComparison.Ordinal));
symbol = CreateSymbol(symbolContent, type, errors);
Buffer.Clear();
CurrentErrors.Clear();
}
return(symbol);
}
protected SyntaxToken EndToken(SyntaxKind type)
{
SyntaxToken token = null;
if (HaveContent)
{
// Perf: Don't allocate a new errors array unless necessary.
var errors = CurrentErrors.Count == 0 ? RazorDiagnostic.EmptyArray : new RazorDiagnostic[CurrentErrors.Count];
for (var i = 0; i < CurrentErrors.Count; i++)
{
errors[i] = CurrentErrors[i];
}
var tokenContent = GetTokenContent(type);
Debug.Assert(string.Equals(tokenContent, Buffer.ToString(), StringComparison.Ordinal));
token = CreateToken(tokenContent, type, errors);
Buffer.Clear();
CurrentErrors.Clear();
}
return(token);
}
private StateResult VerbatimStringLiteral()
{
TakeUntil(c => c == '"');
if (CurrentCharacter == '"')
{
TakeCurrent();
if (CurrentCharacter == '"')
{
TakeCurrent();
// Stay in the literal, this is an escaped "
return(Stay());
}
}
else if (EndOfFile)
{
CurrentErrors.Add(
new RazorError(
LegacyResources.ParseError_Unterminated_String_Literal,
CurrentStart,
length: 1 /* end of file */));
}
return(Transition(CSharpTokenizerState.Data, EndSymbol(CSharpSymbolType.StringLiteral)));
}
// CSharp Spec §2.3.2
private StateResult BlockComment()
{
TakeUntil(c => c == '*');
if (EndOfFile)
{
CurrentErrors.Add(
new RazorError(
LegacyResources.ParseError_BlockComment_Not_Terminated,
CurrentStart,
length: 1 /* end of file */));
return(Transition(CSharpTokenizerState.Data, EndSymbol(CSharpSymbolType.Comment)));
}
if (CurrentCharacter == '*')
{
TakeCurrent();
if (CurrentCharacter == '/')
{
TakeCurrent();
return(Transition(CSharpTokenizerState.Data, EndSymbol(CSharpSymbolType.Comment)));
}
}
return(Stay());
}
protected void StartSymbol()
{
Buffer.Clear();
CurrentStart = CurrentLocation;
CurrentErrors.Clear();
}
/// <summary>
/// Attempts to begin matching the content of a tag.
/// </summary>
/// <param name="raw">True if we are expected a raw tag.</param>
/// <returns>The state result.</returns>
private StateResult BeginTagContent(bool raw)
{
switch (CurrentCharacter)
{
case '~':
{
TakeCurrent();
// We've matched a ~ character - this is for ensuring the tag braces are expanded as whitespace instead of being collapsed.
return(Stay(EndSymbol(HandlebarsSymbolType.Tilde)));
}
case '!':
{
if (raw)
{
// This is an invalid tag, so set and error and exit.
CurrentErrors.Add(new Error("Unexpected '!' in raw tag.", CurrentLocation));
return(Transition(Stop));
}
TakeCurrent();
// We've matched a ! character - this is the start of a comment.
return(Transition(EndSymbol(HandlebarsSymbolType.Bang), BeginComment));
}
case '>':
{
if (raw)
{
// This is an invalid tag, so set and error and exit.
CurrentErrors.Add(new Error("Unexpected '>' in raw tag.", CurrentLocation));
return(Transition(Stop));
}
TakeCurrent();
// We've matched a > character - this is the start of a reference to a partial template.
return(Transition(EndSymbol(HandlebarsSymbolType.RightArrow), () => ContinueTagContent(false)));
}
case '^':
{
if (raw)
{
// This is an invalid tag, so set and error and exit.
CurrentErrors.Add(new Error("Unexpected '^' in raw tag.", CurrentLocation));
return(Transition(Stop));
}
TakeCurrent();
// We've matched a ^ character - this is the start of a negation.
return(Transition(EndSymbol(HandlebarsSymbolType.Negate), () => ContinueTagContent(false)));
}
case '#':
{
if (raw)
{
// This is an invalid tag, so set and error and exit.
CurrentErrors.Add(new Error("Unexpected '#' in raw tag.", CurrentLocation));
return(Transition(Stop));
}
TakeCurrent();
// We've matched a ^ character - this is the start of a negation.
return(Transition(EndSymbol(HandlebarsSymbolType.Hash), () => ContinueTagContent(false)));
}
case '&':
{
TakeCurrent();
// We've matched a & character
return(Transition(EndSymbol(HandlebarsSymbolType.Ampersand), () => ContinueTagContent(false)));
}
case '@':
{
TakeCurrent();
// We've matched a variable reference character.
return(Transition(EndSymbol(HandlebarsSymbolType.At), () => ContinueTagContent(false)));
}
default:
{
// Transition to the tag content.
return(Transition(() => ContinueTagContent(raw)));
}
}
}
/// <summary>
/// Continues the content of the tag.
/// </summary>
/// <param name="raw">True if we are expected a raw tag.</param>
/// <returns>The state result.</returns>
private StateResult ContinueTagContent(bool raw)
{
if (CurrentCharacter == '@')
{
TakeCurrent();
return(Stay(EndSymbol(HandlebarsSymbolType.At)));
}
if (HandlebarsHelpers.IsIdentifierStart(CurrentCharacter))
{
return(Identifier());
}
if (Char.IsDigit(CurrentCharacter))
{
return(NumericLiteral());
}
switch (CurrentCharacter)
{
case '.':
{
TakeCurrent();
if (CurrentCharacter == '/')
{
// We've matched a link to the current context.
TakeCurrent();
return(Stay(EndSymbol(HandlebarsSymbolType.CurrentContext)));
}
if (CurrentCharacter == '.' && Peek() == '/')
{
// We've matched a link to the parent context.
TakeCurrent();
TakeCurrent();
return(Stay(EndSymbol(HandlebarsSymbolType.ParentContext)));
}
// We've matched a dot, which could be part of an expression.
return(Stay(EndSymbol(HandlebarsSymbolType.Dot)));
}
case '/':
{
TakeCurrent();
// We've matched a forward-slash, which could be part of an expression.
return(Stay(EndSymbol(HandlebarsSymbolType.Slash)));
}
case ' ':
{
// Take all the available whitespace.
TakeUntil(c => !ParserHelpers.IsWhiteSpace(c));
return(Stay(EndSymbol(HandlebarsSymbolType.WhiteSpace)));
}
case '~':
{
TakeCurrent();
// We've reached a '~' character, so jump to the end of the tag.
return(Transition(EndSymbol(HandlebarsSymbolType.Tilde), () => EndTag(raw)));
}
case '"':
case '\'':
{
var quote = CurrentCharacter;
TakeCurrent();
// We've reached a quoted literal.
return(QuotedLiteral(quote));
}
case '=':
{
// We're reached a map assignment.
TakeCurrent();
return(Stay(EndSymbol(HandlebarsSymbolType.Assign)));
}
case '}':
{
// We've reached a closing tag, so transition away.
return(Transition(() => EndTag(raw)));
}
default:
{
CurrentErrors.Add(new Error("Unexpected character: " + CurrentCharacter, CurrentLocation));
return(Transition(Stop));
}
}
}
