public static SyntaxNode ConvertToTriviaList(this SyntaxNode[] nodes)
{
var builder = new SyntaxListBuilder(nodes.Length);
for (int i = 0; i < nodes.Length; i++)
{
var node = nodes[i];
var nd = node.GetDiagnostics();
foreach (var token in node.GetTokens())
{
builder.Add(token.GetLeadingTrivia());
if (token.Width > 0)
{
var tk = token.WithLeadingTrivia(null).WithTrailingTrivia(null);
System.Diagnostics.Debug.Assert(tk.HasDiagnostics == token.HasDiagnostics);
if (nd != null && nd.Length > 0)
{
if (token != node)
{
tk = tk.WithAdditionalDiagnostics(nd);
}
nd = null;
}
builder.Add(Syntax.SkippedTokens(tk));
}
builder.Add(token.GetTrailingTrivia());
}
}
return builder.ToListNode();
}
private XmlCommentSyntax ParseXmlComment()
{
var lessThanExclamationMinusMinusToken = this.EatToken(SyntaxKind.XmlCommentStartToken);
var saveMode = this.SetMode(LexMode.XmlCommentText);
var textTokens = new SyntaxListBuilder <SyntaxToken>(10);
while (this.CurrentToken.Kind == SyntaxKind.XmlTextLiteralToken ||
this.CurrentToken.Kind == SyntaxKind.XmlTextLiteralNewLineToken ||
this.CurrentToken.Kind == SyntaxKind.MinusMinusToken)
{
var token = this.EatToken();
if (token.Kind == SyntaxKind.MinusMinusToken)
{
// TODO: It is possible that a non-literal gets in here. --, specifically. Is that ok?
token = this.WithXmlParseError(token, XmlParseErrorCode.XML_IncorrectComment);
}
textTokens.Add(token);
}
var minusMinusGreaterThanToken = this.EatToken(SyntaxKind.XmlCommentEndToken);
this.ResetMode(saveMode);
return(Syntax.XmlComment(lessThanExclamationMinusMinusToken, textTokens, minusMinusGreaterThanToken));
}
public void ParseRemainder(SyntaxListBuilder <XmlNodeSyntax> nodes)
{
bool endTag = this.CurrentToken.Kind == SyntaxKind.LessThanSlashToken;
var saveMode = this.SetMode(LexMode.XmlCDataSectionText);
var textTokens = this.pool.Allocate();
try
{
while (this.CurrentToken.Kind != SyntaxKind.EndOfDocumentationCommentToken)
{
var token = this.EatToken();
// TODO: It is possible that a non-literal gets in here. ]]>, specifically. Is that ok?
textTokens.Add(token);
}
var allRemainderText = Syntax.XmlText(textTokens.ToTokenList());
XmlParseErrorArgument arg = endTag
? new XmlParseErrorArgument(XmlParseErrorCode.XML_EndTagNotExpected)
: new XmlParseErrorArgument(XmlParseErrorCode.XML_ExpectedEndOfXml);
allRemainderText = allRemainderText.WithAdditionalDiagnostics(this.MakeError(0, 1, ErrorCode.WRN_XMLParseError, arg));
nodes.Add(allRemainderText);
}
finally
{
this.pool.Free(textTokens);
}
this.ResetMode(saveMode);
}
private XmlAttributeSyntax ParseXmlAttribute()
{
var attrName = this.ParseXmlName();
if (attrName.LeadingWidth == 0)
{
// The Xml spec requires whitespace here: STag ::= '<' Name (S Attribute)* S? '>'
attrName = this.WithXmlParseError(attrName, XmlParseErrorCode.XML_WhitespaceMissing);
}
var equals = this.EatToken(SyntaxKind.EqualsToken, false);
if (equals.IsMissing)
{
equals = this.WithXmlParseError(equals, XmlParseErrorCode.XML_MissingEqualsAttribute);
}
var attributeTextStyle = this.CurrentToken.Kind == SyntaxKind.SingleQuoteToken
? SyntaxKind.SingleQuoteToken
: SyntaxKind.DoubleQuoteToken;
var startQuote = this.EatToken(attributeTextStyle, false);
if (startQuote.IsMissing)
{
startQuote = this.WithXmlParseError(startQuote, XmlParseErrorCode.XML_StringLiteralNoQuote);
}
var textTokens = new SyntaxListBuilder <SyntaxToken>(10);
{
var saveMode = this.SetMode(attributeTextStyle == SyntaxKind.SingleQuoteToken
? LexMode.XmlAttributeTextQuote
: LexMode.XmlAttributeTextDoubleQuote);
while (this.CurrentToken.Kind == SyntaxKind.XmlTextLiteralToken ||
this.CurrentToken.Kind == SyntaxKind.XmlTextLiteralNewLineToken ||
this.CurrentToken.Kind == SyntaxKind.XmlEntityLiteralToken ||
this.CurrentToken.Kind == SyntaxKind.LessThanToken)
{
var token = this.EatToken();
if (token.Kind == SyntaxKind.LessThanToken)
{
// TODO: It is possible that a non-literal gets in here. <, specifically. Is that ok?
token = this.WithXmlParseError(token, XmlParseErrorCode.XML_LessThanInAttributeValue);
}
textTokens.Add(token);
}
this.ResetMode(saveMode);
}
var endQuote = this.EatToken(attributeTextStyle);
return(Syntax.XmlAttribute(attrName, equals, startQuote, textTokens, endQuote));
}
public static SyntaxNode ConvertToTriviaList(this SyntaxNode[] nodes)
{
var builder = new SyntaxListBuilder(nodes.Length);
for (int i = 0; i < nodes.Length; i++)
{
var node = nodes[i];
var nd = node.GetDiagnostics();
foreach (var token in node.GetTokens())
{
builder.Add(token.GetLeadingTrivia());
if (token.Width > 0)
{
var text = Syntax.SkippedText(token.GetText());
var d = token.GetDiagnostics();
if (d.Length > 0)
{
text = text.WithDiagnostics(d);
}
if (nd != null && nd.Length > 0)
{
if (token != node)
{
text = text.WithAdditionalDiagnostics(nd);
}
nd = null;
}
builder.Add(text);
}
builder.Add(token.GetTrailingTrivia());
}
}
return(builder.ToListNode());
}
private XmlNodeSyntax ParseXmlText()
{
var textTokens = new SyntaxListBuilder(10);
while (this.CurrentToken.Kind == SyntaxKind.XmlTextLiteralToken ||
this.CurrentToken.Kind == SyntaxKind.XmlTextLiteralNewLineToken ||
this.CurrentToken.Kind == SyntaxKind.XmlEntityLiteralToken)
{
textTokens.Add(this.EatToken());
}
return(Syntax.XmlText(textTokens.ToList()));
}
private void ParseXmlNodes(SyntaxListBuilder <XmlNodeSyntax> nodes)
{
while (true)
{
var node = this.ParseXmlNode();
if (node == null)
{
return;
}
nodes.Add(node);
}
}
public static SyntaxNode ConvertToTriviaList(this SyntaxNode[] nodes)
{
var builder = new SyntaxListBuilder(nodes.Length);
for (int i = 0; i < nodes.Length; i++)
{
var node = nodes[i];
var nd = node.GetDiagnostics();
foreach (var token in node.GetTokens())
{
builder.Add(token.GetLeadingTrivia());
if (token.Width > 0)
{
var text = Syntax.SkippedText(token.GetText());
var d = token.GetDiagnostics();
if (d.Length > 0)
{
text = text.WithDiagnostics(d);
}
if (nd != null && nd.Length > 0)
{
if (token != node)
{
text = text.WithAdditionalDiagnostics(nd);
}
nd = null;
}
builder.Add(text);
}
builder.Add(token.GetTrailingTrivia());
}
}
return builder.ToListNode();
}
private TNode CreateForGlobalFailure <TNode>(int position, TNode node) where TNode : LuaSyntaxNode
{
// Turn the complete input into a single skipped token. This avoids running the lexer, and therefore
// the preprocessor directive parser, which may itself run into the same problem that caused the
// original failure.
var builder = new SyntaxListBuilder(1);
builder.Add(SyntaxFactory.BadToken(null, _lexer.Text.ToString(), null));
var fileAsTrivia = SyntaxFactory.SkippedTokensTrivia(builder.ToList <SyntaxToken>());
node = AddLeadingSkippedSyntax(node, fileAsTrivia);
ForceEndOfFile(); // force the scanner to report that it is at the end of the input.
return(AddError(node, position, 0, ErrorCode.ERR_InsufficientStack));
}
private XmlCDataSectionSyntax ParseXmlCDataSection()
{
var startCDataToken = this.EatToken(SyntaxKind.XmlCDataStartToken);
var saveMode = this.SetMode(LexMode.XmlCDataSectionText);
var textTokens = new SyntaxListBuilder <SyntaxToken>(10);
while (this.CurrentToken.Kind == SyntaxKind.XmlTextLiteralToken ||
this.CurrentToken.Kind == SyntaxKind.XmlTextLiteralNewLineToken)
{
textTokens.Add(this.EatToken());
}
var endCDataToken = this.EatToken(SyntaxKind.XmlCDataEndToken);
this.ResetMode(saveMode);
return(Syntax.XmlCDataSection(startCDataToken, textTokens, endCDataToken));
}
static void AddTokens(SyntaxNode current, SyntaxListBuilder <SyntaxToken> tokens)
{
if (current.SlotCount == 0 && current is SyntaxToken token)
{
// Token
tokens.Add(token);
return;
}
for (var i = 0; i < current.SlotCount; i++)
{
var child = current.GetNodeSlot(i);
if (child != null)
{
AddTokens(child, tokens);
}
}
}
private HtmlNodeSyntax GetSyntaxNode(SyntaxKind syntaxKind)
{
if (syntaxKind == SyntaxKind.HtmlText)
{
var textTokens = new SyntaxListBuilder <SyntaxToken>(SyntaxListBuilder.Create());
foreach (var token in Tokens)
{
if (token.SyntaxKind == SyntaxKind.Unknown)
{
Debug.Assert(false, $"Unexpected html token {((HtmlToken)token).Type}");
continue;
}
textTokens.Add(token.SyntaxToken);
}
var textResult = textTokens.ToList();
return(SyntaxFactory.HtmlText(new SyntaxList <SyntaxToken>(textResult.Node)));
}
return(null);
}
private Syntax.GreenNode GetSyntaxNode(SyntaxKind syntaxKind)
{
if (syntaxKind == SyntaxKind.HtmlTextLiteral)
{
var textTokens = new SyntaxListBuilder <SyntaxToken>(SyntaxListBuilder.Create());
foreach (var token in Tokens)
{
if (token.Kind == SyntaxKind.Unknown)
{
Debug.Assert(false, $"Unexpected token {token.Kind}");
continue;
}
textTokens.Add(token);
}
var textResult = textTokens.ToList();
return(SyntaxFactory.HtmlTextLiteral(new SyntaxList <SyntaxToken>(textResult.Node)));
}
return(null);
}
private static SyntaxList <RazorSyntaxNode> GetRewrittenMarkupEndTagChildren(MarkupEndTagSyntax node)
{
// Rewrites the children of the end tag to look like the legacy syntax tree.
if (node.IsMarkupTransition)
{
var tokens = node.DescendantNodes().Where(n => n is SyntaxToken token && !token.IsMissing).Cast <SyntaxToken>().ToArray();
var tokenBuilder = SyntaxListBuilder <SyntaxToken> .Create();
tokenBuilder.AddRange(tokens, 0, tokens.Length);
var markupTransition = SyntaxFactory.MarkupTransition(tokenBuilder.ToList()).Green.CreateRed(node, node.Position);
var spanContext = node.GetSpanContext();
if (spanContext != null)
{
markupTransition = markupTransition.WithSpanContext(spanContext);
}
var builder = new SyntaxListBuilder(1);
builder.Add(markupTransition);
return(new SyntaxList <RazorSyntaxNode>(builder.ToListNode().CreateRed(node, node.Position)));
}
return(node.Children);
}
private SyntaxToken ParseEndOfDirective(bool ignoreErrors, bool afterPragma = false, bool afterLineNumber = false, bool afterReference = false)
{
var skippedTokens = new SyntaxListBuilder<SyntaxToken>();
// Consume all extranous tokens as leading SkippedTokens trivia.
if (this.CurrentToken.Kind != SyntaxKind.EndOfDirectiveToken &&
this.CurrentToken.Kind != SyntaxKind.EndOfFileToken)
{
skippedTokens = new SyntaxListBuilder<SyntaxToken>(10);
if (!ignoreErrors)
{
ErrorCode errorCode = ErrorCode.ERR_EndOfPPLineExpected;
if (afterPragma)
{
errorCode = ErrorCode.WRN_EndOfPPLineExpected;
}
else if (afterLineNumber)
{
errorCode = ErrorCode.ERR_MissingPPFile;
}
else if (afterReference)
{
errorCode = ErrorCode.ERR_ExpectedPPFile;
}
skippedTokens.Add(this.AddError(this.EatToken().WithoutDiagnosticsGreen(), errorCode));
}
while (this.CurrentToken.Kind != SyntaxKind.EndOfDirectiveToken &&
this.CurrentToken.Kind != SyntaxKind.EndOfFileToken)
{
skippedTokens.Add(this.EatToken().WithoutDiagnosticsGreen());
}
}
// attach text from extraneous tokens as trivia to EndOfDirective token
SyntaxToken endOfDirective = this.CurrentToken.Kind == SyntaxKind.EndOfDirectiveToken
? this.EatToken()
: SyntaxFactory.Token(SyntaxKind.EndOfDirectiveToken);
if (!skippedTokens.IsNull)
{
endOfDirective = endOfDirective.WithLeadingTrivia(SyntaxFactory.SkippedTokensTrivia(skippedTokens.ToList()));
}
return endOfDirective;
}
public SyntaxListBuilder <TNode> Add(TNode node)
{
_builder.Add(node);
return(this);
}
private static SyntaxList <RazorSyntaxNode> GetRewrittenMarkupStartTagChildren(MarkupStartTagSyntax node)
{
// Rewrites the children of the start tag to look like the legacy syntax tree.
if (node.IsMarkupTransition)
{
var tokens = node.DescendantNodes().Where(n => n is SyntaxToken token && !token.IsMissing).Cast <SyntaxToken>().ToArray();
var tokenBuilder = SyntaxListBuilder <SyntaxToken> .Create();
tokenBuilder.AddRange(tokens, 0, tokens.Length);
var markupTransition = SyntaxFactory.MarkupTransition(tokenBuilder.ToList()).Green.CreateRed(node, node.Position);
var spanContext = node.GetSpanContext();
if (spanContext != null)
{
markupTransition = markupTransition.WithSpanContext(spanContext);
}
var builder = new SyntaxListBuilder(1);
builder.Add(markupTransition);
return(new SyntaxList <RazorSyntaxNode>(builder.ToListNode().CreateRed(node, node.Position)));
}
SpanContext latestSpanContext = null;
var children = node.Children;
var newChildren = new SyntaxListBuilder(children.Count);
var literals = new List <MarkupTextLiteralSyntax>();
foreach (var child in children)
{
if (child is MarkupTextLiteralSyntax literal)
{
literals.Add(literal);
latestSpanContext = literal.GetSpanContext() ?? latestSpanContext;
}
else if (child is MarkupMiscAttributeContentSyntax miscContent)
{
foreach (var contentChild in miscContent.Children)
{
if (contentChild is MarkupTextLiteralSyntax contentLiteral)
{
literals.Add(contentLiteral);
latestSpanContext = contentLiteral.GetSpanContext() ?? latestSpanContext;
}
else
{
// Pop stack
AddLiteralIfExists();
newChildren.Add(contentChild);
}
}
}
else
{
AddLiteralIfExists();
newChildren.Add(child);
}
}
AddLiteralIfExists();
return(new SyntaxList <RazorSyntaxNode>(newChildren.ToListNode().CreateRed(node, node.Position)));
void AddLiteralIfExists()
{
if (literals.Count > 0)
{
var mergedLiteral = SyntaxUtilities.MergeTextLiterals(literals.ToArray());
mergedLiteral = mergedLiteral.WithSpanContext(latestSpanContext);
literals.Clear();
latestSpanContext = null;
newChildren.Add(mergedLiteral);
}
}
}
private SyntaxList <RazorSyntaxNode> GetLegacyChildren()
{
// This method returns the children of this start tag in legacy format.
// This is needed to generate the same classified spans as the legacy syntax tree.
var builder = new SyntaxListBuilder(5);
var tokens = SyntaxListBuilder <SyntaxToken> .Create();
var context = this.GetSpanContext();
// We want to know if this tag contains non-whitespace attribute content to set the appropriate AcceptedCharacters.
// The prefix of a start tag(E.g '|<foo| attr>') will have 'Any' accepted characters if non-whitespace attribute content exists.
var acceptsAnyContext = new SpanContext(context.ChunkGenerator, SpanEditHandler.CreateDefault());
acceptsAnyContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.Any;
var containsAttributesContent = false;
foreach (var attribute in Attributes)
{
if (!string.IsNullOrWhiteSpace(attribute.GetContent()))
{
containsAttributesContent = true;
break;
}
}
if (!OpenAngle.IsMissing)
{
tokens.Add(OpenAngle);
}
if (Bang != null)
{
builder.Add(SyntaxFactory.MarkupTextLiteral(tokens.Consume()).WithSpanContext(acceptsAnyContext));
tokens.Add(Bang);
var acceptsNoneContext = new SpanContext(context.ChunkGenerator, SpanEditHandler.CreateDefault());
acceptsNoneContext.EditHandler.AcceptedCharacters = AcceptedCharactersInternal.None;
builder.Add(SyntaxFactory.RazorMetaCode(tokens.Consume()).WithSpanContext(acceptsNoneContext));
}
if (!Name.IsMissing)
{
tokens.Add(Name);
}
builder.Add(SyntaxFactory.MarkupTextLiteral(tokens.Consume()).WithSpanContext(containsAttributesContent ? acceptsAnyContext : context));
builder.AddRange(Attributes);
if (ForwardSlash != null)
{
tokens.Add(ForwardSlash);
}
if (!CloseAngle.IsMissing)
{
tokens.Add(CloseAngle);
}
if (tokens.Count > 0)
{
builder.Add(SyntaxFactory.MarkupTextLiteral(tokens.Consume()).WithSpanContext(context));
}
return(new SyntaxList <RazorSyntaxNode>(builder.ToListNode().CreateRed(this, Position)));
}
