public static List <MyToken> Parse(string code)
{
CompilationUnitSyntax tree = SyntaxFactory.ParseCompilationUnit(code);
IEnumerable <SyntaxToken> rawTokens = tree.DescendantTokens();
var malFormedToken = rawTokens.FirstOrDefault(x => x.IsMissing);
if (malFormedToken.Kind() == SyntaxKind.None)
{
malFormedToken = rawTokens.FirstOrDefault(t => t.GetAllTrivia().Any(
x => x.Kind() == SyntaxKind.SkippedTokensTrivia));
}
if (malFormedToken.Kind() != SyntaxKind.None)
{
throw new Exception($"Received code is malformed around: {malFormedToken}");
}
int runningIndex = 0;
var myTokens = new List <MyToken>();
foreach (var rawToken in rawTokens)
{
if (rawToken.HasLeadingTrivia)
{
foreach (var trivia in rawToken.LeadingTrivia)
{
if (trivia.Kind() == SyntaxKind.WhitespaceTrivia ||
trivia.Kind() == SyntaxKind.EndOfLineTrivia)
{
continue;
}
myTokens.Add(MyToken.Create(runningIndex++, trivia));
}
}
myTokens.Add(MyToken.Create(runningIndex++, rawToken));
if (rawToken.HasTrailingTrivia)
{
foreach (var trivia in rawToken.TrailingTrivia)
{
if (trivia.Kind() == SyntaxKind.WhitespaceTrivia ||
trivia.Kind() == SyntaxKind.EndOfLineTrivia)
{
continue;
}
myTokens.Add(MyToken.Create(runningIndex++, trivia));
}
}
}
return(myTokens);
}
internal static MyToken Create(int index, SyntaxToken rawToken)
{
var loc = rawToken.GetLocation();
var lineInfo = loc.GetLineSpan();
var genericTokenType = GetGenericType(rawToken.Kind(), rawToken.IsKeyword());
var t = new MyToken
{
Index = index,
LineNumber = lineInfo.StartLinePosition.Line,
RawType = rawToken.Kind().ToString(),
GenericType = genericTokenType,
Text = rawToken.Text,
StartPos = loc.SourceSpan.Start,
EndPos = loc.SourceSpan.End
};
return(t);
}
internal static MyToken Create(int index, SyntaxTrivia trivia)
{
var loc = trivia.GetLocation();
var lineInfo = loc.GetLineSpan();
var genericTokenType = GetGenericType(trivia.Kind(), false);
var t = new MyToken
{
Index = index,
LineNumber = lineInfo.StartLinePosition.Line,
RawType = trivia.Kind().ToString(),
GenericType = genericTokenType,
Text = trivia.ToString(),
StartPos = loc.SourceSpan.Start,
EndPos = loc.SourceSpan.End,
IsTrivia = true
};
return(t);
}