public static List <MyToken> Parse(string code) { CompilationUnitSyntax tree = SyntaxFactory.ParseCompilationUnit(code); IEnumerable <SyntaxToken> rawTokens = tree.DescendantTokens(); var malFormedToken = rawTokens.FirstOrDefault(x => x.IsMissing); if (malFormedToken.Kind() == SyntaxKind.None) { malFormedToken = rawTokens.FirstOrDefault(t => t.GetAllTrivia().Any( x => x.Kind() == SyntaxKind.SkippedTokensTrivia)); } if (malFormedToken.Kind() != SyntaxKind.None) { throw new Exception($"Received code is malformed around: {malFormedToken}"); } int runningIndex = 0; var myTokens = new List <MyToken>(); foreach (var rawToken in rawTokens) { if (rawToken.HasLeadingTrivia) { foreach (var trivia in rawToken.LeadingTrivia) { if (trivia.Kind() == SyntaxKind.WhitespaceTrivia || trivia.Kind() == SyntaxKind.EndOfLineTrivia) { continue; } myTokens.Add(MyToken.Create(runningIndex++, trivia)); } } myTokens.Add(MyToken.Create(runningIndex++, rawToken)); if (rawToken.HasTrailingTrivia) { foreach (var trivia in rawToken.TrailingTrivia) { if (trivia.Kind() == SyntaxKind.WhitespaceTrivia || trivia.Kind() == SyntaxKind.EndOfLineTrivia) { continue; } myTokens.Add(MyToken.Create(runningIndex++, trivia)); } } } return(myTokens); }
internal static MyToken Create(int index, SyntaxToken rawToken) { var loc = rawToken.GetLocation(); var lineInfo = loc.GetLineSpan(); var genericTokenType = GetGenericType(rawToken.Kind(), rawToken.IsKeyword()); var t = new MyToken { Index = index, LineNumber = lineInfo.StartLinePosition.Line, RawType = rawToken.Kind().ToString(), GenericType = genericTokenType, Text = rawToken.Text, StartPos = loc.SourceSpan.Start, EndPos = loc.SourceSpan.End }; return(t); }
internal static MyToken Create(int index, SyntaxTrivia trivia) { var loc = trivia.GetLocation(); var lineInfo = loc.GetLineSpan(); var genericTokenType = GetGenericType(trivia.Kind(), false); var t = new MyToken { Index = index, LineNumber = lineInfo.StartLinePosition.Line, RawType = trivia.Kind().ToString(), GenericType = genericTokenType, Text = trivia.ToString(), StartPos = loc.SourceSpan.Start, EndPos = loc.SourceSpan.End, IsTrivia = true }; return(t); }