private void anal_Button_Click(object sender, EventArgs e) { richTextBoxOut.Clear(); Lexer.TextData text = new Lexer.TextData(richTextBoxIn.Text.ToString()); var result = Lexer.Lexer.GetTokens(text); foreach (var a in result) { switch (a) { case Lexer.SyntaxValueToken <string> str: richTextBoxOut.AppendText(str.Value.ToString() + " - " + str.SyntaxKind.ToString() + " Позиция - " + str.SourceTextPosition.ToString() + "\n"); break; case Lexer.SyntaxValueToken <int> inti: richTextBoxOut.AppendText(inti.Value.ToString() + " - " + inti.SyntaxKind.ToString() + " Позиция - " + inti.SourceTextPosition.ToString() + "\n"); break; case Lexer.SyntaxValueToken <char> charType: richTextBoxOut.AppendText(charType.Value.ToString() + " - " + charType.SyntaxKind.ToString() + " Позиция - " + charType.SourceTextPosition.ToString() + "\n"); break; case Lexer.SyntaxValueToken <double> doubleType: richTextBoxOut.AppendText(doubleType.Value.ToString() + " - " + doubleType.SyntaxKind.ToString() + " Позиция - " + doubleType.SourceTextPosition.ToString() + "\n"); break; case Lexer.SyntaxIdentifierToken iden: richTextBoxOut.AppendText($"{ iden.IdentifierName} - {iden.SyntaxKind} Позиция - {iden.SourceTextPosition}\n"); break; case Lexer.SyntaxTriviaToken trivia: richTextBoxOut.AppendText($"{trivia.SyntaxKind} Позиция - {trivia.SourceTextPosition}"); break; case Lexer.SyntaxUnknownToken unknown: richTextBoxOut.AppendText($"{unknown.Text} - {unknown.SyntaxKind} Позиция - {unknown.SourceTextPosition}\n"); break; } } }
//private int counter; internal static List <ISyntaxToken> GetTokens(TextData data) { TokenInfo tokenInfo = new TokenInfo { kind = SyntaxKind.None, specialType = SpecialType.None, }; var lexer = new Lexer(data); var tempTokens = new List <ISyntaxToken>(); while (!data.IsEndOfData) { switch (data.PeekChar()) { case ' ': case '\n': case '\r': case '\0': //Here we skip whitespace data.AdvanceChar(); tokenInfo.kind = SyntaxKind.None; break; case '+': tokenInfo.position = data.Position; if (data.PeekChar(1) == '+') { tokenInfo.kind = SyntaxKind.PlusPlusToken; data.AdvanceChar(2); } else if (data.PeekChar(1) == '=') { tokenInfo.kind = SyntaxKind.PlusEqualToken; data.AdvanceChar(2); } else { tokenInfo.kind = SyntaxKind.PlusToken; data.AdvanceChar(); } break; case '-': tokenInfo.position = data.Position; if (data.PeekChar(1) == '-') { tokenInfo.kind = SyntaxKind.MinusMinusToken; data.AdvanceChar(2); } else if (data.PeekChar(1) == '=') { tokenInfo.kind = SyntaxKind.MinusEqualToken; data.AdvanceChar(2); } else { tokenInfo.kind = SyntaxKind.MinusToken; data.AdvanceChar(); } break; case '*': tokenInfo.position = data.Position; if (data.PeekChar(1) == '=') { tokenInfo.kind = SyntaxKind.MultEqualToken; data.AdvanceChar(2); } else { tokenInfo.kind = SyntaxKind.MultToken; data.AdvanceChar(); } break; case '/': tokenInfo.position = data.Position; if (data.PeekChar(1) == '=') { tokenInfo.kind = SyntaxKind.DivEqualToken; data.AdvanceChar(2); } else { tokenInfo.kind = SyntaxKind.DivEqualToken; data.AdvanceChar(); } break; case '=': tokenInfo.position = data.Position; if (data.PeekChar(1) == '=') { tokenInfo.kind = SyntaxKind.EqualEqualToken; data.AdvanceChar(2); } else { tokenInfo.kind = SyntaxKind.EqualToken; data.AdvanceChar(); } break; case '(': tokenInfo.position = data.Position; tokenInfo.kind = SyntaxKind.OpeningBracket; data.AdvanceChar(); break; case '{': tokenInfo.position = data.Position; tokenInfo.kind = SyntaxKind.OpeningBrace; data.AdvanceChar(); break; case ')': tokenInfo.position = data.Position; tokenInfo.kind = SyntaxKind.ClosingBracket; data.AdvanceChar(); break; case '}': tokenInfo.position = data.Position; tokenInfo.kind = SyntaxKind.ClosingBrace; data.AdvanceChar(); break; case '\'': if (data.PeekChar(2) != '\'') { goto default; } tokenInfo.position = data.Position; tokenInfo.kind = SyntaxKind.CharLiteralToken; tokenInfo.specialType = SpecialType.System_Char; tokenInfo.CharValue = data.PeekChar(1); data.AdvanceChar(3); break; case ';': tokenInfo.position = data.Position; tokenInfo.kind = SyntaxKind.SemicolonToken; data.AdvanceChar(); break; case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': lexer.ScanIdentifierOrKeyword(ref tokenInfo); break; case '"': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': lexer.ScanNumericOrLiteralValue(ref tokenInfo); break; default: tokenInfo.position = data.Position; tokenInfo.kind = SyntaxKind.Unknown; tokenInfo.Text = data.PeekChar().ToString(); data.AdvanceChar(); break; } if (tokenInfo.kind != SyntaxKind.None) { tempTokens.Add(GetToken(ref tokenInfo)); } } return(tempTokens); }
private Lexer(TextData data) { _textData = data; _builder = new StringBuilder(); }