private void CheckParseRequest(ParseRequest req) { try { var lexer = new AphidLexer(req.Text); var parser = new AphidParser(lexer.GetTokens()); parser.Parse(); } catch (AphidParserException e) { var lineCol = TokenHelper.GetLineCol(req.Text, e.UnexpectedToken.Index); var span = new TextSpan() { iStartLine = lineCol.Item1, iEndLine = lineCol.Item1, iStartIndex = lineCol.Item2, iEndIndex = lineCol.Item2 + (e.UnexpectedToken.Lexeme != null ? e.UnexpectedToken.Lexeme.Length : 0) }; var msg = string.Format( "Unexpected {0}: {1}, expected {2}", e.UnexpectedToken.TokenType.ToString(), e.UnexpectedToken.Lexeme, e.ExpectedToken); req.Sink.AddError(req.FileName, msg, span, Severity.Error); } }
private static (string, List <AphidToken>) FormatExpression(string code) { var formattedCode = code; var tokens = AphidLexer.GetTokens(code); if (!tokens.Any(x => x.TokenType == AphidTokenType.EndOfStatement)) { //var allTokens = AphidLexer.GetTokens(code); if (tokens.Count > 0) { var t = tokens[tokens.Count - 1]; formattedCode = code.Insert(t.Index + t.Lexeme.Length, ";"); } //Hackier tranformation algorithm: //formattedCode = string.Format( // "{0}{1};", // code, // allTokens // .Where(x => // x.TokenType != AphidTokenType.WhiteSpace || // x.Lexeme.Contains('\n')) // .LastOrDefault() // .If(x => // x.TokenType == AphidTokenType.Comment && // x.Lexeme.Trim().StartsWith("//"), // x => "\r\n") // .Else(x => "")); tokens = AphidLexer.GetTokens(formattedCode); } return(formattedCode, tokens); }
private IEnumerable <ColoredText> HighlightCore(string text) { if (string.IsNullOrEmpty(text)) { return(Array.Empty <ColoredText>()); } var allTokens = AphidLexer.GetAllTokens(text); var tokens = AphidLexer.GetTokens(text); if (tokens.Count == 0) { return(allTokens.Select(x => _theme.GetColoredText(x))); } var peekTable = Enumerable .Range(0, tokens.Count - 1) .ToDictionary(x => tokens[x].Index, x => tokens[x + 1].TokenType); void addNone(in AphidToken t) => peekTable.Add(t.Index, None); addNone(tokens[tokens.Count - 1]); allTokens.Except(tokens).For(x => addNone(in x)); return(allTokens.Select(x => _theme.GetColoredText(x, peekTable[x.Index]))); }
public IEnumerable <ScannerToken> Tokenize(string text) => AphidLexer .GetTokens(text) .Select(x => new ScannerToken( x.TokenType == AphidTokenType.Identifier ? ScannerTokenType.Identifier : x.TokenType == AphidTokenType.MemberOperator ? ScannerTokenType.MemberOperator : ScannerTokenType.Other, x.Index, x.Lexeme));
public void Interpret(string code) { var lexer = new AphidLexer(code); var parser = new AphidParser(lexer.GetTokens()); var ast = parser.Parse(); Interpret(ast); }
public static void Tokenize( [PexAssumeUnderTest] AphidLexer lexer, [PexAssumeUnderTest] string code ) { lexer.Text = code; lexer.GetTokens(); // TODO: add assertions to method AphidInterpreterTest.Interpret(AphidInterpreter, String, Boolean) }
static void Main(string[] args) { var lexer = new AphidLexer(@" using System; Console.WriteLine('Hello world'); "); foreach (var t in lexer.GetTokens()) { Console.WriteLine(t); } }
public static List <AphidExpression> Parse( string code, string filename, bool isTextDocument = false, bool useImplicitReturns = true) { var lexer = new AphidLexer(code); if (isTextDocument) { lexer.SetTextMode(); } return(Parse( lexer.GetTokens(), code, filename, useImplicitReturns: useImplicitReturns)); }
public static AphidObject EvalExpression(AphidInterpreter interpreter, string code) { AphidObject result = null; void action() { if (code.Trim()?.Length == 0) { return; } var lexer = new AphidLexer(code); var tokens = lexer.GetTokens(); var exp = AphidParser.ParseExpression(tokens, code); var retExp = new UnaryOperatorExpression(AphidTokenType.retKeyword, exp); //new AphidCodeVisitor(code).VisitExpression(retExp); interpreter.Interpret(retExp); result = interpreter.GetReturnValue(true); if (result != null && (result.Value != null || result.Any())) { WriteLineHighlighted( SerializingFormatter.Format( interpreter, result, ignoreNull: false, ignoreClrObj: false)); } } if (AphidErrorHandling.HandleErrors) { TryAction(interpreter, code, action); } else { action(); } return(result); }
private static AphidExpression ParseCore(string expression) { var fixedExpression = expression; List <AphidToken> tokens; AphidToken lastToken = default; int state, offset = 0; do { tokens = AphidLexer.GetTokens(fixedExpression); state = 0; for (var i = 0; i < tokens.Count; i++) { lastToken = tokens[i]; if (lastToken.TokenType == _tickType) { offset = lastToken.Index; fixedExpression = fixedExpression.Remove(offset, 1).Insert(offset, "'"); if (++state == 2) { break; } } } if (state == 1) { throw new AphidParserException( $"Unterminated string beginning at {expression.Substring(offset)}") { UnexpectedToken = lastToken, }; } }while (state != 0); return(AphidParser.ParseExpression(tokens, expression)); }
public static AphidExpression ParseExpression(string code) => ParseExpression(AphidLexer.GetTokens(code), code);