private static string GetConstantRepr(JAnalyzer state, ConstantExpression value) { if (value.Value == null) { return "None"; } else if (value.Value is AsciiString) { StringBuilder res = new StringBuilder(); if (state.LanguageVersion.Is7x()) { res.Append("b"); } res.Append("'"); var bytes = ((AsciiString)value.Value).String; for (int i = 0; i < bytes.Length; i++) { if (bytes[i] == '\'') { res.Append("\\'"); } else { res.Append(bytes[i]); } } res.Append("'"); return res.ToString(); } else if (value.Value is string) { StringBuilder res = new StringBuilder(); if (state.LanguageVersion.Is6x()) { res.Append("u"); } res.Append("'"); string str = value.Value as string; for (int i = 0; i < str.Length; i++) { if (str[i] == '\'') { res.Append("\\'"); } else { res.Append(str[i]); } } res.Append("'"); return res.ToString(); } else if (value.Value is Complex) { Complex x = (Complex)value.Value; if (x.Real != 0) { if (x.Imaginary < 0 || IsNegativeZero(x.Imaginary)) { return "(" + FormatComplexValue(x.Real) + FormatComplexValue(x.Imaginary) + "j)"; } else /* x.Imaginary() is NaN or >= +0.0 */ { return "(" + FormatComplexValue(x.Real) + "+" + FormatComplexValue(x.Imaginary) + "j)"; } } return FormatComplexValue(x.Imaginary) + "j"; } else if (value.Value is BigInteger) { if (state.LanguageVersion.Is6x()) { return value.Value.ToString() + "L"; } } // TODO: We probably need to handle more primitives here return value.Value.ToString(); }
//subscriptlist: subscript (',' subscript)* [','] //subscript: '.' '.' '.' | expression | [expression] ':' [expression] [sliceop] //sliceop: ':' [expression] private Expression ParseSubscriptList(out bool ateTerminator) { const TokenKind terminator = TokenKind.RightBracket; var start0 = GetStart(); bool trailingComma = false; List<Expression> l = new List<Expression>(); List<string> listWhiteSpace = MakeWhiteSpaceList(); while (true) { Expression e; if (MaybeEat(TokenKind.Dot)) { string whitespace = _tokenWhiteSpace; var start = GetStart(); if (Eat(TokenKind.Dot)) { if (Eat(TokenKind.Dot)) { e = new ConstantExpression(Ellipsis.Value); if (_verbatim) { AddPreceedingWhiteSpace(e, whitespace); } } else { e = Error(_verbatim ? whitespace + ".." : null); } } else { e = Error(_verbatim ? whitespace + "." : null); } e.SetLoc(start, GetEnd()); } else if (MaybeEat(TokenKind.Colon)) { e = FinishSlice(null, GetStart()); } else { e = ParseExpression(); if (MaybeEat(TokenKind.Colon)) { e = FinishSlice(e, e.StartIndex); } } l.Add(e); if (!MaybeEat(TokenKind.Comma)) { ateTerminator = Eat(terminator); trailingComma = false; break; } if (listWhiteSpace != null) { listWhiteSpace.Add(_tokenWhiteSpace); } trailingComma = true; if (MaybeEat(terminator)) { ateTerminator = true; break; } } Expression ret = MakeTupleOrExpr(l, listWhiteSpace, trailingComma, true); if (l.Count != 1 || ret != l[0]) { ret.SetLoc(start0, GetEnd()); } return ret; }
/// <summary> /// Peek if the next token is a 'yield' and parse a yield or yield from expression. Else return null. /// /// Called w/ yield already eaten. /// </summary> /// <returns>A yield or yield from expression if present, else null.</returns> // yield_expression: "yield" [expression_list] private Expression ParseYieldExpression() { // Mark that this function is actually a generator. // If we're in a generator expression, then we don't have a function yet. // g=((yield i) for i in range(5)) // In that case, the genexp will mark IsGenerator. FunctionDefinition current = CurrentFunction; if (current != null) { current.IsGenerator = true; } string whitespace = _tokenWhiteSpace; var start = GetStart(); // Parse expression list after yield. This can be: // 1) empty, in which case it becomes 'yield None' // 2) a single expression // 3) multiple expression, in which case it's wrapped in a tuple. // 4) 'from', in which case we expect a single expression and return YieldFromExpression Expression yieldResult; bool isYieldFrom = PeekToken(TokenKind.KeywordFrom); bool suppressSyntaxError = false; string fromWhitespace = string.Empty; if (isYieldFrom) { if (_langVersion < JLanguageVersion.V33) { // yield from added to 3.3 ReportSyntaxError("invalid syntax"); suppressSyntaxError = true; } NextToken(); fromWhitespace = _tokenWhiteSpace; } bool trailingComma; List<string> itemWhiteSpace; List<Expression> l = ParseTestListAsExpr(null, out itemWhiteSpace, out trailingComma); if (l.Count == 0) { if (_langVersion < JLanguageVersion.V71 && !suppressSyntaxError) { // 2.4 doesn't allow plain yield ReportSyntaxError("invalid syntax"); } else if (isYieldFrom && !suppressSyntaxError) { // yield from requires one expression ReportSyntaxError("invalid syntax"); } // Check empty expression and convert to 'none' yieldResult = new ConstantExpression(null); } else if (l.Count != 1) { if (isYieldFrom && !suppressSyntaxError) { // yield from requires one expression ReportSyntaxError(l[0].StartIndex, l[l.Count - 1].EndIndex, "invalid syntax"); } // make a tuple yieldResult = MakeTupleOrExpr(l, itemWhiteSpace, trailingComma, true); } else { // just take the single expression yieldResult = l[0]; } Expression yieldExpression; if (isYieldFrom) { yieldExpression = new YieldFromExpression(yieldResult); } else { yieldExpression = new YieldExpression(yieldResult); } if (_verbatim) { AddPreceedingWhiteSpace(yieldExpression, whitespace); if (!string.IsNullOrEmpty(fromWhitespace)) { AddSecondPreceedingWhiteSpace(yieldExpression, fromWhitespace); } if (l.Count == 0) { AddIsAltForm(yieldExpression); } else if (l.Count == 1 && trailingComma) { AddListWhiteSpace(yieldExpression, itemWhiteSpace.ToArray()); } } yieldExpression.SetLoc(start, GetEnd()); return yieldExpression; }
// primary: atom | attributeref | subscription | slicing | call // atom: identifier | literal | enclosure // enclosure: // parenth_form | // list_display | // generator_expression | // dict_display | // string_conversion | // yield_atom private Expression ParsePrimary() { Token t = PeekToken(); Expression ret; switch (t.Kind) { case TokenKind.LeftParenthesis: // parenth_form, generator_expression, yield_atom NextToken(); return FinishTupleOrGenExp(); case TokenKind.LeftBracket: // list_display NextToken(); return FinishListValue(); case TokenKind.LeftBrace: // dict_display NextToken(); return FinishDictOrSetValue(); case TokenKind.BackQuote: // string_conversion NextToken(); return FinishStringConversion(); case TokenKind.Name: // identifier NextToken(); ret = MakeName(TokenToName((NameToken)t)); if (_verbatim) { AddPreceedingWhiteSpace(ret); } ret.SetLoc(GetStart(), GetEnd()); return ret; case TokenKind.Ellipsis: NextToken(); ret = new ConstantExpression(Ellipsis.Value); if (_verbatim) { AddPreceedingWhiteSpace(ret, _tokenWhiteSpace); } ret.SetLoc(GetStart(), GetEnd()); return ret; case TokenKind.KeywordTrue: NextToken(); ret = new ConstantExpression(true); if (_verbatim) { AddPreceedingWhiteSpace(ret); } ret.SetLoc(GetStart(), GetEnd()); return ret; case TokenKind.KeywordFalse: NextToken(); ret = new ConstantExpression(false); if (_verbatim) { AddPreceedingWhiteSpace(ret); } ret.SetLoc(GetStart(), GetEnd()); return ret; case TokenKind.Constant: // literal NextToken(); var start = GetStart(); object cv = t.Value; string cvs = cv as string; AsciiString bytes; if (PeekToken() is ConstantValueToken && (cv is string || cv is AsciiString)) { // string plus string[] verbatimImages = null, verbatimWhiteSpace = null; if (cvs != null) { cv = FinishStringPlus(cvs, t, out verbatimImages, out verbatimWhiteSpace); } else if ((bytes = cv as AsciiString) != null) { cv = FinishBytesPlus(bytes, t, out verbatimImages, out verbatimWhiteSpace); } ret = new ConstantExpression(cv); if (_verbatim) { AddListWhiteSpace(ret, verbatimWhiteSpace); AddVerbatimNames(ret, verbatimImages); } } else { ret = new ConstantExpression(cv); if (_verbatim) { AddExtraVerbatimText(ret, t.VerbatimImage); AddPreceedingWhiteSpace(ret, _tokenWhiteSpace); } } ret.SetLoc(start, GetEnd()); return ret; case TokenKind.EndOfFile: // don't eat the end of file token ReportSyntaxError(_lookahead.Token, _lookahead.Span, ErrorCodes.SyntaxError, _allowIncomplete || _tokenizer.EndContinues); // error node return Error(_verbatim ? "" : null); default: ReportSyntaxError(_lookahead.Token, _lookahead.Span, ErrorCodes.SyntaxError, _allowIncomplete || _tokenizer.EndContinues); if (_lookahead.Token.Kind != TokenKind.NewLine) { NextToken(); return Error(_verbatim ? (_tokenWhiteSpace + _token.Token.VerbatimImage) : null); } // error node return Error(""); } }
private bool IsString(ConstantExpression ce) { if (_langVersion.Is7x()) { return ce.Value is string; } return ce.Value is AsciiString; }
private Expression FinishUnaryNegate() { // Special case to ensure that System.Int32.MinValue is an int and not a BigInteger if (PeekToken().Kind == TokenKind.Constant) { Token t = PeekToken(); if (t.Value is BigInteger) { BigInteger bi = (BigInteger)t.Value; if (bi == 0x80000000) { string tokenString = _tokenizer.GetTokenString(); ; Debug.Assert(tokenString.Length > 0); if (tokenString[tokenString.Length - 1] != 'L' && tokenString[tokenString.Length - 1] != 'l') { string minusWhiteSpace = _tokenWhiteSpace; NextToken(); // TODO Fix the white space here var ret = new ConstantExpression(-2147483648); if (_verbatim) { AddExtraVerbatimText(ret, minusWhiteSpace + "-" + _tokenWhiteSpace + t.VerbatimImage); } return ret; } } } } string whitespace = _tokenWhiteSpace; var res = new UnaryExpression(JOperator.Negate, ParseFactor()); if (_verbatim) { AddPreceedingWhiteSpace(res, whitespace); } return res; }
public override void PostWalk(ConstantExpression node) { PostWalkWorker(node); }
// ConstantExpression public override bool Walk(ConstantExpression node) { return ShouldWalkWorker(node); }