private static string GetConstantRepr(JAnalyzer state, ConstantExpression value)
{
if (value.Value == null) {
return "None";
} else if (value.Value is AsciiString) {
StringBuilder res = new StringBuilder();
if (state.LanguageVersion.Is7x()) {
res.Append("b");
}
res.Append("'");
var bytes = ((AsciiString)value.Value).String;
for (int i = 0; i < bytes.Length; i++) {
if (bytes[i] == '\'') {
res.Append("\\'");
} else {
res.Append(bytes[i]);
}
}
res.Append("'");
return res.ToString();
} else if (value.Value is string) {
StringBuilder res = new StringBuilder();
if (state.LanguageVersion.Is6x()) {
res.Append("u");
}
res.Append("'");
string str = value.Value as string;
for (int i = 0; i < str.Length; i++) {
if (str[i] == '\'') {
res.Append("\\'");
} else {
res.Append(str[i]);
}
}
res.Append("'");
return res.ToString();
} else if (value.Value is Complex) {
Complex x = (Complex)value.Value;
if (x.Real != 0) {
if (x.Imaginary < 0 || IsNegativeZero(x.Imaginary)) {
return "(" + FormatComplexValue(x.Real) + FormatComplexValue(x.Imaginary) + "j)";
} else /* x.Imaginary() is NaN or >= +0.0 */ {
return "(" + FormatComplexValue(x.Real) + "+" + FormatComplexValue(x.Imaginary) + "j)";
}
}
return FormatComplexValue(x.Imaginary) + "j";
} else if (value.Value is BigInteger) {
if (state.LanguageVersion.Is6x()) {
return value.Value.ToString() + "L";
}
}
// TODO: We probably need to handle more primitives here
return value.Value.ToString();
}
//subscriptlist: subscript (',' subscript)* [',']
//subscript: '.' '.' '.' | expression | [expression] ':' [expression] [sliceop]
//sliceop: ':' [expression]
private Expression ParseSubscriptList(out bool ateTerminator)
{
const TokenKind terminator = TokenKind.RightBracket;
var start0 = GetStart();
bool trailingComma = false;
List<Expression> l = new List<Expression>();
List<string> listWhiteSpace = MakeWhiteSpaceList();
while (true) {
Expression e;
if (MaybeEat(TokenKind.Dot)) {
string whitespace = _tokenWhiteSpace;
var start = GetStart();
if (Eat(TokenKind.Dot)) {
if (Eat(TokenKind.Dot)) {
e = new ConstantExpression(Ellipsis.Value);
if (_verbatim) {
AddPreceedingWhiteSpace(e, whitespace);
}
} else {
e = Error(_verbatim ? whitespace + ".." : null);
}
} else {
e = Error(_verbatim ? whitespace + "." : null);
}
e.SetLoc(start, GetEnd());
} else if (MaybeEat(TokenKind.Colon)) {
e = FinishSlice(null, GetStart());
} else {
e = ParseExpression();
if (MaybeEat(TokenKind.Colon)) {
e = FinishSlice(e, e.StartIndex);
}
}
l.Add(e);
if (!MaybeEat(TokenKind.Comma)) {
ateTerminator = Eat(terminator);
trailingComma = false;
break;
}
if (listWhiteSpace != null) {
listWhiteSpace.Add(_tokenWhiteSpace);
}
trailingComma = true;
if (MaybeEat(terminator)) {
ateTerminator = true;
break;
}
}
Expression ret = MakeTupleOrExpr(l, listWhiteSpace, trailingComma, true);
if (l.Count != 1 || ret != l[0]) {
ret.SetLoc(start0, GetEnd());
}
return ret;
}
/// <summary>
/// Peek if the next token is a 'yield' and parse a yield or yield from expression. Else return null.
///
/// Called w/ yield already eaten.
/// </summary>
/// <returns>A yield or yield from expression if present, else null.</returns>
// yield_expression: "yield" [expression_list]
private Expression ParseYieldExpression()
{
// Mark that this function is actually a generator.
// If we're in a generator expression, then we don't have a function yet.
// g=((yield i) for i in range(5))
// In that case, the genexp will mark IsGenerator.
FunctionDefinition current = CurrentFunction;
if (current != null) {
current.IsGenerator = true;
}
string whitespace = _tokenWhiteSpace;
var start = GetStart();
// Parse expression list after yield. This can be:
// 1) empty, in which case it becomes 'yield None'
// 2) a single expression
// 3) multiple expression, in which case it's wrapped in a tuple.
// 4) 'from', in which case we expect a single expression and return YieldFromExpression
Expression yieldResult;
bool isYieldFrom = PeekToken(TokenKind.KeywordFrom);
bool suppressSyntaxError = false;
string fromWhitespace = string.Empty;
if (isYieldFrom) {
if (_langVersion < JLanguageVersion.V33) {
// yield from added to 3.3
ReportSyntaxError("invalid syntax");
suppressSyntaxError = true;
}
NextToken();
fromWhitespace = _tokenWhiteSpace;
}
bool trailingComma;
List<string> itemWhiteSpace;
List<Expression> l = ParseTestListAsExpr(null, out itemWhiteSpace, out trailingComma);
if (l.Count == 0) {
if (_langVersion < JLanguageVersion.V71 && !suppressSyntaxError) {
// 2.4 doesn't allow plain yield
ReportSyntaxError("invalid syntax");
} else if (isYieldFrom && !suppressSyntaxError) {
// yield from requires one expression
ReportSyntaxError("invalid syntax");
}
// Check empty expression and convert to 'none'
yieldResult = new ConstantExpression(null);
} else if (l.Count != 1) {
if (isYieldFrom && !suppressSyntaxError) {
// yield from requires one expression
ReportSyntaxError(l[0].StartIndex, l[l.Count - 1].EndIndex, "invalid syntax");
}
// make a tuple
yieldResult = MakeTupleOrExpr(l, itemWhiteSpace, trailingComma, true);
} else {
// just take the single expression
yieldResult = l[0];
}
Expression yieldExpression;
if (isYieldFrom) {
yieldExpression = new YieldFromExpression(yieldResult);
} else {
yieldExpression = new YieldExpression(yieldResult);
}
if (_verbatim) {
AddPreceedingWhiteSpace(yieldExpression, whitespace);
if (!string.IsNullOrEmpty(fromWhitespace)) {
AddSecondPreceedingWhiteSpace(yieldExpression, fromWhitespace);
}
if (l.Count == 0) {
AddIsAltForm(yieldExpression);
} else if (l.Count == 1 && trailingComma) {
AddListWhiteSpace(yieldExpression, itemWhiteSpace.ToArray());
}
}
yieldExpression.SetLoc(start, GetEnd());
return yieldExpression;
}
// primary: atom | attributeref | subscription | slicing | call
// atom: identifier | literal | enclosure
// enclosure:
// parenth_form |
// list_display |
// generator_expression |
// dict_display |
// string_conversion |
// yield_atom
private Expression ParsePrimary()
{
Token t = PeekToken();
Expression ret;
switch (t.Kind) {
case TokenKind.LeftParenthesis: // parenth_form, generator_expression, yield_atom
NextToken();
return FinishTupleOrGenExp();
case TokenKind.LeftBracket: // list_display
NextToken();
return FinishListValue();
case TokenKind.LeftBrace: // dict_display
NextToken();
return FinishDictOrSetValue();
case TokenKind.BackQuote: // string_conversion
NextToken();
return FinishStringConversion();
case TokenKind.Name: // identifier
NextToken();
ret = MakeName(TokenToName((NameToken)t));
if (_verbatim) {
AddPreceedingWhiteSpace(ret);
}
ret.SetLoc(GetStart(), GetEnd());
return ret;
case TokenKind.Ellipsis:
NextToken();
ret = new ConstantExpression(Ellipsis.Value);
if (_verbatim) {
AddPreceedingWhiteSpace(ret, _tokenWhiteSpace);
}
ret.SetLoc(GetStart(), GetEnd());
return ret;
case TokenKind.KeywordTrue:
NextToken();
ret = new ConstantExpression(true);
if (_verbatim) {
AddPreceedingWhiteSpace(ret);
}
ret.SetLoc(GetStart(), GetEnd());
return ret;
case TokenKind.KeywordFalse:
NextToken();
ret = new ConstantExpression(false);
if (_verbatim) {
AddPreceedingWhiteSpace(ret);
}
ret.SetLoc(GetStart(), GetEnd());
return ret;
case TokenKind.Constant: // literal
NextToken();
var start = GetStart();
object cv = t.Value;
string cvs = cv as string;
AsciiString bytes;
if (PeekToken() is ConstantValueToken && (cv is string || cv is AsciiString)) {
// string plus
string[] verbatimImages = null, verbatimWhiteSpace = null;
if (cvs != null) {
cv = FinishStringPlus(cvs, t, out verbatimImages, out verbatimWhiteSpace);
} else if ((bytes = cv as AsciiString) != null) {
cv = FinishBytesPlus(bytes, t, out verbatimImages, out verbatimWhiteSpace);
}
ret = new ConstantExpression(cv);
if (_verbatim) {
AddListWhiteSpace(ret, verbatimWhiteSpace);
AddVerbatimNames(ret, verbatimImages);
}
} else {
ret = new ConstantExpression(cv);
if (_verbatim) {
AddExtraVerbatimText(ret, t.VerbatimImage);
AddPreceedingWhiteSpace(ret, _tokenWhiteSpace);
}
}
ret.SetLoc(start, GetEnd());
return ret;
case TokenKind.EndOfFile:
// don't eat the end of file token
ReportSyntaxError(_lookahead.Token, _lookahead.Span, ErrorCodes.SyntaxError, _allowIncomplete || _tokenizer.EndContinues);
// error node
return Error(_verbatim ? "" : null);
default:
ReportSyntaxError(_lookahead.Token, _lookahead.Span, ErrorCodes.SyntaxError, _allowIncomplete || _tokenizer.EndContinues);
if (_lookahead.Token.Kind != TokenKind.NewLine) {
NextToken();
return Error(_verbatim ? (_tokenWhiteSpace + _token.Token.VerbatimImage) : null);
}
// error node
return Error("");
}
}
private bool IsString(ConstantExpression ce)
{
if (_langVersion.Is7x()) {
return ce.Value is string;
}
return ce.Value is AsciiString;
}
private Expression FinishUnaryNegate()
{
// Special case to ensure that System.Int32.MinValue is an int and not a BigInteger
if (PeekToken().Kind == TokenKind.Constant) {
Token t = PeekToken();
if (t.Value is BigInteger) {
BigInteger bi = (BigInteger)t.Value;
if (bi == 0x80000000) {
string tokenString = _tokenizer.GetTokenString(); ;
Debug.Assert(tokenString.Length > 0);
if (tokenString[tokenString.Length - 1] != 'L' &&
tokenString[tokenString.Length - 1] != 'l') {
string minusWhiteSpace = _tokenWhiteSpace;
NextToken();
// TODO Fix the white space here
var ret = new ConstantExpression(-2147483648);
if (_verbatim) {
AddExtraVerbatimText(ret, minusWhiteSpace + "-" + _tokenWhiteSpace + t.VerbatimImage);
}
return ret;
}
}
}
}
string whitespace = _tokenWhiteSpace;
var res = new UnaryExpression(JOperator.Negate, ParseFactor());
if (_verbatim) {
AddPreceedingWhiteSpace(res, whitespace);
}
return res;
}
public override void PostWalk(ConstantExpression node)
{
PostWalkWorker(node);
}
// ConstantExpression
public override bool Walk(ConstantExpression node)
{
return ShouldWalkWorker(node);
}
