private bool PeekToken(Token check) { return PeekToken() == check; }
private PythonOperator GetAssignOperator(Token t) { switch (t.Kind) { case TokenKind.AddEqual: return PythonOperator.Add; case TokenKind.SubtractEqual: return PythonOperator.Subtract; case TokenKind.MultiplyEqual: return PythonOperator.Multiply; case TokenKind.MatMultiplyEqual: return PythonOperator.MatMultiply; case TokenKind.DivideEqual: return TrueDivision ? PythonOperator.TrueDivide : PythonOperator.Divide; case TokenKind.ModEqual: return PythonOperator.Mod; case TokenKind.BitwiseAndEqual: return PythonOperator.BitwiseAnd; case TokenKind.BitwiseOrEqual: return PythonOperator.BitwiseOr; case TokenKind.ExclusiveOrEqual: return PythonOperator.Xor; case TokenKind.LeftShiftEqual: return PythonOperator.LeftShift; case TokenKind.RightShiftEqual: return PythonOperator.RightShift; case TokenKind.PowerEqual: return PythonOperator.Power; case TokenKind.FloorDivideEqual: return PythonOperator.FloorDivide; default: return PythonOperator.None; } }
private Name TokenToName(Token t) { if (!AllowAsyncAwaitSyntax) { if (t.Kind == TokenKind.KeywordAwait) { return Name.Await; } else if (t.Kind == TokenKind.KeywordAsync) { return Name.Async; } } var n = t as NameToken; if (n != null) { return new Name(FixName(n.Name), n.Name); } return Name.Empty; }
private static bool NeverTestToken(Token t) { switch (t.Kind) { case TokenKind.AddEqual: case TokenKind.SubtractEqual: case TokenKind.MultiplyEqual: case TokenKind.DivideEqual: case TokenKind.ModEqual: case TokenKind.BitwiseAndEqual: case TokenKind.BitwiseOrEqual: case TokenKind.ExclusiveOrEqual: case TokenKind.LeftShiftEqual: case TokenKind.RightShiftEqual: case TokenKind.PowerEqual: case TokenKind.FloorDivideEqual: case TokenKind.Indent: case TokenKind.Dedent: case TokenKind.NewLine: case TokenKind.EndOfFile: case TokenKind.Semicolon: case TokenKind.Assign: case TokenKind.RightBrace: case TokenKind.RightBracket: case TokenKind.RightParenthesis: case TokenKind.Comma: case TokenKind.Colon: case TokenKind.KeywordFor: case TokenKind.KeywordIn: case TokenKind.KeywordIf: return true; default: return false; } }
private object FinishBytesPlus(AsciiString s, Token initialToken, out string[] verbatimImages, out string[] verbatimWhiteSpace) { List<string> verbatimImagesList = null; List<string> verbatimWhiteSpaceList = null; if (_verbatim) { verbatimWhiteSpaceList = new List<string>(); verbatimImagesList = new List<string>(); verbatimWhiteSpaceList.Add(_tokenWhiteSpace); verbatimImagesList.Add(initialToken.VerbatimImage); } var res = FinishBytesPlus(s, verbatimImagesList, verbatimWhiteSpaceList); if (_verbatim) { verbatimWhiteSpace = verbatimWhiteSpaceList.ToArray(); verbatimImages = verbatimImagesList.ToArray(); } else { verbatimWhiteSpace = verbatimImages = null; } return res; }
private static string GetErrorMessage(Token t, int errorCode) { string msg; if ((errorCode & ~ErrorCodes.IncompleteMask) == ErrorCodes.IndentationError) { msg = "expected an indented block"; } else if (t.Kind != TokenKind.EndOfFile) { msg = "unexpected token '{0}'"; } else { msg = "unexpected EOF while parsing"; } return msg; }
private void ReportSyntaxError(Token t, IndexSpan span, int errorCode, bool allowIncomplete) { var start = span.Start; var end = span.End; if (allowIncomplete && (t.Kind == TokenKind.EndOfFile || (_tokenizer.IsEndOfFile && (t.Kind == TokenKind.Dedent || t.Kind == TokenKind.NLToken)))) { errorCode |= ErrorCodes.IncompleteStatement; } string msg = String.Format(System.Globalization.CultureInfo.InvariantCulture, GetErrorMessage(t, errorCode), t.Image); ReportSyntaxError(start, end, msg, errorCode); }
public TokenWithSpan(Token token, IndexSpan span) { _token = token; _span = span; }
private static bool IsEndOfLineToken(Token t) { switch (t.Kind) { case TokenKind.Comment: case TokenKind.NewLine: case TokenKind.NLToken: case TokenKind.EndOfFile: return true; } return false; }