private static bool NeverTestToken(Token t) { switch (t.Kind) { case TokenKind.AddEqual: case TokenKind.SubtractEqual: case TokenKind.MultiplyEqual: case TokenKind.DivideEqual: case TokenKind.ModEqual: case TokenKind.BitwiseAndEqual: case TokenKind.BitwiseOrEqual: case TokenKind.ExclusiveOrEqual: case TokenKind.LeftShiftEqual: case TokenKind.RightShiftEqual: case TokenKind.PowerEqual: case TokenKind.FloorDivideEqual: case TokenKind.Indent: case TokenKind.Dedent: case TokenKind.NewLine: case TokenKind.EndOfFile: case TokenKind.Semicolon: case TokenKind.Assign: case TokenKind.RightBrace: case TokenKind.RightBracket: case TokenKind.RightParenthesis: case TokenKind.Comma: case TokenKind.Colon: case TokenKind.KeywordFor: case TokenKind.KeywordIn: case TokenKind.KeywordIf: return true; default: return false; } }
public TokenWithSpan(Token token, IndexSpan span) { _token = token; _span = span; }
private void ReportSyntaxError(Token t, IndexSpan span, int errorCode, bool allowIncomplete) { var start = span.Start; var end = span.End; if (allowIncomplete && (t.Kind == TokenKind.EndOfFile || (_tokenizer.IsEndOfFile && (t.Kind == TokenKind.Dedent || t.Kind == TokenKind.NLToken)))) { errorCode |= ErrorCodes.IncompleteStatement; } string msg = String.Format(System.Globalization.CultureInfo.InvariantCulture, GetErrorMessage(t, errorCode), t.Image); ReportSyntaxError(start, end, msg, errorCode); }
private bool PeekToken(Token check) { return PeekToken() == check; }
private static string GetErrorMessage(Token t, int errorCode) { string msg; if ((errorCode & ~ErrorCodes.IncompleteMask) == ErrorCodes.IndentationError) { msg = "expected an indented block"; } else if (t.Kind != TokenKind.EndOfFile) { msg = "unexpected token '{0}'"; } else { msg = "unexpected EOF while parsing"; } return msg; }
private JOperator GetAssignOperator(Token t) { switch (t.Kind) { case TokenKind.AddEqual: return JOperator.Add; case TokenKind.SubtractEqual: return JOperator.Subtract; case TokenKind.MultiplyEqual: return JOperator.Multiply; case TokenKind.DivideEqual: return TrueDivision ? JOperator.TrueDivide : JOperator.Divide; case TokenKind.ModEqual: return JOperator.Mod; case TokenKind.BitwiseAndEqual: return JOperator.BitwiseAnd; case TokenKind.BitwiseOrEqual: return JOperator.BitwiseOr; case TokenKind.ExclusiveOrEqual: return JOperator.Xor; case TokenKind.LeftShiftEqual: return JOperator.LeftShift; case TokenKind.RightShiftEqual: return JOperator.RightShift; case TokenKind.PowerEqual: return JOperator.Power; case TokenKind.FloorDivideEqual: return JOperator.FloorDivide; default: return JOperator.None; } }
private string FinishStringPlus(string s, Token initialToken, out string[] verbatimImages, out string[] verbatimWhiteSpace) { List<string> verbatimImagesList = null; List<string> verbatimWhiteSpaceList = null; if (_verbatim) { verbatimWhiteSpaceList = new List<string>(); verbatimImagesList = new List<string>(); verbatimWhiteSpaceList.Add(_tokenWhiteSpace); verbatimImagesList.Add(initialToken.VerbatimImage); } var res = FinishStringPlus(s, verbatimImagesList, verbatimWhiteSpaceList); if (_verbatim) { verbatimWhiteSpace = verbatimWhiteSpaceList.ToArray(); verbatimImages = verbatimImagesList.ToArray(); } else { verbatimWhiteSpace = verbatimImages = null; } return res; }