private bool PeekToken(Token check) {
return PeekToken() == check;
}
private PythonOperator GetAssignOperator(Token t) {
switch (t.Kind) {
case TokenKind.AddEqual: return PythonOperator.Add;
case TokenKind.SubtractEqual: return PythonOperator.Subtract;
case TokenKind.MultiplyEqual: return PythonOperator.Multiply;
case TokenKind.MatMultiplyEqual: return PythonOperator.MatMultiply;
case TokenKind.DivideEqual: return TrueDivision ? PythonOperator.TrueDivide : PythonOperator.Divide;
case TokenKind.ModEqual: return PythonOperator.Mod;
case TokenKind.BitwiseAndEqual: return PythonOperator.BitwiseAnd;
case TokenKind.BitwiseOrEqual: return PythonOperator.BitwiseOr;
case TokenKind.ExclusiveOrEqual: return PythonOperator.Xor;
case TokenKind.LeftShiftEqual: return PythonOperator.LeftShift;
case TokenKind.RightShiftEqual: return PythonOperator.RightShift;
case TokenKind.PowerEqual: return PythonOperator.Power;
case TokenKind.FloorDivideEqual: return PythonOperator.FloorDivide;
default: return PythonOperator.None;
}
}
private Name TokenToName(Token t) {
if (!AllowAsyncAwaitSyntax) {
if (t.Kind == TokenKind.KeywordAwait) {
return Name.Await;
} else if (t.Kind == TokenKind.KeywordAsync) {
return Name.Async;
}
}
var n = t as NameToken;
if (n != null) {
return new Name(FixName(n.Name), n.Name);
}
return Name.Empty;
}
private static bool NeverTestToken(Token t) {
switch (t.Kind) {
case TokenKind.AddEqual:
case TokenKind.SubtractEqual:
case TokenKind.MultiplyEqual:
case TokenKind.DivideEqual:
case TokenKind.ModEqual:
case TokenKind.BitwiseAndEqual:
case TokenKind.BitwiseOrEqual:
case TokenKind.ExclusiveOrEqual:
case TokenKind.LeftShiftEqual:
case TokenKind.RightShiftEqual:
case TokenKind.PowerEqual:
case TokenKind.FloorDivideEqual:
case TokenKind.Indent:
case TokenKind.Dedent:
case TokenKind.NewLine:
case TokenKind.EndOfFile:
case TokenKind.Semicolon:
case TokenKind.Assign:
case TokenKind.RightBrace:
case TokenKind.RightBracket:
case TokenKind.RightParenthesis:
case TokenKind.Comma:
case TokenKind.Colon:
case TokenKind.KeywordFor:
case TokenKind.KeywordIn:
case TokenKind.KeywordIf:
return true;
default: return false;
}
}
private object FinishBytesPlus(AsciiString s, Token initialToken, out string[] verbatimImages, out string[] verbatimWhiteSpace) {
List<string> verbatimImagesList = null;
List<string> verbatimWhiteSpaceList = null;
if (_verbatim) {
verbatimWhiteSpaceList = new List<string>();
verbatimImagesList = new List<string>();
verbatimWhiteSpaceList.Add(_tokenWhiteSpace);
verbatimImagesList.Add(initialToken.VerbatimImage);
}
var res = FinishBytesPlus(s, verbatimImagesList, verbatimWhiteSpaceList);
if (_verbatim) {
verbatimWhiteSpace = verbatimWhiteSpaceList.ToArray();
verbatimImages = verbatimImagesList.ToArray();
} else {
verbatimWhiteSpace = verbatimImages = null;
}
return res;
}
private static string GetErrorMessage(Token t, int errorCode) {
string msg;
if ((errorCode & ~ErrorCodes.IncompleteMask) == ErrorCodes.IndentationError) {
msg = "expected an indented block";
} else if (t.Kind != TokenKind.EndOfFile) {
msg = "unexpected token '{0}'";
} else {
msg = "unexpected EOF while parsing";
}
return msg;
}
private void ReportSyntaxError(Token t, IndexSpan span, int errorCode, bool allowIncomplete) {
var start = span.Start;
var end = span.End;
if (allowIncomplete && (t.Kind == TokenKind.EndOfFile || (_tokenizer.IsEndOfFile && (t.Kind == TokenKind.Dedent || t.Kind == TokenKind.NLToken)))) {
errorCode |= ErrorCodes.IncompleteStatement;
}
string msg = String.Format(System.Globalization.CultureInfo.InvariantCulture, GetErrorMessage(t, errorCode), t.Image);
ReportSyntaxError(start, end, msg, errorCode);
}
public TokenWithSpan(Token token, IndexSpan span) {
_token = token;
_span = span;
}
private static bool IsEndOfLineToken(Token t) {
switch (t.Kind) {
case TokenKind.Comment:
case TokenKind.NewLine:
case TokenKind.NLToken:
case TokenKind.EndOfFile:
return true;
}
return false;
}
