private static bool NeverTestToken(Token t)
{
switch (t.Kind) {
case TokenKind.AddEqual:
case TokenKind.SubtractEqual:
case TokenKind.MultiplyEqual:
case TokenKind.DivideEqual:
case TokenKind.ModEqual:
case TokenKind.BitwiseAndEqual:
case TokenKind.BitwiseOrEqual:
case TokenKind.ExclusiveOrEqual:
case TokenKind.LeftShiftEqual:
case TokenKind.RightShiftEqual:
case TokenKind.PowerEqual:
case TokenKind.FloorDivideEqual:
case TokenKind.Indent:
case TokenKind.Dedent:
case TokenKind.NewLine:
case TokenKind.EndOfFile:
case TokenKind.Semicolon:
case TokenKind.Assign:
case TokenKind.RightBrace:
case TokenKind.RightBracket:
case TokenKind.RightParenthesis:
case TokenKind.Comma:
case TokenKind.Colon:
case TokenKind.KeywordFor:
case TokenKind.KeywordIn:
case TokenKind.KeywordIf:
return true;
default: return false;
}
}
public TokenWithSpan(Token token, IndexSpan span)
{
_token = token;
_span = span;
}
private void ReportSyntaxError(Token t, IndexSpan span, int errorCode, bool allowIncomplete)
{
var start = span.Start;
var end = span.End;
if (allowIncomplete && (t.Kind == TokenKind.EndOfFile || (_tokenizer.IsEndOfFile && (t.Kind == TokenKind.Dedent || t.Kind == TokenKind.NLToken)))) {
errorCode |= ErrorCodes.IncompleteStatement;
}
string msg = String.Format(System.Globalization.CultureInfo.InvariantCulture, GetErrorMessage(t, errorCode), t.Image);
ReportSyntaxError(start, end, msg, errorCode);
}
private bool PeekToken(Token check)
{
return PeekToken() == check;
}
private static string GetErrorMessage(Token t, int errorCode)
{
string msg;
if ((errorCode & ~ErrorCodes.IncompleteMask) == ErrorCodes.IndentationError) {
msg = "expected an indented block";
} else if (t.Kind != TokenKind.EndOfFile) {
msg = "unexpected token '{0}'";
} else {
msg = "unexpected EOF while parsing";
}
return msg;
}
private JOperator GetAssignOperator(Token t)
{
switch (t.Kind) {
case TokenKind.AddEqual: return JOperator.Add;
case TokenKind.SubtractEqual: return JOperator.Subtract;
case TokenKind.MultiplyEqual: return JOperator.Multiply;
case TokenKind.DivideEqual: return TrueDivision ? JOperator.TrueDivide : JOperator.Divide;
case TokenKind.ModEqual: return JOperator.Mod;
case TokenKind.BitwiseAndEqual: return JOperator.BitwiseAnd;
case TokenKind.BitwiseOrEqual: return JOperator.BitwiseOr;
case TokenKind.ExclusiveOrEqual: return JOperator.Xor;
case TokenKind.LeftShiftEqual: return JOperator.LeftShift;
case TokenKind.RightShiftEqual: return JOperator.RightShift;
case TokenKind.PowerEqual: return JOperator.Power;
case TokenKind.FloorDivideEqual: return JOperator.FloorDivide;
default: return JOperator.None;
}
}
private string FinishStringPlus(string s, Token initialToken, out string[] verbatimImages, out string[] verbatimWhiteSpace)
{
List<string> verbatimImagesList = null;
List<string> verbatimWhiteSpaceList = null;
if (_verbatim) {
verbatimWhiteSpaceList = new List<string>();
verbatimImagesList = new List<string>();
verbatimWhiteSpaceList.Add(_tokenWhiteSpace);
verbatimImagesList.Add(initialToken.VerbatimImage);
}
var res = FinishStringPlus(s, verbatimImagesList, verbatimWhiteSpaceList);
if (_verbatim) {
verbatimWhiteSpace = verbatimWhiteSpaceList.ToArray();
verbatimImages = verbatimImagesList.ToArray();
} else {
verbatimWhiteSpace = verbatimImages = null;
}
return res;
}
