/// <summary> /// 初始化预定义运算符表达式。 /// </summary> /// <param name="line">指定预定义运算符表达式所在位置的行数。</param> /// <param name="col">指定预定义运算符表达式所在位置的列数。</param> /// <param name="lhs">指定预定义运算符表达式的左侧值。</param> /// <param name="op">指定预定义运算符表达式的运算符。</param> /// <param name="rhs">指定预定义运算符表达式的右侧值。</param> public BinaryExpression(int line, int col, Expression lhs, TokenKind op, Expression rhs) : base(line, col) { _Lhs = lhs; _Rhs = rhs; _op = op; }
public BinaryOperatorTerminal(string @operator, TokenKind tokenKind) : base("-" + @operator, "(?<_operator_" + @operator + ">" + dash_pattern + @operator + ")") { this.Operator = @operator; this.TokenKind = tokenKind; Priority = TerminalPriority.ReservedWords; }
public UnexpectedTokenException(Token current, TokenKind[] tokenKinds) : base(string.Format("On line {0} expected {1} but got {2}.", current.Line, string.Join(",",tokenKinds), current.Kind )) { EncounteredToken = current; ExpectedTokenKinds = tokenKinds; }
public Token(TokenKind kind, string value, int line, int column) { this.Kind = kind; this.Value = value; this.Line = line; this.Column = column; }
public Token(int lineNum, int columnNum, string literal, TokenKind tokenKind) { line = lineNum; column = columnNum; token_literal = literal; kind = tokenKind; }
public void Reset() { this.kind = TokenKind.EOF; this.text = string.Empty; this.value = -1L; this.pos = -1; }
public Token(TokenKind kind, string data, int line, int col) { this.tokenKind = kind; this.line = line; this.col = col; this.data = data; }
public Token Scan() { _currentContent = string.Empty; _currentKind = ScanToken(); return new Token(_currentKind, _currentContent); }
public TokenKindTests() { lower = new Pattern("Lowercase", @"[a-z]+"); upper = new Pattern("Uppercase", @"[A-Z]+"); caseInsensitive = new Pattern("Case Insensitive", @"[a-z]+", RegexOptions.IgnoreCase); abcDEF = new Text("abcDEF"); }
public void Ensure(TokenKind kind, string readerName) { if (current.Kind != kind) { throw new ParserException($"Expected {kind} while reading {readerName}). Was {current}."); } }
public BinaryExpression(int line, int col, Expression lhs, TokenKind op, Expression rhs) : base(line, col) { this.lhs = lhs; this.rhs = rhs; this.op = op; }
public Token(TokenKind kind, string text, int line, int column) { this.kind = kind; this.line = line; this.column = column; this.text = text; }
public Token(int line, int col, string data,TokenKind tokenKind) { this.Line = line; this.Col = col; this.Text = data; this.Kind = tokenKind; }
public Token(TokenKind kind, string data, int line, int col) { _tokenKind = kind; _line = line; _col = col; _data = data; }
public Token Scan() { this._currentSpelling.Clear(); this._currentKind = this.ScanToken(); return new Token(this._currentKind, this._currentSpelling.ToString()); }
public Token(TokenKind kind, string data, int line, int col) { this.TokenKind = kind; this.Data = data; this.Line = line; this.Col = col; }
private Token Match(TokenKind kind) { if (_lookahead.Kind == kind) return ReadToken(); throw new ParserError(_lookahead.Position, "Expected '" + kind.ToString() + "', found '" + _lookahead.ToString() + "'"); }
public Token(TokenKind kind, string value, int line, int column) { this.kind = kind; this.value = value; this.line = line; this.column = column; }
public AssignmentStatementAst(IScriptExtent extent, ExpressionAst left, TokenKind @operator, StatementAst right, IScriptExtent errorPosition) : base(extent) { if (((left == null) || (right == null)) || (errorPosition == null)) { throw PSTraceSource.NewArgumentNullException((left == null) ? "left" : ((right == null) ? "right" : "errorPosition")); } if ((@operator.GetTraits() & TokenFlags.AssignmentOperator) == TokenFlags.None) { throw PSTraceSource.NewArgumentException("operator"); } PipelineAst ast = right as PipelineAst; if ((ast != null) && (ast.PipelineElements.Count == 1)) { CommandExpressionAst ast2 = ast.PipelineElements[0] as CommandExpressionAst; if (ast2 != null) { right = ast2; right.ClearParent(); } } this.Operator = @operator; this.Left = left; base.SetParent(left); this.Right = right; base.SetParent(right); this.ErrorPosition = errorPosition; }
/// <summary> /// 初始化 Token 对象。 /// </summary> /// <param name="kind">指定 Token 类型。</param> /// <param name="data">指定 Token 内容。</param> /// <param name="line">指定 Token 所在的行数</param> /// <param name="col">指定 Token 所在的列数</param> public Token(TokenKind kind, string data, int line, int col) { _TokenKind = kind; _Line = line; _Col = col; _Data = data; }
internal void SetIsCommandArgument() { if (this._kind != TokenKind.Identifier) { this._kind = TokenKind.Generic; } }
public BinaryExpression(TokenKind type, Expression left, Expression right) { ContractUtils.RequiresNotNull(left, "left"); ContractUtils.RequiresNotNull(right, "right"); switch (type) { case TokenKind.Add: _op = MSAst.ExpressionType.Add; break; case TokenKind.Subtract: _op = MSAst.ExpressionType.Subtract; break; case TokenKind.Mod: _op = MSAst.ExpressionType.Modulo; break; case TokenKind.Multiply: _op = MSAst.ExpressionType.Multiply; break; case TokenKind.Divide: _op = MSAst.ExpressionType.Divide; break; case TokenKind.LessThan: _op = MSAst.ExpressionType.LessThan; break; case TokenKind.LessThanOrEqual: _op = MSAst.ExpressionType.LessThanOrEqual; break; case TokenKind.GreaterThan: _op = MSAst.ExpressionType.GreaterThan; break; case TokenKind.GreaterThanOrEqual: _op = MSAst.ExpressionType.GreaterThanOrEqual; break; case TokenKind.Equals: _op = MSAst.ExpressionType.Equal; break; case TokenKind.NotEquals: _op = MSAst.ExpressionType.NotEqual; break; case TokenKind.LogicalAnd: _op = MSAst.ExpressionType.And; break; case TokenKind.LogicalOr: _op = MSAst.ExpressionType.Or; break; default: throw Assert.Unreachable; } _left = left; _right = right; StartIndex = left.StartIndex; EndIndex = right.EndIndex; }
public Token(TokenKind kind, string value, int line, int column) { _kind = kind; _value = value; _line = line; _column = column; }
public TokenNode(TokenKind kind, int level, double score, string firstExample) { Kind = kind; Level = level; Score = score; NextTokens = new List<TokenNode>(); Examples = new HashSet<string> { firstExample }; }
internal StringExpandableToken(InternalScriptExtent scriptExtent, TokenKind tokenKind, string value, string formatString, List<Token> nestedTokens, TokenFlags flags) : base(scriptExtent, tokenKind, flags, value) { if ((nestedTokens != null) && nestedTokens.Any<Token>()) { this._nestedTokens = new ReadOnlyCollection<Token>(nestedTokens.ToArray()); } this._formatString = formatString; }
public BinaryExpressionAst(IScriptExtent extent, ExpressionAst left, TokenKind @operator, ExpressionAst right, IScriptExtent errorPosition) : base(extent) { this.Left = left; this.Operator = @operator; this.Right = @right; this.ErrorPosition = errorPosition; }
internal void DoVisit(ISemanticModel semanticModel, CommonSyntaxNode token, TokenParameter parameter, TokenKind searchTokenKind, Action<int> symbolFoundDelegate) { this.semanticModel = semanticModel as SemanticModel; this.symbolFoundDelegate = symbolFoundDelegate; this.parameter = parameter; this.searchTokenKind = searchTokenKind; Visit(token as SyntaxNode); }
public AssignmentStatementAst(IScriptExtent extent, ExpressionAst left, TokenKind @operator, StatementAst right, IScriptExtent errorPosition) : base(extent) { this.Left = left; this.Operator = @operator; this.Right = right; this.ErrorPosition = errorPosition; }
private void TestReservedWord(string expression, object value, TokenKind expectedTokenKind) { var lexer = new Tokenizer(expression); var token1 = lexer.NextToken(); Assert.That(token1.Kind, Is.EqualTo(expectedTokenKind)); Assert.That(token1.Value, Is.EqualTo(value)); var token2 = lexer.NextToken(); Assert.That(token2.Kind, Is.EqualTo(TokenKind.Eof)); }
private void TestStringLiteral(string value, string expected, TokenKind expectedTokenKind) { var lexer = new Tokenizer(value); var token1 = lexer.NextToken(); Assert.That(token1.Kind, Is.EqualTo(expectedTokenKind)); Assert.That(token1.Value, Is.EqualTo(expected)); var token2 = lexer.NextToken(); Assert.That(token2.Kind, Is.EqualTo(TokenKind.Eof)); }
/// <summary> /// Helper method to deparent/parent a <see cref="SyntaxToken"/> to this instance with an expected kind of token. /// </summary> /// <typeparam name="TSyntaxNode">Type of the node</typeparam> /// <param name="set">The previous child node parented to this instance</param> /// <param name="node">The new child node to parent to this instance</param> /// <param name="expectedKind">The expected kind of token</param> protected void ParentToThis <TSyntaxNode>(ref TSyntaxNode set, TSyntaxNode node, TokenKind expectedKind) where TSyntaxNode : SyntaxToken { ParentToThis(ref set, node, node.TokenKind == expectedKind, expectedKind); }
internal JsonToken(TokenKind kind, string value) { Kind = kind; Value = value; }
public Token(TokenKind kind) : this(kind, string.Empty) { }
public Token(TokenKind kind, string text) { Kind = kind; Text = text; }
public static String?GetText(TokenKind kind) => to.TryGetValue(kind, out var result) ? result : null;
public TokenInfo(TokenKind kind, int line, int column, int width) : this(kind, line, column, width, string.Empty) { }
/// <summary> /// TOKEN /// </summary> /// <param name="kind"></param> /// <param name="text"></param> public Token(TokenKind kind, String text) { this._tokenkind = kind; this._text = text; }
public OPINFO(TokenKind t, PredefinedName pn, ExpressionKind e, int c) { iToken = t; methodName = pn; expressionKind = e; }
public Token(TokenKind kind) { Kind = kind; }
public Token(TokenKind kind, string value, PositionInfo location) { Kind = kind; Content = value; Location = location; }
public Token(TokenKind kind, string value) { Kind = kind; Content = value; }
public Token(Span span, TokenKind kind) { Span = span; Kind = kind; }
public Token(string str, TokenKind kind) { this.str = str; this.kind = kind; }
internal static bool IsOpenGrouping(this TokenKind tokKind) { return(tokKind == TokenKind.LeftBracket || tokKind == TokenKind.LeftBrace || tokKind == TokenKind.LeftParenthesis); }
public Token(TokenKind kind, PositionInfo location) { Kind = kind; Location = location; }
void ReadString(int quote) { Start = Location; Read(); // consume the opening quote while (true) { var c = Read(); if (c == -1) { HandleError(JsonErrorCode.UnterminatedString, Start); return; } if (c == quote) { End = Location; TokenKind = TokenKind.String; return; } if (c == '\r') { if (Peek() == '\n') { Read(); } NewLine(); } if (c == '\n') { NewLine(); } if (c == '\\') { var n = Read(); switch (n) { case '\"': case '\\': case '/': case 'b': case 'f': case 'n': case 'r': case 't': break; case 'u': for (int i = 0; i < 4; i++) { var d = Peek(); if (IsHexDigit(d)) { Read(); } else { HandleError(JsonErrorCode.UnexpectedCharacter); break; } } break; } } } }
public Token(TokenKind kind, Position position, string value) { Kind = kind; Position = position; Literal = value; }
private static Token CreateToken(int line, int column, string lexeme, TokenKind tokenKind) { return(new Token(tokenKind, new Position(line, column), lexeme)); }
internal static bool IsCloseGrouping(this TokenKind tokKind) { return(tokKind == TokenKind.RightBracket || tokKind == TokenKind.RightBrace || tokKind == TokenKind.RightParenthesis); }
/// <summary> /// Creates a new instance of <see cref="ReturnTokenLexemeParser"/>. /// </summary> /// <param name="kind">The kind of token. Not null.</param> public ReturnTokenLexemeParser(TokenKind kind) { _kind = kind ?? throw new ArgumentNullException(nameof(kind)); }
private static int GetParameterValue(List <Token> parameterValueTokens, int tokenIndex, out object parameterValue) { int count = parameterValueTokens.Count; parameterValue = null; if (tokenIndex >= count) { return(tokenIndex); } Token token = parameterValueTokens[tokenIndex]; TokenKind kind = token.Kind; if (kind <= TokenKind.Number) { if (kind != TokenKind.Variable) { if (kind == TokenKind.Number) { parameterValue = ((NumberToken)token).Value; goto IL_23B; } } else { bool flag; if (bool.TryParse(((VariableToken)token).Name, out flag)) { parameterValue = flag; goto IL_23B; } parameterValue = token.Text; goto IL_23B; } } else { switch (kind) { case TokenKind.StringLiteral: parameterValue = ((StringToken)token).Value; goto IL_23B; case TokenKind.StringExpandable: parameterValue = ((StringExpandableToken)token).Value; goto IL_23B; default: switch (kind) { case TokenKind.LBracket: case TokenKind.RBracket: case TokenKind.Comma: parameterValue = null; goto IL_23B; case TokenKind.AtParen: { List <object> list = new List <object>(); tokenIndex++; while (tokenIndex < count && parameterValueTokens[tokenIndex].Kind != TokenKind.RParen) { if (parameterValueTokens[tokenIndex].Kind == TokenKind.Comma) { tokenIndex++; } else { object obj = null; tokenIndex = CmdletValidator.GetParameterValue(parameterValueTokens, tokenIndex, out obj); if (obj != null) { list.Add(obj); } } } parameterValue = list.ToArray(); goto IL_23B; } case TokenKind.AtCurly: { Dictionary <object, object> dictionary = new Dictionary <object, object>(); tokenIndex++; while (tokenIndex < count && parameterValueTokens[tokenIndex].Kind != TokenKind.RCurly) { if (parameterValueTokens[tokenIndex].Kind == TokenKind.Semi) { tokenIndex++; } else { object obj2 = null; tokenIndex = CmdletValidator.GetParameterValue(parameterValueTokens, tokenIndex, out obj2); if (tokenIndex >= count || parameterValueTokens[tokenIndex].Kind != TokenKind.Equals) { throw new InvalidOperationException(string.Format("Cmdlet parsing failed at index {0}. Expected Equals. Actual {1}", tokenIndex, parameterValueTokens[tokenIndex].Text)); } if (++tokenIndex >= count) { throw new InvalidOperationException(string.Format("Cmdlet parsing failed. No value for key {0}", obj2)); } object value = null; tokenIndex = CmdletValidator.GetParameterValue(parameterValueTokens, tokenIndex, out value); dictionary.Add(obj2, value); } } parameterValue = dictionary; goto IL_23B; } case TokenKind.Semi: case TokenKind.Pipe: throw new InvalidOperationException(string.Format("Cmdlet parsing failed at index {0}. Encountered unexpected {1}", tokenIndex, token.Kind)); } break; } } parameterValue = token.Text; IL_23B: return(++tokenIndex); }
private static TokenSpec ToTokenSpec(TokenKind kind) { return(TokenSpec.A(kind)); }
public Token(TokenKind kind, string value) { this.Kind = kind; this.Value = value; }
public static bool IsOpenerOrCloser(TokenKind tt) { return(tt >= TokenKind.LParen && tt < (TokenKind)((int)TokenKind.ROther + (1 << TokenKindShift))); }
internal static Boolean?IsRightAssociative(TokenKind kind) { return(false); }
public static bool IsCloser(TokenKind tt) { return(IsOpenerOrCloser(tt) && ((int)tt & 0x0100) != 0); }
public Token(TokenKind kind, char ch) : this(kind, ch.ToString()) { }
/// <summary> /// Helper method to deparent/parent a <see cref="SyntaxToken"/> to this instance with an expected kind of token. /// </summary> /// <typeparam name="TSyntaxNode">Type of the node</typeparam> /// <param name="set">The previous child node parented to this instance</param> /// <param name="node">The new child node to parent to this instance</param> /// <param name="expectedKind1">The expected kind of token (option1)</param> /// <param name="expectedKind2">The expected kind of token (option2)</param> protected void ParentToThis <TSyntaxNode>(ref TSyntaxNode set, TSyntaxNode node, TokenKind expectedKind1, TokenKind expectedKind2) where TSyntaxNode : SyntaxToken { ParentToThis(ref set, node, node.TokenKind == expectedKind1 || node.TokenKind == expectedKind2, new ExpectedTuple2 <TokenKind, TokenKind>(expectedKind1, expectedKind2)); }
public bool NextToken() { Start = default; End = default; TokenKind = TokenKind.None; while (true) { bufferTokenStart = bufferPos; var c = Peek(); switch (c) { case -1: Start = End = Location; TokenKind = TokenKind.None; return(false); case '\"': case '\'': ReadString(c); return(true); case '{': SingleCharacterToken(TokenKind.StartObject); return(true); case '}': SingleCharacterToken(TokenKind.EndObject); return(true); case '[': SingleCharacterToken(TokenKind.StartArray); return(true); case ']': SingleCharacterToken(TokenKind.EndArray); return(true); case ':': SingleCharacterToken(TokenKind.Colon); return(true); case ',': SingleCharacterToken(TokenKind.Comma); return(true); case '/': var location = Location; Read(); var n = Peek(); switch (n) { case '/': Read(); ReadLineComment(); return(true); case '*': Read(); ReadBlockComment(); return(true); } HandleError(JsonErrorCode.UnexpectedCharacter, location); continue; case '\r': Read(); if (Peek() == '\n') { Read(); } NewLine(); break; case '\n': Read(); NewLine(); break; default: if (IsDigit((char)c) || c == '-') { ReadNumber(); return(true); } if (char.IsWhiteSpace((char)c)) { Read(); continue; } if (IsNameStart(c)) { ReadName(); return(true); } HandleError(JsonErrorCode.UnexpectedCharacter, Location); Read(); continue; } } }
public bool Check(TokenKind required) { return(this.kind == required); }