/// <summary>
/// 初始化预定义运算符表达式。
/// </summary>
/// <param name="line">指定预定义运算符表达式所在位置的行数。</param>
/// <param name="col">指定预定义运算符表达式所在位置的列数。</param>
/// <param name="lhs">指定预定义运算符表达式的左侧值。</param>
/// <param name="op">指定预定义运算符表达式的运算符。</param>
/// <param name="rhs">指定预定义运算符表达式的右侧值。</param>
public BinaryExpression(int line, int col, Expression lhs, TokenKind op, Expression rhs)
: base(line, col)
{
_Lhs = lhs;
_Rhs = rhs;
_op = op;
}
public BinaryOperatorTerminal(string @operator, TokenKind tokenKind)
: base("-" + @operator, "(?<_operator_" + @operator + ">" + dash_pattern + @operator + ")")
{
this.Operator = @operator;
this.TokenKind = tokenKind;
Priority = TerminalPriority.ReservedWords;
}
public UnexpectedTokenException(Token current, TokenKind[] tokenKinds)
: base(string.Format("On line {0} expected {1} but got {2}.",
current.Line, string.Join(",",tokenKinds), current.Kind ))
{
EncounteredToken = current;
ExpectedTokenKinds = tokenKinds;
}
public Token(TokenKind kind, string value, int line, int column)
{
this.Kind = kind;
this.Value = value;
this.Line = line;
this.Column = column;
}
public Token(int lineNum, int columnNum, string literal, TokenKind tokenKind)
{
line = lineNum;
column = columnNum;
token_literal = literal;
kind = tokenKind;
}
public void Reset()
{
this.kind = TokenKind.EOF;
this.text = string.Empty;
this.value = -1L;
this.pos = -1;
}
public Token(TokenKind kind, string data, int line, int col)
{
this.tokenKind = kind;
this.line = line;
this.col = col;
this.data = data;
}
public Token Scan()
{
_currentContent = string.Empty;
_currentKind = ScanToken();
return new Token(_currentKind, _currentContent);
}
public TokenKindTests()
{
lower = new Pattern("Lowercase", @"[a-z]+");
upper = new Pattern("Uppercase", @"[A-Z]+");
caseInsensitive = new Pattern("Case Insensitive", @"[a-z]+", RegexOptions.IgnoreCase);
abcDEF = new Text("abcDEF");
}
public void Ensure(TokenKind kind, string readerName)
{
if (current.Kind != kind)
{
throw new ParserException($"Expected {kind} while reading {readerName}). Was {current}.");
}
}
public BinaryExpression(int line, int col, Expression lhs, TokenKind op, Expression rhs)
: base(line, col)
{
this.lhs = lhs;
this.rhs = rhs;
this.op = op;
}
public Token(TokenKind kind, string text, int line, int column)
{
this.kind = kind;
this.line = line;
this.column = column;
this.text = text;
}
public Token(int line, int col, string data,TokenKind tokenKind)
{
this.Line = line;
this.Col = col;
this.Text = data;
this.Kind = tokenKind;
}
public Token(TokenKind kind, string data, int line, int col)
{
_tokenKind = kind;
_line = line;
_col = col;
_data = data;
}
public Token Scan()
{
this._currentSpelling.Clear();
this._currentKind = this.ScanToken();
return new Token(this._currentKind, this._currentSpelling.ToString());
}
public Token(TokenKind kind, string data, int line, int col)
{
this.TokenKind = kind;
this.Data = data;
this.Line = line;
this.Col = col;
}
private Token Match(TokenKind kind)
{
if (_lookahead.Kind == kind)
return ReadToken();
throw new ParserError(_lookahead.Position, "Expected '" + kind.ToString() + "', found '" + _lookahead.ToString() + "'");
}
public Token(TokenKind kind, string value, int line, int column)
{
this.kind = kind;
this.value = value;
this.line = line;
this.column = column;
}
public AssignmentStatementAst(IScriptExtent extent, ExpressionAst left, TokenKind @operator, StatementAst right, IScriptExtent errorPosition) : base(extent)
{
if (((left == null) || (right == null)) || (errorPosition == null))
{
throw PSTraceSource.NewArgumentNullException((left == null) ? "left" : ((right == null) ? "right" : "errorPosition"));
}
if ((@operator.GetTraits() & TokenFlags.AssignmentOperator) == TokenFlags.None)
{
throw PSTraceSource.NewArgumentException("operator");
}
PipelineAst ast = right as PipelineAst;
if ((ast != null) && (ast.PipelineElements.Count == 1))
{
CommandExpressionAst ast2 = ast.PipelineElements[0] as CommandExpressionAst;
if (ast2 != null)
{
right = ast2;
right.ClearParent();
}
}
this.Operator = @operator;
this.Left = left;
base.SetParent(left);
this.Right = right;
base.SetParent(right);
this.ErrorPosition = errorPosition;
}
/// <summary>
/// 初始化 Token 对象。
/// </summary>
/// <param name="kind">指定 Token 类型。</param>
/// <param name="data">指定 Token 内容。</param>
/// <param name="line">指定 Token 所在的行数</param>
/// <param name="col">指定 Token 所在的列数</param>
public Token(TokenKind kind, string data, int line, int col)
{
_TokenKind = kind;
_Line = line;
_Col = col;
_Data = data;
}
internal void SetIsCommandArgument()
{
if (this._kind != TokenKind.Identifier)
{
this._kind = TokenKind.Generic;
}
}
public BinaryExpression(TokenKind type, Expression left, Expression right)
{
ContractUtils.RequiresNotNull(left, "left");
ContractUtils.RequiresNotNull(right, "right");
switch (type)
{
case TokenKind.Add: _op = MSAst.ExpressionType.Add; break;
case TokenKind.Subtract: _op = MSAst.ExpressionType.Subtract; break;
case TokenKind.Mod: _op = MSAst.ExpressionType.Modulo; break;
case TokenKind.Multiply: _op = MSAst.ExpressionType.Multiply; break;
case TokenKind.Divide: _op = MSAst.ExpressionType.Divide; break;
case TokenKind.LessThan: _op = MSAst.ExpressionType.LessThan; break;
case TokenKind.LessThanOrEqual: _op = MSAst.ExpressionType.LessThanOrEqual; break;
case TokenKind.GreaterThan: _op = MSAst.ExpressionType.GreaterThan; break;
case TokenKind.GreaterThanOrEqual: _op = MSAst.ExpressionType.GreaterThanOrEqual; break;
case TokenKind.Equals: _op = MSAst.ExpressionType.Equal; break;
case TokenKind.NotEquals: _op = MSAst.ExpressionType.NotEqual; break;
case TokenKind.LogicalAnd: _op = MSAst.ExpressionType.And; break;
case TokenKind.LogicalOr: _op = MSAst.ExpressionType.Or; break;
default: throw Assert.Unreachable;
}
_left = left;
_right = right;
StartIndex = left.StartIndex;
EndIndex = right.EndIndex;
}
public Token(TokenKind kind, string value, int line, int column)
{
_kind = kind;
_value = value;
_line = line;
_column = column;
}
public TokenNode(TokenKind kind, int level, double score, string firstExample)
{
Kind = kind;
Level = level;
Score = score;
NextTokens = new List<TokenNode>();
Examples = new HashSet<string> { firstExample };
}
internal StringExpandableToken(InternalScriptExtent scriptExtent, TokenKind tokenKind, string value, string formatString, List<Token> nestedTokens, TokenFlags flags) : base(scriptExtent, tokenKind, flags, value)
{
if ((nestedTokens != null) && nestedTokens.Any<Token>())
{
this._nestedTokens = new ReadOnlyCollection<Token>(nestedTokens.ToArray());
}
this._formatString = formatString;
}
public BinaryExpressionAst(IScriptExtent extent, ExpressionAst left, TokenKind @operator, ExpressionAst right, IScriptExtent errorPosition)
: base(extent)
{
this.Left = left;
this.Operator = @operator;
this.Right = @right;
this.ErrorPosition = errorPosition;
}
internal void DoVisit(ISemanticModel semanticModel, CommonSyntaxNode token, TokenParameter parameter, TokenKind searchTokenKind, Action<int> symbolFoundDelegate)
{
this.semanticModel = semanticModel as SemanticModel;
this.symbolFoundDelegate = symbolFoundDelegate;
this.parameter = parameter;
this.searchTokenKind = searchTokenKind;
Visit(token as SyntaxNode);
}
public AssignmentStatementAst(IScriptExtent extent, ExpressionAst left, TokenKind @operator, StatementAst right, IScriptExtent errorPosition)
: base(extent)
{
this.Left = left;
this.Operator = @operator;
this.Right = right;
this.ErrorPosition = errorPosition;
}
private void TestReservedWord(string expression, object value, TokenKind expectedTokenKind) {
var lexer = new Tokenizer(expression);
var token1 = lexer.NextToken();
Assert.That(token1.Kind, Is.EqualTo(expectedTokenKind));
Assert.That(token1.Value, Is.EqualTo(value));
var token2 = lexer.NextToken();
Assert.That(token2.Kind, Is.EqualTo(TokenKind.Eof));
}
private void TestStringLiteral(string value, string expected, TokenKind expectedTokenKind) {
var lexer = new Tokenizer(value);
var token1 = lexer.NextToken();
Assert.That(token1.Kind, Is.EqualTo(expectedTokenKind));
Assert.That(token1.Value, Is.EqualTo(expected));
var token2 = lexer.NextToken();
Assert.That(token2.Kind, Is.EqualTo(TokenKind.Eof));
}
/// <summary>
/// Helper method to deparent/parent a <see cref="SyntaxToken"/> to this instance with an expected kind of token.
/// </summary>
/// <typeparam name="TSyntaxNode">Type of the node</typeparam>
/// <param name="set">The previous child node parented to this instance</param>
/// <param name="node">The new child node to parent to this instance</param>
/// <param name="expectedKind">The expected kind of token</param>
protected void ParentToThis <TSyntaxNode>(ref TSyntaxNode set, TSyntaxNode node, TokenKind expectedKind) where TSyntaxNode : SyntaxToken
{
ParentToThis(ref set, node, node.TokenKind == expectedKind, expectedKind);
}
internal JsonToken(TokenKind kind, string value)
{
Kind = kind;
Value = value;
}
public Token(TokenKind kind) : this(kind, string.Empty)
{
}
public Token(TokenKind kind, string text)
{
Kind = kind;
Text = text;
}
public static String?GetText(TokenKind kind) => to.TryGetValue(kind, out var result) ? result : null;
public TokenInfo(TokenKind kind, int line, int column, int width)
: this(kind, line, column, width, string.Empty)
{
}
/// <summary>
/// TOKEN
/// </summary>
/// <param name="kind"></param>
/// <param name="text"></param>
public Token(TokenKind kind, String text)
{
this._tokenkind = kind;
this._text = text;
}
public OPINFO(TokenKind t, PredefinedName pn, ExpressionKind e, int c)
{
iToken = t;
methodName = pn;
expressionKind = e;
}
public Token(TokenKind kind)
{
Kind = kind;
}
public Token(TokenKind kind, string value, PositionInfo location)
{
Kind = kind;
Content = value;
Location = location;
}
public Token(TokenKind kind, string value)
{
Kind = kind;
Content = value;
}
public Token(Span span, TokenKind kind)
{
Span = span;
Kind = kind;
}
public Token(string str, TokenKind kind)
{
this.str = str;
this.kind = kind;
}
internal static bool IsOpenGrouping(this TokenKind tokKind)
{
return(tokKind == TokenKind.LeftBracket || tokKind == TokenKind.LeftBrace || tokKind == TokenKind.LeftParenthesis);
}
public Token(TokenKind kind, PositionInfo location)
{
Kind = kind;
Location = location;
}
void ReadString(int quote)
{
Start = Location;
Read(); // consume the opening quote
while (true)
{
var c = Read();
if (c == -1)
{
HandleError(JsonErrorCode.UnterminatedString, Start);
return;
}
if (c == quote)
{
End = Location;
TokenKind = TokenKind.String;
return;
}
if (c == '\r')
{
if (Peek() == '\n')
{
Read();
}
NewLine();
}
if (c == '\n')
{
NewLine();
}
if (c == '\\')
{
var n = Read();
switch (n)
{
case '\"':
case '\\':
case '/':
case 'b':
case 'f':
case 'n':
case 'r':
case 't':
break;
case 'u':
for (int i = 0; i < 4; i++)
{
var d = Peek();
if (IsHexDigit(d))
{
Read();
}
else
{
HandleError(JsonErrorCode.UnexpectedCharacter);
break;
}
}
break;
}
}
}
}
public Token(TokenKind kind, Position position, string value)
{
Kind = kind;
Position = position;
Literal = value;
}
private static Token CreateToken(int line, int column, string lexeme, TokenKind tokenKind)
{
return(new Token(tokenKind, new Position(line, column), lexeme));
}
internal static bool IsCloseGrouping(this TokenKind tokKind)
{
return(tokKind == TokenKind.RightBracket || tokKind == TokenKind.RightBrace || tokKind == TokenKind.RightParenthesis);
}
/// <summary>
/// Creates a new instance of <see cref="ReturnTokenLexemeParser"/>.
/// </summary>
/// <param name="kind">The kind of token. Not null.</param>
public ReturnTokenLexemeParser(TokenKind kind)
{
_kind = kind ?? throw new ArgumentNullException(nameof(kind));
}
private static int GetParameterValue(List <Token> parameterValueTokens, int tokenIndex, out object parameterValue)
{
int count = parameterValueTokens.Count;
parameterValue = null;
if (tokenIndex >= count)
{
return(tokenIndex);
}
Token token = parameterValueTokens[tokenIndex];
TokenKind kind = token.Kind;
if (kind <= TokenKind.Number)
{
if (kind != TokenKind.Variable)
{
if (kind == TokenKind.Number)
{
parameterValue = ((NumberToken)token).Value;
goto IL_23B;
}
}
else
{
bool flag;
if (bool.TryParse(((VariableToken)token).Name, out flag))
{
parameterValue = flag;
goto IL_23B;
}
parameterValue = token.Text;
goto IL_23B;
}
}
else
{
switch (kind)
{
case TokenKind.StringLiteral:
parameterValue = ((StringToken)token).Value;
goto IL_23B;
case TokenKind.StringExpandable:
parameterValue = ((StringExpandableToken)token).Value;
goto IL_23B;
default:
switch (kind)
{
case TokenKind.LBracket:
case TokenKind.RBracket:
case TokenKind.Comma:
parameterValue = null;
goto IL_23B;
case TokenKind.AtParen:
{
List <object> list = new List <object>();
tokenIndex++;
while (tokenIndex < count && parameterValueTokens[tokenIndex].Kind != TokenKind.RParen)
{
if (parameterValueTokens[tokenIndex].Kind == TokenKind.Comma)
{
tokenIndex++;
}
else
{
object obj = null;
tokenIndex = CmdletValidator.GetParameterValue(parameterValueTokens, tokenIndex, out obj);
if (obj != null)
{
list.Add(obj);
}
}
}
parameterValue = list.ToArray();
goto IL_23B;
}
case TokenKind.AtCurly:
{
Dictionary <object, object> dictionary = new Dictionary <object, object>();
tokenIndex++;
while (tokenIndex < count && parameterValueTokens[tokenIndex].Kind != TokenKind.RCurly)
{
if (parameterValueTokens[tokenIndex].Kind == TokenKind.Semi)
{
tokenIndex++;
}
else
{
object obj2 = null;
tokenIndex = CmdletValidator.GetParameterValue(parameterValueTokens, tokenIndex, out obj2);
if (tokenIndex >= count || parameterValueTokens[tokenIndex].Kind != TokenKind.Equals)
{
throw new InvalidOperationException(string.Format("Cmdlet parsing failed at index {0}. Expected Equals. Actual {1}", tokenIndex, parameterValueTokens[tokenIndex].Text));
}
if (++tokenIndex >= count)
{
throw new InvalidOperationException(string.Format("Cmdlet parsing failed. No value for key {0}", obj2));
}
object value = null;
tokenIndex = CmdletValidator.GetParameterValue(parameterValueTokens, tokenIndex, out value);
dictionary.Add(obj2, value);
}
}
parameterValue = dictionary;
goto IL_23B;
}
case TokenKind.Semi:
case TokenKind.Pipe:
throw new InvalidOperationException(string.Format("Cmdlet parsing failed at index {0}. Encountered unexpected {1}", tokenIndex, token.Kind));
}
break;
}
}
parameterValue = token.Text;
IL_23B:
return(++tokenIndex);
}
private static TokenSpec ToTokenSpec(TokenKind kind)
{
return(TokenSpec.A(kind));
}
public Token(TokenKind kind, string value)
{
this.Kind = kind;
this.Value = value;
}
public static bool IsOpenerOrCloser(TokenKind tt)
{
return(tt >= TokenKind.LParen && tt < (TokenKind)((int)TokenKind.ROther + (1 << TokenKindShift)));
}
internal static Boolean?IsRightAssociative(TokenKind kind)
{
return(false);
}
public static bool IsCloser(TokenKind tt)
{
return(IsOpenerOrCloser(tt) && ((int)tt & 0x0100) != 0);
}
public Token(TokenKind kind, char ch) : this(kind, ch.ToString())
{
}
/// <summary>
/// Helper method to deparent/parent a <see cref="SyntaxToken"/> to this instance with an expected kind of token.
/// </summary>
/// <typeparam name="TSyntaxNode">Type of the node</typeparam>
/// <param name="set">The previous child node parented to this instance</param>
/// <param name="node">The new child node to parent to this instance</param>
/// <param name="expectedKind1">The expected kind of token (option1)</param>
/// <param name="expectedKind2">The expected kind of token (option2)</param>
protected void ParentToThis <TSyntaxNode>(ref TSyntaxNode set, TSyntaxNode node, TokenKind expectedKind1, TokenKind expectedKind2) where TSyntaxNode : SyntaxToken
{
ParentToThis(ref set, node, node.TokenKind == expectedKind1 || node.TokenKind == expectedKind2, new ExpectedTuple2 <TokenKind, TokenKind>(expectedKind1, expectedKind2));
}
public bool NextToken()
{
Start = default;
End = default;
TokenKind = TokenKind.None;
while (true)
{
bufferTokenStart = bufferPos;
var c = Peek();
switch (c)
{
case -1:
Start = End = Location;
TokenKind = TokenKind.None;
return(false);
case '\"':
case '\'':
ReadString(c);
return(true);
case '{':
SingleCharacterToken(TokenKind.StartObject);
return(true);
case '}':
SingleCharacterToken(TokenKind.EndObject);
return(true);
case '[':
SingleCharacterToken(TokenKind.StartArray);
return(true);
case ']':
SingleCharacterToken(TokenKind.EndArray);
return(true);
case ':':
SingleCharacterToken(TokenKind.Colon);
return(true);
case ',':
SingleCharacterToken(TokenKind.Comma);
return(true);
case '/':
var location = Location;
Read();
var n = Peek();
switch (n)
{
case '/':
Read();
ReadLineComment();
return(true);
case '*':
Read();
ReadBlockComment();
return(true);
}
HandleError(JsonErrorCode.UnexpectedCharacter, location);
continue;
case '\r':
Read();
if (Peek() == '\n')
{
Read();
}
NewLine();
break;
case '\n':
Read();
NewLine();
break;
default:
if (IsDigit((char)c) || c == '-')
{
ReadNumber();
return(true);
}
if (char.IsWhiteSpace((char)c))
{
Read();
continue;
}
if (IsNameStart(c))
{
ReadName();
return(true);
}
HandleError(JsonErrorCode.UnexpectedCharacter, Location);
Read();
continue;
}
}
}
public bool Check(TokenKind required)
{
return(this.kind == required);
}
