public int FindNextIndexAtSameLevel(ExpressionTokenType targetExpressionTokenType) { int i = 0; int level = 0; while (HasTokenAt(i)) { ExpressionTokenType expressionToken = Peek(i); if (expressionToken == ExpressionTokenType.ParenOpen || expressionToken == ExpressionTokenType.ArrayAccessOpen || expressionToken == ExpressionTokenType.LessThan) { level++; } else if (expressionToken == ExpressionTokenType.ParenClose || expressionToken == ExpressionTokenType.ArrayAccessClose || expressionToken == ExpressionTokenType.GreaterThan) { level--; } else if (expressionToken == targetExpressionTokenType && level == 0) { return(i); } i++; } return(-1); }
private static void MergeToken(List <ExpressionToken> tokens, ExpressionTokenType type, int end) { var previous = tokens[tokens.Count - 1]; previous.Type = type; previous.End = end; }
public void Tokenize_SingleToken_ReturnsExpectedToken(string path, ExpressionTokenType expectedType) { var tokenizer = new ExpressionTokenizer(); var token = tokenizer.Tokenize(path).Single(); Assert.AreEqual(expectedType, token.Type); }
public int FindMatchingIndexNoAdvance(ExpressionTokenType braceOpen, ExpressionTokenType braceClose) { if (Current != braceOpen) { return(-1); } int i = ptr; int counter = 1; while (i != lastTokenIndex - 1) { i++; if (tokens.array[i].expressionTokenType == braceOpen) { counter++; } if (tokens.array[i].expressionTokenType == braceClose) { counter--; if (counter == 0) { return(i); } } } return(-1); }
private static void AddMergeableToken(List <ExpressionToken> tokens, ExpressionTokenType type, int start, int end) { // This handles merging names separated by whitespace so "Thing 2" becomes a single Identifier token // instead of an Identifier followed by an Int. if (tokens.Count > 0 && IsMergeable(tokens[tokens.Count - 1].Type)) { if (IsMergeable(type)) { MergeToken(tokens, ExpressionTokenType.Identifier, end); } else if (type == ExpressionTokenType.Command) { MergeToken(tokens, ExpressionTokenType.Command, end); } else { AddToken(tokens, type, start, end, 0); } } else { AddToken(tokens, type, start, end, 0); } }
/// <summary> /// Lève une exception si le jeton e n'est pas du type type. /// </summary> void AssertType(ExpressionToken e, ExpressionTokenType type) { if (e.TkType != type) { throw new InvalidOperationException(); } }
/// <summary> /// Initializes a new instance of the <see cref="Token"/> class. /// </summary> /// <param name="currentToken">The text from the expression.</param> /// <param name="startIndex">The index where it was discovered in the text.</param> /// <param name="length">The length of the token in the original expression</param> public Token(string currentToken, int startIndex, int length, ExpressionTokenType type) { this.CurrentToken = currentToken; this.StartIndex = startIndex; this.Length = length; this.Type = type; }
private static void AddSingleToken(List <ExpressionToken> tokens, ExpressionTokenType type, ref int start) { tokens.Add(new ExpressionToken { Location = start, Type = type, Start = start, End = start + 1 }); ++start; }
public ExpressionToken(ExpressionTokenType expressionTokenType, string value, int line, int column) { this.expressionTokenType = expressionTokenType; this.value = value; this.line = line; this.column = column; }
private static void AddTokenInfo(string tokenRegex, ExpressionTokenType propertyPathTokenType, RegexOptions options = RegexOptions.None) { var regex = ConstructTokenRegex(tokenRegex, options); var tokenInfo = new ExpressionTokenInfo(regex, propertyPathTokenType); TokenInfos.Add(tokenInfo); }
public int FindNextIndex(ExpressionTokenType targetExpressionTokenType) { int i = 0; int counter = 0; while (HasTokenAt(i)) { ExpressionTokenType expressionToken = Peek(i); if (expressionToken == ExpressionTokenType.ParenOpen) { counter++; } else if (expressionToken == ExpressionTokenType.ParenClose) { counter--; } else if (expressionToken == targetExpressionTokenType && counter == 0) { return(i); } i++; } return(-1); }
public TokenStream AdvanceAndGetSubStreamBetween(ExpressionTokenType open, ExpressionTokenType close) { int index = FindMatchingIndexNoAdvance(open, close); if (index == -1) { return(default);
public bool NextTokenIs(ExpressionTokenType tokenType) { if (ptr + 1 >= lastTokenIndex) { return(false); } return(Next == tokenType); }
private void SkipNextToken(ExpressionTokenType type) { var token = TakeNextToken(); if (token.Type != type) { throw new ExpressionParseException(token); } }
public bool Skip(ExpressionTokenType tokenType) { if (Current == null || Current.Type != tokenType) { return(false); } Pop(); return(true); }
public bool TryTokenType(ExpressionTokenType tokenType, out ExpressionToken token) { token = null; if (Current == null || Current.Type != tokenType) { return(false); } token = Pop(); return(true); }
private static bool CheckState(ExpressionTokenType state, ExpressionToken token) { switch (state) { case ExpressionTokenType.Identifier: return(token.Type == ExpressionTokenType.Identifier); case ExpressionTokenType.Command: return(token.Type == ExpressionTokenType.Identifier); case ExpressionTokenType.Int: return(token.Type == ExpressionTokenType.Int); case ExpressionTokenType.Operator: return(token.Type == ExpressionTokenType.Operator || token.Type == ExpressionTokenType.StartLookup); case ExpressionTokenType.EndLookup: return(token.Type == ExpressionTokenType.EndLookup); default: return(false); } }
private bool ParseListExpression(ref LightList <ASTNode> retn, ExpressionTokenType openExpressionToken, ExpressionTokenType closeExpressionToken) { if (tokenStream.Current != openExpressionToken) { return(false); } int range = tokenStream.FindMatchingIndex(openExpressionToken, closeExpressionToken); tokenStream.Save(); if (range == 1) { tokenStream.Advance(2); retn = LightList <ASTNode> .Get(); return(true); } if (retn != null) { LightList <ASTNode> .Release(ref retn); } retn = LightList <ASTNode> .Get(); //todo find next comma at same level (meaning not inside [ or ( or < ExpressionParser parser = CreateSubParser(range); bool valid = parser.ParseListExpressionStep(ref retn); parser.Release(); if (!valid) { tokenStream.Restore(); ReleaseList(retn); return(false); } return(true); }
public int FindMatchingTernaryColon() { int i = 0; int level = 0; while (HasTokenAt(i)) { ExpressionTokenType expressionToken = Peek(i); // find paren open, recurse if (expressionToken == ExpressionTokenType.ParenOpen) { Save(); Set(CurrentIndex + i); i = FindMatchingIndex(ExpressionTokenType.ParenOpen, ExpressionTokenType.ParenClose); // level++; Restore(); } else if (expressionToken == ExpressionTokenType.ParenClose) { // level--; } // avoid false positives from ?. and ?[ else if (expressionToken == ExpressionTokenType.QuestionMark && HasTokenAt(i + 1) && Peek(i + 1) != ExpressionTokenType.QuestionMark && Peek(i + 1) != ExpressionTokenType.Dot && Peek(i + 1) != ExpressionTokenType.ArrayAccessOpen) { level++; } else if (expressionToken == ExpressionTokenType.Colon && level != 0) { level--; } else if (expressionToken == ExpressionTokenType.Colon && level == 0) { return(i); } i++; } return(-1); }
public int FindMatchingTernaryColon() { int i = 0; int level = 0; int store = ptr; while (HasTokenAt(i)) { ExpressionTokenType expressionToken = Peek(i); // find paren open, recurse if (expressionToken == ExpressionTokenType.ParenOpen) { store = ptr; ptr += i; // FindMatchingIndex returns an advance value to the closing paren i += FindMatchingIndex(ExpressionTokenType.ParenOpen, ExpressionTokenType.ParenClose); ptr = store; } // avoid false positives from ?. and ?[ else if (expressionToken == ExpressionTokenType.QuestionMark && HasTokenAt(i + 1) && Peek(i + 1) != ExpressionTokenType.QuestionMark && Peek(i + 1) != ExpressionTokenType.Dot && Peek(i + 1) != ExpressionTokenType.ArrayAccessOpen) { level++; } else if (expressionToken == ExpressionTokenType.Colon && level != 0) { level--; } else if (expressionToken == ExpressionTokenType.Colon && level == 0) { return(i); } i++; } return(-1); }
public int FindMatchingIndex(ExpressionTokenType braceOpen, ExpressionTokenType braceClose) { if (Current != braceOpen) { return(-1); } Save(); int i = -1; int counter = 0; while (ptr != lastTokenIndex) { i++; if (Current == braceOpen) { counter++; } if (Current == braceClose) { counter--; if (counter == 0) { Restore(); return(i); } } Advance(); } Restore(); return(-1); }
/// <summary> /// Adds a new token with the position, length and type to the token list /// </summary> /// <param name="start"></param> /// <param name="length"></param> /// <param name="value"></param> /// <param name="type"></param> protected virtual void AddNewToken(int start, int length, string value, ExpressionTokenType type) { Token t = new Token(value, start, length, type); this.Root.Add(t); }
ExpressionToken ReadToLiteralAs(ListReader<RawToken> reader, string literalValue, ExpressionTokenType tokenType) { ExpressionToken lastExpToken = new ExpressionToken { TokenType = tokenType }; _currentTokens.Add(lastExpToken); while (!reader.IsEnd) { var next = reader.Read(); if (next.TokenType == RawTokenType.Symbol && next.Value == literalValue) { break; } lastExpToken.Append(next); } return lastExpToken; }
public ExpressionToken(ExpressionTokenType type, double value) { Value = value; TokenType = type; }
private static void AddToken(List <ExpressionToken> tokens, ExpressionTokenType type, int start, int end, int offset) { tokens.Add(new ExpressionToken { Location = start, Type = type, Start = start + offset, End = end }); }
/// <summary> /// Initializes a new instance of the <see cref="Token"/> class. /// </summary> /// <param name="currentToken">The text from the expression.</param> /// <param name="startIndex">The index where it was discovered in the text.</param> public Token(string currentToken, int startIndex, ExpressionTokenType type) : this(currentToken, startIndex, currentToken?.Length ?? 0, type) { }
internal ExpressionToken(int value) { Type = ExpressionTokenType.Numeric; Value = value; }
internal ExpressionToken(ExpressionTokenType type, string content) { Type = type; Content = content; }
private static bool IsMergeable(ExpressionTokenType type) { return(type == ExpressionTokenType.Int || type == ExpressionTokenType.Float || type == ExpressionTokenType.Identifier || type == ExpressionTokenType.Type); }
public ExpressionValueToken(ExpressionTokenType tokenType, string value, CharacterLocation location) { TokenType = tokenType; Value = value; Location = location; }
ExpressionToken ReadToLiteralAs(ListReader <RawToken> reader, string literalValue, ExpressionTokenType tokenType) { ExpressionToken lastExpToken = new ExpressionToken { TokenType = tokenType }; _currentTokens.Add(lastExpToken); while (!reader.IsEnd) { var next = reader.Read(); if (next.TokenType == RawTokenType.Symbol && next.Value == literalValue) { break; } lastExpToken.Append(next); } return(lastExpToken); }