public int FindNextIndexAtSameLevel(ExpressionTokenType targetExpressionTokenType)
{
int i = 0;
int level = 0;
while (HasTokenAt(i))
{
ExpressionTokenType expressionToken = Peek(i);
if (expressionToken == ExpressionTokenType.ParenOpen || expressionToken == ExpressionTokenType.ArrayAccessOpen || expressionToken == ExpressionTokenType.LessThan)
{
level++;
}
else if (expressionToken == ExpressionTokenType.ParenClose || expressionToken == ExpressionTokenType.ArrayAccessClose || expressionToken == ExpressionTokenType.GreaterThan)
{
level--;
}
else if (expressionToken == targetExpressionTokenType && level == 0)
{
return(i);
}
i++;
}
return(-1);
}
private static void MergeToken(List <ExpressionToken> tokens, ExpressionTokenType type, int end)
{
var previous = tokens[tokens.Count - 1];
previous.Type = type;
previous.End = end;
}
public void Tokenize_SingleToken_ReturnsExpectedToken(string path, ExpressionTokenType expectedType)
{
var tokenizer = new ExpressionTokenizer();
var token = tokenizer.Tokenize(path).Single();
Assert.AreEqual(expectedType, token.Type);
}
public int FindMatchingIndexNoAdvance(ExpressionTokenType braceOpen, ExpressionTokenType braceClose)
{
if (Current != braceOpen)
{
return(-1);
}
int i = ptr;
int counter = 1;
while (i != lastTokenIndex - 1)
{
i++;
if (tokens.array[i].expressionTokenType == braceOpen)
{
counter++;
}
if (tokens.array[i].expressionTokenType == braceClose)
{
counter--;
if (counter == 0)
{
return(i);
}
}
}
return(-1);
}
private static void AddMergeableToken(List <ExpressionToken> tokens, ExpressionTokenType type, int start, int end)
{
// This handles merging names separated by whitespace so "Thing 2" becomes a single Identifier token
// instead of an Identifier followed by an Int.
if (tokens.Count > 0 && IsMergeable(tokens[tokens.Count - 1].Type))
{
if (IsMergeable(type))
{
MergeToken(tokens, ExpressionTokenType.Identifier, end);
}
else if (type == ExpressionTokenType.Command)
{
MergeToken(tokens, ExpressionTokenType.Command, end);
}
else
{
AddToken(tokens, type, start, end, 0);
}
}
else
{
AddToken(tokens, type, start, end, 0);
}
}
/// <summary>
/// Lève une exception si le jeton e n'est pas du type type.
/// </summary>
void AssertType(ExpressionToken e, ExpressionTokenType type)
{
if (e.TkType != type)
{
throw new InvalidOperationException();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="Token"/> class.
/// </summary>
/// <param name="currentToken">The text from the expression.</param>
/// <param name="startIndex">The index where it was discovered in the text.</param>
/// <param name="length">The length of the token in the original expression</param>
public Token(string currentToken, int startIndex, int length, ExpressionTokenType type)
{
this.CurrentToken = currentToken;
this.StartIndex = startIndex;
this.Length = length;
this.Type = type;
}
private static void AddSingleToken(List <ExpressionToken> tokens, ExpressionTokenType type, ref int start)
{
tokens.Add(new ExpressionToken {
Location = start, Type = type, Start = start, End = start + 1
});
++start;
}
public ExpressionToken(ExpressionTokenType expressionTokenType, string value, int line, int column)
{
this.expressionTokenType = expressionTokenType;
this.value = value;
this.line = line;
this.column = column;
}
private static void AddTokenInfo(string tokenRegex, ExpressionTokenType propertyPathTokenType, RegexOptions options = RegexOptions.None)
{
var regex = ConstructTokenRegex(tokenRegex, options);
var tokenInfo = new ExpressionTokenInfo(regex, propertyPathTokenType);
TokenInfos.Add(tokenInfo);
}
public int FindNextIndex(ExpressionTokenType targetExpressionTokenType)
{
int i = 0;
int counter = 0;
while (HasTokenAt(i))
{
ExpressionTokenType expressionToken = Peek(i);
if (expressionToken == ExpressionTokenType.ParenOpen)
{
counter++;
}
else if (expressionToken == ExpressionTokenType.ParenClose)
{
counter--;
}
else if (expressionToken == targetExpressionTokenType && counter == 0)
{
return(i);
}
i++;
}
return(-1);
}
public TokenStream AdvanceAndGetSubStreamBetween(ExpressionTokenType open, ExpressionTokenType close)
{
int index = FindMatchingIndexNoAdvance(open, close);
if (index == -1)
{
return(default);
public bool NextTokenIs(ExpressionTokenType tokenType)
{
if (ptr + 1 >= lastTokenIndex)
{
return(false);
}
return(Next == tokenType);
}
private void SkipNextToken(ExpressionTokenType type)
{
var token = TakeNextToken();
if (token.Type != type)
{
throw new ExpressionParseException(token);
}
}
public bool Skip(ExpressionTokenType tokenType)
{
if (Current == null || Current.Type != tokenType)
{
return(false);
}
Pop();
return(true);
}
public bool TryTokenType(ExpressionTokenType tokenType, out ExpressionToken token)
{
token = null;
if (Current == null || Current.Type != tokenType)
{
return(false);
}
token = Pop();
return(true);
}
private static bool CheckState(ExpressionTokenType state, ExpressionToken token)
{
switch (state)
{
case ExpressionTokenType.Identifier: return(token.Type == ExpressionTokenType.Identifier);
case ExpressionTokenType.Command: return(token.Type == ExpressionTokenType.Identifier);
case ExpressionTokenType.Int: return(token.Type == ExpressionTokenType.Int);
case ExpressionTokenType.Operator: return(token.Type == ExpressionTokenType.Operator || token.Type == ExpressionTokenType.StartLookup);
case ExpressionTokenType.EndLookup: return(token.Type == ExpressionTokenType.EndLookup);
default: return(false);
}
}
private bool ParseListExpression(ref LightList <ASTNode> retn, ExpressionTokenType openExpressionToken, ExpressionTokenType closeExpressionToken)
{
if (tokenStream.Current != openExpressionToken)
{
return(false);
}
int range = tokenStream.FindMatchingIndex(openExpressionToken, closeExpressionToken);
tokenStream.Save();
if (range == 1)
{
tokenStream.Advance(2);
retn = LightList <ASTNode> .Get();
return(true);
}
if (retn != null)
{
LightList <ASTNode> .Release(ref retn);
}
retn = LightList <ASTNode> .Get();
//todo find next comma at same level (meaning not inside [ or ( or <
ExpressionParser parser = CreateSubParser(range);
bool valid = parser.ParseListExpressionStep(ref retn);
parser.Release();
if (!valid)
{
tokenStream.Restore();
ReleaseList(retn);
return(false);
}
return(true);
}
public int FindMatchingTernaryColon()
{
int i = 0;
int level = 0;
while (HasTokenAt(i))
{
ExpressionTokenType expressionToken = Peek(i);
// find paren open, recurse
if (expressionToken == ExpressionTokenType.ParenOpen)
{
Save();
Set(CurrentIndex + i);
i = FindMatchingIndex(ExpressionTokenType.ParenOpen, ExpressionTokenType.ParenClose);
// level++;
Restore();
}
else if (expressionToken == ExpressionTokenType.ParenClose)
{
// level--;
}
// avoid false positives from ?. and ?[
else if (expressionToken == ExpressionTokenType.QuestionMark && HasTokenAt(i + 1) && Peek(i + 1) != ExpressionTokenType.QuestionMark && Peek(i + 1) != ExpressionTokenType.Dot && Peek(i + 1) != ExpressionTokenType.ArrayAccessOpen)
{
level++;
}
else if (expressionToken == ExpressionTokenType.Colon && level != 0)
{
level--;
}
else if (expressionToken == ExpressionTokenType.Colon && level == 0)
{
return(i);
}
i++;
}
return(-1);
}
public int FindMatchingTernaryColon()
{
int i = 0;
int level = 0;
int store = ptr;
while (HasTokenAt(i))
{
ExpressionTokenType expressionToken = Peek(i);
// find paren open, recurse
if (expressionToken == ExpressionTokenType.ParenOpen)
{
store = ptr;
ptr += i;
// FindMatchingIndex returns an advance value to the closing paren
i += FindMatchingIndex(ExpressionTokenType.ParenOpen, ExpressionTokenType.ParenClose);
ptr = store;
}
// avoid false positives from ?. and ?[
else if (expressionToken == ExpressionTokenType.QuestionMark && HasTokenAt(i + 1) && Peek(i + 1) != ExpressionTokenType.QuestionMark && Peek(i + 1) != ExpressionTokenType.Dot && Peek(i + 1) != ExpressionTokenType.ArrayAccessOpen)
{
level++;
}
else if (expressionToken == ExpressionTokenType.Colon && level != 0)
{
level--;
}
else if (expressionToken == ExpressionTokenType.Colon && level == 0)
{
return(i);
}
i++;
}
return(-1);
}
public int FindMatchingIndex(ExpressionTokenType braceOpen, ExpressionTokenType braceClose)
{
if (Current != braceOpen)
{
return(-1);
}
Save();
int i = -1;
int counter = 0;
while (ptr != lastTokenIndex)
{
i++;
if (Current == braceOpen)
{
counter++;
}
if (Current == braceClose)
{
counter--;
if (counter == 0)
{
Restore();
return(i);
}
}
Advance();
}
Restore();
return(-1);
}
/// <summary>
/// Adds a new token with the position, length and type to the token list
/// </summary>
/// <param name="start"></param>
/// <param name="length"></param>
/// <param name="value"></param>
/// <param name="type"></param>
protected virtual void AddNewToken(int start, int length, string value, ExpressionTokenType type)
{
Token t = new Token(value, start, length, type);
this.Root.Add(t);
}
ExpressionToken ReadToLiteralAs(ListReader<RawToken> reader, string literalValue, ExpressionTokenType tokenType)
{
ExpressionToken lastExpToken = new ExpressionToken { TokenType = tokenType };
_currentTokens.Add(lastExpToken);
while (!reader.IsEnd)
{
var next = reader.Read();
if (next.TokenType == RawTokenType.Symbol && next.Value == literalValue)
{
break;
}
lastExpToken.Append(next);
}
return lastExpToken;
}
public ExpressionToken(ExpressionTokenType type, double value)
{
Value = value;
TokenType = type;
}
private static void AddToken(List <ExpressionToken> tokens, ExpressionTokenType type, int start, int end, int offset)
{
tokens.Add(new ExpressionToken {
Location = start, Type = type, Start = start + offset, End = end
});
}
/// <summary>
/// Initializes a new instance of the <see cref="Token"/> class.
/// </summary>
/// <param name="currentToken">The text from the expression.</param>
/// <param name="startIndex">The index where it was discovered in the text.</param>
public Token(string currentToken, int startIndex, ExpressionTokenType type)
: this(currentToken, startIndex, currentToken?.Length ?? 0, type)
{
}
internal ExpressionToken(int value)
{
Type = ExpressionTokenType.Numeric;
Value = value;
}
internal ExpressionToken(ExpressionTokenType type, string content)
{
Type = type;
Content = content;
}
private static bool IsMergeable(ExpressionTokenType type)
{
return(type == ExpressionTokenType.Int || type == ExpressionTokenType.Float || type == ExpressionTokenType.Identifier || type == ExpressionTokenType.Type);
}
public ExpressionValueToken(ExpressionTokenType tokenType, string value, CharacterLocation location)
{
TokenType = tokenType;
Value = value;
Location = location;
}
ExpressionToken ReadToLiteralAs(ListReader <RawToken> reader, string literalValue, ExpressionTokenType tokenType)
{
ExpressionToken lastExpToken = new ExpressionToken {
TokenType = tokenType
};
_currentTokens.Add(lastExpToken);
while (!reader.IsEnd)
{
var next = reader.Read();
if (next.TokenType == RawTokenType.Symbol && next.Value == literalValue)
{
break;
}
lastExpToken.Append(next);
}
return(lastExpToken);
}
