private bool nextIsTerminator()
{
if (!_context.HasNext)
{
return(true);
}
if (TokenFilter.IsWhitespace(_context.Next.Value))
{
return(true);
}
bool nextIsSpecial = StringEscaper.SpecialChars.ContainsString(_context.Next.Value);
bool currentIsSpecial = StringEscaper.SpecialChars.ContainsString(_context.Current.Value);
if (!nextIsSpecial && !currentIsSpecial)
{
return(false);
}
if (_context.Next.Value != _context.Current.Value)
{
// There are no operators constructed from different special chars
return(true);
}
if (StringEscaper.TwoSymbolOperators.ContainsString(_context.Current.Value))
{
return(false);
}
return(true);
}
private bool nextIsColon()
{
return(_context.HasNext && TokenFilter.GetTokenType(_context.Next.Value)?.IsAny(TokenType.Colon) == true);
}
public void Parse()
{
_substring = string.Empty;
while (_context.MoveNext())
{
_position = _context.Current.Position;
_substring += _context.Current.Value;
bool beforeTerminator = nextIsTerminator();
var tokenTypeNullable = TokenFilter.GetTokenType(_substring);
if (_isRegexOpen)
{
// empty regex body
if (tokenTypeNullable == TokenType.RegexDelimiter)
{
// close quote
var token = addToken(TokenType.RegexDelimiter);
_openOperators.Pop();
updateCurrentField();
token.NextTokenField = _currentField;
_isRegexOpen = false;
}
// regex body token lasts until regex delimiter or End Of String
else if (!_context.HasNext || TokenFilter.GetTokenType(_context.Next.Value) == TokenType.RegexDelimiter)
{
var token = addToken(TokenType.RegexBody);
token.NextTokenField = _currentField;
_isRegexOpen = false;
}
}
else if (tokenTypeNullable.HasValue)
{
var tokenType = tokenTypeNullable.Value;
if (tokenType.IsAny(TokenType.Open))
{
var token = addToken(tokenType);
_openOperators.Push(token);
token.NextTokenField = _currentField;
}
else if (tokenType.IsAny(TokenType.Close))
{
if (tokenType.IsLegalCloserOf(_openOperators.TryPeek()?.Type))
{
// close parenthesis
var token = addToken(tokenType);
_openOperators.Pop();
updateCurrentField();
token.NextTokenField = _currentField;
}
else
{
if (_openOperators.Count == 0)
{
SyntaxErrors.Add($"Unmatched {_substring} at {_start}");
}
else
{
SyntaxErrors.Add($"Unexpected {_substring} at {_start} closing {_openOperators.Peek().Value} at {_openOperators.Peek().Position}");
}
var token = addToken(tokenType);
token.NextTokenField = _currentField;
}
}
else if (tokenType.IsAny(TokenType.Quote | TokenType.RegexDelimiter))
{
TokenType generalType;
TokenType openType;
TokenType closeType;
bool isRegex;
if (tokenType.IsAny(TokenType.Quote))
{
generalType = TokenType.Quote;
openType = TokenType.OpenQuote;
closeType = TokenType.CloseQuote;
isRegex = false;
}
else
{
generalType = TokenType.RegexDelimiter;
openType = TokenType.OpenRegex;
closeType = TokenType.CloseRegex;
isRegex = true;
}
if (_openOperators.Count > 0 && _openOperators.Peek().Type.IsAny(generalType))
{
// close quote
var token = addToken(closeType);
_openOperators.Pop();
updateCurrentField();
token.NextTokenField = _currentField;
}
else
{
var token = addToken(openType);
_openOperators.Push(token);
if (isRegex)
{
_isRegexOpen = true;
}
token.NextTokenField = _currentField;
}
}
else if (tokenType.IsAny(TokenType.Modifier | TokenType.Colon) || tokenType.IsAny(TokenType.Boolean) &&
// To avoid recognizing AND in ANDY
(StringEscaper.SpecialChars.Contains(_substring[0]) || beforeTerminator))
{
var token = addToken(tokenType);
token.NextTokenField = _currentField;
}
else if (tokenType.IsAny(TokenType.Wildcard))
{
if (tokenType.IsAny(TokenType.AnyString) && nextIsColon())
{
// add field
var token = addToken(TokenType.Field);
_currentField = Tokens[Tokens.Count - 1].ParentField;
token.NextTokenField = _currentField;
}
else
{
var previous = Tokens.TryGetLast();
// adjacent wildcard tokens are related to the same field
if (previous != null && previous.Position + previous.Value.Length == _start)
{
_currentField = previous.ParentField;
}
var token = addToken(tokenType);
token.NextTokenField = _currentField;
}
}
else if (_openOperators.TryPeek()?.Type.IsAny(TokenType.OpenRange) == true && tokenType.IsAny(TokenType.To))
{
// interval extremes separator
var token = addToken(tokenType);
token.NextTokenField = _currentField;
}
}
else if (TokenFilter.IsWhitespace(_substring))
{
// ignore whitespace token
_start = _position;
_substring = string.Empty;
}
else if (nextIsColon())
{
// add field
var token = addToken(TokenType.Field);
_currentField = Tokens[Tokens.Count - 1].ParentField;
token.NextTokenField = _currentField;
}
else if (beforeTerminator && prevIsModifier())
{
var token = addToken(TokenType.ModifierValue);
updateCurrentField();
token.NextTokenField = _currentField;
}
else if (beforeTerminator)
{
var token = addToken(TokenType.FieldValue);
updateCurrentField();
token.NextTokenField = _currentField;
}
else if (isCjk())
{
var token = addToken(TokenType.FieldValue);
token.NextTokenField = _currentField;
}
}
}