public void VerifyTokenTrieNotEnoughBufferLeft()
{
byte[] hello = Encoding.UTF8.GetBytes("hello");
byte[] helloBang = Encoding.UTF8.GetBytes("hello!");
TokenTrie t = new TokenTrie();
t.AddToken(hello);
t.AddToken(helloBang);
byte[] source1 = Encoding.UTF8.GetBytes("hi");
byte[] source2 = Encoding.UTF8.GetBytes(" hello");
int token;
int pos = 0;
Assert.False(t.GetOperation(source1, source1.Length, ref pos, out token));
Assert.Equal(-1, token);
pos = 1;
Assert.True(t.GetOperation(source2, source2.Length, ref pos, out token));
Assert.Equal(0, token);
pos = 2;
Assert.False(t.GetOperation(source2, source2.Length, ref pos, out token));
Assert.Equal(-1, token);
}
public IOperation GetOperation(Encoding encoding, IProcessorState processorState)
{
TokenTrie structureTrie = new TokenTrie();
TokenTrie closeConditionTrie = new TokenTrie();
TokenTrie scanBackTrie = new TokenTrie();
IToken openOpenElementTokenBytes = Tokens.OpenOpenElementToken.ToToken(processorState.Encoding);
scanBackTrie.AddToken(openOpenElementTokenBytes);
int openOpenElementToken = structureTrie.AddToken(openOpenElementTokenBytes);
int openCloseElementToken = structureTrie.AddToken(Tokens.OpenCloseElementToken.ToToken(processorState.Encoding));
int closeCloseElementToken = structureTrie.AddToken(Tokens.CloseElementTagToken.ToToken(processorState.Encoding));
int selfClosingElementEndToken = -1;
if (Tokens.SelfClosingElementEndToken != null)
{
selfClosingElementEndToken = structureTrie.AddToken(Tokens.SelfClosingElementEndToken.ToToken(processorState.Encoding));
}
closeConditionTrie.AddToken(Tokens.CloseConditionExpression.ToToken(processorState.Encoding));
MarkupTokenMapping mapping = new MarkupTokenMapping(
openOpenElementToken,
openCloseElementToken,
closeCloseElementToken,
selfClosingElementEndToken
);
IReadOnlyList <IToken> start = new[] { Tokens.OpenConditionExpression.ToToken(processorState.Encoding) };
return(new Impl(this, start, structureTrie, closeConditionTrie, scanBackTrie, mapping, _id, _initialState));
}
public ScopeBuilder(IProcessorState processor, ITokenTrie tokens, IOperatorMap <TOperator, TToken> operatorMap, bool dereferenceInLiterals)
{
TokenTrie trie = new TokenTrie();
trie.Append(tokens);
_badSyntaxTokens = operatorMap.BadSyntaxTokens;
_noops = operatorMap.NoOpTokens;
_openGroup = operatorMap.OpenGroupToken;
_closeGroup = operatorMap.CloseGroupToken;
_literal = operatorMap.LiteralToken;
_identity = operatorMap.Identity;
_literalSequenceBoundsMarkers = operatorMap.LiteralSequenceBoundsMarkers;
_terminators = operatorMap.Terminators;
_isSymbolDereferenceInLiteralSequenceRequired = dereferenceInLiterals;
_processor = processor;
_knownTokensCount = tokens.Count;
_valueEncoder = operatorMap.Encode;
_valueDecoder = operatorMap.Decode;
List <object> symbolValues = new List <object>();
foreach (KeyValuePair <string, object> variable in processor.Config.Variables)
{
trie.AddToken(processor.Encoding.GetBytes(string.Format(processor.Config.VariableFormatString, variable.Key)));
symbolValues.Add(variable.Value);
}
_symbolValues = symbolValues;
_tokenToOperatorMap = operatorMap.TokensToOperatorsMap;
_operatorScopeFactory = operatorMap.OperatorScopeLookupFactory;
_tokens = trie;
}
public void VerifyTokenTrieAtBegin()
{
byte[] hello = Encoding.UTF8.GetBytes("hello");
byte[] helloBang = Encoding.UTF8.GetBytes("hello!");
byte[] hi = Encoding.UTF8.GetBytes("hi");
TokenTrie t = new TokenTrie();
t.AddToken(hello);
t.AddToken(helloBang);
t.AddToken(hi);
byte[] source1 = Encoding.UTF8.GetBytes("hello there");
byte[] source2 = Encoding.UTF8.GetBytes("hello1 there");
byte[] source3 = Encoding.UTF8.GetBytes("hello! there");
byte[] source4 = Encoding.UTF8.GetBytes("hi there");
byte[] source5 = Encoding.UTF8.GetBytes("hi");
byte[] source6 = Encoding.UTF8.GetBytes("he");
int token;
int pos = 0;
Assert.True(t.GetOperation(source1, source1.Length, ref pos, out token));
Assert.Equal(0, token);
pos = 0;
Assert.True(t.GetOperation(source2, source2.Length, ref pos, out token));
Assert.Equal(0, token);
pos = 0;
Assert.True(t.GetOperation(source3, source3.Length, ref pos, out token));
Assert.Equal(1, token);
pos = 0;
Assert.True(t.GetOperation(source4, source4.Length, ref pos, out token));
Assert.Equal(2, token);
pos = 0;
Assert.True(t.GetOperation(source5, source5.Length, ref pos, out token));
Assert.Equal(2, token);
pos = 0;
Assert.False(t.GetOperation(source6, source6.Length, ref pos, out token));
Assert.Equal(-1, token);
}
public void VerifyTokenTrieCombine()
{
byte[] hello = Encoding.UTF8.GetBytes("hello");
byte[] helloBang = Encoding.UTF8.GetBytes("hello!");
byte[] hi = Encoding.UTF8.GetBytes("hi");
byte[] there = Encoding.UTF8.GetBytes("there!");
TokenTrie t = new TokenTrie();
t.AddToken(hello);
t.AddToken(helloBang);
TokenTrie t2 = new TokenTrie();
t.AddToken(hi);
t.AddToken(there);
TokenTrie combined = new TokenTrie();
combined.Append(t);
combined.Append(t2);
byte[] source1 = Encoding.UTF8.GetBytes("hello there");
byte[] source2 = Encoding.UTF8.GetBytes("hello! there");
byte[] source3 = Encoding.UTF8.GetBytes("hi there");
byte[] source4 = Encoding.UTF8.GetBytes("there!");
int token;
int pos = 0;
Assert.True(t.GetOperation(source1, source1.Length, ref pos, out token));
Assert.Equal(0, token);
pos = 0;
Assert.True(t.GetOperation(source2, source2.Length, ref pos, out token));
Assert.Equal(1, token);
pos = 0;
Assert.True(t.GetOperation(source3, source3.Length, ref pos, out token));
Assert.Equal(2, token);
pos = 0;
Assert.True(t.GetOperation(source4, source4.Length, ref pos, out token));
Assert.Equal(3, token);
}
public IOperation GetOperation(Encoding encoding, IProcessorState processorState)
{
IToken tokenBytes = StartToken.ToToken(encoding);
IToken endTokenBytes = EndToken.ToToken(encoding);
TokenTrie endTokenMatcher = new TokenTrie();
endTokenMatcher.AddToken(endTokenBytes);
return(new Impl(tokenBytes, endTokenMatcher, this, _id, _initialState));
}
public IOperation GetOperation(Encoding encoding, IProcessorState processorState)
{
byte[] tokenBytes = encoding.GetBytes(StartToken);
byte[] endTokenBytes = encoding.GetBytes(EndToken);
TokenTrie endTokenMatcher = new TokenTrie();
endTokenMatcher.AddToken(endTokenBytes);
return(new Impl(tokenBytes, endTokenMatcher, this, _id));
}
/// <summary>
/// Puts the tokensOfType into the tokenMasterList at indexes which are congruent to typeRemainder mod TokenTypeModulus
/// </summary>
/// <param name="trie"></param>
/// <param name="tokenMasterList"></param>
/// <param name="tokensOfType"></param>
/// <param name="typeRemainder"></param>
/// <param name="encoding"></param>
private void AddTokensOfTypeToTokenListAndTrie(TokenTrie trie, List <byte[]> tokenMasterList, IReadOnlyList <string> tokensOfType, int typeRemainder, Encoding encoding)
{
int tokenIndex = typeRemainder;
for (int i = 0; i < tokensOfType.Count; i++)
{
byte[] byteToken = encoding.GetBytes(tokensOfType[i]);
tokenMasterList[tokenIndex] = byteToken;
trie.AddToken(byteToken, typeRemainder);
tokenIndex += TokenTypeModulus;
}
}
static GlobbingPatternMatcher()
{
Trie = new TokenTrie();
byte[] anyNumberOfPathParts = Encoding.UTF8.GetBytes("**/");
byte[] onePathPart = Encoding.UTF8.GetBytes("*");
byte[] wildcard = Encoding.UTF8.GetBytes("?");
byte[] openCharSet = Encoding.UTF8.GetBytes("[");
byte[] closeCharSet = Encoding.UTF8.GetBytes("]");
byte[] separatorChar = Encoding.UTF8.GetBytes("/");
byte[] separatorChar2 = Encoding.UTF8.GetBytes("\\");
Trie.AddToken(anyNumberOfPathParts);
Trie.AddToken(onePathPart);
Trie.AddToken(wildcard);
Trie.AddToken(openCharSet);
Trie.AddToken(closeCharSet);
Trie.AddToken(separatorChar);
Trie.AddToken(separatorChar2);
}
public void VerifyTokenTrieLookArounds(string original, int checkPosition, int expectedPosition, bool success, string after, string value, string before)
{
byte[] data = Encoding.UTF8.GetBytes(original);
TokenTrie t = new TokenTrie();
TokenConfig builder = (value ?? "").TokenConfigBuilder();
if (!string.IsNullOrEmpty(after))
{
builder = builder.OnlyIfAfter(after);
}
if (!string.IsNullOrEmpty(before))
{
builder = builder.OnlyIfBefore(before);
}
t.AddToken(builder.ToToken(Encoding.UTF8));
int pos = checkPosition;
Assert.Equal(success, t.GetOperation(data, data.Length, ref pos, out int token));
Assert.Equal(expectedPosition, pos);
}
protected override ITokenTrie GetSymbols(IProcessorState processor)
{
if (!TokenCache.TryGetValue(processor.Encoding, out ITokenTrie tokens))
{
TokenTrie trie = new TokenTrie();
//Logic
trie.AddToken(processor.Encoding.GetBytes("AND"));
trie.AddToken(processor.Encoding.GetBytes("OR"));
trie.AddToken(processor.Encoding.GetBytes("!"));
trie.AddToken(processor.Encoding.GetBytes(">"));
trie.AddToken(processor.Encoding.GetBytes(">="));
trie.AddToken(processor.Encoding.GetBytes("<"));
trie.AddToken(processor.Encoding.GetBytes("<="));
trie.AddToken(processor.Encoding.GetBytes("=="));
trie.AddToken(processor.Encoding.GetBytes("!="));
//Braces
trie.AddToken(processor.Encoding.GetBytes("("));
trie.AddToken(processor.Encoding.GetBytes(")"));
//Whitespace
trie.AddToken(processor.Encoding.GetBytes(" "));
trie.AddToken(processor.Encoding.GetBytes("\t"));
//EOLs
trie.AddToken(processor.Encoding.GetBytes("\r\n"));
trie.AddToken(processor.Encoding.GetBytes("\n"));
trie.AddToken(processor.Encoding.GetBytes("\r"));
// quotes
trie.AddToken(processor.Encoding.GetBytes("'"));
// variable start
trie.AddToken(processor.Encoding.GetBytes("$("));
TokenCache[processor.Encoding] = tokens = trie;
}
return(tokens);
}
public static bool Evaluate(IProcessorState processor, ref int bufferLength, ref int currentBufferPosition, out bool faulted)
{
faulted = false;
TokenTrie trie = new TokenTrie();
//Logic
trie.AddToken(processor.Encoding.GetBytes("&&"), 0);
trie.AddToken(processor.Encoding.GetBytes("||"), 1);
trie.AddToken(processor.Encoding.GetBytes("^"), 2);
trie.AddToken(processor.Encoding.GetBytes("!"), 3);
trie.AddToken(processor.Encoding.GetBytes(">"), 4);
trie.AddToken(processor.Encoding.GetBytes(">="), 5);
trie.AddToken(processor.Encoding.GetBytes("<"), 6);
trie.AddToken(processor.Encoding.GetBytes("<="), 7);
trie.AddToken(processor.Encoding.GetBytes("=="), 8);
trie.AddToken(processor.Encoding.GetBytes("="), 9);
trie.AddToken(processor.Encoding.GetBytes("!="), 10);
//Bitwise
trie.AddToken(processor.Encoding.GetBytes("&"), 11);
trie.AddToken(processor.Encoding.GetBytes("|"), 12);
trie.AddToken(processor.Encoding.GetBytes("<<"), 13);
trie.AddToken(processor.Encoding.GetBytes(">>"), 14);
//Braces
trie.AddToken(processor.Encoding.GetBytes("("), 15);
trie.AddToken(processor.Encoding.GetBytes(")"), 16);
//Whitespace
trie.AddToken(processor.Encoding.GetBytes(" "), 17);
trie.AddToken(processor.Encoding.GetBytes("\t"), 18);
//EOLs
trie.AddToken(processor.Encoding.GetBytes("\r\n"), 19);
trie.AddToken(processor.Encoding.GetBytes("\n"), 20);
trie.AddToken(processor.Encoding.GetBytes("\r"), 21);
// quotes
trie.AddToken(processor.Encoding.GetBytes("\""), 22);
trie.AddToken(processor.Encoding.GetBytes("'"), 23);
//Tokens
trie.Append(processor.EncodingConfig.Variables);
//Run forward to EOL and collect args
TokenFamily currentTokenFamily;
List <byte> currentTokenBytes = new List <byte>();
List <TokenRef> tokens = new List <TokenRef>();
if (!trie.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out int token))
{
currentTokenFamily = TokenFamily.Literal;
currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]);
}
else if (token > ReservedTokenMaxIndex)
{
currentTokenFamily = TokenFamily.Reference | (TokenFamily)token;
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
currentTokenFamily = (TokenFamily)token;
if (currentTokenFamily != TokenFamily.WindowsEOL && currentTokenFamily != TokenFamily.LegacyMacEOL && currentTokenFamily != TokenFamily.UnixEOL)
{
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues));
}
}
int braceDepth = 0;
if (tokens[0].Family == TokenFamily.OpenBrace)
{
++braceDepth;
}
bool first = true;
QuotedRegionKind inQuoteType = QuotedRegionKind.None;
while ((first || braceDepth > 0) && bufferLength > 0)
{
int targetLen = Math.Min(bufferLength, trie.MaxLength);
for (; currentBufferPosition < bufferLength - targetLen + 1;)
{
int oldBufferPos = currentBufferPosition;
if (trie.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out token))
{
if (braceDepth == 0)
{
switch (tokens[tokens.Count - 1].Family)
{
case TokenFamily.Whitespace:
case TokenFamily.Tab:
case TokenFamily.CloseBrace:
case TokenFamily.WindowsEOL:
case TokenFamily.UnixEOL:
case TokenFamily.LegacyMacEOL:
TokenFamily thisFamily = (TokenFamily)token;
if (thisFamily == TokenFamily.WindowsEOL || thisFamily == TokenFamily.UnixEOL || thisFamily == TokenFamily.LegacyMacEOL)
{
currentBufferPosition = oldBufferPos;
}
break;
default:
currentBufferPosition = oldBufferPos;
first = false;
break;
}
if (!first)
{
break;
}
}
// We matched an item, so whatever this is, it's not a literal.
// if the current token is a literal, end it.
if (currentTokenFamily == TokenFamily.Literal)
{
string literal = processor.Encoding.GetString(currentTokenBytes.ToArray());
tokens.Add(new TokenRef
{
Family = TokenFamily.Literal,
Literal = literal
});
currentTokenBytes.Clear();
}
TokenFamily foundTokenFamily = (TokenFamily)token;
if (foundTokenFamily == TokenFamily.QuotedLiteral || foundTokenFamily == TokenFamily.SingleQuotedLiteral)
{
QuotedRegionKind incomingQuoteKind;
switch (foundTokenFamily)
{
case TokenFamily.QuotedLiteral:
incomingQuoteKind = QuotedRegionKind.DoubleQuoteRegion;
break;
case TokenFamily.SingleQuotedLiteral:
incomingQuoteKind = QuotedRegionKind.SingleQuoteRegion;
break;
default:
incomingQuoteKind = QuotedRegionKind.None;
break;
}
if (inQuoteType == QuotedRegionKind.None)
{
// starting quote found
currentTokenBytes.AddRange(trie.Tokens[token].Value);
inQuoteType = incomingQuoteKind;
}
else if (incomingQuoteKind == inQuoteType)
{
// end quote found
currentTokenBytes.AddRange(trie.Tokens[token].Value);
tokens.Add(new TokenRef
{
Family = TokenFamily.Literal,
Literal = processor.Encoding.GetString(currentTokenBytes.ToArray())
});
currentTokenBytes.Clear();
inQuoteType = QuotedRegionKind.None;
}
else
{
// this is a different quote type. Treat it like a non-match, just add the token to the currentTokenBytes
currentTokenBytes.AddRange(trie.Tokens[token].Value);
}
}
else if (inQuoteType != QuotedRegionKind.None)
{
// we're inside a quoted literal, the token found by the trie should not be processed, just included with the literal
currentTokenBytes.AddRange(trie.Tokens[token].Value);
}
else if (token > ReservedTokenMaxIndex)
{
currentTokenFamily = TokenFamily.Reference | (TokenFamily)token;
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
//If we have a normal token...
currentTokenFamily = (TokenFamily)token;
if (currentTokenFamily != TokenFamily.WindowsEOL && currentTokenFamily != TokenFamily.LegacyMacEOL && currentTokenFamily != TokenFamily.UnixEOL)
{
switch (currentTokenFamily)
{
case TokenFamily.OpenBrace:
++braceDepth;
break;
case TokenFamily.CloseBrace:
--braceDepth;
break;
}
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues));
}
}
}
else if (inQuoteType != QuotedRegionKind.None)
{
// we're in a quoted literal but did not match a token at the current position.
// so just add the current byte to the currentTokenBytes
currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]);
}
else if (braceDepth > 0)
{
currentTokenFamily = TokenFamily.Literal;
currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]);
}
else
{
first = false;
break;
}
}
processor.AdvanceBuffer(currentBufferPosition);
currentBufferPosition = processor.CurrentBufferPosition;
bufferLength = processor.CurrentBufferLength;
}
#if DEBUG
Debug.Assert(
inQuoteType == QuotedRegionKind.None,
$"Malformed predicate due to unmatched quotes. InitialBuffer = {processor.Encoding.GetString(processor.CurrentBuffer)} currentTokenFamily = {currentTokenFamily} | TokenFamily.QuotedLiteral = {TokenFamily.QuotedLiteral} | TokenFamily.SingleQuotedLiteral = {TokenFamily.SingleQuotedLiteral}");
#endif
return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues));
}
public static bool CppStyleEvaluator(IProcessorState processor, ref int bufferLength, ref int currentBufferPosition)
{
TokenTrie trie = new TokenTrie();
//Logic
trie.AddToken(processor.Encoding.GetBytes("&&"), 0);
trie.AddToken(processor.Encoding.GetBytes("||"), 1);
trie.AddToken(processor.Encoding.GetBytes("^"), 2);
trie.AddToken(processor.Encoding.GetBytes("!"), 3);
trie.AddToken(processor.Encoding.GetBytes(">"), 4);
trie.AddToken(processor.Encoding.GetBytes(">="), 5);
trie.AddToken(processor.Encoding.GetBytes("<"), 6);
trie.AddToken(processor.Encoding.GetBytes("<="), 7);
trie.AddToken(processor.Encoding.GetBytes("=="), 8);
trie.AddToken(processor.Encoding.GetBytes("="), 9);
trie.AddToken(processor.Encoding.GetBytes("!="), 10);
//Bitwise
trie.AddToken(processor.Encoding.GetBytes("&"), 11);
trie.AddToken(processor.Encoding.GetBytes("|"), 12);
trie.AddToken(processor.Encoding.GetBytes("<<"), 13);
trie.AddToken(processor.Encoding.GetBytes(">>"), 14);
//Braces
trie.AddToken(processor.Encoding.GetBytes("("), 15);
trie.AddToken(processor.Encoding.GetBytes(")"), 16);
//Whitespace
trie.AddToken(processor.Encoding.GetBytes(" "), 17);
trie.AddToken(processor.Encoding.GetBytes("\t"), 18);
//EOLs
trie.AddToken(processor.Encoding.GetBytes("\r\n"), 19);
trie.AddToken(processor.Encoding.GetBytes("\n"), 20);
trie.AddToken(processor.Encoding.GetBytes("\r"), 21);
//Tokens
trie.Append(processor.EncodingConfig.Variables);
//Run forward to EOL and collect args
TokenFamily currentTokenFamily;
List <byte> currentTokenBytes = new List <byte>();
List <TokenRef> tokens = new List <TokenRef>();
int token;
if (!trie.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out token))
{
currentTokenFamily = TokenFamily.Literal;
currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]);
}
else if (token > ReservedTokenMaxIndex)
{
currentTokenFamily = TokenFamily.Reference | (TokenFamily)token;
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
currentTokenFamily = (TokenFamily)token;
if (currentTokenFamily != TokenFamily.WindowsEOL && currentTokenFamily != TokenFamily.LegacyMacEOL && currentTokenFamily != TokenFamily.UnixEOL)
{
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues));
}
}
int braceDepth = 0;
if (tokens[0].Family == TokenFamily.OpenBrace)
{
++braceDepth;
}
bool first = true;
while ((first || braceDepth > 0) && bufferLength > 0)
{
int targetLen = Math.Min(bufferLength, trie.MaxLength);
for (; currentBufferPosition < bufferLength - targetLen + 1;)
{
int oldBufferPos = currentBufferPosition;
if (trie.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out token))
{
if (braceDepth == 0)
{
switch (tokens[tokens.Count - 1].Family)
{
case TokenFamily.Whitespace:
case TokenFamily.Tab:
case TokenFamily.CloseBrace:
case TokenFamily.WindowsEOL:
case TokenFamily.UnixEOL:
case TokenFamily.LegacyMacEOL:
TokenFamily thisFamily = (TokenFamily)token;
if (thisFamily == TokenFamily.WindowsEOL || thisFamily == TokenFamily.UnixEOL || thisFamily == TokenFamily.LegacyMacEOL)
{
currentBufferPosition = oldBufferPos;
}
break;
default:
currentBufferPosition = oldBufferPos;
first = false;
break;
}
if (!first)
{
break;
}
}
//We matched an item, so whatever this is, it's not a literal, end the current literal if that's
// what we currently have
if (currentTokenFamily == TokenFamily.Literal)
{
string literal = processor.Encoding.GetString(currentTokenBytes.ToArray());
tokens.Add(new TokenRef
{
Family = TokenFamily.Literal,
Literal = literal
});
currentTokenBytes.Clear();
}
//If we have a token from the args...
if (token > ReservedTokenMaxIndex)
{
if (currentTokenFamily == TokenFamily.Literal)
{
TokenRef previous = tokens[tokens.Count - 1];
previous.Literal += processor.Encoding.GetString(currentTokenBytes.ToArray());
currentTokenBytes = processor.Encoding.GetBytes(previous.Literal).ToList();
tokens.RemoveAt(tokens.Count - 1);
}
else
{
currentTokenFamily = TokenFamily.Reference | (TokenFamily)token;
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
}
//If we have a normal token...
else
{
currentTokenFamily = (TokenFamily)token;
if (currentTokenFamily != TokenFamily.WindowsEOL && currentTokenFamily != TokenFamily.LegacyMacEOL && currentTokenFamily != TokenFamily.UnixEOL)
{
if (currentTokenFamily == TokenFamily.OpenBrace)
{
++braceDepth;
}
else if (currentTokenFamily == TokenFamily.CloseBrace)
{
--braceDepth;
}
tokens.Add(new TokenRef
{
Family = currentTokenFamily
});
}
else
{
return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues));
}
}
}
else if (braceDepth > 0)
{
currentTokenFamily = TokenFamily.Literal;
currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]);
}
else
{
first = false;
break;
}
}
processor.AdvanceBuffer(currentBufferPosition);
currentBufferPosition = processor.CurrentBufferPosition;
bufferLength = processor.CurrentBufferLength;
}
return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues));
}
private static ITokenTrie GetSymbols(IProcessorState processor)
{
if (!TokenCache.TryGetValue(processor.Encoding, out ITokenTrie tokens))
{
TokenTrie trie = new TokenTrie();
//Logic
trie.AddToken(processor.Encoding.GetBytes("&&"));
trie.AddToken(processor.Encoding.GetBytes("||"));
trie.AddToken(processor.Encoding.GetBytes("!"));
trie.AddToken(processor.Encoding.GetBytes(">"));
trie.AddToken(processor.Encoding.GetBytes(">="));
trie.AddToken(processor.Encoding.GetBytes("<"));
trie.AddToken(processor.Encoding.GetBytes("<="));
trie.AddToken(processor.Encoding.GetBytes("=="));
trie.AddToken(processor.Encoding.GetBytes("!="));
//Braces
trie.AddToken(processor.Encoding.GetBytes("("));
trie.AddToken(processor.Encoding.GetBytes(")"));
//Whitespace
trie.AddToken(processor.Encoding.GetBytes(" "));
trie.AddToken(processor.Encoding.GetBytes("\t"));
//EOLs
trie.AddToken(processor.Encoding.GetBytes("\r\n"));
trie.AddToken(processor.Encoding.GetBytes("\n"));
trie.AddToken(processor.Encoding.GetBytes("\r"));
// quotes
trie.AddToken(processor.Encoding.GetBytes("'"));
//Shifts
trie.AddToken(processor.Encoding.GetBytes("<<"));
trie.AddToken(processor.Encoding.GetBytes(">>"));
//Maths
trie.AddToken(processor.Encoding.GetBytes("+"));
trie.AddToken(processor.Encoding.GetBytes("-"));
trie.AddToken(processor.Encoding.GetBytes("*"));
trie.AddToken(processor.Encoding.GetBytes("/"));
//Bitwise operators
trie.AddToken(processor.Encoding.GetBytes("&"));
trie.AddToken(processor.Encoding.GetBytes("|"));
// quotes
trie.AddToken(processor.Encoding.GetBytes("'"));
trie.AddToken(processor.Encoding.GetBytes("\""));
TokenCache[processor.Encoding] = tokens = trie;
}
return(tokens);
}
protected override ITokenTrie GetSymbols(IProcessorState processor)
{
if (!TokenCache.TryGetValue(processor.Encoding, out ITokenTrie tokens))
{
TokenTrie trie = new TokenTrie();
//Logic
trie.AddToken(processor.Encoding.GetBytes("And"));
trie.AddToken(processor.Encoding.GetBytes("AndAlso"));
trie.AddToken(processor.Encoding.GetBytes("Or"));
trie.AddToken(processor.Encoding.GetBytes("OrElse"));
trie.AddToken(processor.Encoding.GetBytes("Not"));
trie.AddToken(processor.Encoding.GetBytes(">"));
trie.AddToken(processor.Encoding.GetBytes(">="));
trie.AddToken(processor.Encoding.GetBytes("<"));
trie.AddToken(processor.Encoding.GetBytes("<="));
trie.AddToken(processor.Encoding.GetBytes("="));
trie.AddToken(processor.Encoding.GetBytes("<>"));
trie.AddToken(processor.Encoding.GetBytes("Xor"));
//Braces
trie.AddToken(processor.Encoding.GetBytes("("));
trie.AddToken(processor.Encoding.GetBytes(")"));
//Whitespace
trie.AddToken(processor.Encoding.GetBytes(" "));
trie.AddToken(processor.Encoding.GetBytes("\t"));
//EOLs
trie.AddToken(processor.Encoding.GetBytes("\r\n"));
trie.AddToken(processor.Encoding.GetBytes("\n"));
trie.AddToken(processor.Encoding.GetBytes("\r"));
// quotes
trie.AddToken(processor.Encoding.GetBytes("'"));
//Shifts
trie.AddToken(processor.Encoding.GetBytes("<<"));
trie.AddToken(processor.Encoding.GetBytes(">>"));
//Maths
trie.AddToken(processor.Encoding.GetBytes("+"));
trie.AddToken(processor.Encoding.GetBytes("-"));
trie.AddToken(processor.Encoding.GetBytes("*"));
trie.AddToken(processor.Encoding.GetBytes("/"));
trie.AddToken(processor.Encoding.GetBytes("^"));
// quotes
trie.AddToken(processor.Encoding.GetBytes("\""));
TokenCache[processor.Encoding] = tokens = trie;
}
return(tokens);
}