public ScopeBuilder(IProcessorState processor, ITokenTrie tokens, IOperatorMap <TOperator, TToken> operatorMap, bool dereferenceInLiterals) { TokenTrie trie = new TokenTrie(); trie.Append(tokens); _badSyntaxTokens = operatorMap.BadSyntaxTokens; _noops = operatorMap.NoOpTokens; _openGroup = operatorMap.OpenGroupToken; _closeGroup = operatorMap.CloseGroupToken; _literal = operatorMap.LiteralToken; _identity = operatorMap.Identity; _literalSequenceBoundsMarkers = operatorMap.LiteralSequenceBoundsMarkers; _terminators = operatorMap.Terminators; _isSymbolDereferenceInLiteralSequenceRequired = dereferenceInLiterals; _processor = processor; _knownTokensCount = tokens.Count; _valueEncoder = operatorMap.Encode; _valueDecoder = operatorMap.Decode; List <object> symbolValues = new List <object>(); foreach (KeyValuePair <string, object> variable in processor.Config.Variables) { trie.AddToken(processor.Encoding.GetBytes(string.Format(processor.Config.VariableFormatString, variable.Key))); symbolValues.Add(variable.Value); } _symbolValues = symbolValues; _tokenToOperatorMap = operatorMap.TokensToOperatorsMap; _operatorScopeFactory = operatorMap.OperatorScopeLookupFactory; _tokens = trie; }
public void VerifyTokenTrieCombine() { byte[] hello = Encoding.UTF8.GetBytes("hello"); byte[] helloBang = Encoding.UTF8.GetBytes("hello!"); byte[] hi = Encoding.UTF8.GetBytes("hi"); byte[] there = Encoding.UTF8.GetBytes("there!"); TokenTrie t = new TokenTrie(); t.AddToken(hello); t.AddToken(helloBang); TokenTrie t2 = new TokenTrie(); t.AddToken(hi); t.AddToken(there); TokenTrie combined = new TokenTrie(); combined.Append(t); combined.Append(t2); byte[] source1 = Encoding.UTF8.GetBytes("hello there"); byte[] source2 = Encoding.UTF8.GetBytes("hello! there"); byte[] source3 = Encoding.UTF8.GetBytes("hi there"); byte[] source4 = Encoding.UTF8.GetBytes("there!"); int token; int pos = 0; Assert.True(t.GetOperation(source1, source1.Length, ref pos, out token)); Assert.Equal(0, token); pos = 0; Assert.True(t.GetOperation(source2, source2.Length, ref pos, out token)); Assert.Equal(1, token); pos = 0; Assert.True(t.GetOperation(source3, source3.Length, ref pos, out token)); Assert.Equal(2, token); pos = 0; Assert.True(t.GetOperation(source4, source4.Length, ref pos, out token)); Assert.Equal(3, token); }
public static bool Evaluate(IProcessorState processor, ref int bufferLength, ref int currentBufferPosition, out bool faulted) { faulted = false; TokenTrie trie = new TokenTrie(); //Logic trie.AddToken(processor.Encoding.GetBytes("&&"), 0); trie.AddToken(processor.Encoding.GetBytes("||"), 1); trie.AddToken(processor.Encoding.GetBytes("^"), 2); trie.AddToken(processor.Encoding.GetBytes("!"), 3); trie.AddToken(processor.Encoding.GetBytes(">"), 4); trie.AddToken(processor.Encoding.GetBytes(">="), 5); trie.AddToken(processor.Encoding.GetBytes("<"), 6); trie.AddToken(processor.Encoding.GetBytes("<="), 7); trie.AddToken(processor.Encoding.GetBytes("=="), 8); trie.AddToken(processor.Encoding.GetBytes("="), 9); trie.AddToken(processor.Encoding.GetBytes("!="), 10); //Bitwise trie.AddToken(processor.Encoding.GetBytes("&"), 11); trie.AddToken(processor.Encoding.GetBytes("|"), 12); trie.AddToken(processor.Encoding.GetBytes("<<"), 13); trie.AddToken(processor.Encoding.GetBytes(">>"), 14); //Braces trie.AddToken(processor.Encoding.GetBytes("("), 15); trie.AddToken(processor.Encoding.GetBytes(")"), 16); //Whitespace trie.AddToken(processor.Encoding.GetBytes(" "), 17); trie.AddToken(processor.Encoding.GetBytes("\t"), 18); //EOLs trie.AddToken(processor.Encoding.GetBytes("\r\n"), 19); trie.AddToken(processor.Encoding.GetBytes("\n"), 20); trie.AddToken(processor.Encoding.GetBytes("\r"), 21); // quotes trie.AddToken(processor.Encoding.GetBytes("\""), 22); trie.AddToken(processor.Encoding.GetBytes("'"), 23); //Tokens trie.Append(processor.EncodingConfig.Variables); //Run forward to EOL and collect args TokenFamily currentTokenFamily; List <byte> currentTokenBytes = new List <byte>(); List <TokenRef> tokens = new List <TokenRef>(); if (!trie.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out int token)) { currentTokenFamily = TokenFamily.Literal; currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]); } else if (token > ReservedTokenMaxIndex) { currentTokenFamily = TokenFamily.Reference | (TokenFamily)token; tokens.Add(new TokenRef { Family = currentTokenFamily }); } else { currentTokenFamily = (TokenFamily)token; if (currentTokenFamily != TokenFamily.WindowsEOL && currentTokenFamily != TokenFamily.LegacyMacEOL && currentTokenFamily != TokenFamily.UnixEOL) { tokens.Add(new TokenRef { Family = currentTokenFamily }); } else { return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues)); } } int braceDepth = 0; if (tokens[0].Family == TokenFamily.OpenBrace) { ++braceDepth; } bool first = true; QuotedRegionKind inQuoteType = QuotedRegionKind.None; while ((first || braceDepth > 0) && bufferLength > 0) { int targetLen = Math.Min(bufferLength, trie.MaxLength); for (; currentBufferPosition < bufferLength - targetLen + 1;) { int oldBufferPos = currentBufferPosition; if (trie.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out token)) { if (braceDepth == 0) { switch (tokens[tokens.Count - 1].Family) { case TokenFamily.Whitespace: case TokenFamily.Tab: case TokenFamily.CloseBrace: case TokenFamily.WindowsEOL: case TokenFamily.UnixEOL: case TokenFamily.LegacyMacEOL: TokenFamily thisFamily = (TokenFamily)token; if (thisFamily == TokenFamily.WindowsEOL || thisFamily == TokenFamily.UnixEOL || thisFamily == TokenFamily.LegacyMacEOL) { currentBufferPosition = oldBufferPos; } break; default: currentBufferPosition = oldBufferPos; first = false; break; } if (!first) { break; } } // We matched an item, so whatever this is, it's not a literal. // if the current token is a literal, end it. if (currentTokenFamily == TokenFamily.Literal) { string literal = processor.Encoding.GetString(currentTokenBytes.ToArray()); tokens.Add(new TokenRef { Family = TokenFamily.Literal, Literal = literal }); currentTokenBytes.Clear(); } TokenFamily foundTokenFamily = (TokenFamily)token; if (foundTokenFamily == TokenFamily.QuotedLiteral || foundTokenFamily == TokenFamily.SingleQuotedLiteral) { QuotedRegionKind incomingQuoteKind; switch (foundTokenFamily) { case TokenFamily.QuotedLiteral: incomingQuoteKind = QuotedRegionKind.DoubleQuoteRegion; break; case TokenFamily.SingleQuotedLiteral: incomingQuoteKind = QuotedRegionKind.SingleQuoteRegion; break; default: incomingQuoteKind = QuotedRegionKind.None; break; } if (inQuoteType == QuotedRegionKind.None) { // starting quote found currentTokenBytes.AddRange(trie.Tokens[token].Value); inQuoteType = incomingQuoteKind; } else if (incomingQuoteKind == inQuoteType) { // end quote found currentTokenBytes.AddRange(trie.Tokens[token].Value); tokens.Add(new TokenRef { Family = TokenFamily.Literal, Literal = processor.Encoding.GetString(currentTokenBytes.ToArray()) }); currentTokenBytes.Clear(); inQuoteType = QuotedRegionKind.None; } else { // this is a different quote type. Treat it like a non-match, just add the token to the currentTokenBytes currentTokenBytes.AddRange(trie.Tokens[token].Value); } } else if (inQuoteType != QuotedRegionKind.None) { // we're inside a quoted literal, the token found by the trie should not be processed, just included with the literal currentTokenBytes.AddRange(trie.Tokens[token].Value); } else if (token > ReservedTokenMaxIndex) { currentTokenFamily = TokenFamily.Reference | (TokenFamily)token; tokens.Add(new TokenRef { Family = currentTokenFamily }); } else { //If we have a normal token... currentTokenFamily = (TokenFamily)token; if (currentTokenFamily != TokenFamily.WindowsEOL && currentTokenFamily != TokenFamily.LegacyMacEOL && currentTokenFamily != TokenFamily.UnixEOL) { switch (currentTokenFamily) { case TokenFamily.OpenBrace: ++braceDepth; break; case TokenFamily.CloseBrace: --braceDepth; break; } tokens.Add(new TokenRef { Family = currentTokenFamily }); } else { return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues)); } } } else if (inQuoteType != QuotedRegionKind.None) { // we're in a quoted literal but did not match a token at the current position. // so just add the current byte to the currentTokenBytes currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]); } else if (braceDepth > 0) { currentTokenFamily = TokenFamily.Literal; currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]); } else { first = false; break; } } processor.AdvanceBuffer(currentBufferPosition); currentBufferPosition = processor.CurrentBufferPosition; bufferLength = processor.CurrentBufferLength; } #if DEBUG Debug.Assert( inQuoteType == QuotedRegionKind.None, $"Malformed predicate due to unmatched quotes. InitialBuffer = {processor.Encoding.GetString(processor.CurrentBuffer)} currentTokenFamily = {currentTokenFamily} | TokenFamily.QuotedLiteral = {TokenFamily.QuotedLiteral} | TokenFamily.SingleQuotedLiteral = {TokenFamily.SingleQuotedLiteral}"); #endif return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues)); }
public static bool CppStyleEvaluator(IProcessorState processor, ref int bufferLength, ref int currentBufferPosition) { TokenTrie trie = new TokenTrie(); //Logic trie.AddToken(processor.Encoding.GetBytes("&&"), 0); trie.AddToken(processor.Encoding.GetBytes("||"), 1); trie.AddToken(processor.Encoding.GetBytes("^"), 2); trie.AddToken(processor.Encoding.GetBytes("!"), 3); trie.AddToken(processor.Encoding.GetBytes(">"), 4); trie.AddToken(processor.Encoding.GetBytes(">="), 5); trie.AddToken(processor.Encoding.GetBytes("<"), 6); trie.AddToken(processor.Encoding.GetBytes("<="), 7); trie.AddToken(processor.Encoding.GetBytes("=="), 8); trie.AddToken(processor.Encoding.GetBytes("="), 9); trie.AddToken(processor.Encoding.GetBytes("!="), 10); //Bitwise trie.AddToken(processor.Encoding.GetBytes("&"), 11); trie.AddToken(processor.Encoding.GetBytes("|"), 12); trie.AddToken(processor.Encoding.GetBytes("<<"), 13); trie.AddToken(processor.Encoding.GetBytes(">>"), 14); //Braces trie.AddToken(processor.Encoding.GetBytes("("), 15); trie.AddToken(processor.Encoding.GetBytes(")"), 16); //Whitespace trie.AddToken(processor.Encoding.GetBytes(" "), 17); trie.AddToken(processor.Encoding.GetBytes("\t"), 18); //EOLs trie.AddToken(processor.Encoding.GetBytes("\r\n"), 19); trie.AddToken(processor.Encoding.GetBytes("\n"), 20); trie.AddToken(processor.Encoding.GetBytes("\r"), 21); //Tokens trie.Append(processor.EncodingConfig.Variables); //Run forward to EOL and collect args TokenFamily currentTokenFamily; List <byte> currentTokenBytes = new List <byte>(); List <TokenRef> tokens = new List <TokenRef>(); int token; if (!trie.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out token)) { currentTokenFamily = TokenFamily.Literal; currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]); } else if (token > ReservedTokenMaxIndex) { currentTokenFamily = TokenFamily.Reference | (TokenFamily)token; tokens.Add(new TokenRef { Family = currentTokenFamily }); } else { currentTokenFamily = (TokenFamily)token; if (currentTokenFamily != TokenFamily.WindowsEOL && currentTokenFamily != TokenFamily.LegacyMacEOL && currentTokenFamily != TokenFamily.UnixEOL) { tokens.Add(new TokenRef { Family = currentTokenFamily }); } else { return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues)); } } int braceDepth = 0; if (tokens[0].Family == TokenFamily.OpenBrace) { ++braceDepth; } bool first = true; while ((first || braceDepth > 0) && bufferLength > 0) { int targetLen = Math.Min(bufferLength, trie.MaxLength); for (; currentBufferPosition < bufferLength - targetLen + 1;) { int oldBufferPos = currentBufferPosition; if (trie.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out token)) { if (braceDepth == 0) { switch (tokens[tokens.Count - 1].Family) { case TokenFamily.Whitespace: case TokenFamily.Tab: case TokenFamily.CloseBrace: case TokenFamily.WindowsEOL: case TokenFamily.UnixEOL: case TokenFamily.LegacyMacEOL: TokenFamily thisFamily = (TokenFamily)token; if (thisFamily == TokenFamily.WindowsEOL || thisFamily == TokenFamily.UnixEOL || thisFamily == TokenFamily.LegacyMacEOL) { currentBufferPosition = oldBufferPos; } break; default: currentBufferPosition = oldBufferPos; first = false; break; } if (!first) { break; } } //We matched an item, so whatever this is, it's not a literal, end the current literal if that's // what we currently have if (currentTokenFamily == TokenFamily.Literal) { string literal = processor.Encoding.GetString(currentTokenBytes.ToArray()); tokens.Add(new TokenRef { Family = TokenFamily.Literal, Literal = literal }); currentTokenBytes.Clear(); } //If we have a token from the args... if (token > ReservedTokenMaxIndex) { if (currentTokenFamily == TokenFamily.Literal) { TokenRef previous = tokens[tokens.Count - 1]; previous.Literal += processor.Encoding.GetString(currentTokenBytes.ToArray()); currentTokenBytes = processor.Encoding.GetBytes(previous.Literal).ToList(); tokens.RemoveAt(tokens.Count - 1); } else { currentTokenFamily = TokenFamily.Reference | (TokenFamily)token; tokens.Add(new TokenRef { Family = currentTokenFamily }); } } //If we have a normal token... else { currentTokenFamily = (TokenFamily)token; if (currentTokenFamily != TokenFamily.WindowsEOL && currentTokenFamily != TokenFamily.LegacyMacEOL && currentTokenFamily != TokenFamily.UnixEOL) { if (currentTokenFamily == TokenFamily.OpenBrace) { ++braceDepth; } else if (currentTokenFamily == TokenFamily.CloseBrace) { --braceDepth; } tokens.Add(new TokenRef { Family = currentTokenFamily }); } else { return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues)); } } } else if (braceDepth > 0) { currentTokenFamily = TokenFamily.Literal; currentTokenBytes.Add(processor.CurrentBuffer[currentBufferPosition++]); } else { first = false; break; } } processor.AdvanceBuffer(currentBufferPosition); currentBufferPosition = processor.CurrentBufferPosition; bufferLength = processor.CurrentBufferLength; } return(EvaluateCondition(tokens, processor.EncodingConfig.VariableValues)); }