public void AddToken(byte[] token, int index) { ++Count; SimpleTrie current = this; _tokenLength.Add(token.Length); _tokens.Add(token); MaxLength = Math.Max(MaxLength, token.Length); MinLength = MinLength == 0 ? token.Length : Math.Min(MinLength, token.Length); for (int i = 0; i < token.Length; ++i) { SimpleTrie child; if (!current._map.TryGetValue(token[i], out child)) { child = new SimpleTrie(); current._map[token[i]] = child; } if (i == token.Length - 1) { child.Index = index; } current = child; } }
public Impl(Conditional definition, IReadOnlyList <byte[]> tokens, int elseIfTokenIndex, SimpleTrie trie) { _trie = trie; _elseIfTokenIndex = elseIfTokenIndex; _definition = definition; Tokens = tokens; }
public void Append(SimpleTrie trie) { foreach (byte[] token in trie._tokens) { AddToken(token); } }
public EncodingConfig(EngineConfig config, Encoding encoding) { _variableKeys = new List <byte[]>(); Encoding = encoding; LineEndings = new SimpleTrie(); Whitespace = new SimpleTrie(); WhitespaceOrLineEnding = new SimpleTrie(); Variables = new SimpleTrie(); foreach (string token in config.LineEndings) { byte[] tokenBytes = encoding.GetBytes(token); LineEndings.AddToken(tokenBytes); WhitespaceOrLineEnding.AddToken(tokenBytes); } foreach (string token in config.Whitespaces) { byte[] tokenBytes = encoding.GetBytes(token); Whitespace.AddToken(tokenBytes); WhitespaceOrLineEnding.AddToken(tokenBytes); } _values = new Func <object> [config.Variables.Count]; ExpandVariables(config, encoding); }
public IOperation GetOperation(Encoding encoding, IProcessorState processorState) { byte[] ifToken = encoding.GetBytes(_ifToken); byte[] endToken = encoding.GetBytes(_endIfToken); List <byte[]> tokens = new List <byte[]> { ifToken, endToken }; SimpleTrie trie = new SimpleTrie(); trie.AddToken(ifToken, 0); trie.AddToken(endToken, 1); int elseIfTokenIndex = -1; if (!string.IsNullOrEmpty(_elseToken)) { byte[] elseToken = encoding.GetBytes(_elseToken); trie.AddToken(elseToken, tokens.Count); tokens.Add(elseToken); } if (!string.IsNullOrEmpty(_elseIfToken)) { byte[] elseIfToken = encoding.GetBytes(_elseIfToken); elseIfTokenIndex = tokens.Count; trie.AddToken(elseIfToken, elseIfTokenIndex); tokens.Add(elseIfToken); } return(new Impl(this, tokens, elseIfTokenIndex, trie)); }
public void SeekForwardThrough(SimpleTrie match, ref int bufferLength, ref int currentBufferPosition) { while (bufferLength >= match.MinLength) { //Try to get at least the max length of the tree into the buffer if (bufferLength - currentBufferPosition < match.MaxLength) { AdvanceBuffer(currentBufferPosition); currentBufferPosition = CurrentBufferPosition; bufferLength = CurrentBufferLength; } int sz = bufferLength == CurrentBuffer.Length ? match.MaxLength : match.MinLength; for (; currentBufferPosition < bufferLength - sz + 1; ++currentBufferPosition) { if (bufferLength == 0) { currentBufferPosition = 0; return; } int token; if (match.GetOperation(CurrentBuffer, bufferLength, ref currentBufferPosition, out token)) { return; } } } //Ran out of places to check and haven't reached the actual match, consume all the way to the end currentBufferPosition = bufferLength; }
public IOperation GetOperation(Encoding encoding, IProcessorState processorState) { byte[] tokenBytes = encoding.GetBytes(StartToken); byte[] endTokenBytes = encoding.GetBytes(EndToken); SimpleTrie endTokenMatcher = new SimpleTrie(); endTokenMatcher.AddToken(endTokenBytes); return(new Impl(tokenBytes, endTokenMatcher, this)); }
public void SeekBackWhile(SimpleTrie match) { byte[] buffer = new byte[match.MaxLength]; while (_target.Position > 0) { if (_target.Position < buffer.Length) { _target.Position = 0; } else { _target.Position -= buffer.Length; } int nRead = _target.Read(buffer, 0, buffer.Length); bool anyMatch = false; int token = -1; int i = nRead - match.MinLength; for (; i >= 0; --i) { if (match.GetOperation(buffer, nRead, ref i, out token)) { i -= match.TokenLength[token]; anyMatch = true; break; } } if (!anyMatch || (token != -1 && i + match.TokenLength[token] != nRead)) { _target.SetLength(_target.Position); return; } //Back up the amount we already read to get a new window of data in if (_target.Position < buffer.Length) { _target.Position = 0; } else { _target.Position -= buffer.Length; } } if (_target.Position == 0) { _target.SetLength(0); } }
public void SeekBackUntil(SimpleTrie match, bool consume) { byte[] buffer = new byte[match.MaxLength]; while (_target.Position > 0) { if (_target.Position < buffer.Length) { _target.Position = 0; } else { _target.Position -= buffer.Length; } int nRead = _target.Read(buffer, 0, buffer.Length); int best = -1; int bestPos = -1; for (int i = nRead - match.MaxLength; i >= 0; --i) { int token; int ic = i; if (match.GetOperation(buffer, nRead, ref ic, out token)) { bestPos = ic; best = token; } } if (best != -1) { _target.Position -= nRead - bestPos + (consume ? match.TokenLength[best] : 0); _target.SetLength(_target.Position); return; } //Back up the amount we already read to get a new window of data in if (_target.Position < buffer.Length) { _target.Position = 0; } else { _target.Position -= buffer.Length; } } if (_target.Position == 0) { _target.SetLength(0); } }
public bool GetOperation(byte[] buffer, int bufferLength, ref int currentBufferPosition, out int token) { if (bufferLength < MinLength) { token = -1; return(false); } int i = currentBufferPosition; SimpleTrie current = this; int index = -1; int offsetToMatch = 0; while (i < bufferLength) { if (!current._map.TryGetValue(buffer[i], out current)) { token = index; if (index != -1) { currentBufferPosition = i - offsetToMatch; token = index; return(true); } return(false); } if (current.Index != -1) { index = current.Index; offsetToMatch = 0; } else { ++offsetToMatch; } ++i; } if (index != -1) { currentBufferPosition = i; } token = index; return(index != -1); }
public void SeekForwardWhile(SimpleTrie match, ref int bufferLength, ref int currentBufferPosition) { while (bufferLength > match.MinLength) { while (currentBufferPosition < bufferLength - match.MinLength + 1) { if (bufferLength == 0) { currentBufferPosition = 0; return; } int token; if (!match.GetOperation(CurrentBuffer, bufferLength, ref currentBufferPosition, out token)) { return; } } AdvanceBuffer(currentBufferPosition); currentBufferPosition = CurrentBufferPosition; bufferLength = CurrentBufferLength; } }
public Impl(byte[] token, SimpleTrie endTokenMatcher, Include source) { Tokens = new[] { token }; _source = source; _endTokenMatcher = endTokenMatcher; }
public void SeekBackUntil(SimpleTrie match) { SeekBackUntil(match, false); }