public bool GetOperation(byte[] buffer, int bufferLength, ref int currentBufferPosition, bool mustMatchPosition, out int token) { int originalPosition = currentBufferPosition; TrieEvaluator <Token> evaluator = new TrieEvaluator <Token>(this); TrieEvaluationDriver <Token> driver = new TrieEvaluationDriver <Token>(evaluator); if (mustMatchPosition) { bufferLength = Math.Min(bufferLength, currentBufferPosition + MaxLength); } TerminalLocation <Token> location = driver.Evaluate(buffer, bufferLength, true, 0, ref currentBufferPosition); if (location != null && (!mustMatchPosition || (currentBufferPosition - location.Terminal.Length == originalPosition))) { token = location.Terminal.Index; currentBufferPosition = location.Location + location.Terminal.End - location.Terminal.Start + 1; return(true); } if (mustMatchPosition) { currentBufferPosition = originalPosition; } else { currentBufferPosition = bufferLength - MaxLength + 1; } token = -1; return(false); }
public bool GetOperation(byte[] buffer, int bufferLength, ref int currentBufferPosition, out int token) { int originalPosition = currentBufferPosition; TrieEvaluator <Token> evaluator = new TrieEvaluator <Token>(this); TrieEvaluationDriver <Token> driver = new TrieEvaluationDriver <Token>(evaluator); TerminalLocation <Token> location = driver.Evaluate(buffer, bufferLength, true, 0, ref currentBufferPosition); if (location != null && currentBufferPosition - location.Terminal.Length == originalPosition) { token = location.Terminal.Index; currentBufferPosition = location.Location + location.Terminal.End - location.Terminal.Start + 1; return(true); } currentBufferPosition = originalPosition; token = -1; return(false); }
public IOperation GetOperation(byte[] buffer, int bufferLength, ref int bufferPosition, out int token) { int originalPosition = bufferPosition; TrieEvaluator <OperationTerminal> evaluator = new TrieEvaluator <OperationTerminal>(this); int sn = originalPosition; for (; bufferPosition < bufferLength; ++bufferPosition) { if (evaluator.Accept(buffer[bufferPosition], ref sn, out TerminalLocation <OperationTerminal> terminal)) { if (terminal.Location == originalPosition) { bufferPosition -= sn - terminal.Location - terminal.Terminal.End; token = terminal.Terminal.Token; return(terminal.Terminal.Operation); } else { token = -1; bufferPosition = originalPosition; return(null); } } ++sn; } if (bufferPosition == bufferLength) { evaluator.FinalizeMatchesInProgress(ref sn, out TerminalLocation <OperationTerminal> terminal); if (terminal != null) { bufferPosition -= sn - terminal.Location - terminal.Terminal.End; token = terminal.Terminal.Token; return(terminal.Terminal.Operation); } } bufferPosition = originalPosition; token = -1; return(null); }
public ProcessorState(Stream source, Stream target, int bufferSize, int flushThreshold, IEngineConfig config, IReadOnlyList <IOperationProvider> operationProviders) { if (source == null) { throw new ArgumentNullException(nameof(source)); } if (target == null) { throw new ArgumentNullException(nameof(target)); } if (config == null) { throw new ArgumentNullException(nameof(config)); } if (operationProviders == null) { throw new ArgumentNullException(nameof(operationProviders)); } if (source.CanSeek) { try { if (source.Length < bufferSize) { bufferSize = (int)source.Length; } } catch { //The stream may not support getting the length property (in NetworkStream for instance, which throw a NotSupportedException), suppress any errors in // accessing the property and continue with the specified buffer size } } //Buffer has to be at least as large as the largest BOM we could expect else if (bufferSize < 4) { bufferSize = 4; } _source = source; _target = target; Config = config; _flushThreshold = flushThreshold; CurrentBuffer = new byte[bufferSize]; CurrentBufferLength = ReadExactBytes(source, CurrentBuffer, 0, CurrentBuffer.Length); Encoding encoding = EncodingUtil.Detect(CurrentBuffer, CurrentBufferLength, out byte[] bom); EncodingConfig = new EncodingConfig(Config, encoding); _bomSize = bom.Length; CurrentBufferPosition = _bomSize; CurrentSequenceNumber = _bomSize; target.Write(bom, 0, _bomSize); bool explicitOnConfigurationRequired = false; Dictionary <Encoding, Trie <OperationTerminal> > byEncoding = TrieLookup.GetOrAdd(operationProviders, x => new Dictionary <Encoding, Trie <OperationTerminal> >()); List <string> turnOnByDefault = OperationsToExplicitlySetOnByDefault.GetOrAdd(operationProviders, x => { explicitOnConfigurationRequired = true; return(new List <string>()); }); if (!byEncoding.TryGetValue(encoding, out Trie <OperationTerminal> trie)) { trie = new Trie <OperationTerminal>(); for (int i = 0; i < operationProviders.Count; ++i) { IOperation op = operationProviders[i].GetOperation(encoding, this); if (op != null) { for (int j = 0; j < op.Tokens.Count; ++j) { if (op.Tokens[j] != null) { trie.AddPath(op.Tokens[j].Value, new OperationTerminal(op, j, op.Tokens[j].Length, op.Tokens[j].Start, op.Tokens[j].End)); } } if (explicitOnConfigurationRequired && op.IsInitialStateOn && !string.IsNullOrEmpty(op.Id)) { turnOnByDefault.Add(op.Id); } } } byEncoding[encoding] = trie; } foreach (string state in turnOnByDefault) { config.Flags[state] = true; } _trie = new TrieEvaluator <OperationTerminal>(trie); if (bufferSize < _trie.MaxLength + 1) { byte[] tmp = new byte[_trie.MaxLength + 1]; Buffer.BlockCopy(CurrentBuffer, CurrentBufferPosition, tmp, 0, CurrentBufferLength - CurrentBufferPosition); int nRead = ReadExactBytes(_source, tmp, CurrentBufferLength - CurrentBufferPosition, tmp.Length - CurrentBufferLength); CurrentBuffer = tmp; CurrentBufferLength += nRead - _bomSize; CurrentBufferPosition = 0; CurrentSequenceNumber = 0; } }
public TrieEvaluationDriver(TrieEvaluator <T> trie) { _evaluator = trie; }