public bool GetOperation(byte[] buffer, int bufferLength, ref int currentBufferPosition, bool mustMatchPosition, out int token)
{
int originalPosition = currentBufferPosition;
TrieEvaluator <Token> evaluator = new TrieEvaluator <Token>(this);
TrieEvaluationDriver <Token> driver = new TrieEvaluationDriver <Token>(evaluator);
if (mustMatchPosition)
{
bufferLength = Math.Min(bufferLength, currentBufferPosition + MaxLength);
}
TerminalLocation <Token> location = driver.Evaluate(buffer, bufferLength, true, 0, ref currentBufferPosition);
if (location != null && (!mustMatchPosition || (currentBufferPosition - location.Terminal.Length == originalPosition)))
{
token = location.Terminal.Index;
currentBufferPosition = location.Location + location.Terminal.End - location.Terminal.Start + 1;
return(true);
}
if (mustMatchPosition)
{
currentBufferPosition = originalPosition;
}
else
{
currentBufferPosition = bufferLength - MaxLength + 1;
}
token = -1;
return(false);
}
public bool GetOperation(byte[] buffer, int bufferLength, ref int currentBufferPosition, out int token)
{
int originalPosition = currentBufferPosition;
TrieEvaluator <Token> evaluator = new TrieEvaluator <Token>(this);
TrieEvaluationDriver <Token> driver = new TrieEvaluationDriver <Token>(evaluator);
TerminalLocation <Token> location = driver.Evaluate(buffer, bufferLength, true, 0, ref currentBufferPosition);
if (location != null && currentBufferPosition - location.Terminal.Length == originalPosition)
{
token = location.Terminal.Index;
currentBufferPosition = location.Location + location.Terminal.End - location.Terminal.Start + 1;
return(true);
}
currentBufferPosition = originalPosition;
token = -1;
return(false);
}
public IOperation GetOperation(byte[] buffer, int bufferLength, ref int bufferPosition, out int token)
{
int originalPosition = bufferPosition;
TrieEvaluator <OperationTerminal> evaluator = new TrieEvaluator <OperationTerminal>(this);
int sn = originalPosition;
for (; bufferPosition < bufferLength; ++bufferPosition)
{
if (evaluator.Accept(buffer[bufferPosition], ref sn, out TerminalLocation <OperationTerminal> terminal))
{
if (terminal.Location == originalPosition)
{
bufferPosition -= sn - terminal.Location - terminal.Terminal.End;
token = terminal.Terminal.Token;
return(terminal.Terminal.Operation);
}
else
{
token = -1;
bufferPosition = originalPosition;
return(null);
}
}
++sn;
}
if (bufferPosition == bufferLength)
{
evaluator.FinalizeMatchesInProgress(ref sn, out TerminalLocation <OperationTerminal> terminal);
if (terminal != null)
{
bufferPosition -= sn - terminal.Location - terminal.Terminal.End;
token = terminal.Terminal.Token;
return(terminal.Terminal.Operation);
}
}
bufferPosition = originalPosition;
token = -1;
return(null);
}
public ProcessorState(Stream source, Stream target, int bufferSize, int flushThreshold, IEngineConfig config, IReadOnlyList <IOperationProvider> operationProviders)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
if (target == null)
{
throw new ArgumentNullException(nameof(target));
}
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (operationProviders == null)
{
throw new ArgumentNullException(nameof(operationProviders));
}
if (source.CanSeek)
{
try
{
if (source.Length < bufferSize)
{
bufferSize = (int)source.Length;
}
}
catch
{
//The stream may not support getting the length property (in NetworkStream for instance, which throw a NotSupportedException), suppress any errors in
// accessing the property and continue with the specified buffer size
}
}
//Buffer has to be at least as large as the largest BOM we could expect
else if (bufferSize < 4)
{
bufferSize = 4;
}
_source = source;
_target = target;
Config = config;
_flushThreshold = flushThreshold;
CurrentBuffer = new byte[bufferSize];
CurrentBufferLength = ReadExactBytes(source, CurrentBuffer, 0, CurrentBuffer.Length);
Encoding encoding = EncodingUtil.Detect(CurrentBuffer, CurrentBufferLength, out byte[] bom);
EncodingConfig = new EncodingConfig(Config, encoding);
_bomSize = bom.Length;
CurrentBufferPosition = _bomSize;
CurrentSequenceNumber = _bomSize;
target.Write(bom, 0, _bomSize);
bool explicitOnConfigurationRequired = false;
Dictionary <Encoding, Trie <OperationTerminal> > byEncoding = TrieLookup.GetOrAdd(operationProviders, x => new Dictionary <Encoding, Trie <OperationTerminal> >());
List <string> turnOnByDefault = OperationsToExplicitlySetOnByDefault.GetOrAdd(operationProviders, x =>
{
explicitOnConfigurationRequired = true;
return(new List <string>());
});
if (!byEncoding.TryGetValue(encoding, out Trie <OperationTerminal> trie))
{
trie = new Trie <OperationTerminal>();
for (int i = 0; i < operationProviders.Count; ++i)
{
IOperation op = operationProviders[i].GetOperation(encoding, this);
if (op != null)
{
for (int j = 0; j < op.Tokens.Count; ++j)
{
if (op.Tokens[j] != null)
{
trie.AddPath(op.Tokens[j].Value, new OperationTerminal(op, j, op.Tokens[j].Length, op.Tokens[j].Start, op.Tokens[j].End));
}
}
if (explicitOnConfigurationRequired && op.IsInitialStateOn && !string.IsNullOrEmpty(op.Id))
{
turnOnByDefault.Add(op.Id);
}
}
}
byEncoding[encoding] = trie;
}
foreach (string state in turnOnByDefault)
{
config.Flags[state] = true;
}
_trie = new TrieEvaluator <OperationTerminal>(trie);
if (bufferSize < _trie.MaxLength + 1)
{
byte[] tmp = new byte[_trie.MaxLength + 1];
Buffer.BlockCopy(CurrentBuffer, CurrentBufferPosition, tmp, 0, CurrentBufferLength - CurrentBufferPosition);
int nRead = ReadExactBytes(_source, tmp, CurrentBufferLength - CurrentBufferPosition, tmp.Length - CurrentBufferLength);
CurrentBuffer = tmp;
CurrentBufferLength += nRead - _bomSize;
CurrentBufferPosition = 0;
CurrentSequenceNumber = 0;
}
}
public TrieEvaluationDriver(TrieEvaluator <T> trie)
{
_evaluator = trie;
}