public void AddToken(byte[] token, int index)
{
++Count;
SimpleTrie current = this;
_tokenLength.Add(token.Length);
_tokens.Add(token);
MaxLength = Math.Max(MaxLength, token.Length);
MinLength = MinLength == 0
? token.Length
: Math.Min(MinLength, token.Length);
for (int i = 0; i < token.Length; ++i)
{
SimpleTrie child;
if (!current._map.TryGetValue(token[i], out child))
{
child = new SimpleTrie();
current._map[token[i]] = child;
}
if (i == token.Length - 1)
{
child.Index = index;
}
current = child;
}
}
public Impl(Conditional definition, IReadOnlyList <byte[]> tokens, int elseIfTokenIndex, SimpleTrie trie)
{
_trie = trie;
_elseIfTokenIndex = elseIfTokenIndex;
_definition = definition;
Tokens = tokens;
}
public void Append(SimpleTrie trie)
{
foreach (byte[] token in trie._tokens)
{
AddToken(token);
}
}
public EncodingConfig(EngineConfig config, Encoding encoding)
{
_variableKeys = new List <byte[]>();
Encoding = encoding;
LineEndings = new SimpleTrie();
Whitespace = new SimpleTrie();
WhitespaceOrLineEnding = new SimpleTrie();
Variables = new SimpleTrie();
foreach (string token in config.LineEndings)
{
byte[] tokenBytes = encoding.GetBytes(token);
LineEndings.AddToken(tokenBytes);
WhitespaceOrLineEnding.AddToken(tokenBytes);
}
foreach (string token in config.Whitespaces)
{
byte[] tokenBytes = encoding.GetBytes(token);
Whitespace.AddToken(tokenBytes);
WhitespaceOrLineEnding.AddToken(tokenBytes);
}
_values = new Func <object> [config.Variables.Count];
ExpandVariables(config, encoding);
}
public IOperation GetOperation(Encoding encoding, IProcessorState processorState)
{
byte[] ifToken = encoding.GetBytes(_ifToken);
byte[] endToken = encoding.GetBytes(_endIfToken);
List <byte[]> tokens = new List <byte[]>
{
ifToken,
endToken
};
SimpleTrie trie = new SimpleTrie();
trie.AddToken(ifToken, 0);
trie.AddToken(endToken, 1);
int elseIfTokenIndex = -1;
if (!string.IsNullOrEmpty(_elseToken))
{
byte[] elseToken = encoding.GetBytes(_elseToken);
trie.AddToken(elseToken, tokens.Count);
tokens.Add(elseToken);
}
if (!string.IsNullOrEmpty(_elseIfToken))
{
byte[] elseIfToken = encoding.GetBytes(_elseIfToken);
elseIfTokenIndex = tokens.Count;
trie.AddToken(elseIfToken, elseIfTokenIndex);
tokens.Add(elseIfToken);
}
return(new Impl(this, tokens, elseIfTokenIndex, trie));
}
public void SeekForwardThrough(SimpleTrie match, ref int bufferLength, ref int currentBufferPosition)
{
while (bufferLength >= match.MinLength)
{
//Try to get at least the max length of the tree into the buffer
if (bufferLength - currentBufferPosition < match.MaxLength)
{
AdvanceBuffer(currentBufferPosition);
currentBufferPosition = CurrentBufferPosition;
bufferLength = CurrentBufferLength;
}
int sz = bufferLength == CurrentBuffer.Length ? match.MaxLength : match.MinLength;
for (; currentBufferPosition < bufferLength - sz + 1; ++currentBufferPosition)
{
if (bufferLength == 0)
{
currentBufferPosition = 0;
return;
}
int token;
if (match.GetOperation(CurrentBuffer, bufferLength, ref currentBufferPosition, out token))
{
return;
}
}
}
//Ran out of places to check and haven't reached the actual match, consume all the way to the end
currentBufferPosition = bufferLength;
}
public IOperation GetOperation(Encoding encoding, IProcessorState processorState)
{
byte[] tokenBytes = encoding.GetBytes(StartToken);
byte[] endTokenBytes = encoding.GetBytes(EndToken);
SimpleTrie endTokenMatcher = new SimpleTrie();
endTokenMatcher.AddToken(endTokenBytes);
return(new Impl(tokenBytes, endTokenMatcher, this));
}
public void SeekBackWhile(SimpleTrie match)
{
byte[] buffer = new byte[match.MaxLength];
while (_target.Position > 0)
{
if (_target.Position < buffer.Length)
{
_target.Position = 0;
}
else
{
_target.Position -= buffer.Length;
}
int nRead = _target.Read(buffer, 0, buffer.Length);
bool anyMatch = false;
int token = -1;
int i = nRead - match.MinLength;
for (; i >= 0; --i)
{
if (match.GetOperation(buffer, nRead, ref i, out token))
{
i -= match.TokenLength[token];
anyMatch = true;
break;
}
}
if (!anyMatch || (token != -1 && i + match.TokenLength[token] != nRead))
{
_target.SetLength(_target.Position);
return;
}
//Back up the amount we already read to get a new window of data in
if (_target.Position < buffer.Length)
{
_target.Position = 0;
}
else
{
_target.Position -= buffer.Length;
}
}
if (_target.Position == 0)
{
_target.SetLength(0);
}
}
public void SeekBackUntil(SimpleTrie match, bool consume)
{
byte[] buffer = new byte[match.MaxLength];
while (_target.Position > 0)
{
if (_target.Position < buffer.Length)
{
_target.Position = 0;
}
else
{
_target.Position -= buffer.Length;
}
int nRead = _target.Read(buffer, 0, buffer.Length);
int best = -1;
int bestPos = -1;
for (int i = nRead - match.MaxLength; i >= 0; --i)
{
int token;
int ic = i;
if (match.GetOperation(buffer, nRead, ref ic, out token))
{
bestPos = ic;
best = token;
}
}
if (best != -1)
{
_target.Position -= nRead - bestPos + (consume ? match.TokenLength[best] : 0);
_target.SetLength(_target.Position);
return;
}
//Back up the amount we already read to get a new window of data in
if (_target.Position < buffer.Length)
{
_target.Position = 0;
}
else
{
_target.Position -= buffer.Length;
}
}
if (_target.Position == 0)
{
_target.SetLength(0);
}
}
public bool GetOperation(byte[] buffer, int bufferLength, ref int currentBufferPosition, out int token)
{
if (bufferLength < MinLength)
{
token = -1;
return(false);
}
int i = currentBufferPosition;
SimpleTrie current = this;
int index = -1;
int offsetToMatch = 0;
while (i < bufferLength)
{
if (!current._map.TryGetValue(buffer[i], out current))
{
token = index;
if (index != -1)
{
currentBufferPosition = i - offsetToMatch;
token = index;
return(true);
}
return(false);
}
if (current.Index != -1)
{
index = current.Index;
offsetToMatch = 0;
}
else
{
++offsetToMatch;
}
++i;
}
if (index != -1)
{
currentBufferPosition = i;
}
token = index;
return(index != -1);
}
public void SeekForwardWhile(SimpleTrie match, ref int bufferLength, ref int currentBufferPosition)
{
while (bufferLength > match.MinLength)
{
while (currentBufferPosition < bufferLength - match.MinLength + 1)
{
if (bufferLength == 0)
{
currentBufferPosition = 0;
return;
}
int token;
if (!match.GetOperation(CurrentBuffer, bufferLength, ref currentBufferPosition, out token))
{
return;
}
}
AdvanceBuffer(currentBufferPosition);
currentBufferPosition = CurrentBufferPosition;
bufferLength = CurrentBufferLength;
}
}
public Impl(byte[] token, SimpleTrie endTokenMatcher, Include source)
{
Tokens = new[] { token };
_source = source;
_endTokenMatcher = endTokenMatcher;
}
public void SeekBackUntil(SimpleTrie match)
{
SeekBackUntil(match, false);
}