public void SeekBackUntil(ITokenTrie match, bool consume) { byte[] buffer = new byte[match.MaxLength]; while (_target.Position > _bomSize) { if (_target.Position - _bomSize < buffer.Length) { _target.Position = _bomSize; } else { _target.Position -= buffer.Length; } int nRead = ReadExactBytes(_target, buffer, 0, buffer.Length); int best = -1; int bestPos = -1; for (int i = nRead - match.MinLength; i >= 0; --i) { int token; int ic = i; if (match.GetOperation(buffer, nRead, ref ic, out token) && ic >= bestPos) { bestPos = ic; best = token; } } if (best != -1) { _target.Position -= nRead - bestPos + (consume ? match.TokenLength[best] : 0); _target.SetLength(_target.Position); return; } //Back up the amount we already read to get a new window of data in if (_target.Position - _bomSize < buffer.Length) { _target.Position = _bomSize; } else { _target.Position -= buffer.Length; } } if (_target.Position == _bomSize) { _target.SetLength(_bomSize); } }
public void SeekBackWhile(ITokenTrie match) { byte[] buffer = new byte[match.MaxLength]; while (_target.Position > _bomSize) { if (_target.Position - _bomSize < buffer.Length) { _target.Position = _bomSize; } else { _target.Position -= buffer.Length; } int nRead = ReadExactBytes(_target, buffer, 0, buffer.Length); bool anyMatch = false; int token = -1; int i = nRead - match.MinLength; for (; i >= 0; --i) { if (match.GetOperation(buffer, nRead, ref i, out token)) { i -= match.TokenLength[token]; anyMatch = true; break; } } if (!anyMatch || (token != -1 && i + match.TokenLength[token] != nRead)) { _target.SetLength(_target.Position); return; } //Back up the amount we already read to get a new window of data in if (_target.Position - _bomSize < buffer.Length) { _target.Position = _bomSize; } else { _target.Position -= buffer.Length; } } if (_target.Position == _bomSize) { _target.SetLength(_bomSize); } }
private void ScanToCloseCondition(IProcessorState processorState, List <byte> conditionBytes, ref int bufferLength, ref int currentBufferPosition) { int previousPosition = currentBufferPosition; while (bufferLength >= _closeConditionTrie.MinLength) { //Try to get at least the max length of the tree into the buffer if (bufferLength - currentBufferPosition < _closeConditionTrie.MaxLength) { conditionBytes.AddRange(processorState.CurrentBuffer.Skip(previousPosition).Take(currentBufferPosition - previousPosition)); processorState.AdvanceBuffer(currentBufferPosition); currentBufferPosition = processorState.CurrentBufferPosition; bufferLength = processorState.CurrentBufferLength; previousPosition = 0; } int sz = bufferLength == processorState.CurrentBuffer.Length ? _closeConditionTrie.MaxLength : _closeConditionTrie.MinLength; for (; currentBufferPosition < bufferLength - sz + 1; ++currentBufferPosition) { if (bufferLength == 0) { currentBufferPosition = 0; return; } int token; if (_closeConditionTrie.GetOperation(processorState.CurrentBuffer, bufferLength, ref currentBufferPosition, out token)) { conditionBytes.AddRange(processorState.CurrentBuffer.Skip(previousPosition).Take(currentBufferPosition - previousPosition - _closeConditionTrie.Tokens[token].Length)); return; } } } //Ran out of places to check and haven't reached the actual match, consume all the way to the end currentBufferPosition = bufferLength; }
private void BaseSeekForward(ITokenTrie match, ref int bufferLength, ref int currentBufferPosition, bool consumeToken) { while (bufferLength >= match.MinLength) { //Try to get at least the max length of the tree into the buffer if (bufferLength - currentBufferPosition < match.MaxLength) { AdvanceBuffer(currentBufferPosition); currentBufferPosition = CurrentBufferPosition; bufferLength = CurrentBufferLength; } int sz = bufferLength == CurrentBuffer.Length ? match.MaxLength : match.MinLength; for (; currentBufferPosition < bufferLength - sz + 1; ++currentBufferPosition) { if (bufferLength == 0) { currentBufferPosition = 0; return; } int token; if (match.GetOperation(CurrentBuffer, bufferLength, ref currentBufferPosition, out token)) { if (!consumeToken) { currentBufferPosition -= match.Tokens[token].Length; } return; } } } //Ran out of places to check and haven't reached the actual match, consume all the way to the end currentBufferPosition = bufferLength; }
public void SeekForwardWhile(ITokenTrie trie, ref int bufferLength, ref int currentBufferPosition) { while (bufferLength > trie.MinLength) { while (currentBufferPosition < bufferLength - trie.MinLength + 1) { if (bufferLength == 0) { currentBufferPosition = 0; return; } int token; if (!trie.GetOperation(CurrentBuffer, bufferLength, ref currentBufferPosition, out token)) { return; } } AdvanceBuffer(currentBufferPosition); currentBufferPosition = CurrentBufferPosition; bufferLength = CurrentBufferLength; } }
private void FindEnd(IProcessorState processorState, ref int bufferLength, ref int currentBufferPosition) { int depth = 1; bool inElement = true; while (bufferLength >= _structureTrie.MinLength) { //Try to get at least the max length of the tree into the buffer if (bufferLength - currentBufferPosition < _structureTrie.MaxLength) { processorState.AdvanceBuffer(currentBufferPosition); currentBufferPosition = processorState.CurrentBufferPosition; bufferLength = processorState.CurrentBufferLength; } int sz = bufferLength == processorState.CurrentBuffer.Length ? _structureTrie.MaxLength : _structureTrie.MinLength; for (; currentBufferPosition < bufferLength - sz + 1; ++currentBufferPosition) { if (bufferLength == 0) { currentBufferPosition = 0; return; } int token; if (_structureTrie.GetOperation(processorState.CurrentBuffer, bufferLength, ref currentBufferPosition, out token)) { if (token == _mapping.OpenOpenElementToken) { ++depth; inElement = true; } else if (token == _mapping.SelfClosingElementEndToken) { --depth; inElement = false; } else if (token == _mapping.CloseElementTagToken) { if (inElement) { inElement = false; } else { --depth; } } else if (token == _mapping.OpenCloseElementToken) { inElement = false; } if (depth == 0) { return; } } } } //Ran out of places to check and haven't reached the actual match, consume all the way to the end currentBufferPosition = bufferLength; }
public int HandleMatch(IProcessorState processor, int bufferLength, ref int currentBufferPosition, int token, Stream target) { bool flag; if (processor.Config.Flags.TryGetValue(OperationName, out flag) && !flag) { target.Write(Tokens[token].Value, Tokens[token].Start, Tokens[token].Length); return(Tokens[token].Length); } List <byte> pathBytes = new List <byte>(); while (!_endTokenMatcher.GetOperation(processor.CurrentBuffer, bufferLength, ref currentBufferPosition, out token)) { pathBytes.Add(processor.CurrentBuffer[currentBufferPosition++]); if (bufferLength - currentBufferPosition < _endTokenMatcher.MinLength) { processor.AdvanceBuffer(currentBufferPosition); bufferLength = processor.CurrentBufferLength; currentBufferPosition = 0; if (bufferLength == 0) { break; } } } byte[] pathBytesArray = pathBytes.ToArray(); string sourceLocation = processor.Encoding.GetString(pathBytesArray).Trim(); const int pageSize = 65536; //Start off with a 64K buffer, we'll keep adding chunks to this byte[] composite = new byte[pageSize]; int totalLength; using (Stream data = _source.SourceStreamOpener(sourceLocation)) { int index = composite.Length - pageSize; int nRead = data.Read(composite, index, pageSize); //As long as we're reading whole pages, keep allocating more space ahead while (nRead == pageSize) { byte[] newBuffer = new byte[composite.Length + pageSize]; Buffer.BlockCopy(composite, 0, newBuffer, 0, composite.Length); composite = newBuffer; nRead = data.Read(composite, index, pageSize); } totalLength = composite.Length - (pageSize - nRead); } byte[] bom; Encoding realEncoding = EncodingUtil.Detect(composite, totalLength, out bom); if (!Equals(realEncoding, processor.Encoding)) { composite = Encoding.Convert(realEncoding, processor.Encoding, composite, bom.Length, totalLength - bom.Length); totalLength = composite.Length; } target.Write(composite, 0, totalLength - bom.Length); return(composite.Length); }