HTMLchunk ParseTextWithEntities()
{
// okay, now that we got our first entity we will need to continue
// parsing by copying data into temporary buffer and when finished
// convert it to string
while (_curPos < _dataLength)
{
var c = _html[_curPos++];
// ok we got tag, but all we had before it was spaces, most likely end of lines
// so we will return compact representation of that text data
if (c == (byte)'<')
{
_curPos--;
break;
}
// check if we got entity again
if (c == (byte)'&')
{
var newEntityChar = _e.CheckForEntity(_html, ref _curPos, _dataLength);
// restore current symbol
if (newEntityChar != 0)
{
if (newEntityChar == (byte)'<')
{
_chunk.LtEntity = true;
}
_text.Append(newEntityChar);
// we continue here since we fully parsed entity
continue;
}
// ok we did not parse entity in which case we add & char and continue along the way
_text._buffer[_text._bufPos++] = c;
continue;
}
_text._buffer[_text._bufPos++] = c;
}
_chunk.ChunkLength = _curPos - _chunk.ChunkOffset;
_chunk.Type = HTMLchunkType.Text;
_chunk.Html = _text.SetToString();
return(_chunk);
}
/// <summary>
/// Internal: parses tag that started from current position
/// </summary>
/// <returns>HTMLchunk with tag information</returns>
internal HTMLchunk ParseTag(ref int curPos)
{
/*
* WARNING: this code was optimised for performance rather than for readability,
* so be extremely careful at changing it -- your changes could easily result in wrongly parsed HTML
*
* This routine takes about 60% of CPU time, in theory its the best place to gain extra speed,
* but I've spent plenty of time doing it, so it won't be easy... and if it is easy then please post
* your changes for everyone to enjoy!
* */
//var whiteSpaceHere = false;
//var paramValue = false;
byte c = 0;
byte cPeek = 0;
// if true it means we have parsed complete tag
//var gotTag = false;
//var equalIdx = 0;
// we reach this function immediately after tag's byte (<) was
// detected, so we need to save it in order to keep correct HTML copy
// _hunk.Append((byte)'<'); // (byte)'<'
/*
* _chunk.Buffer[0] = 60;
* _chunk.BufPos = 1;
* _chunk.HTMLen = 1;
*/
// initialise peeked char - this will point to the next after < character
if (curPos < _dataLength)
{
cPeek = _html[curPos];
// in case of comments ! must follow immediately after <
if (cPeek == (byte)'!')
{
if (curPos + 2 < _dataLength &&
_html[curPos + 1] == (byte)'-' && _html[curPos + 2] == (byte)'-')
{
// we detected start of comments here, instead of parsing the rest here we will
// call special function tuned to do the job much more effectively
_chunk.Tag = "!--";
_chunk.Type = HTMLchunkType.Comment;
_chunk.Comments = true;
// _chunk.Append((byte)'!');
// _chunk.Append((byte)'-');
// _chunk.Append((byte)'-');
curPos += 3;
_chunk = ParseComments(ref curPos, out bool fullTag);
_chunk.ChunkLength = curPos - _chunk.ChunkOffset;
if (_p.AutoKeepComments || _p.KeepRawHTML)
{
if (!_p.AutoExtractBetweenTagsOnly)
{
_chunk.Html = GetString(_chunk.ChunkOffset, _chunk.ChunkLength);
}
else
{
_chunk.Html = GetString(_chunk.ChunkOffset + 4, _chunk.ChunkLength - fullTag ? 7 : 4);
}
}
return(_chunk);
}
// ok we might have here CDATA element of XML:
// ref: http://www.w3schools.com/xml/xml_cdata.asp
if (curPos + 7 < _dataLength &&
_html[curPos + 1] == (byte)'[' &&
_html[curPos + 2] == (byte)'C' &&
_html[curPos + 3] == (byte)'D' &&
_html[curPos + 4] == (byte)'A' &&
_html[curPos + 5] == (byte)'T' &&
_html[curPos + 6] == (byte)'A' &&
_html[curPos + 7] == (byte)'[')
{
// we detected start of comments here, instead of parsing the rest here we will
// call special function tuned to do the job much more effectively
_chunk.Tag = "![CDATA[";
_chunk.Type = HTMLchunkType.Comment;
_chunk.Comments = true;
// _chunk.Append((byte)'!');
// _chunk.Append((byte)'-');
// _chunk.Append((byte)'-');
curPos += 8;
_chunk = ParseCDATA(ref curPos, out bool fullTag);
_chunk.ChunkLength = curPos - _chunk.ChunkOffset;
if (_p.AutoKeepComments || _p.KeepRawHTML)
{
if (!_p.AutoExtractBetweenTagsOnly)
{
_chunk.Html = GetString(_chunk.ChunkOffset, _chunk.ChunkLength);
}
else
{
_chunk.Html = GetString(_chunk.ChunkOffset + 4 + 5, _chunk.ChunkLength - fullTag ? 7 + 5 : 4 + 5);
}
}
return(_chunk);
}
}
}
else
{
// empty tag but its not closed, so we will call it open...
_chunk.Type = HTMLchunkType.OpenTag;
// end of data... before it started
return(_chunk);
}
// tag ID, non-zero if matched by heuristics engine
var tagId = 0;
// STAGE 0: lets try some heuristics to see if we can quickly identify most common tags
// that should be present most of the time, this should save a lot of looping and string creation
if (EnableHeuristics && curPos < _maxHeuDataLength)
{
// check if we have got closure of the tag
if (cPeek == (byte)'/')
{
_chunk.Closure = true;
_chunk.EndClosure = false;
_chunk.Type = HTMLchunkType.CloseTag;
curPos++;
cPeek = _html[curPos];
}
c = _html[curPos + 1];
// probability of having a match is very high (or so we expect)
tagId = _he.MatchTag(cPeek, c);
if (tagId != 0)
{
if (tagId < 0)
{
tagId *= -1;
// single character tag
_chunk.Tag = _he.GetString(tagId);
// see if we got fully closed tag
if (c == (byte)'>')
{
curPos += 2;
goto ReturnChunk;
}
cPeek = c;
curPos++;
// everything else means we need to continue scanning as we may have params and stuff
goto AttributeParsing;
}
else
{
// ok, we have here 2 or more character string that we need to check further
// often when we have full 2 char match the next char will be >, if that's the case
// then we definately matched our tag
var nextChar = _html[curPos + 2];
if (nextChar == (byte)'>')
{
//oChunk.sTag=oHE.GetString(iTagID);
_chunk.Tag = _he.GetTwoCharString(cPeek, c);
curPos += 3;
goto ReturnChunk;
}
// ok, check next char for space, if that's the case we still got our tag
// but need to skip to attribute parsing
if (nextChar == (byte)' ')
{
//_chunk.Tag = _he.GetString(tagId);
_chunk.Tag = _he.GetTwoCharString(cPeek, c);
curPos += 2;
cPeek = nextChar;
goto AttributeParsing;
}
// ok, we are not very lucky, but it is still worth fighting for
// now we need to check fully long string against what we have matched, maybe
// we got exact match and we can avoid full parsing of the tag
var tag = _he.GetStringData(tagId);
if (curPos + tag.Length + 5 >= _dataLength)
{
goto TagParsing;
}
// in a loop (and this is not an ideal solution, but still)
for (int i = 2; i < tag.Length; i++)
{
// if a single char is not matched, then we
if (tag[i] != _html[curPos + i])
{
goto TagParsing;
}
}
// ok we matched full long word, but we need to be sure that char
// after the word is ' ' or '>' as otherwise we may have matched prefix of even longer word
nextChar = _html[curPos + tag.Length];
if (nextChar == (byte)'>')
{
_chunk.Tag = _he.GetString(tagId);
curPos += tag.Length + 1;
goto ReturnChunk;
}
if (nextChar == (byte)' ')
{
cPeek = nextChar;
_chunk.Tag = _he.GetString(tagId);
curPos += tag.Length;
goto AttributeParsing;
}
// no luck: we need to parse tag fully as our heuristical matching failed miserably :'o(
}
}
}
TagParsing:
_text.Clear();
var charType = 0;
// STAGE 1: parse tag (anything until > or /> or whitespace leading to start of attribute)
while (cPeek != 0)
{
charType = _tagCharTypes[cPeek];
//if (cPeek <= 32 && whiteSpace[cPeek] == 1)
if (charType == (byte)TagCharType.WhiteSpace)
{
curPos++;
// speculative loop unroll -- we have a very good chance of seeing non-space char next
// so instead of setting up loop we will just read it directly, this should save ticks
// on having to prepare while() loop
if (curPos < _dataLength)
{
c = _html[curPos++];
}
else
{
c = 0;
}
charType = _tagCharTypes[c];
//if (c == ' ' || c == '\t' || c == 13 || c == 10)
//if (c <= 32 && whiteSpace[c] == 1)
if (charType == (byte)TagCharType.WhiteSpace)
{
while (curPos < _dataLength)
{
c = _html[curPos++];
charType = _tagCharTypes[c];
if (charType == (byte)TagCharType.WhiteSpace)
//if(c != ' ' && c != '\t' && c != 13 && c != 10)
{
//cPeek = _html[curPos];
continue;
}
break;
}
if (curPos >= _dataLength)
{
c = 0;
}
}
//whiteSpaceHere = true;
// now, if we have already got tag it means that we are most likely
// going to need to parse tag attributes
if (_text._bufPos > 0)
{
_chunk.Tag = _text.SetToStringASCII();
// _chunk.Append((byte)' ');
curPos--;
if (curPos < _dataLength)
{
cPeek = _html[curPos];
}
else
{
cPeek = 0;
}
break;
}
}
else
{
// reuse Peeked char from previous run
//c = cPeek; curPos++;
if (curPos < _dataLength)
{
c = _html[curPos++];
}
else
{
c = 0;
}
}
if (curPos < _dataLength)
{
cPeek = _html[curPos];
}
else
{
cPeek = 0;
}
// most likely we should have lower-cased ASCII char
if (charType == (byte)TagCharType.LowerCasedASCIIorDigit)
{
_text._buffer[_text._bufPos++] = c;
// _chunk.Append(c);
continue;
}
// tag end - we did not have any params
if (c == (byte)'>')
{
if (_text._bufPos > 0)
{
_chunk.Tag = _text.SetToStringASCII();
}
if (!_chunk.Closure)
{
_chunk.Type = HTMLchunkType.OpenTag;
}
return(_chunk);
}
// closure of tag sign
if (c == (byte)'/')
{
_chunk.Closure = true;
_chunk.EndClosure = (_text._bufPos > 0);
_chunk.Type = HTMLchunkType.CloseTag;
continue;
}
// 03/08/08 XML support: ?xml tags - grrr
if (c == (byte)'?')
{
_text._buffer[_text._bufPos++] = c;
continue;
}
// nope, we have got upper cased ASCII char - this seems to be LESS likely than > and /
//if (c >= 65 && c <= 90)
if (charType > 32)
{
// bCharType in this case contains already lower-cased char
_text._buffer[_text._bufPos++] = charType;
// _chunk.Append(bCharType);
continue;
}
// we might have namespace : sign here - all text before would have to be
// saved as namespace and we will need to continue parsing actual tag
if (charType == (byte)TagCharType.NameSpaceColon)
{
// ok here we got a choice - we can just continue and treat the whole
// thing as a single tag with namespace stuff prefixed, OR
// we can separate first part into namespace and keep tag as normal
_text._buffer[_text._bufPos++] = (byte)':';
continue;
}
// ok, we have got some other char - we break out to deal with it in attributes part
break;
}
if (cPeek == 0)
{
return(_chunk);
}
// if true then equal sign was found
//var equalsSign = false;
// STAGE 2: parse attributes (if any available)
// attribute name can be standalone or with value after =
// attribute itself can't have entities or anything like this - we expect it to be in ASCII characters
AttributeParsing:
string attrName;
if (tagId != 0)
{
// first, skip whitespace:
if (cPeek <= 32 && _tagCharTypes[cPeek] == (byte)TagCharType.WhiteSpace)
{
// most likely next char is not-whitespace
curPos++;
if (curPos >= _dataLength)
{
goto ReturnChunk;
}
cPeek = _html[curPos];
if (cPeek <= 32 && _tagCharTypes[cPeek] == (byte)TagCharType.WhiteSpace)
{
// ok long loop here then
while (curPos < _dataLength)
{
cPeek = _html[curPos++];
if (cPeek <= 32 && _tagCharTypes[cPeek] == (byte)TagCharType.WhiteSpace)
{
continue;
}
break;
}
if (cPeek == (byte)'>')
{
goto ReturnChunk;
}
curPos--;
if (curPos >= _dataLength)
{
goto ReturnChunk;
}
}
if (curPos >= _dataLength)
{
goto ReturnChunk;
}
}
// ok we have got matched tag, it is possible that we might be able to quickly match
// attribute name known to be used for that tag:
var attrId = _he.MatchAttr(cPeek, tagId);
if (attrId > 0)
{
var attr = _he.GetAttrData(attrId);
if (curPos + attr.Length + 2 >= _dataLength)
{
goto ActualAttributeParsing;
}
// in a loop (and this is not an ideal solution, but still)
for (var i = 1; i < attr.Length; i++)
{
// if a single char is not matched, then we
if (attr[i] != _html[curPos + i])
{
goto ActualAttributeParsing;
}
}
var nextChar = _html[curPos + attr.Length];
// ok, we expect next symbol to be =
if (nextChar == (byte)'=')
{
attrName = _he.GetAttr(attrId);
curPos += attr.Length + 1;
cPeek = _html[curPos];
goto AttributeValueParsing;
}
}
}
ActualAttributeParsing:
_text.Clear();
// doing exactly the same thing as in tag parsing
while (cPeek != 0)
{
charType = _tagCharTypes[cPeek];
//if (cPeek <= 32 && whiteSpace[cPeek] == 1)
if (charType == (byte)TagCharType.WhiteSpace)
{
curPos++;
// speculative loop unroll -- we have a very good chance of seeing non-space char next
// so instead of setting up loop we will just read it directly, this should save ticks
// on having to prepare while() loop
if (curPos < _dataLength)
{
c = _html[curPos++];
}
else
{
cPeek = 0;
break;
}
charType = _tagCharTypes[c];
//if (c == ' ' || c == '\t' || c == 13 || c == 10)
//if (c <= 32 && whiteSpace[c] == 1)
if (charType == (byte)TagCharType.WhiteSpace)
{
while (curPos < _dataLength)
{
c = _html[curPos++];
charType = _tagCharTypes[c];
if (charType == (byte)TagCharType.WhiteSpace)
//if(c != ' ' && c != '\t' && c != 13 && c != 10)
{
//cPeek = _html[curPos];
continue;
}
//if (c == (byte)'>')
// goto ReturnChunk;
//curPos--;
break;
}
if (curPos >= _dataLength)
{
c = 0;
cPeek = 0;
break;
}
}
//whiteSpaceHere = true;
// now, if we have already got attribute name it means that we need to go to parse value (which may not be present)
if (_text._bufPos > 0)
{
// _chunk.Append((byte)' ');
curPos--;
if (curPos < _dataLength)
{
cPeek = _html[curPos];
}
else
{
cPeek = 0;
}
// ok, we have got attribute name and now we have got next char there
// most likely we have got = here and then value
if (cPeek == (byte)'=')
{
//equalsSign = true;
// move forward one char
curPos++;
if (curPos < _dataLength)
{
cPeek = _html[curPos];
}
else
{
cPeek = 0;
}
break;
}
// or we can have end of tag itself, doh!
if (cPeek == (byte)'>')
{
// move forward one char
curPos++;
if (_text._bufPos > 0)
{
_chunk.AddParam(_text.SetToStringASCII(), "", (byte)' ');
}
if (!_chunk.Closure)
{
_chunk.Type = HTMLchunkType.OpenTag;
}
return(_chunk);
}
// closure
if (cPeek == (byte)'/')
{
_chunk.Closure = true;
_chunk.EndClosure = true;
_chunk.Type = HTMLchunkType.CloseTag;
continue;
}
// ok, we have got new char starting after current attribute name is fully parsed
// this means the attribute name is on its own and the char we found is start
// of a new attribute
_chunk.AddParam(_text.SetToStringASCII(), "", (byte)' ');
_text.Clear();
goto AttributeParsing;
}
}
else
{
// reuse Peeked char from previous run
//c = cPeek; curPos++;
if (curPos < _dataLength)
{
c = _html[curPos++];
}
else
{
c = 0;
}
}
if (curPos < _dataLength)
{
cPeek = _html[curPos];
}
else
{
cPeek = 0;
}
// most likely we should have lower-cased ASCII char here
if (charType == (byte)TagCharType.LowerCasedASCIIorDigit)
{
_text._buffer[_text._bufPos++] = c;
// _chunk.Append(cChar);
continue;
}
// = with attribute value to follow
if (c == (byte)'=')
{
//equalsSign=true;
break;
}
// nope, we have got upper cased ASCII char - this seems to be LESS likely than > and /
//if(c >= 65 && c <= 90)
if (charType > 32)
{
// bCharType in this case contains already lower-cased char
_text._buffer[_text._bufPos++] = charType;
// _chunk.Append(bCharType);
continue;
}
// tag end - we did not have any params
if (c == (byte)'>')
{
if (_text._bufPos > 0)
{
_chunk.AddParam(_text.SetToStringASCII(), "", (byte)' ');
}
if (!_chunk.Closure)
{
_chunk.Type = HTMLchunkType.OpenTag;
}
return(_chunk);
}
// closure of tag sign
if (c == (byte)'/')
{
_chunk.Closure = true;
_chunk.EndClosure = true;
_chunk.Type = HTMLchunkType.CloseTag;
continue;
}
// some other char
_text._buffer[_text._bufPos++] = c;
// _chunk.Append(cChar);
}
if (cPeek == 0)
{
if (_text._bufPos > 0)
{
_chunk.AddParam(_text.SetToStringASCII(), "", (byte)' ');
}
if (!_chunk.Closure)
{
_chunk.Type = HTMLchunkType.OpenTag;
}
return(_chunk);
}
attrName = _text.SetToStringASCII();
AttributeValueParsing:
/// ***********************************************************************
/// STAGE 3: parse attribute value
/// ***********************************************************************
// the value could be just string, or in quotes (single or double)
// or we can have next attribute name start, in which case we will jump back to attribute parsing
// for tracking quotes purposes
var quotes = cPeek;
int valueStartOffset;
// skip whitespace if any
if (cPeek <= 32 && _tagCharTypes[cPeek] == (byte)TagCharType.WhiteSpace)
{
curPos++;
// speculative loop unroll -- we have a very good chance of seeing non-space char next
// so instead of setting up loop we will just read it directly, this should save ticks
// on having to prepare while() loop
if (curPos < _dataLength)
{
cPeek = _html[curPos];
}
else
{
valueStartOffset = curPos - 1;
goto AttributeValueEnd;
}
//if (c == ' ' || c == '\t' || c == 13 || c == 10)
//if (c <= 32 && whiteSpace[c] == 1)
if (cPeek <= 32 && _tagCharTypes[cPeek] == (byte)TagCharType.WhiteSpace)
{
while (curPos < _dataLength)
{
cPeek = _html[curPos++];
if (cPeek <= 32 && _tagCharTypes[cPeek] == (byte)TagCharType.WhiteSpace)
//if(c != ' ' && c != '\t' && c != 13 && c != 10)
{
//cPeek = _html[curPos];
continue;
}
curPos--;
break;
}
if (curPos >= _dataLength)
{
valueStartOffset = curPos - 1;
goto AttributeValueEnd;
}
}
quotes = cPeek;
}
// because we deal with VALUE of the attribute it means we can't lower-case it,
// or skip whitespace (if in quotes), which in practice means that we don't need to copy
// it to temporary string buffer, we can just remember starting offset and then create string from
// data in bHTML
// ok, first char can be one of the quote chars or something else
if (cPeek != '\"' && cPeek != '\'')
{
valueStartOffset = curPos;
quotes = (byte)' ';
// any other char here means we have value up until next whitespace or end of tag
// this gives us good opportunity to scan fairly quickly without otherwise redundant
// checks - this should happen fairly rarely, however loop dealing with data between quotes
// will happen often enough and its best to eliminate as much stuff from it as possible
//sText.bBuffer[sText.iBufPos++]=cPeek;
// move to next char
if (curPos < _dataLength)
{
cPeek = _html[curPos++];
}
else
{
goto AttributeValueEnd;
}
while (cPeek != 0)
{
// if whitespace then we got our value and need to go back to param
if (cPeek <= 32 && _tagCharTypes[cPeek] == (byte)TagCharType.WhiteSpace)
{
_chunk.AddParam(attrName, GetString(valueStartOffset, curPos - valueStartOffset - 1), (byte)' ');
curPos--;
goto AttributeParsing;
}
// end of tag?
if (cPeek == (byte)'>')
{
//curPos--;
break;
}
if (curPos < _dataLength)
{
cPeek = _html[curPos++];
}
else
{
curPos = _dataLength + 1;
goto AttributeValueEnd;
}
}
// ok we are done, add outstanding attribute
_chunk.AddParam(attrName, GetString(valueStartOffset, curPos - valueStartOffset - 1), (byte)' ');
goto ReturnChunk;
}
// move one step forward
curPos++;
valueStartOffset = curPos;
if (curPos < _dataLength)
{
cPeek = _html[curPos++];
}
else
{
goto AttributeValueEnd;
}
// attribute value parsing from between two quotes
while (cPeek != 0)
{
// check whether we have got possible entity (can be anything starting with &)
if (cPeek == 38)
{
var prevPos = curPos;
var entityChar = _e.CheckForEntity(_html, ref curPos, _dataLength);
// restore current symbol
if (entityChar == 0)
{
if (curPos < _dataLength)
{
cPeek = _html[curPos++];
}
else
{
break;
}
//_text.Buffer[_text.BufPos++] = 38; //(byte)'&';;
continue;
}
else
{
// okay we have got an entity, our hope of not having to copy stuff into variable
// is over, we have to continue in a slower fashion :(
// but thankfully this should happen very rarely, so, annoying to code, but
// most codepaths will run very fast!
var preEntLen = prevPos - valueStartOffset - 1;
// 14/05/08 need to clear text - it contains attribute name text
_text.Clear();
// copy previous data
if (preEntLen > 0)
{
Array.Copy(_html, valueStartOffset, _text._buffer, 0, preEntLen);
_text._bufPos = preEntLen;
}
// we have to skip now to next byte, since
// some converted chars might well be control chars like >
_chunk.Entities = true;
if (c == (byte)'<')
{
_chunk.LtEntity = true;
}
// unless is space we will ignore it
// note that this won't work if is defined as it should
// byte int value of 160, rather than 32.
//if (c != ' ')
_text.Append(entityChar);
if (curPos < _dataLength)
{
cPeek = _html[curPos++];
}
else
{
goto AttributeValueEnd;
}
// okay, we continue here using in effect new inside loop as we might have more entities here
// attribute value parsing from between two quotes
while (cPeek != 0)
{
// check whether we have got possible entity (can be anything starting with &)
if (cPeek == 38)
{
var newEntityChar = _e.CheckForEntity(_html, ref curPos, _dataLength);
// restore current symbol
if (newEntityChar != 0)
{
if (newEntityChar == (byte)'<')
{
_chunk.LtEntity = true;
}
_text.Append(newEntityChar);
if (curPos < _dataLength)
{
cPeek = _html[curPos++];
}
else
{
goto AttributeValueEnd;
}
continue;
}
}
// check if is end of quotes
if (cPeek == quotes)
{
// ok we finished scanning it: add param with value and then go back to param name parsing
_chunk.AddParam(attrName, _text.SetToString(), quotes);
if (curPos < _dataLength)
{
cPeek = _html[curPos];
}
else
{
break;
}
goto AttributeParsing;
}
_text._buffer[_text._bufPos++] = cPeek;
//_text.Append(cPeek);
if (curPos < _dataLength)
{
cPeek = _html[curPos++];
}
else
{
break;
}
}
_chunk.AddParam(attrName, _text.SetToString(), quotes);
goto ReturnChunk;
}
}
// check if is end of quotes
if (cPeek == quotes)
{
// ok we finished scanning it: add param with value and then go back to param name parsing
//_text.Clear();
_chunk.AddParam(attrName, GetString(valueStartOffset, curPos - valueStartOffset - 1), quotes);
if (curPos < _dataLength)
{
cPeek = _html[curPos];
}
else /*curPos++;*/ break {
;
}
goto AttributeParsing;
}
if (curPos < _dataLength)
{
cPeek = _html[curPos++];
}
else /*curPos++;*/ break {
;
}
