/* off points to character following "<" */
private TOK scanLt(byte[] buf, int off, int end, ContentToken token)
{
if (off == end)
throw new PartialTokenException();
switch (byteType(buf, off))
{
case BT_NMSTRT:
off += minBPC;
break;
case BT_LEAD2:
if (end - off < 2)
throw new PartialCharException(off);
if (byteType2(buf, off) != BT_NMSTRT)
throw new InvalidTokenException(off);
off += 2;
break;
case BT_LEAD3:
if (end - off < 3)
throw new PartialCharException(off);
if (byteType3(buf, off) != BT_NMSTRT)
throw new InvalidTokenException(off);
off += 3;
break;
case BT_LEAD4:
if (end - off < 4)
throw new PartialCharException(off);
if (byteType4(buf, off) != BT_NMSTRT)
throw new InvalidTokenException(off);
off += 4;
break;
case BT_EXCL:
if ((off += minBPC) == end)
throw new PartialTokenException();
switch (byteType(buf, off))
{
case BT_MINUS:
return scanComment(buf, off + minBPC, end, token);
case BT_LSQB:
return scanCdataSection(buf, off + minBPC, end, token);
}
throw new InvalidTokenException(off);
case BT_QUEST:
return scanPi(buf, off + minBPC, end, token);
case BT_SOL:
return scanEndTag(buf, off + minBPC, end, token);
default:
throw new InvalidTokenException(off);
}
/* we have a start-tag */
token.NameEnd = -1;
token.clearAttributes();
while (off != end)
{
switch (byteType(buf, off))
{
case BT_NMSTRT:
case BT_NAME:
case BT_MINUS:
off += minBPC;
break;
case BT_LEAD2:
if (end - off < 2)
throw new PartialCharException(off);
if (!isNameChar2(buf, off))
throw new InvalidTokenException(off);
off += 2;
break;
case BT_LEAD3:
if (end - off < 3)
throw new PartialCharException(off);
if (!isNameChar3(buf, off))
throw new InvalidTokenException(off);
off += 3;
break;
case BT_LEAD4:
if (end - off < 4)
throw new PartialCharException(off);
if (!isNameChar4(buf, off))
throw new InvalidTokenException(off);
off += 4;
break;
case BT_S:
case BT_CR:
case BT_LF:
token.NameEnd = off;
off += minBPC;
for (;;)
{
if (off == end)
throw new PartialTokenException();
switch (byteType(buf, off))
{
case BT_NMSTRT:
return scanAtts(off, buf, off + minBPC, end, token);
case BT_LEAD2:
if (end - off < 2)
throw new PartialCharException(off);
if (byteType2(buf, off) != BT_NMSTRT)
throw new InvalidTokenException(off);
return scanAtts(off, buf, off + 2, end, token);
case BT_LEAD3:
if (end - off < 3)
throw new PartialCharException(off);
if (byteType3(buf, off) != BT_NMSTRT)
throw new InvalidTokenException(off);
return scanAtts(off, buf, off + 3, end, token);
case BT_LEAD4:
if (end - off < 4)
throw new PartialCharException(off);
if (byteType4(buf, off) != BT_NMSTRT)
throw new InvalidTokenException(off);
return scanAtts(off, buf, off + 4, end, token);
case BT_GT:
case BT_SOL:
goto loop;
case BT_S:
case BT_CR:
case BT_LF:
off += minBPC;
break;
default:
throw new InvalidTokenException(off);
}
}
loop:
break;
case BT_GT:
if (token.NameEnd < 0)
token.NameEnd = off;
token.TokenEnd = off + minBPC;
return TOK.START_TAG_NO_ATTS;
case BT_SOL:
if (token.NameEnd < 0)
token.NameEnd = off;
off += minBPC;
if (off == end)
throw new PartialTokenException();
checkCharMatches(buf, off, '>');
token.TokenEnd = off + minBPC;
return TOK.EMPTY_ELEMENT_NO_ATTS;
default:
throw new InvalidTokenException(off);
}
}
throw new PartialTokenException();
}
/* off points to character following first character of
attribute name */
private TOK scanAtts(int nameStart, byte[] buf, int off, int end,
ContentToken token)
{
int NameEnd = -1;
while (off != end)
{
switch (byteType(buf, off))
{
case BT_NMSTRT:
case BT_NAME:
case BT_MINUS:
off += minBPC;
break;
case BT_LEAD2:
if (end - off < 2)
throw new PartialCharException(off);
if (!isNameChar2(buf, off))
throw new InvalidTokenException(off);
off += 2;
break;
case BT_LEAD3:
if (end - off < 3)
throw new PartialCharException(off);
if (!isNameChar3(buf, off))
throw new InvalidTokenException(off);
off += 3;
break;
case BT_LEAD4:
if (end - off < 4)
throw new PartialCharException(off);
if (!isNameChar4(buf, off))
throw new InvalidTokenException(off);
off += 4;
break;
case BT_S:
case BT_CR:
case BT_LF:
NameEnd = off;
for (;;)
{
off += minBPC;
if (off == end)
throw new PartialTokenException();
switch (byteType(buf, off))
{
case BT_EQUALS:
goto loop;
case BT_S:
case BT_LF:
case BT_CR:
break;
default:
throw new InvalidTokenException(off);
}
}
loop: ;
/* fall through */
goto case BT_EQUALS;
case BT_EQUALS:
{
if (NameEnd < 0)
NameEnd = off;
int open;
for (;;)
{
off += minBPC;
if (off == end)
throw new PartialTokenException();
open = byteType(buf, off);
if (open == BT_QUOT || open == BT_APOS)
break;
switch (open)
{
case BT_S:
case BT_LF:
case BT_CR:
break;
default:
throw new InvalidTokenException(off);
}
}
off += minBPC;
int valueStart = off;
bool normalized = true;
int t;
/* in attribute value */
for (;;)
{
if (off == end)
throw new PartialTokenException();
t = byteType(buf, off);
if (t == open)
break;
switch (t)
{
case BT_NONXML:
case BT_MALFORM:
throw new InvalidTokenException(off);
case BT_LEAD2:
if (end - off < 2)
throw new PartialCharException(off);
check2(buf, off);
off += 2;
break;
case BT_LEAD3:
if (end - off < 3)
throw new PartialCharException(off);
check3(buf, off);
off += 3;
break;
case BT_LEAD4:
if (end - off < 4)
throw new PartialCharException(off);
check4(buf, off);
off += 4;
break;
case BT_AMP:
{
normalized = false;
int saveNameEnd = token.NameEnd;
scanRef(buf, off + minBPC, end, token);
token.NameEnd = saveNameEnd;
off = token.TokenEnd;
break;
}
case BT_S:
if (normalized
&& (off == valueStart
|| byteToAscii(buf, off) != ' '
|| (off + minBPC != end
&& (byteToAscii(buf, off + minBPC) == ' '
|| byteType(buf, off + minBPC) == open))))
normalized = false;
off += minBPC;
break;
case BT_LT:
throw new InvalidTokenException(off);
case BT_LF:
case BT_CR:
normalized = false;
/* fall through */
goto default;
default:
off += minBPC;
break;
}
}
token.appendAttribute(nameStart, NameEnd, valueStart,
off,
normalized);
off += minBPC;
if (off == end)
throw new PartialTokenException();
t = byteType(buf, off);
switch (t)
{
case BT_S:
case BT_CR:
case BT_LF:
off += minBPC;
if (off == end)
throw new PartialTokenException();
t = byteType(buf, off);
break;
case BT_GT:
case BT_SOL:
break;
default:
throw new InvalidTokenException(off);
}
/* off points to closing quote */
for (;;)
{
switch (t)
{
case BT_NMSTRT:
nameStart = off;
off += minBPC;
goto skipToName;
case BT_LEAD2:
if (end - off < 2)
throw new PartialCharException(off);
if (byteType2(buf, off) != BT_NMSTRT)
throw new InvalidTokenException(off);
nameStart = off;
off += 2;
goto skipToName;
case BT_LEAD3:
if (end - off < 3)
throw new PartialCharException(off);
if (byteType3(buf, off) != BT_NMSTRT)
throw new InvalidTokenException(off);
nameStart = off;
off += 3;
goto skipToName;
case BT_LEAD4:
if (end - off < 4)
throw new PartialCharException(off);
if (byteType4(buf, off) != BT_NMSTRT)
throw new InvalidTokenException(off);
nameStart = off;
off += 4;
goto skipToName;
case BT_S:
case BT_CR:
case BT_LF:
break;
case BT_GT:
token.checkAttributeUniqueness(buf);
token.TokenEnd = off + minBPC;
return TOK.START_TAG_WITH_ATTS;
case BT_SOL:
off += minBPC;
if (off == end)
throw new PartialTokenException();
checkCharMatches(buf, off, '>');
token.checkAttributeUniqueness(buf);
token.TokenEnd = off + minBPC;
return TOK.EMPTY_ELEMENT_WITH_ATTS;
default:
throw new InvalidTokenException(off);
}
off += minBPC;
if (off == end)
throw new PartialTokenException();
t = byteType(buf, off);
}
skipToName:
NameEnd = -1;
break;
}
default:
throw new InvalidTokenException(off);
}
}
throw new PartialTokenException();
}
/**
* Scans the first token of a byte subarrary that contains content.
* Returns one of the following integers according to the type of token
* that the subarray starts with:
* <ul>
* <li><code>TOK.START_TAG_NO_ATTS</code></li>
* <li><code>TOK.START_TAG_WITH_ATTS</code></li>
* <li><code>TOK.EMPTY_ELEMENT_NO_ATTS</code></li>
* <li><code>TOK.EMPTY_ELEMENT_WITH_ATTS</code></li>
* <li><code>TOK.END_TAG</code></li>
* <li><code>TOK.DATA_CHARS</code></li>
* <li><code>TOK.DATA_NEWLINE</code></li>
* <li><code>TOK.CDATA_SECT_OPEN</code></li>
* <li><code>TOK.ENTITY_REF</code></li>
* <li><code>TOK.MAGIC_ENTITY_REF</code></li>
* <li><code>TOK.CHAR_REF</code></li>
* <li><code>TOK.CHAR_PAIR_REF</code></li>
* <li><code>TOK.PI</code></li>
* <li><code>TOK.XML_DECL</code></li>
* <li><code>TOK.COMMENT</code></li>
* </ul>
* <p>
* Information about the token is stored in <code>token</code>.
* </p>
* When <code>TOK.CDATA_SECT_OPEN</code> is returned,
* <code>tokenizeCdataSection</code> should be called until
* it returns <code>TOK.CDATA_SECT</code>.
*
* @exception EmptyTokenException if the subarray is empty
* @exception PartialTokenException if the subarray contains only part of
* a legal token
* @exception InvalidTokenException if the subarrary does not start
* with a legal token or part of one
* @exception ExtensibleTokenException if the subarray encodes just a carriage
* return ('\r')
*
* @see #TOK.START_TAG_NO_ATTS
* @see #TOK.START_TAG_WITH_ATTS
* @see #TOK.EMPTY_ELEMENT_NO_ATTS
* @see #TOK.EMPTY_ELEMENT_WITH_ATTS
* @see #TOK.END_TAG
* @see #TOK.DATA_CHARS
* @see #TOK.DATA_NEWLINE
* @see #TOK.CDATA_SECT_OPEN
* @see #TOK.ENTITY_REF
* @see #TOK.MAGIC_ENTITY_REF
* @see #TOK.CHAR_REF
* @see #TOK.CHAR_PAIR_REF
* @see #TOK.PI
* @see #TOK.XML_DECL
* @see #TOK.COMMENT
* @see ContentToken
* @see EmptyTokenException
* @see PartialTokenException
* @see InvalidTokenException
* @see ExtensibleTokenException
* @see #tokenizeCdataSection
*/
public TOK tokenizeContent(byte[] buf, int off, int end,
ContentToken token)
{
if (minBPC > 1)
end = adjustEnd(off, end);
if (off == end)
throw new EmptyTokenException();
switch (byteType(buf, off))
{
case BT_LT:
return scanLt(buf, off + minBPC, end, token);
case BT_AMP:
return scanRef(buf, off + minBPC, end, token);
case BT_CR:
off += minBPC;
if (off == end)
throw new ExtensibleTokenException(TOK.DATA_NEWLINE);
if (byteType(buf, off) == BT_LF)
off += minBPC;
token.TokenEnd = off;
return TOK.DATA_NEWLINE;
case BT_LF:
token.TokenEnd = off + minBPC;
return TOK.DATA_NEWLINE;
case BT_RSQB:
off += minBPC;
if (off == end)
throw new ExtensibleTokenException(TOK.DATA_CHARS);
if (!charMatches(buf, off, ']'))
break;
off += minBPC;
if (off == end)
throw new ExtensibleTokenException(TOK.DATA_CHARS);
if (!charMatches(buf, off, '>'))
{
off -= minBPC;
break;
}
throw new InvalidTokenException(off);
case BT_NONXML:
case BT_MALFORM:
throw new InvalidTokenException(off);
case BT_LEAD2:
if (end - off < 2)
throw new PartialCharException(off);
check2(buf, off);
off += 2;
break;
case BT_LEAD3:
if (end - off < 3)
throw new PartialCharException(off);
check3(buf, off);
off += 3;
break;
case BT_LEAD4:
if (end - off < 4)
throw new PartialCharException(off);
check4(buf, off);
off += 4;
break;
default:
off += minBPC;
break;
}
token.TokenEnd = extendData(buf, off, end);
return TOK.DATA_CHARS;
}
/// <summary>
/// Put bytes into the parser.
/// </summary>
/// <param name="buf">The bytes to put into the parse stream</param>
/// <param name="offset">Offset into buf to start at</param>
/// <param name="length">Number of bytes to write</param>
public void Push(byte[] buf, int offset, int length)
{
// or assert, really, but this is a little nicer.
if (length == 0)
return;
// No locking is required. Read() won't get called again
// until this method returns. Keep in mind that we're
// already on a thread in a ThreadPool, which is created
// and managed by System.IO at the end of the day.
// TODO: only do this copy if we have a partial token at the
// end of parsing.
byte[] copy = new byte[length];
System.Buffer.BlockCopy(buf, offset, copy, 0, length);
m_buf.Write(copy);
byte[] b = m_buf.GetBuffer();
int off = 0;
TOK tok = TOK.END_TAG;
ContentToken ct = new ContentToken();
try
{
while (off < b.Length)
{
if (m_cdata)
tok = m_enc.tokenizeCdataSection(b, off, b.Length, ct);
else
tok = m_enc.tokenizeContent(b, off, b.Length, ct);
switch (tok)
{
case TOK.EMPTY_ELEMENT_NO_ATTS:
case TOK.EMPTY_ELEMENT_WITH_ATTS:
StartTag(b, off, ct, tok);
EndTag(b, off, ct, tok);
break;
case TOK.START_TAG_NO_ATTS:
case TOK.START_TAG_WITH_ATTS:
StartTag(b, off, ct, tok);
break;
case TOK.END_TAG:
EndTag(b, off, ct, tok);
break;
case TOK.DATA_CHARS:
case TOK.DATA_NEWLINE:
AddText(utf.GetString(b, off, ct.TokenEnd - off));
break;
case TOK.CHAR_REF:
case TOK.MAGIC_ENTITY_REF:
AddText(new string(new char[] { ct.RefChar1 }));
break;
case TOK.CHAR_PAIR_REF:
AddText(new string(new char[] {ct.RefChar1,
ct.RefChar2}));
break;
case TOK.COMMENT:
if (m_elem != null)
{
// <!-- 4
// --> 3
int start = off + 4*m_enc.MinBytesPerChar;
int end = ct.TokenEnd - off -
7*m_enc.MinBytesPerChar;
string text = utf.GetString(b, start, end);
m_elem.AppendChild(m_doc.CreateComment(text));
}
break;
case TOK.CDATA_SECT_OPEN:
m_cdata = true;
break;
case TOK.CDATA_SECT_CLOSE:
m_cdata = false;
break;
case TOK.XML_DECL:
// thou shalt use UTF8, and XML version 1.
// i shall ignore evidence to the contrary...
// TODO: Throw an exception if these assuptions are
// wrong
break;
case TOK.ENTITY_REF:
case TOK.PI:
throw new System.NotImplementedException("Token type not implemented: " + tok);
}
off = ct.TokenEnd;
ct.clearAttributes();
}
}
catch (PartialTokenException)
{
// Console.WriteLine("PartialTokenException: " + System.Text.Encoding.UTF8.GetString(copy));
// ignored;
}
catch (ExtensibleTokenException)
{
// ignored;
}
catch (xpnet.InvalidTokenException e)
{
throw new XMLParseException(e, this, buf, offset, length);
}
catch (Exception e)
{
throw new Exception("Unexpected exception", e);
}
finally
{
m_buf.Clear(off);
ct.clearAttributes();
}
}
private void StartTag(byte[] buf, int offset,
ContentToken ct, TOK tok)
{
int colon;
string name;
string prefix;
Hashtable ht = new Hashtable();
m_ns.PushScope();
// if i have attributes
if ((tok == TOK.START_TAG_WITH_ATTS) ||
(tok == TOK.EMPTY_ELEMENT_WITH_ATTS))
{
int start;
int end;
string val;
for (int i=0; i<ct.getAttributeSpecifiedCount(); i++)
{
start = ct.getAttributeNameStart(i);
end = ct.getAttributeNameEnd(i);
name = utf.GetString(buf, start, end - start);
start = ct.getAttributeValueStart(i);
end = ct.getAttributeValueEnd(i);
val = utf.GetString(buf, start, end - start);
// <foo b='&'/>
// <foo b='&amp;'
// TODO: if val includes &, it gets double-escaped
if (name.StartsWith("xmlns:"))
{
colon = name.IndexOf(':');
prefix = name.Substring(colon+1);
m_ns.AddNamespace(prefix, val);
}
else if (name == "xmlns")
{
m_ns.AddNamespace(string.Empty, val);
}
ht.Add(name, val);
}
}
name = utf.GetString(buf,
offset + m_enc.MinBytesPerChar,
ct.NameEnd - offset - m_enc.MinBytesPerChar);
colon = name.IndexOf(':');
string ns = "";
prefix = "";
if (colon > 0)
{
prefix = name.Substring(0, colon);
name = name.Substring(colon + 1);
ns = m_ns.LookupNamespace(prefix);
}
else
{
ns = m_ns.DefaultNamespace;
}
XmlQualifiedName q = new XmlQualifiedName(name, ns);
XmlElement elem = m_factory.GetElement(prefix, q, m_doc);
foreach (string attrname in ht.Keys)
{
colon = attrname.IndexOf(':');
if (colon > 0)
{
prefix = attrname.Substring(0, colon);
name = attrname.Substring(colon+1);
XmlAttribute attr = m_doc.CreateAttribute(prefix,
name,
m_ns.LookupNamespace(prefix));
attr.InnerXml = (string)ht[attrname];
elem.SetAttributeNode(attr);
}
else
{
XmlAttribute attr = m_doc.CreateAttribute(attrname);
attr.InnerXml = (string)ht[attrname];
elem.SetAttributeNode(attr);
}
}
if (m_root == null)
{
m_root = elem;
FireOnDocumentStart(m_root);
}
else
{
if (m_elem != null)
m_elem.AppendChild(elem);
m_elem = elem;
}
}
private void EndTag(byte[] buf, int offset,
ContentToken ct, TOK tok)
{
m_ns.PopScope();
if (m_elem == null)
{// end of doc
FireOnDocumentEnd();
return;
}
string name = null;
if ((tok == TOK.EMPTY_ELEMENT_WITH_ATTS) ||
(tok == TOK.EMPTY_ELEMENT_NO_ATTS))
name = utf.GetString(buf,
offset + m_enc.MinBytesPerChar,
ct.NameEnd - offset -
m_enc.MinBytesPerChar);
else
name = utf.GetString(buf,
offset + m_enc.MinBytesPerChar*2,
ct.NameEnd - offset -
m_enc.MinBytesPerChar*2);
if (m_elem.Name != name)
throw new XmlException("Invalid end tag: " + name +
" != " + m_elem.Name);
XmlElement parent = (XmlElement)m_elem.ParentNode;
if (parent == null)
{
FireOnElement(m_elem);
}
m_elem = parent;
}
