/// <summary>
/// </summary>
/// <param name="buf"> </param>
/// <param name="off"> </param>
/// <param name="end"> </param>
/// <param name="token"> </param>
/// <returns> </returns>
/// <exception cref="EmptyTokenException"></exception>
/// <exception cref="ExtensibleTokenException"></exception>
/// <exception cref="InvalidTokenException"></exception>
/// <exception cref="PartialCharException"></exception>
public TOK tokenizeContent(byte[] buf, int off, int end, ContentToken token)
{
if (minBPC > 1)
{
end = adjustEnd(off, end);
}
if (off == end)
{
throw new EmptyTokenException();
}
switch (byteType(buf, off))
{
case BT_LT:
return scanLt(buf, off + minBPC, end, token);
case BT_AMP:
return scanRef(buf, off + minBPC, end, token);
case BT_CR:
off += minBPC;
if (off == end)
{
throw new ExtensibleTokenException(TOK.DATA_NEWLINE);
}
if (byteType(buf, off) == BT_LF)
{
off += minBPC;
}
token.TokenEnd = off;
return TOK.DATA_NEWLINE;
case BT_LF:
token.TokenEnd = off + minBPC;
return TOK.DATA_NEWLINE;
case BT_RSQB:
off += minBPC;
if (off == end)
{
throw new ExtensibleTokenException(TOK.DATA_CHARS);
}
if (!charMatches(buf, off, ']'))
{
break;
}
off += minBPC;
if (off == end)
{
throw new ExtensibleTokenException(TOK.DATA_CHARS);
}
if (!charMatches(buf, off, '>'))
{
off -= minBPC;
break;
}
throw new InvalidTokenException(off);
case BT_NONXML:
case BT_MALFORM:
throw new InvalidTokenException(off);
case BT_LEAD2:
if (end - off < 2)
{
throw new PartialCharException(off);
}
check2(buf, off);
off += 2;
break;
case BT_LEAD3:
if (end - off < 3)
{
throw new PartialCharException(off);
}
check3(buf, off);
off += 3;
break;
case BT_LEAD4:
if (end - off < 4)
{
throw new PartialCharException(off);
}
check4(buf, off);
off += 4;
break;
default:
off += minBPC;
break;
}
token.TokenEnd = extendData(buf, off, end);
return TOK.DATA_CHARS;
}
/* off points to character following first character of
attribute name */
/// <summary>
/// </summary>
/// <param name="nameStart"> </param>
/// <param name="buf"> </param>
/// <param name="off"> </param>
/// <param name="end"> </param>
/// <param name="token"> </param>
/// <returns> </returns>
/// <exception cref="PartialCharException"></exception>
/// <exception cref="InvalidTokenException"></exception>
/// <exception cref="PartialTokenException"></exception>
private TOK scanAtts(int nameStart, byte[] buf, int off, int end, ContentToken token)
{
int NameEnd = -1;
while (off != end)
{
switch (byteType(buf, off))
{
case BT_NMSTRT:
case BT_NAME:
case BT_MINUS:
off += minBPC;
break;
case BT_LEAD2:
if (end - off < 2)
{
throw new PartialCharException(off);
}
if (!isNameChar2(buf, off))
{
throw new InvalidTokenException(off);
}
off += 2;
break;
case BT_LEAD3:
if (end - off < 3)
{
throw new PartialCharException(off);
}
if (!isNameChar3(buf, off))
{
throw new InvalidTokenException(off);
}
off += 3;
break;
case BT_LEAD4:
if (end - off < 4)
{
throw new PartialCharException(off);
}
if (!isNameChar4(buf, off))
{
throw new InvalidTokenException(off);
}
off += 4;
break;
case BT_S:
case BT_CR:
case BT_LF:
NameEnd = off;
for (;;)
{
off += minBPC;
if (off == end)
{
throw new PartialTokenException();
}
switch (byteType(buf, off))
{
case BT_EQUALS:
goto loop;
case BT_S:
case BT_LF:
case BT_CR:
break;
default:
throw new InvalidTokenException(off);
}
}
loop:
;
/* fall through */
goto case BT_EQUALS;
case BT_EQUALS:
{
if (NameEnd < 0)
{
NameEnd = off;
}
int open;
for (;;)
{
off += minBPC;
if (off == end)
{
throw new PartialTokenException();
}
open = byteType(buf, off);
if (open == BT_QUOT || open == BT_APOS)
{
break;
}
switch (open)
{
case BT_S:
case BT_LF:
case BT_CR:
break;
default:
throw new InvalidTokenException(off);
}
}
off += minBPC;
int valueStart = off;
bool normalized = true;
int t;
/* in attribute value */
for (;;)
{
if (off == end)
{
throw new PartialTokenException();
}
t = byteType(buf, off);
if (t == open)
{
break;
}
switch (t)
{
case BT_NONXML:
case BT_MALFORM:
throw new InvalidTokenException(off);
case BT_LEAD2:
if (end - off < 2)
{
throw new PartialCharException(off);
}
check2(buf, off);
off += 2;
break;
case BT_LEAD3:
if (end - off < 3)
{
throw new PartialCharException(off);
}
check3(buf, off);
off += 3;
break;
case BT_LEAD4:
if (end - off < 4)
{
throw new PartialCharException(off);
}
check4(buf, off);
off += 4;
break;
case BT_AMP:
{
normalized = false;
int saveNameEnd = token.NameEnd;
scanRef(buf, off + minBPC, end, token);
token.NameEnd = saveNameEnd;
off = token.TokenEnd;
break;
}
case BT_S:
if (normalized &&
(off == valueStart || byteToAscii(buf, off) != ' ' ||
(off + minBPC != end &&
(byteToAscii(buf, off + minBPC) == ' ' ||
byteType(buf, off + minBPC) == open))))
{
normalized = false;
}
off += minBPC;
break;
case BT_LT:
throw new InvalidTokenException(off);
case BT_LF:
case BT_CR:
normalized = false;
/* fall through */
goto default;
default:
off += minBPC;
break;
}
}
token.appendAttribute(nameStart, NameEnd, valueStart, off, normalized);
off += minBPC;
if (off == end)
{
throw new PartialTokenException();
}
t = byteType(buf, off);
switch (t)
{
case BT_S:
case BT_CR:
case BT_LF:
off += minBPC;
if (off == end)
{
throw new PartialTokenException();
}
t = byteType(buf, off);
break;
case BT_GT:
case BT_SOL:
break;
default:
throw new InvalidTokenException(off);
}
/* off points to closing quote */
for (;;)
{
switch (t)
{
case BT_NMSTRT:
nameStart = off;
off += minBPC;
goto skipToName;
case BT_LEAD2:
if (end - off < 2)
{
throw new PartialCharException(off);
}
if (byteType2(buf, off) != BT_NMSTRT)
{
throw new InvalidTokenException(off);
}
nameStart = off;
off += 2;
goto skipToName;
case BT_LEAD3:
if (end - off < 3)
{
throw new PartialCharException(off);
}
if (byteType3(buf, off) != BT_NMSTRT)
{
throw new InvalidTokenException(off);
}
nameStart = off;
off += 3;
goto skipToName;
case BT_LEAD4:
if (end - off < 4)
{
throw new PartialCharException(off);
}
if (byteType4(buf, off) != BT_NMSTRT)
{
throw new InvalidTokenException(off);
}
nameStart = off;
off += 4;
goto skipToName;
case BT_S:
case BT_CR:
case BT_LF:
break;
case BT_GT:
token.checkAttributeUniqueness(buf);
token.TokenEnd = off + minBPC;
return TOK.START_TAG_WITH_ATTS;
case BT_SOL:
off += minBPC;
if (off == end)
{
throw new PartialTokenException();
}
checkCharMatches(buf, off, '>');
token.checkAttributeUniqueness(buf);
token.TokenEnd = off + minBPC;
return TOK.EMPTY_ELEMENT_WITH_ATTS;
default:
throw new InvalidTokenException(off);
}
off += minBPC;
if (off == end)
{
throw new PartialTokenException();
}
t = byteType(buf, off);
}
skipToName:
NameEnd = -1;
break;
}
default:
throw new InvalidTokenException(off);
}
}
throw new PartialTokenException();
}
/* off points to character following "<" */
/// <summary>
/// </summary>
/// <param name="buf"> </param>
/// <param name="off"> </param>
/// <param name="end"> </param>
/// <param name="token"> </param>
/// <returns> </returns>
/// <exception cref="PartialTokenException"></exception>
/// <exception cref="PartialCharException"></exception>
/// <exception cref="InvalidTokenException"></exception>
private TOK scanLt(byte[] buf, int off, int end, ContentToken token)
{
if (off == end)
{
throw new PartialTokenException();
}
switch (byteType(buf, off))
{
case BT_NMSTRT:
off += minBPC;
break;
case BT_LEAD2:
if (end - off < 2)
{
throw new PartialCharException(off);
}
if (byteType2(buf, off) != BT_NMSTRT)
{
throw new InvalidTokenException(off);
}
off += 2;
break;
case BT_LEAD3:
if (end - off < 3)
{
throw new PartialCharException(off);
}
if (byteType3(buf, off) != BT_NMSTRT)
{
throw new InvalidTokenException(off);
}
off += 3;
break;
case BT_LEAD4:
if (end - off < 4)
{
throw new PartialCharException(off);
}
if (byteType4(buf, off) != BT_NMSTRT)
{
throw new InvalidTokenException(off);
}
off += 4;
break;
case BT_EXCL:
if ((off += minBPC) == end)
{
throw new PartialTokenException();
}
switch (byteType(buf, off))
{
case BT_MINUS:
return scanComment(buf, off + minBPC, end, token);
case BT_LSQB:
return scanCdataSection(buf, off + minBPC, end, token);
}
throw new InvalidTokenException(off);
case BT_QUEST:
return scanPi(buf, off + minBPC, end, token);
case BT_SOL:
return scanEndTag(buf, off + minBPC, end, token);
default:
throw new InvalidTokenException(off);
}
/* we have a start-tag */
token.NameEnd = -1;
token.clearAttributes();
while (off != end)
{
switch (byteType(buf, off))
{
case BT_NMSTRT:
case BT_NAME:
case BT_MINUS:
off += minBPC;
break;
case BT_LEAD2:
if (end - off < 2)
{
throw new PartialCharException(off);
}
if (!isNameChar2(buf, off))
{
throw new InvalidTokenException(off);
}
off += 2;
break;
case BT_LEAD3:
if (end - off < 3)
{
throw new PartialCharException(off);
}
if (!isNameChar3(buf, off))
{
throw new InvalidTokenException(off);
}
off += 3;
break;
case BT_LEAD4:
if (end - off < 4)
{
throw new PartialCharException(off);
}
if (!isNameChar4(buf, off))
{
throw new InvalidTokenException(off);
}
off += 4;
break;
case BT_S:
case BT_CR:
case BT_LF:
token.NameEnd = off;
off += minBPC;
for (;;)
{
if (off == end)
{
throw new PartialTokenException();
}
switch (byteType(buf, off))
{
case BT_NMSTRT:
return scanAtts(off, buf, off + minBPC, end, token);
case BT_LEAD2:
if (end - off < 2)
{
throw new PartialCharException(off);
}
if (byteType2(buf, off) != BT_NMSTRT)
{
throw new InvalidTokenException(off);
}
return scanAtts(off, buf, off + 2, end, token);
case BT_LEAD3:
if (end - off < 3)
{
throw new PartialCharException(off);
}
if (byteType3(buf, off) != BT_NMSTRT)
{
throw new InvalidTokenException(off);
}
return scanAtts(off, buf, off + 3, end, token);
case BT_LEAD4:
if (end - off < 4)
{
throw new PartialCharException(off);
}
if (byteType4(buf, off) != BT_NMSTRT)
{
throw new InvalidTokenException(off);
}
return scanAtts(off, buf, off + 4, end, token);
case BT_GT:
case BT_SOL:
goto loop;
case BT_S:
case BT_CR:
case BT_LF:
off += minBPC;
break;
default:
throw new InvalidTokenException(off);
}
}
loop:
break;
case BT_GT:
if (token.NameEnd < 0)
{
token.NameEnd = off;
}
token.TokenEnd = off + minBPC;
return TOK.START_TAG_NO_ATTS;
case BT_SOL:
if (token.NameEnd < 0)
{
token.NameEnd = off;
}
off += minBPC;
if (off == end)
{
throw new PartialTokenException();
}
checkCharMatches(buf, off, '>');
token.TokenEnd = off + minBPC;
return TOK.EMPTY_ELEMENT_NO_ATTS;
default:
throw new InvalidTokenException(off);
}
}
throw new PartialTokenException();
}
/// <summary>
/// </summary>
/// <param name="buf"> </param>
/// <param name="offset"> </param>
/// <param name="length"> </param>
/// <returns> </returns>
/// <exception cref="NotImplementedException"></exception>
private string NormalizeAttributeValue(byte[] buf, int offset, int length)
{
if (length == 0)
{
return null;
}
string val = null;
var buffer = new BufferAggregate();
var copy = new byte[length];
Buffer.BlockCopy(buf, offset, copy, 0, length);
buffer.Write(copy);
byte[] b = buffer.GetBuffer();
int off = 0;
TOK tok = TOK.END_TAG;
var ct = new ContentToken();
try
{
while (off < b.Length)
{
// tok = m_enc.tokenizeContent(b, off, b.Length, ct);
tok = m_enc.tokenizeAttributeValue(b, off, b.Length, ct);
switch (tok)
{
case TOK.ATTRIBUTE_VALUE_S:
case TOK.DATA_CHARS:
case TOK.DATA_NEWLINE:
val += utf.GetString(b, off, ct.TokenEnd - off);
break;
case TOK.CHAR_REF:
case TOK.MAGIC_ENTITY_REF:
val += new string(new[] {ct.RefChar1});
break;
case TOK.CHAR_PAIR_REF:
val += new string(new[] {ct.RefChar1, ct.RefChar2});
break;
case TOK.ENTITY_REF:
#if CF
throw new util.NotImplementedException("Token type not implemented: " + tok);
#else
throw new NotImplementedException("Token type not implemented: " + tok);
#endif
}
off = ct.TokenEnd;
}
}
catch (PartialTokenException)
{
// ignored;
}
catch (ExtensibleTokenException)
{
// ignored;
}
catch (Exception ex)
{
if (OnStreamError != null)
{
OnStreamError(this, ex);
}
}
finally
{
buffer.Clear(off);
}
return val;
}
/// <summary>
/// </summary>
/// <param name="buf"> </param>
/// <param name="offset"> </param>
/// <param name="ct"> </param>
/// <param name="tok"> </param>
private void EndTag(byte[] buf, int offset, ContentToken ct, TOK tok)
{
m_Depth--;
m_ns.PopScope();
if (current == null)
{
// end of doc
if (OnStreamEnd != null)
{
OnStreamEnd(this, m_root);
}
// FireOnDocumentEnd();
return;
}
// if (current.Name != name)
// throw new Exception("Invalid end tag: " + name +
// " != " + current.Name);
var parent = (Element) current.Parent;
if (parent == null)
{
DoRaiseOnStreamElement(current);
// if (OnStreamElement!=null)
// OnStreamElement(this, current);
// FireOnElement(current);
}
current = parent;
}
/// <summary>
/// </summary>
/// <param name="buf"> </param>
/// <param name="offset"> </param>
/// <param name="ct"> </param>
/// <param name="tok"> </param>
private void StartTag(byte[] buf, int offset, ContentToken ct, TOK tok)
{
m_Depth++;
int colon;
string name;
string prefix;
var ht = new Hashtable();
m_ns.PushScope();
// if i have attributes
if ((tok == TOK.START_TAG_WITH_ATTS) || (tok == TOK.EMPTY_ELEMENT_WITH_ATTS))
{
int start;
int end;
string val;
for (int i = 0; i < ct.getAttributeSpecifiedCount(); i++)
{
start = ct.getAttributeNameStart(i);
end = ct.getAttributeNameEnd(i);
name = utf.GetString(buf, start, end - start);
start = ct.getAttributeValueStart(i);
end = ct.getAttributeValueEnd(i);
// val = utf.GetString(buf, start, end - start);
val = NormalizeAttributeValue(buf, start, end - start);
// <foo b='&'/>
// <foo b='&amp;'
// TODO: if val includes &, it gets double-escaped
if (name.StartsWith("xmlns:"))
{
colon = name.IndexOf(':');
prefix = name.Substring(colon + 1);
m_ns.AddNamespace(prefix, val);
}
else if (name == "xmlns")
{
m_ns.AddNamespace(string.Empty, val);
}
else
{
ht.Add(name, val);
}
}
}
name = utf.GetString(buf,
offset + m_enc.MinBytesPerChar,
ct.NameEnd - offset - m_enc.MinBytesPerChar);
colon = name.IndexOf(':');
string ns = string.Empty;
prefix = null;
if (colon > 0)
{
prefix = name.Substring(0, colon);
name = name.Substring(colon + 1);
ns = m_ns.LookupNamespace(prefix);
}
else
{
ns = m_ns.DefaultNamespace;
}
Element newel = ElementFactory.GetElement(prefix, name, ns);
foreach (string attrname in ht.Keys)
{
newel.SetAttribute(attrname, (string) ht[attrname]);
}
if (m_root == null)
{
m_root = newel;
// FireOnDocumentStart(m_root);
if (OnStreamStart != null)
{
OnStreamStart(this, m_root, m_ns.DefaultNamespace ?? "");
}
}
else
{
if (current != null)
{
current.AddChild(newel);
}
current = newel;
}
}
/// <summary>
/// Put bytes into the parser.
/// </summary>
/// <param name="buf"> The bytes to put into the parse stream </param>
/// <param name="offset"> Offset into buf to start at </param>
/// <param name="length"> Number of bytes to write </param>
public void Push(byte[] buf, int offset, int length)
{
// or assert, really, but this is a little nicer.
if (length == 0)
{
return;
}
// No locking is required. Read() won't get called again
// until this method returns.
// TODO: only do this copy if we have a partial token at the
// end of parsing.
var copy = new byte[length];
Buffer.BlockCopy(buf, offset, copy, 0, length);
m_buf.Write(copy);
byte[] b = m_buf.GetBuffer();
int off = 0;
TOK tok = TOK.END_TAG;
var ct = new ContentToken();
try
{
while (off < b.Length)
{
if (m_cdata)
{
tok = m_enc.tokenizeCdataSection(b, off, b.Length, ct);
}
else
{
tok = m_enc.tokenizeContent(b, off, b.Length, ct);
}
switch (tok)
{
case TOK.EMPTY_ELEMENT_NO_ATTS:
case TOK.EMPTY_ELEMENT_WITH_ATTS:
StartTag(b, off, ct, tok);
EndTag(b, off, ct, tok);
break;
case TOK.START_TAG_NO_ATTS:
case TOK.START_TAG_WITH_ATTS:
StartTag(b, off, ct, tok);
break;
case TOK.END_TAG:
EndTag(b, off, ct, tok);
break;
case TOK.DATA_CHARS:
case TOK.DATA_NEWLINE:
AddText(utf.GetString(b, off, ct.TokenEnd - off));
break;
case TOK.CHAR_REF:
case TOK.MAGIC_ENTITY_REF:
AddText(new string(new[] {ct.RefChar1}));
break;
case TOK.CHAR_PAIR_REF:
AddText(new string(new[] {ct.RefChar1, ct.RefChar2}));
break;
case TOK.COMMENT:
if (current != null)
{
// <!-- 4
// --> 3
int start = off + 4*m_enc.MinBytesPerChar;
int end = ct.TokenEnd - off - 7*m_enc.MinBytesPerChar;
string text = utf.GetString(b, start, end);
current.AddChild(new Comment(text));
}
break;
case TOK.CDATA_SECT_OPEN:
m_cdata = true;
break;
case TOK.CDATA_SECT_CLOSE:
m_cdata = false;
break;
case TOK.XML_DECL:
// thou shalt use UTF8, and XML version 1.
// i shall ignore evidence to the contrary...
// TODO: Throw an exception if these assuptions are
// wrong
break;
case TOK.ENTITY_REF:
case TOK.PI:
#if CF
throw new util.NotImplementedException("Token type not implemented: " + tok);
#else
throw new NotImplementedException("Token type not implemented: " + tok);
#endif
}
off = ct.TokenEnd;
}
}
catch (PartialTokenException)
{
// ignored;
}
catch (ExtensibleTokenException)
{
// ignored;
}
catch (Exception ex)
{
if (OnStreamError != null)
{
OnStreamError(this, ex);
}
}
finally
{
m_buf.Clear(off);
}
}
/// <summary>
/// </summary>
/// <param name="buf"> </param>
/// <param name="offset"> </param>
/// <param name="ct"> </param>
/// <param name="tok"> </param>
private void EndTag(byte[] buf, int offset, ContentToken ct, TOK tok)
{
m_Depth--;
m_ns.PopScope();
if (current == null)
{
// end of doc
if (OnStreamEnd != null)
{
OnStreamEnd(this, m_root);
}
// FireOnDocumentEnd();
return;
}
string name = null;
if ((tok == TOK.EMPTY_ELEMENT_WITH_ATTS) || (tok == TOK.EMPTY_ELEMENT_NO_ATTS))
{
name = utf.GetString(buf,
offset + m_enc.MinBytesPerChar,
ct.NameEnd - offset - m_enc.MinBytesPerChar);
}
else
{
name = utf.GetString(buf,
offset + m_enc.MinBytesPerChar*2,
ct.NameEnd - offset - m_enc.MinBytesPerChar*2);
}
// if (current.Name != name)
// throw new Exception("Invalid end tag: " + name +
// " != " + current.Name);
var parent = (Element) current.Parent;
if (parent == null)
{
DoRaiseOnStreamElement(current);
// if (OnStreamElement!=null)
// OnStreamElement(this, current);
// FireOnElement(current);
}
current = parent;
}
