/// <summary> /// </summary> /// <param name="buf"> </param> /// <param name="off"> </param> /// <param name="end"> </param> /// <param name="token"> </param> /// <returns> </returns> /// <exception cref="EmptyTokenException"></exception> /// <exception cref="ExtensibleTokenException"></exception> /// <exception cref="InvalidTokenException"></exception> /// <exception cref="PartialCharException"></exception> public TOK tokenizeContent(byte[] buf, int off, int end, ContentToken token) { if (minBPC > 1) { end = adjustEnd(off, end); } if (off == end) { throw new EmptyTokenException(); } switch (byteType(buf, off)) { case BT_LT: return scanLt(buf, off + minBPC, end, token); case BT_AMP: return scanRef(buf, off + minBPC, end, token); case BT_CR: off += minBPC; if (off == end) { throw new ExtensibleTokenException(TOK.DATA_NEWLINE); } if (byteType(buf, off) == BT_LF) { off += minBPC; } token.TokenEnd = off; return TOK.DATA_NEWLINE; case BT_LF: token.TokenEnd = off + minBPC; return TOK.DATA_NEWLINE; case BT_RSQB: off += minBPC; if (off == end) { throw new ExtensibleTokenException(TOK.DATA_CHARS); } if (!charMatches(buf, off, ']')) { break; } off += minBPC; if (off == end) { throw new ExtensibleTokenException(TOK.DATA_CHARS); } if (!charMatches(buf, off, '>')) { off -= minBPC; break; } throw new InvalidTokenException(off); case BT_NONXML: case BT_MALFORM: throw new InvalidTokenException(off); case BT_LEAD2: if (end - off < 2) { throw new PartialCharException(off); } check2(buf, off); off += 2; break; case BT_LEAD3: if (end - off < 3) { throw new PartialCharException(off); } check3(buf, off); off += 3; break; case BT_LEAD4: if (end - off < 4) { throw new PartialCharException(off); } check4(buf, off); off += 4; break; default: off += minBPC; break; } token.TokenEnd = extendData(buf, off, end); return TOK.DATA_CHARS; }
/* off points to character following first character of attribute name */ /// <summary> /// </summary> /// <param name="nameStart"> </param> /// <param name="buf"> </param> /// <param name="off"> </param> /// <param name="end"> </param> /// <param name="token"> </param> /// <returns> </returns> /// <exception cref="PartialCharException"></exception> /// <exception cref="InvalidTokenException"></exception> /// <exception cref="PartialTokenException"></exception> private TOK scanAtts(int nameStart, byte[] buf, int off, int end, ContentToken token) { int NameEnd = -1; while (off != end) { switch (byteType(buf, off)) { case BT_NMSTRT: case BT_NAME: case BT_MINUS: off += minBPC; break; case BT_LEAD2: if (end - off < 2) { throw new PartialCharException(off); } if (!isNameChar2(buf, off)) { throw new InvalidTokenException(off); } off += 2; break; case BT_LEAD3: if (end - off < 3) { throw new PartialCharException(off); } if (!isNameChar3(buf, off)) { throw new InvalidTokenException(off); } off += 3; break; case BT_LEAD4: if (end - off < 4) { throw new PartialCharException(off); } if (!isNameChar4(buf, off)) { throw new InvalidTokenException(off); } off += 4; break; case BT_S: case BT_CR: case BT_LF: NameEnd = off; for (;;) { off += minBPC; if (off == end) { throw new PartialTokenException(); } switch (byteType(buf, off)) { case BT_EQUALS: goto loop; case BT_S: case BT_LF: case BT_CR: break; default: throw new InvalidTokenException(off); } } loop: ; /* fall through */ goto case BT_EQUALS; case BT_EQUALS: { if (NameEnd < 0) { NameEnd = off; } int open; for (;;) { off += minBPC; if (off == end) { throw new PartialTokenException(); } open = byteType(buf, off); if (open == BT_QUOT || open == BT_APOS) { break; } switch (open) { case BT_S: case BT_LF: case BT_CR: break; default: throw new InvalidTokenException(off); } } off += minBPC; int valueStart = off; bool normalized = true; int t; /* in attribute value */ for (;;) { if (off == end) { throw new PartialTokenException(); } t = byteType(buf, off); if (t == open) { break; } switch (t) { case BT_NONXML: case BT_MALFORM: throw new InvalidTokenException(off); case BT_LEAD2: if (end - off < 2) { throw new PartialCharException(off); } check2(buf, off); off += 2; break; case BT_LEAD3: if (end - off < 3) { throw new PartialCharException(off); } check3(buf, off); off += 3; break; case BT_LEAD4: if (end - off < 4) { throw new PartialCharException(off); } check4(buf, off); off += 4; break; case BT_AMP: { normalized = false; int saveNameEnd = token.NameEnd; scanRef(buf, off + minBPC, end, token); token.NameEnd = saveNameEnd; off = token.TokenEnd; break; } case BT_S: if (normalized && (off == valueStart || byteToAscii(buf, off) != ' ' || (off + minBPC != end && (byteToAscii(buf, off + minBPC) == ' ' || byteType(buf, off + minBPC) == open)))) { normalized = false; } off += minBPC; break; case BT_LT: throw new InvalidTokenException(off); case BT_LF: case BT_CR: normalized = false; /* fall through */ goto default; default: off += minBPC; break; } } token.appendAttribute(nameStart, NameEnd, valueStart, off, normalized); off += minBPC; if (off == end) { throw new PartialTokenException(); } t = byteType(buf, off); switch (t) { case BT_S: case BT_CR: case BT_LF: off += minBPC; if (off == end) { throw new PartialTokenException(); } t = byteType(buf, off); break; case BT_GT: case BT_SOL: break; default: throw new InvalidTokenException(off); } /* off points to closing quote */ for (;;) { switch (t) { case BT_NMSTRT: nameStart = off; off += minBPC; goto skipToName; case BT_LEAD2: if (end - off < 2) { throw new PartialCharException(off); } if (byteType2(buf, off) != BT_NMSTRT) { throw new InvalidTokenException(off); } nameStart = off; off += 2; goto skipToName; case BT_LEAD3: if (end - off < 3) { throw new PartialCharException(off); } if (byteType3(buf, off) != BT_NMSTRT) { throw new InvalidTokenException(off); } nameStart = off; off += 3; goto skipToName; case BT_LEAD4: if (end - off < 4) { throw new PartialCharException(off); } if (byteType4(buf, off) != BT_NMSTRT) { throw new InvalidTokenException(off); } nameStart = off; off += 4; goto skipToName; case BT_S: case BT_CR: case BT_LF: break; case BT_GT: token.checkAttributeUniqueness(buf); token.TokenEnd = off + minBPC; return TOK.START_TAG_WITH_ATTS; case BT_SOL: off += minBPC; if (off == end) { throw new PartialTokenException(); } checkCharMatches(buf, off, '>'); token.checkAttributeUniqueness(buf); token.TokenEnd = off + minBPC; return TOK.EMPTY_ELEMENT_WITH_ATTS; default: throw new InvalidTokenException(off); } off += minBPC; if (off == end) { throw new PartialTokenException(); } t = byteType(buf, off); } skipToName: NameEnd = -1; break; } default: throw new InvalidTokenException(off); } } throw new PartialTokenException(); }
/* off points to character following "<" */ /// <summary> /// </summary> /// <param name="buf"> </param> /// <param name="off"> </param> /// <param name="end"> </param> /// <param name="token"> </param> /// <returns> </returns> /// <exception cref="PartialTokenException"></exception> /// <exception cref="PartialCharException"></exception> /// <exception cref="InvalidTokenException"></exception> private TOK scanLt(byte[] buf, int off, int end, ContentToken token) { if (off == end) { throw new PartialTokenException(); } switch (byteType(buf, off)) { case BT_NMSTRT: off += minBPC; break; case BT_LEAD2: if (end - off < 2) { throw new PartialCharException(off); } if (byteType2(buf, off) != BT_NMSTRT) { throw new InvalidTokenException(off); } off += 2; break; case BT_LEAD3: if (end - off < 3) { throw new PartialCharException(off); } if (byteType3(buf, off) != BT_NMSTRT) { throw new InvalidTokenException(off); } off += 3; break; case BT_LEAD4: if (end - off < 4) { throw new PartialCharException(off); } if (byteType4(buf, off) != BT_NMSTRT) { throw new InvalidTokenException(off); } off += 4; break; case BT_EXCL: if ((off += minBPC) == end) { throw new PartialTokenException(); } switch (byteType(buf, off)) { case BT_MINUS: return scanComment(buf, off + minBPC, end, token); case BT_LSQB: return scanCdataSection(buf, off + minBPC, end, token); } throw new InvalidTokenException(off); case BT_QUEST: return scanPi(buf, off + minBPC, end, token); case BT_SOL: return scanEndTag(buf, off + minBPC, end, token); default: throw new InvalidTokenException(off); } /* we have a start-tag */ token.NameEnd = -1; token.clearAttributes(); while (off != end) { switch (byteType(buf, off)) { case BT_NMSTRT: case BT_NAME: case BT_MINUS: off += minBPC; break; case BT_LEAD2: if (end - off < 2) { throw new PartialCharException(off); } if (!isNameChar2(buf, off)) { throw new InvalidTokenException(off); } off += 2; break; case BT_LEAD3: if (end - off < 3) { throw new PartialCharException(off); } if (!isNameChar3(buf, off)) { throw new InvalidTokenException(off); } off += 3; break; case BT_LEAD4: if (end - off < 4) { throw new PartialCharException(off); } if (!isNameChar4(buf, off)) { throw new InvalidTokenException(off); } off += 4; break; case BT_S: case BT_CR: case BT_LF: token.NameEnd = off; off += minBPC; for (;;) { if (off == end) { throw new PartialTokenException(); } switch (byteType(buf, off)) { case BT_NMSTRT: return scanAtts(off, buf, off + minBPC, end, token); case BT_LEAD2: if (end - off < 2) { throw new PartialCharException(off); } if (byteType2(buf, off) != BT_NMSTRT) { throw new InvalidTokenException(off); } return scanAtts(off, buf, off + 2, end, token); case BT_LEAD3: if (end - off < 3) { throw new PartialCharException(off); } if (byteType3(buf, off) != BT_NMSTRT) { throw new InvalidTokenException(off); } return scanAtts(off, buf, off + 3, end, token); case BT_LEAD4: if (end - off < 4) { throw new PartialCharException(off); } if (byteType4(buf, off) != BT_NMSTRT) { throw new InvalidTokenException(off); } return scanAtts(off, buf, off + 4, end, token); case BT_GT: case BT_SOL: goto loop; case BT_S: case BT_CR: case BT_LF: off += minBPC; break; default: throw new InvalidTokenException(off); } } loop: break; case BT_GT: if (token.NameEnd < 0) { token.NameEnd = off; } token.TokenEnd = off + minBPC; return TOK.START_TAG_NO_ATTS; case BT_SOL: if (token.NameEnd < 0) { token.NameEnd = off; } off += minBPC; if (off == end) { throw new PartialTokenException(); } checkCharMatches(buf, off, '>'); token.TokenEnd = off + minBPC; return TOK.EMPTY_ELEMENT_NO_ATTS; default: throw new InvalidTokenException(off); } } throw new PartialTokenException(); }
/// <summary> /// </summary> /// <param name="buf"> </param> /// <param name="offset"> </param> /// <param name="length"> </param> /// <returns> </returns> /// <exception cref="NotImplementedException"></exception> private string NormalizeAttributeValue(byte[] buf, int offset, int length) { if (length == 0) { return null; } string val = null; var buffer = new BufferAggregate(); var copy = new byte[length]; Buffer.BlockCopy(buf, offset, copy, 0, length); buffer.Write(copy); byte[] b = buffer.GetBuffer(); int off = 0; TOK tok = TOK.END_TAG; var ct = new ContentToken(); try { while (off < b.Length) { // tok = m_enc.tokenizeContent(b, off, b.Length, ct); tok = m_enc.tokenizeAttributeValue(b, off, b.Length, ct); switch (tok) { case TOK.ATTRIBUTE_VALUE_S: case TOK.DATA_CHARS: case TOK.DATA_NEWLINE: val += utf.GetString(b, off, ct.TokenEnd - off); break; case TOK.CHAR_REF: case TOK.MAGIC_ENTITY_REF: val += new string(new[] {ct.RefChar1}); break; case TOK.CHAR_PAIR_REF: val += new string(new[] {ct.RefChar1, ct.RefChar2}); break; case TOK.ENTITY_REF: #if CF throw new util.NotImplementedException("Token type not implemented: " + tok); #else throw new NotImplementedException("Token type not implemented: " + tok); #endif } off = ct.TokenEnd; } } catch (PartialTokenException) { // ignored; } catch (ExtensibleTokenException) { // ignored; } catch (Exception ex) { if (OnStreamError != null) { OnStreamError(this, ex); } } finally { buffer.Clear(off); } return val; }
/// <summary> /// </summary> /// <param name="buf"> </param> /// <param name="offset"> </param> /// <param name="ct"> </param> /// <param name="tok"> </param> private void EndTag(byte[] buf, int offset, ContentToken ct, TOK tok) { m_Depth--; m_ns.PopScope(); if (current == null) { // end of doc if (OnStreamEnd != null) { OnStreamEnd(this, m_root); } // FireOnDocumentEnd(); return; } // if (current.Name != name) // throw new Exception("Invalid end tag: " + name + // " != " + current.Name); var parent = (Element) current.Parent; if (parent == null) { DoRaiseOnStreamElement(current); // if (OnStreamElement!=null) // OnStreamElement(this, current); // FireOnElement(current); } current = parent; }
/// <summary> /// </summary> /// <param name="buf"> </param> /// <param name="offset"> </param> /// <param name="ct"> </param> /// <param name="tok"> </param> private void StartTag(byte[] buf, int offset, ContentToken ct, TOK tok) { m_Depth++; int colon; string name; string prefix; var ht = new Hashtable(); m_ns.PushScope(); // if i have attributes if ((tok == TOK.START_TAG_WITH_ATTS) || (tok == TOK.EMPTY_ELEMENT_WITH_ATTS)) { int start; int end; string val; for (int i = 0; i < ct.getAttributeSpecifiedCount(); i++) { start = ct.getAttributeNameStart(i); end = ct.getAttributeNameEnd(i); name = utf.GetString(buf, start, end - start); start = ct.getAttributeValueStart(i); end = ct.getAttributeValueEnd(i); // val = utf.GetString(buf, start, end - start); val = NormalizeAttributeValue(buf, start, end - start); // <foo b='&'/> // <foo b='&amp;' // TODO: if val includes &, it gets double-escaped if (name.StartsWith("xmlns:")) { colon = name.IndexOf(':'); prefix = name.Substring(colon + 1); m_ns.AddNamespace(prefix, val); } else if (name == "xmlns") { m_ns.AddNamespace(string.Empty, val); } else { ht.Add(name, val); } } } name = utf.GetString(buf, offset + m_enc.MinBytesPerChar, ct.NameEnd - offset - m_enc.MinBytesPerChar); colon = name.IndexOf(':'); string ns = string.Empty; prefix = null; if (colon > 0) { prefix = name.Substring(0, colon); name = name.Substring(colon + 1); ns = m_ns.LookupNamespace(prefix); } else { ns = m_ns.DefaultNamespace; } Element newel = ElementFactory.GetElement(prefix, name, ns); foreach (string attrname in ht.Keys) { newel.SetAttribute(attrname, (string) ht[attrname]); } if (m_root == null) { m_root = newel; // FireOnDocumentStart(m_root); if (OnStreamStart != null) { OnStreamStart(this, m_root, m_ns.DefaultNamespace ?? ""); } } else { if (current != null) { current.AddChild(newel); } current = newel; } }
/// <summary> /// Put bytes into the parser. /// </summary> /// <param name="buf"> The bytes to put into the parse stream </param> /// <param name="offset"> Offset into buf to start at </param> /// <param name="length"> Number of bytes to write </param> public void Push(byte[] buf, int offset, int length) { // or assert, really, but this is a little nicer. if (length == 0) { return; } // No locking is required. Read() won't get called again // until this method returns. // TODO: only do this copy if we have a partial token at the // end of parsing. var copy = new byte[length]; Buffer.BlockCopy(buf, offset, copy, 0, length); m_buf.Write(copy); byte[] b = m_buf.GetBuffer(); int off = 0; TOK tok = TOK.END_TAG; var ct = new ContentToken(); try { while (off < b.Length) { if (m_cdata) { tok = m_enc.tokenizeCdataSection(b, off, b.Length, ct); } else { tok = m_enc.tokenizeContent(b, off, b.Length, ct); } switch (tok) { case TOK.EMPTY_ELEMENT_NO_ATTS: case TOK.EMPTY_ELEMENT_WITH_ATTS: StartTag(b, off, ct, tok); EndTag(b, off, ct, tok); break; case TOK.START_TAG_NO_ATTS: case TOK.START_TAG_WITH_ATTS: StartTag(b, off, ct, tok); break; case TOK.END_TAG: EndTag(b, off, ct, tok); break; case TOK.DATA_CHARS: case TOK.DATA_NEWLINE: AddText(utf.GetString(b, off, ct.TokenEnd - off)); break; case TOK.CHAR_REF: case TOK.MAGIC_ENTITY_REF: AddText(new string(new[] {ct.RefChar1})); break; case TOK.CHAR_PAIR_REF: AddText(new string(new[] {ct.RefChar1, ct.RefChar2})); break; case TOK.COMMENT: if (current != null) { // <!-- 4 // --> 3 int start = off + 4*m_enc.MinBytesPerChar; int end = ct.TokenEnd - off - 7*m_enc.MinBytesPerChar; string text = utf.GetString(b, start, end); current.AddChild(new Comment(text)); } break; case TOK.CDATA_SECT_OPEN: m_cdata = true; break; case TOK.CDATA_SECT_CLOSE: m_cdata = false; break; case TOK.XML_DECL: // thou shalt use UTF8, and XML version 1. // i shall ignore evidence to the contrary... // TODO: Throw an exception if these assuptions are // wrong break; case TOK.ENTITY_REF: case TOK.PI: #if CF throw new util.NotImplementedException("Token type not implemented: " + tok); #else throw new NotImplementedException("Token type not implemented: " + tok); #endif } off = ct.TokenEnd; } } catch (PartialTokenException) { // ignored; } catch (ExtensibleTokenException) { // ignored; } catch (Exception ex) { if (OnStreamError != null) { OnStreamError(this, ex); } } finally { m_buf.Clear(off); } }
/// <summary> /// </summary> /// <param name="buf"> </param> /// <param name="offset"> </param> /// <param name="ct"> </param> /// <param name="tok"> </param> private void EndTag(byte[] buf, int offset, ContentToken ct, TOK tok) { m_Depth--; m_ns.PopScope(); if (current == null) { // end of doc if (OnStreamEnd != null) { OnStreamEnd(this, m_root); } // FireOnDocumentEnd(); return; } string name = null; if ((tok == TOK.EMPTY_ELEMENT_WITH_ATTS) || (tok == TOK.EMPTY_ELEMENT_NO_ATTS)) { name = utf.GetString(buf, offset + m_enc.MinBytesPerChar, ct.NameEnd - offset - m_enc.MinBytesPerChar); } else { name = utf.GetString(buf, offset + m_enc.MinBytesPerChar*2, ct.NameEnd - offset - m_enc.MinBytesPerChar*2); } // if (current.Name != name) // throw new Exception("Invalid end tag: " + name + // " != " + current.Name); var parent = (Element) current.Parent; if (parent == null) { DoRaiseOnStreamElement(current); // if (OnStreamElement!=null) // OnStreamElement(this, current); // FireOnElement(current); } current = parent; }