public abstract CodePointAt ( char chars, int offset, int limit ) : int | ||
chars | char | /// a character array |
offset | int | /// the offset to the char values in the chars array to be converted |
limit | int | the index afer the last element that should be used to calculate /// codepoint. /// |
Результат | int |
public override sealed bool IncrementToken() { ClearAttributes(); int length = 0; int start = -1; // this variable is always initialized int end = -1; char[] buffer = termAtt.Buffer; while (true) { if (bufferIndex >= dataLen) { offset += dataLen; charUtils.Fill(ioBuffer, m_input); // read supplementary char aware with CharacterUtils if (ioBuffer.Length == 0) { dataLen = 0; // so next offset += dataLen won't decrement offset if (length > 0) { break; } else { finalOffset = CorrectOffset(offset); return(false); } } dataLen = ioBuffer.Length; bufferIndex = 0; } // use CharacterUtils here to support < 3.1 UTF-16 code unit behavior if the char based methods are gone int c = charUtils.CodePointAt(ioBuffer.Buffer, bufferIndex, ioBuffer.Length); int charCount = Character.CharCount(c); bufferIndex += charCount; if (IsTokenChar(c)) // if it's a token char { if (length == 0) // start of token { Debug.Assert(start == -1); start = offset + bufferIndex - charCount; end = start; } // check if a supplementary could run out of bounds else if (length >= buffer.Length - 1) { buffer = termAtt.ResizeBuffer(2 + length); // make sure a supplementary fits in the buffer } end += charCount; length += Character.ToChars(Normalize(c), buffer, length); // buffer it, normalized if (length >= MAX_WORD_LEN) // buffer overflow! make sure to check for >= surrogate pair could break == test { break; } } // at non-Letter w/ chars else if (length > 0) { break; // return 'em } } termAtt.Length = length; Debug.Assert(start != -1); offsetAtt.SetOffset(CorrectOffset(start), finalOffset = CorrectOffset(end)); return(true); }