private void Init(int bufferSize)
{
this.done = false;
termAtt = (TermAttribute)AddAttribute(typeof(TermAttribute));
offsetAtt = (OffsetAttribute)AddAttribute(typeof(OffsetAttribute));
termAtt.ResizeTermBuffer(bufferSize);
}
public override bool IncrementToken()
{
if (!done)
{
ClearAttributes();
done = true;
int upto = 0;
char[] buffer = termAtt.TermBuffer();
while (true)
{
int length = input.Read(buffer, upto, buffer.Length - upto);
if (length == 0)
{
break;
}
upto += length;
if (upto == buffer.Length)
{
buffer = termAtt.ResizeTermBuffer(1 + buffer.Length);
}
}
termAtt.SetTermLength(upto);
finalOffset = CorrectOffset(upto);
offsetAtt.SetOffset(CorrectOffset(0), finalOffset);
return(true);
}
return(false);
}
private void Init(int bufferSize)
{
this.done = false;
termAtt = (TermAttribute) AddAttribute(typeof(TermAttribute));
offsetAtt = (OffsetAttribute) AddAttribute(typeof(OffsetAttribute));
termAtt.ResizeTermBuffer(bufferSize);
}
public override bool IncrementToken()
{
ClearAttributes();
int length = 0;
int start = bufferIndex;
char[] buffer = termAtt.TermBuffer();
while (true)
{
if (bufferIndex >= dataLen)
{
offset += dataLen;
dataLen = input.Read((System.Char[])ioBuffer, 0, ioBuffer.Length);
if (dataLen <= 0)
{
dataLen = 0; // so next offset += dataLen won't decrement offset
if (length > 0)
{
break;
}
else
{
return(false);
}
}
bufferIndex = 0;
}
char c = ioBuffer[bufferIndex++];
if (IsTokenChar(c))
{
// if it's a token char
if (length == 0)
{
// start of token
start = offset + bufferIndex - 1;
}
else if (length == buffer.Length)
{
buffer = termAtt.ResizeTermBuffer(1 + length);
}
buffer[length++] = Normalize(c); // buffer it, normalized
if (length == MAX_WORD_LEN)
{
// buffer overflow!
break;
}
}
else if (length > 0)
{
// at non-Letter w/ chars
break; // return 'em
}
}
termAtt.SetTermLength(length);
offsetAtt.SetOffset(CorrectOffset(start), CorrectOffset(start + length));
return(true);
}