/// <summary> /// Returns <c>true</c> if there is a token from the buffer, or <c>false</c> if it is exhausted. /// </summary> /// <returns><c>true</c> if there is a token from the buffer, or <c>false</c> if it is exhausted.</returns> private bool IncrementTokenBuffer() { int start = breaker.Current; if (start == BreakIterator.Done) { return(false); // BreakIterator exhausted } // find the next set of boundaries, skipping over non-tokens (rule status 0) int end = breaker.Next(); while (end != BreakIterator.Done && breaker.RuleStatus == 0) { start = end; end = breaker.Next(); } if (end == BreakIterator.Done) { return(false); // BreakIterator exhausted } termAtt.CopyBuffer(buffer, start, end - start); offsetAtt.SetOffset(CorrectOffset(offset + start), CorrectOffset(offset + end)); typeAtt.Type = config.GetType(breaker.ScriptCode, breaker.RuleStatus); scriptAtt.Code = breaker.ScriptCode; return(true); }
/// <summary> /// Returns true if there is a token from the buffer, or null if it is exhausted. /// </summary> /// <returns>true if there is a token from the buffer, or null if it is exhausted.</returns> private bool IncrementTokenBuffer() { int start = breaker.Current; if (start == BreakIterator.Done) { return(false); // BreakIterator exhausted } // find the next set of boundaries, skipping over non-tokens (rule status 0) int end = breaker.Next(); // LUCENENET specific - ICU 60.1 does not set the rule status back to 0, // so we need to explicitly check whether we went out of bounds. // This is more efficient anyway, since we don't call Next() twice in // this case. if (end == BreakIterator.Done) { return(false); // BreakIterator exhausted } while (start != BreakIterator.Done && breaker.RuleStatus == 0) { start = end; end = breaker.Next(); } if (start == BreakIterator.Done) { return(false); // BreakIterator exhausted } termAtt.CopyBuffer(buffer, start, end - start); offsetAtt.SetOffset(CorrectOffset(offset + start), CorrectOffset(offset + end)); typeAtt.Type = config.GetType(breaker.ScriptCode, breaker.RuleStatus); scriptAtt.Code = breaker.ScriptCode; return(true); }