//-------------------------------------------------------------------------
// primary token (it is a subject of possible transformations later) is a smth
// delimited by the set of delimiter symbols OR is the rest of string
// till EOB
//-------------------------------------------------------------------------
protected int FindRightBorderOfToken(out int NumberOfSpecials, out bool hasUpperCaseChars)
{
char charCode;
UnicodeCategory charCategory;
NumberOfSpecials = 0;
iEndOffset = iOffset;
hasUpperCaseChars = Char.IsUpper(strBuffer, iEndOffset);
while (++iEndOffset < BufferLength)
{
charCode = strBuffer[iEndOffset];
if (charCode < 128)
{
charCategory = ASCIIUnicodeCategory[charCode];
}
else
{
charCategory = Char.GetUnicodeCategory(charCode);
}
if (TextDelimitingCategories.IsDelimiter((int)charCategory))
{
break;
}
if (TextDelimitingCategories.IsCleaningCat((int)charCategory))
{
NumberOfSpecials++;
}
hasUpperCaseChars = hasUpperCaseChars || Char.IsUpper(charCode);
}
return(--iEndOffset);
}
public static bool isDelimiter(char ch)
{
UnicodeCategory category = Char.GetUnicodeCategory(ch);
return(TextDelimitingCategories.IsDelimiter((int)category));
}