//------------------------------------------------------------------------- // primary token (it is a subject of possible transformations later) is a smth // delimited by the set of delimiter symbols OR is the rest of string // till EOB //------------------------------------------------------------------------- protected int FindRightBorderOfToken(out int NumberOfSpecials, out bool hasUpperCaseChars) { char charCode; UnicodeCategory charCategory; NumberOfSpecials = 0; iEndOffset = iOffset; hasUpperCaseChars = Char.IsUpper(strBuffer, iEndOffset); while (++iEndOffset < BufferLength) { charCode = strBuffer[iEndOffset]; if (charCode < 128) { charCategory = ASCIIUnicodeCategory[charCode]; } else { charCategory = Char.GetUnicodeCategory(charCode); } if (TextDelimitingCategories.IsDelimiter((int)charCategory)) { break; } if (TextDelimitingCategories.IsCleaningCat((int)charCategory)) { NumberOfSpecials++; } hasUpperCaseChars = hasUpperCaseChars || Char.IsUpper(charCode); } return(--iEndOffset); }
protected bool isDelimiterAt(int i_Offset) { Debug.Assert(i_Offset < BufferLength); UnicodeCategory category = Char.GetUnicodeCategory(strBuffer[i_Offset]); return(TextDelimitingCategories.IsCleaningCat((int)category)); }
public static bool isDelimiter(char ch) { UnicodeCategory category = Char.GetUnicodeCategory(ch); return(TextDelimitingCategories.IsDelimiter((int)category)); }