[System.Security.SecurityCritical] // auto-generated
internal bool Tokenize(TokenType TokenMask, out TokenType tokenType, out int tokenValue, ref __DTString str) {
tokenType = TokenType.UnknownToken;
tokenValue = 0;
TokenHashValue value;
Contract.Assert(str.Index < str.Value.Length, "DateTimeFormatInfo.Tokenize(): start < value.Length");
char ch = str.m_current;
bool isLetter = Char.IsLetter(ch);
if (isLetter) {
ch = Char.ToLower(ch, this.Culture);
if (IsHebrewChar(ch) && TokenMask == TokenType.RegularTokenMask) {
bool badFormat;
if (TryParseHebrewNumber(ref str, out badFormat, out tokenValue)) {
if (badFormat) {
tokenType = TokenType.UnknownToken;
return (false);
}
// This is a Hebrew number.
// Do nothing here. TryParseHebrewNumber() will update token accordingly.
tokenType = TokenType.HebrewNumber;
return (true);
}
}
}
int hashcode = ch % TOKEN_HASH_SIZE;
int hashProbe = 1 + ch % SECOND_PRIME;
int remaining = str.len - str.Index;
int i = 0;
TokenHashValue[] hashTable = m_dtfiTokenHash;
if (hashTable == null) {
hashTable = CreateTokenHashTable();
}
do {
value = hashTable[hashcode];
if (value == null) {
// Not found.
break;
}
// Check this value has the right category (regular token or separator token) that we are looking for.
if (((int)value.tokenType & (int)TokenMask) > 0 && value.tokenString.Length <= remaining) {
if (String.Compare(str.Value, str.Index, value.tokenString, 0, value.tokenString.Length, this.Culture, CompareOptions.IgnoreCase)==0) {
if (isLetter) {
// If this token starts with a letter, make sure that we won't allow partial match. So you can't tokenize "MarchWed" separately.
int nextCharIndex;
if ((nextCharIndex = str.Index + value.tokenString.Length) < str.len) {
// Check word boundary. The next character should NOT be a letter.
char nextCh = str.Value[nextCharIndex];
if (Char.IsLetter(nextCh)) {
return (false);
}
}
}
tokenType = value.tokenType & TokenMask;
tokenValue = value.tokenValue;
str.Advance(value.tokenString.Length);
return (true);
} else if (value.tokenType == TokenType.MonthToken && HasSpacesInMonthNames) {
// For month token, we will match the month names which have spaces.
int matchStrLen = 0;
if (str.MatchSpecifiedWords(value.tokenString, true, ref matchStrLen)) {
tokenType = value.tokenType & TokenMask;
tokenValue = value.tokenValue;
str.Advance(matchStrLen);
return (true);
}
} else if (value.tokenType == TokenType.DayOfWeekToken && HasSpacesInDayNames) {
// For month token, we will match the month names which have spaces.
int matchStrLen = 0;
if (str.MatchSpecifiedWords(value.tokenString, true, ref matchStrLen)) {
tokenType = value.tokenType & TokenMask;
tokenValue = value.tokenValue;
str.Advance(matchStrLen);
return (true);
}
}
}
i++;
hashcode += hashProbe;
if (hashcode >= TOKEN_HASH_SIZE) hashcode -= TOKEN_HASH_SIZE;
}while (i < TOKEN_HASH_SIZE);
return (false);
}
////////////////////////////////////////////////////////////////////////
//
// Actions:
// Try to parse the current word to see if it is a Hebrew number.
// Tokens will be updated accordingly.
// This is called by the Lexer of DateTime.Parse().
//
// Unlike most of the functions in this class, the return value indicates
// whether or not it started to parse. The badFormat parameter indicates
// if parsing began, but the format was bad.
//
////////////////////////////////////////////////////////////////////////
private static bool TryParseHebrewNumber(
ref __DTString str,
out Boolean badFormat,
out int number) {
number = -1;
badFormat = false;
int i = str.Index;
if (!HebrewNumber.IsDigit(str.Value[i])) {
// If the current character is not a Hebrew digit, just return false.
// There is no chance that we can parse a valid Hebrew number from here.
return (false);
}
// The current character is a Hebrew digit. Try to parse this word as a Hebrew number.
HebrewNumberParsingContext context = new HebrewNumberParsingContext(0);
HebrewNumberParsingState state;
do {
state = HebrewNumber.ParseByChar(str.Value[i++], ref context);
switch (state) {
case HebrewNumberParsingState.InvalidHebrewNumber: // Not a valid Hebrew number.
case HebrewNumberParsingState.NotHebrewDigit: // The current character is not a Hebrew digit character.
// Break out so that we don't continue to try parse this as a Hebrew number.
return (false);
}
} while (i < str.Value.Length && (state != HebrewNumberParsingState.FoundEndOfHebrewNumber));
// When we are here, we are either at the end of the string, or we find a valid Hebrew number.
Contract.Assert(state == HebrewNumberParsingState.ContinueParsing || state == HebrewNumberParsingState.FoundEndOfHebrewNumber,
"Invalid returned state from HebrewNumber.ParseByChar()");
if (state != HebrewNumberParsingState.FoundEndOfHebrewNumber) {
// We reach end of the string but we can't find a terminal state in parsing Hebrew number.
return (false);
}
// We have found a valid Hebrew number. Update the index.
str.Advance(i - str.Index);
// Get the final Hebrew number value from the HebrewNumberParsingContext.
number = context.result;
return (true);
}
internal bool Tokenize(TokenType TokenMask, out TokenType tokenType, out int tokenValue, ref __DTString str)
{
tokenType = TokenType.UnknownToken;
tokenValue = 0;
char ch = str.m_current;
bool flag = char.IsLetter(ch);
if (flag)
{
ch = char.ToLower(ch, this.Culture);
bool badFormat;
if (DateTimeFormatInfo.IsHebrewChar(ch) && TokenMask == TokenType.RegularTokenMask && DateTimeFormatInfo.TryParseHebrewNumber(ref str, out badFormat, out tokenValue))
{
if (badFormat)
{
tokenType = TokenType.UnknownToken;
return false;
}
else
{
tokenType = TokenType.HebrewNumber;
return true;
}
}
}
int index1 = (int) ch % 199;
int num1 = 1 + (int) ch % 197;
int num2 = str.len - str.Index;
int num3 = 0;
TokenHashValue[] tokenHashValueArray = this.m_dtfiTokenHash ?? this.CreateTokenHashTable();
do
{
TokenHashValue tokenHashValue = tokenHashValueArray[index1];
if (tokenHashValue != null)
{
if ((tokenHashValue.tokenType & TokenMask) > (TokenType) 0 && tokenHashValue.tokenString.Length <= num2)
{
if (string.Compare(str.Value, str.Index, tokenHashValue.tokenString, 0, tokenHashValue.tokenString.Length, this.Culture, CompareOptions.IgnoreCase) == 0)
{
int index2;
if (flag && (index2 = str.Index + tokenHashValue.tokenString.Length) < str.len && char.IsLetter(str.Value[index2]))
return false;
tokenType = tokenHashValue.tokenType & TokenMask;
tokenValue = tokenHashValue.tokenValue;
str.Advance(tokenHashValue.tokenString.Length);
return true;
}
else if (tokenHashValue.tokenType == TokenType.MonthToken && this.HasSpacesInMonthNames)
{
int matchLength = 0;
if (str.MatchSpecifiedWords(tokenHashValue.tokenString, true, ref matchLength))
{
tokenType = tokenHashValue.tokenType & TokenMask;
tokenValue = tokenHashValue.tokenValue;
str.Advance(matchLength);
return true;
}
}
else if (tokenHashValue.tokenType == TokenType.DayOfWeekToken && this.HasSpacesInDayNames)
{
int matchLength = 0;
if (str.MatchSpecifiedWords(tokenHashValue.tokenString, true, ref matchLength))
{
tokenType = tokenHashValue.tokenType & TokenMask;
tokenValue = tokenHashValue.tokenValue;
str.Advance(matchLength);
return true;
}
}
}
++num3;
index1 += num1;
if (index1 >= 199)
index1 -= 199;
}
else
break;
}
while (num3 < 199);
return false;
}
private static bool TryParseHebrewNumber(ref __DTString str, out bool badFormat, out int number)
{
number = -1;
badFormat = false;
int index = str.Index;
if (!HebrewNumber.IsDigit(str.Value[index]))
return false;
HebrewNumberParsingContext context = new HebrewNumberParsingContext(0);
HebrewNumberParsingState numberParsingState;
do
{
numberParsingState = HebrewNumber.ParseByChar(str.Value[index++], ref context);
switch (numberParsingState)
{
case HebrewNumberParsingState.InvalidHebrewNumber:
case HebrewNumberParsingState.NotHebrewDigit:
return false;
default:
continue;
}
}
while (index < str.Value.Length && numberParsingState != HebrewNumberParsingState.FoundEndOfHebrewNumber);
if (numberParsingState != HebrewNumberParsingState.FoundEndOfHebrewNumber)
return false;
str.Advance(index - str.Index);
number = context.result;
return true;
}
internal bool Tokenize(TokenType TokenMask, out TokenType tokenType, out int tokenValue, ref __DTString str)
{
tokenType = TokenType.UnknownToken;
tokenValue = 0;
char current = str.m_current;
bool flag = char.IsLetter(current);
if (flag)
{
bool flag2;
current = char.ToLower(current, this.Culture);
if ((IsHebrewChar(current) && (TokenMask == TokenType.RegularTokenMask)) && TryParseHebrewNumber(ref str, out flag2, out tokenValue))
{
if (flag2)
{
tokenType = TokenType.UnknownToken;
return false;
}
tokenType = TokenType.HebrewNumber;
return true;
}
}
int index = current % '\x00c7';
int num2 = 1 + (current % '\x00c5');
int num3 = str.len - str.Index;
int num4 = 0;
TokenHashValue[] dtfiTokenHash = this.m_dtfiTokenHash;
if (dtfiTokenHash == null)
{
dtfiTokenHash = this.CreateTokenHashTable();
}
do
{
TokenHashValue value2 = dtfiTokenHash[index];
if (value2 == null)
{
break;
}
if (((value2.tokenType & TokenMask) > ((TokenType) 0)) && (value2.tokenString.Length <= num3))
{
if (string.Compare(str.Value, str.Index, value2.tokenString, 0, value2.tokenString.Length, this.Culture, CompareOptions.IgnoreCase) == 0)
{
int num5;
if (flag && ((num5 = str.Index + value2.tokenString.Length) < str.len))
{
char c = str.Value[num5];
if (char.IsLetter(c))
{
return false;
}
}
tokenType = value2.tokenType & TokenMask;
tokenValue = value2.tokenValue;
str.Advance(value2.tokenString.Length);
return true;
}
if ((value2.tokenType == TokenType.MonthToken) && this.HasSpacesInMonthNames)
{
int matchLength = 0;
if (str.MatchSpecifiedWords(value2.tokenString, true, ref matchLength))
{
tokenType = value2.tokenType & TokenMask;
tokenValue = value2.tokenValue;
str.Advance(matchLength);
return true;
}
}
else if ((value2.tokenType == TokenType.DayOfWeekToken) && this.HasSpacesInDayNames)
{
int num7 = 0;
if (str.MatchSpecifiedWords(value2.tokenString, true, ref num7))
{
tokenType = value2.tokenType & TokenMask;
tokenValue = value2.tokenValue;
str.Advance(num7);
return true;
}
}
}
num4++;
index += num2;
if (index >= 0xc7)
{
index -= 0xc7;
}
}
while (num4 < 0xc7);
return false;
}
