internal bool Tokenize(TokenType TokenMask, out TokenType tokenType, out int tokenValue, ref FormatProvider.__DTString str)
{
tokenType = TokenType.UnknownToken;
tokenValue = 0;
TokenHashValue value;
Contract.Assert(str.Index < str.Value.Length, "DateTimeFormatInfo.Tokenize(): start < value.Length");
char ch = str.m_current;
bool isLetter = Char.IsLetter(ch);
if (isLetter)
{
ch = this.Culture.TextInfo.ToLower(ch);
if (IsHebrewChar(ch) && TokenMask == TokenType.RegularTokenMask)
{
bool badFormat;
if (TryParseHebrewNumber(ref str, out badFormat, out tokenValue))
{
if (badFormat)
{
tokenType = TokenType.UnknownToken;
return (false);
}
// This is a Hebrew number.
// Do nothing here. TryParseHebrewNumber() will update token accordingly.
tokenType = TokenType.HebrewNumber;
return (true);
}
}
}
int hashcode = ch % TOKEN_HASH_SIZE;
int hashProbe = 1 + ch % SECOND_PRIME;
int remaining = str.len - str.Index;
int i = 0;
TokenHashValue[] hashTable = _dtfiTokenHash;
if (hashTable == null)
{
hashTable = CreateTokenHashTable();
}
do
{
value = hashTable[hashcode];
if (value == null)
{
// Not found.
break;
}
// Check this value has the right category (regular token or separator token) that we are looking for.
if (((int)value.tokenType & (int)TokenMask) > 0 && value.tokenString.Length <= remaining)
{
if (this.Culture.CompareInfo.Compare(str.Value, str.Index, value.tokenString.Length, value.tokenString, 0, value.tokenString.Length, CompareOptions.IgnoreCase) == 0)
{
if (isLetter)
{
// If this token starts with a letter, make sure that we won't allow partial match. So you can't tokenize "MarchWed" separately.
int nextCharIndex;
if ((nextCharIndex = str.Index + value.tokenString.Length) < str.len)
{
// Check word boundary. The next character should NOT be a letter.
char nextCh = str.Value[nextCharIndex];
if (Char.IsLetter(nextCh))
{
return (false);
}
}
}
tokenType = value.tokenType & TokenMask;
tokenValue = value.tokenValue;
str.Advance(value.tokenString.Length);
return (true);
}
else if (value.tokenType == TokenType.MonthToken && HasSpacesInMonthNames)
{
// For month token, we will match the month names which have spaces.
int matchStrLen = 0;
if (str.MatchSpecifiedWords(value.tokenString, true, ref matchStrLen))
{
tokenType = value.tokenType & TokenMask;
tokenValue = value.tokenValue;
str.Advance(matchStrLen);
return (true);
}
}
else if (value.tokenType == TokenType.DayOfWeekToken && HasSpacesInDayNames)
{
// For month token, we will match the month names which have spaces.
int matchStrLen = 0;
if (str.MatchSpecifiedWords(value.tokenString, true, ref matchStrLen))
{
tokenType = value.tokenType & TokenMask;
tokenValue = value.tokenValue;
str.Advance(matchStrLen);
return (true);
}
}
}
i++;
hashcode += hashProbe;
if (hashcode >= TOKEN_HASH_SIZE) hashcode -= TOKEN_HASH_SIZE;
} while (i < TOKEN_HASH_SIZE);
return (false);
}
////////////////////////////////////////////////////////////////////////
//
// Actions:
// Try to parse the current word to see if it is a Hebrew number.
// Tokens will be updated accordingly.
// This is called by the Lexer of DateTime.Parse().
//
// Unlike most of the functions in this class, the return value indicates
// whether or not it started to parse. The badFormat parameter indicates
// if parsing began, but the format was bad.
//
////////////////////////////////////////////////////////////////////////
private static bool TryParseHebrewNumber(
ref FormatProvider.__DTString str,
out Boolean badFormat,
out int number)
{
number = -1;
badFormat = false;
int i = str.Index;
if (!HebrewNumber.IsDigit(str.Value[i]))
{
// If the current character is not a Hebrew digit, just return false.
// There is no chance that we can parse a valid Hebrew number from here.
return (false);
}
// The current character is a Hebrew digit. Try to parse this word as a Hebrew number.
HebrewNumberParsingContext context = new HebrewNumberParsingContext(0);
HebrewNumberParsingState state;
do
{
state = HebrewNumber.ParseByChar(str.Value[i++], ref context);
switch (state)
{
case HebrewNumberParsingState.InvalidHebrewNumber: // Not a valid Hebrew number.
case HebrewNumberParsingState.NotHebrewDigit: // The current character is not a Hebrew digit character.
// Break out so that we don't continue to try parse this as a Hebrew number.
return (false);
}
} while (i < str.Value.Length && (state != HebrewNumberParsingState.FoundEndOfHebrewNumber));
// When we are here, we are either at the end of the string, or we find a valid Hebrew number.
Contract.Assert(state == HebrewNumberParsingState.ContinueParsing || state == HebrewNumberParsingState.FoundEndOfHebrewNumber,
"Invalid returned state from HebrewNumber.ParseByChar()");
if (state != HebrewNumberParsingState.FoundEndOfHebrewNumber)
{
// We reach end of the string but we can't find a terminal state in parsing Hebrew number.
return (false);
}
// We have found a valid Hebrew number. Update the index.
str.Advance(i - str.Index);
// Get the final Hebrew number value from the HebrewNumberParsingContext.
number = context.result;
return (true);
}