}//method
private void AssignTypeCodes(CompoundTokenDetails details)
{
//Type could be assigned when we read suffix; if so, just exit
if (details.TypeCodes != null)
{
return;
}
//Decide on float types
var hasDot = details.IsSet((short)(NumberFlagsInternal.HasDot));
var hasExp = details.IsSet((short)(NumberFlagsInternal.HasExp));
var isFloat = (hasDot || hasExp);
if (!isFloat)
{
details.TypeCodes = DefaultIntTypes;
return;
}
//so we have a float. If we have exponent symbol then use it to select type
if (hasExp)
{
TypeCode code;
if (_exponentsTable.TryGetValue(details.ExponentSymbol[0], out code))
{
details.TypeCodes = new TypeCode[] { code };
return;
}
}//if hasExp
//Finally assign default float type
details.TypeCodes = new TypeCode[] { DefaultFloatType };
}
//Override to assign IsKeyword flag to keyword tokens
protected override Token CreateToken(CompilerContext context, ISourceStream source, CompoundTokenDetails details)
{
if (details.IsSet((short)IdFlags.NameIncludesPrefix) && !string.IsNullOrEmpty(details.Prefix))
{
details.Value = details.Prefix + details.Body;
}
Token token = base.CreateToken(context, source, details);
if (details.IsSet((short)IdFlags.IsNotKeyword))
{
return(token);
}
//check if it is keyword
CheckReservedWord(token);
return(token);
}
private int GetRadix(CompoundTokenDetails details)
{
if (details.IsSet((short)NumberOptions.Hex))
{
return(16);
}
if (details.IsSet((short)NumberOptions.Octal))
{
return(8);
}
if (details.IsSet((short)NumberOptions.Binary))
{
return(2);
}
return(10);
}
private int GetSafeWordLength(CompoundTokenDetails details)
{
if (details.IsSet((short)NumberOptions.Hex))
{
return(15);
}
if (details.IsSet((short)NumberOptions.Octal))
{
return(21); //maxWordLength 22
}
if (details.IsSet((short)NumberOptions.Binary))
{
return(63);
}
return(19); //maxWordLength 20
}
private string GetDigits(CompoundTokenDetails details)
{
if (details.IsSet((short)NumberOptions.Hex))
{
return(Strings.HexDigits);
}
if (details.IsSet((short)NumberOptions.Octal))
{
return(Strings.OctalDigits);
}
if (details.IsSet((short)NumberOptions.Binary))
{
return(Strings.BinaryDigits);
}
return(Strings.DecimalDigits);
}
//radix^safeWordLength
private ulong GetSafeWordRadix(CompoundTokenDetails details)
{
if (details.IsSet((short)NumberOptions.Hex))
{
return(1152921504606846976);
}
if (details.IsSet((short)NumberOptions.Octal))
{
return(9223372036854775808);
}
if (details.IsSet((short)NumberOptions.Binary))
{
return(9223372036854775808);
}
return(10000000000000000000);
}
}//method
protected override void ReadSuffix(ISourceStream source, CompoundTokenDetails details)
{
base.ReadSuffix(source, details);
if (string.IsNullOrEmpty(details.Suffix))
{
details.TypeCodes = details.IsSet((short)(NumberFlags.HasDot | NumberFlags.HasExp)) ? _defaultFloatTypes : DefaultIntTypes;
}
}
protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details)
{
int start = source.PreviewPosition;
bool allowEscapes = details.IsSet((short)IdOptions.AllowsEscapes);
CharList outputChars = new CharList();
while (!source.EOF())
{
char current = source.PreviewChar;
if (Grammar.IsWhitespaceOrDelimiter(current))
{
break;
}
if (allowEscapes && current == this.EscapeChar)
{
current = ReadUnicodeEscape(source, details);
//We need to back off the position. ReadUnicodeEscape sets the position to symbol right after escape digits.
//This is the char that we should process in next iteration, so we must backup one char, to pretend the escaped
// char is at position of last digit of escape sequence.
source.PreviewPosition--;
if (details.Error != null)
{
return(false);
}
}
//Check if current character is OK
if (!CharOk(current, source.PreviewPosition == start))
{
break;
}
//Check if we need to skip this char
#if NETSTANDARD
UnicodeCategory currCat = CharUnicodeInfo.GetUnicodeCategory(current);
#else
UnicodeCategory currCat = char.GetUnicodeCategory(current); //I know, it suxx, we do it twice, fix it later
#endif
if (!this.CharsToRemoveCategories.Contains(currCat))
{
outputChars.Add(current); //add it to output (identifier)
}
source.PreviewPosition++;
}//while
if (outputChars.Count == 0)
{
return(false);
}
//Convert collected chars to string
details.Body = new string(outputChars.ToArray());
if (!CheckCaseRestriction(details.Body))
{
return(false);
}
return(!string.IsNullOrEmpty(details.Body));
}
private bool QuickConvertToDouble(CompoundTokenDetails details)
{
if (details.IsSet((short)(NumberOptions.Binary | NumberOptions.Octal | NumberOptions.Hex)))
{
return(false);
}
if (details.IsSet((short)(NumberFlagsInternal.HasExp)))
{
return(false);
}
if (DecimalSeparator != '.')
{
return(false);
}
if (!double.TryParse(details.Body, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out double dvalue))
{
return(false);
}
details.Value = dvalue;
return(true);
}
protected override bool ConvertValue(CompoundTokenDetails details)
{
if (details.IsSet((short)IdOptions.NameIncludesPrefix))
{
details.Value = details.Prefix + details.Body;
}
else
{
details.Value = details.Body;
}
return(true);
}
//Override to assign IsKeyword flag to keyword tokens
protected override Token CreateToken(ParsingContext context, ISourceStream source, CompoundTokenDetails details)
{
Token token = base.CreateToken(context, source, details);
if (details.IsSet((short)IdOptions.IsNotKeyword))
{
return(token);
}
//check if it is keyword
CheckReservedWord(token);
return(token);
}
protected override void ReadSuffix(ISourceStream source, CompoundTokenDetails details)
{
base.ReadSuffix(source, details);
//"char" type can be identified by suffix (like VB where c suffix identifies char)
// in this case we have details.TypeCodes[0] == char and we need to set the IsChar flag
if (details.TypeCodes != null && details.TypeCodes[0] == TypeCode.Char)
{
details.Flags |= (int)StringOptions.IsChar;
}
else if (details.IsSet((short)StringOptions.IsChar))
{
//we may have IsChar flag set (from startEndSymbol, like in c# single quote identifies char)
// in this case set type code
details.TypeCodes = new[] { TypeCode.Char };
}
}
//Extract the string content from lexeme, adjusts the escaped and double-end symbols
protected override bool ConvertValue(CompoundTokenDetails details)
{
var value = details.Body;
var escapeEnabled = !details.IsSet((short)HereDocOptions.NoEscapes);
//Fix all escapes
if (escapeEnabled && value.IndexOf(EscapeChar) >= 0)
{
details.Flags |= (int)StringFlagsInternal.HasEscapes;
var arr = value.Split(EscapeChar);
var ignoreNext = false;
//we skip the 0 element as it is not preceeded by "\"
for (var i = 1; i < arr.Length; i++)
{
if (ignoreNext)
{
ignoreNext = false;
continue;
}
var s = arr[i];
if (string.IsNullOrEmpty(s))
{
//it is "\\" - escaped escape symbol.
arr[i] = @"\";
ignoreNext = true;
continue;
}
//The char is being escaped is the first one; replace it with char in Escapes table
var first = s[0];
if (Escapes.TryGetValue(first, out char newFirst))
{
arr[i] = newFirst + s.Substring(1);
}
else
{
arr[i] = HandleSpecialEscape(arr[i], details);
}
}
value = string.Join(string.Empty, arr);
}
details.TypeCodes = new[] { TypeCode.String };
details.Value = value;
return(true);
}
private string GetDigits(CompoundTokenDetails details) {
if (details.IsSet((short)NumberFlags.Hex))
return Strings.HexDigits;
if (details.IsSet((short)NumberFlags.Octal))
return Strings.OctalDigits;
if (details.IsSet((short)NumberFlags.Binary))
return Strings.BinaryDigits;
return Strings.DecimalDigits;
}
private bool CompleteReadBody(ISourceStream source, CompoundTokenDetails details)
{
var escapeEnabled = !details.IsSet((short)HereDocOptions.NoEscapes);
var start = source.PreviewPosition;
var endQuoteSymbol = details.TokenName;
//1. Find the string end
// first get the position of the next line break; we are interested in it to detect malformed string,
// therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
// MIC: in heredoc, multiline is always allowed.
//fix by ashmind for EOF right after opening symbol
while (true)
{
var endPos = source.Text.IndexOf(endQuoteSymbol, source.PreviewPosition, StringComparison.InvariantCulture);
//Check for partial token in line-scanning mode
if (endPos < 0 && details.PartialOk)
{
ProcessPartialBody(source, details);
return(true);
}
//Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
var malformed = endPos < 0;
if (malformed)
{
if (endPos > 0)
{
source.PreviewPosition = endPos + 1;
}
details.Error = Resources.ErrBadStrLiteral; // "Mal-formed string literal - cannot find termination symbol.";
return(true); //we did find start symbol, so it is definitely string, only malformed
}
if (source.EOF)
{
return(true);
}
//We found EndSymbol - check if it is escaped; if yes, skip it and continue search
if (escapeEnabled && IsEndQuoteEscaped(source.Text, endPos))
{
source.PreviewPosition = endPos + endQuoteSymbol.Length;
continue; //searching for end symbol
}
// Check if not correctly indented.
if (source.Text[endPos - 1] != '\n' && !details.IsSet((short)HereDocOptions.AllowIndentedEndToken))
{
return(false);
}
//Ok, this is normal endSymbol that terminates the string.
// Advance source position and get out from the loop
source.PreviewPosition = endPos + endQuoteSymbol.Length;
var indent = 0;
// Remove the last newline.
while (source.Text[endPos] != '\n')
{
// Remove the indentations.
--endPos;
++indent;
}
// Compensation for the extra '\n'.
--indent;
// Text[endPos] is now always \n.
if (source.Text[endPos - 1] == '\r')
{
--endPos;
}
if (details.IsSet((short)HereDocOptions.RemoveBeginningNewLine))
{
if (source.Text[start] == '\r')
{
start += 2;
}
else
{
start += 1;
}
}
var body = source.Text.Substring(start, endPos - start);
if (details.IsSet((short)HereDocOptions.AllowIndentedEndToken) && details.IsSet((short)HereDocOptions.RemoveIndents) && indent > 0)
{
// Remove indentations.
var undented = false;
var lines = body.Split('\n');
for (var i = 0; i < lines.Length; ++i)
{
var line = lines[i];
if (line.Length == 0 || !Grammar.IsWhitespace(line[0]))
{
continue;
}
var n = 0;
while (n < indent && n < line.Length && Grammar.IsWhitespace(line[n]))
{
++n;
}
if (n != 0)
{
line = n == line.Length ? string.Empty : line.Substring(n, line.Length - n);
lines[i] = line;
undented = true;
}
}
if (undented)
{
body = string.Join("\n", lines);
}
}
details.Body = body;
//if we come here it means we're done - we found string end.
return(true);
}
}
}//method
protected override void ReadSuffix(ISourceStream source, CompoundTokenDetails details) {
base.ReadSuffix(source, details);
if (string.IsNullOrEmpty(details.Suffix))
details.TypeCodes = details.IsSet((short) (NumberFlags.HasDot | NumberFlags.HasExp)) ? _defaultFloatTypes : DefaultIntTypes;
}
private bool ConvertToFloat(TypeCode typeCode, CompoundTokenDetails details, ParsingContext context)
{
//only decimal numbers can be fractions
if (details.IsSet((short)(NumberOptions.Binary | NumberOptions.Octal | NumberOptions.Hex)))
{
details.Error = Resources.ErrInvNumber; // "Invalid number.";
return(false);
}
string body = details.Body;
//Some languages allow exp symbols other than E. Check if it is the case, and change it to E
// - otherwise .NET conversion methods may fail
if (details.IsSet((short)NumberFlagsInternal.HasExp) && details.ExponentSymbol.ToUpper() != "E")
{
body = body.Replace(details.ExponentSymbol, "E");
}
//'.' decimal seperator required by invariant culture
if (details.IsSet((short)NumberFlagsInternal.HasDot) && DecimalSeparator != '.')
{
body = body.Replace(DecimalSeparator, '.');
}
switch (typeCode)
{
case TypeCode.Double:
case TypeCodeImaginary:
double dValue;
if (!Double.TryParse(body, NumberStyles.Float, context.Culture, out dValue))
{
return(false);
}
if (typeCode == TypeCodeImaginary)
{
details.Value = new Complex64(0, dValue);
}
else
{
details.Value = dValue;
}
return(true);
case TypeCode.Single:
float fValue;
if (!Single.TryParse(body, NumberStyles.Float, context.Culture, out fValue))
{
return(false);
}
details.Value = fValue;
return(true);
case TypeCode.Decimal:
decimal decValue;
if (!Decimal.TryParse(body, NumberStyles.Float, context.Culture, out decValue))
{
return(false);
}
details.Value = decValue;
return(true);
}//switch
return(false);
}
protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details) {
//remember start - it may be different from source.TokenStart, we may have skipped prefix
int start = source.Position;
char current = source.CurrentChar;
if (current == '-' || current == '+') {
details.Sign = current.ToString();
source.Position++;
}
//Figure out digits set
string digits = GetDigits(details);
bool isDecimal = !details.IsSet((short) (NumberFlags.Binary | NumberFlags.Octal | NumberFlags.Hex));
bool allowFloat = !IsSet(NumberFlags.IntOnly);
bool foundDigits = false;
while (!source.EOF()) {
current = source.CurrentChar;
//1. If it is a digit, just continue going
if (digits.IndexOf(current) >= 0) {
source.Position++;
foundDigits = true;
continue;
}
//2. Check if it is a dot in float number
bool isDot = current == DecimalSeparator;
if (allowFloat && isDot) {
//If we had seen already a dot or exponent, don't accept this one;
bool hasDotOrExp = details.IsSet((short) (NumberFlags.HasDot | NumberFlags.HasExp));
if (hasDotOrExp) break; //from while loop
//In python number literals (NumberAllowPointFloat) a point can be the first and last character,
//We accept dot only if it is followed by a digit
if (digits.IndexOf(source.NextChar) < 0 && !IsSet(NumberFlags.AllowStartEndDot))
break; //from while loop
details.Flags |= (int) NumberFlags.HasDot;
source.Position++;
continue;
}
//3. Check if it is int number followed by dot or exp symbol
bool isExpSymbol = (details.ExponentSymbol == null) && ExponentSymbols.IndexOf(current) >= 0;
if (!allowFloat && foundDigits && (isDot || isExpSymbol)) {
//If no partial float allowed then return false - it is not integer, let float terminal recognize it as float
if (IsSet(NumberFlags.AvoidPartialFloat)) return false;
//otherwise break, it is integer and we're done reading digits
break;
}
//4. Only for decimals - check if it is (the first) exponent symbol
if (allowFloat && isDecimal && isExpSymbol) {
char next = source.NextChar;
bool nextIsSign = next == '-' || next == '+';
bool nextIsDigit = digits.IndexOf(next) >= 0;
if (!nextIsSign && !nextIsDigit)
break; //Exponent should be followed by either sign or digit
//ok, we've got real exponent
details.ExponentSymbol = current.ToString(); //remember the exp char
details.Flags |= (int) NumberFlags.HasExp;
source.Position++;
if (nextIsSign)
source.Position++; //skip +/- explicitly so we don't have to deal with them on the next iteration
continue;
}
//4. It is something else (not digit, not dot or exponent) - we're done
break; //from while loop
}//while
int end = source.Position;
if (!foundDigits)
return false;
details.Body = source.Text.Substring(start, end - start);
return true;
}
}//method
private bool QuickConvertToDouble(CompoundTokenDetails details) {
if (details.IsSet((short)(NumberFlags.Binary | NumberFlags.Octal | NumberFlags.Hex | NumberFlags.HasExp))) return false;
if (DecimalSeparator != '.') return false;
double dvalue;
if (!double.TryParse(details.Body, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out dvalue)) return false;
details.Value = dvalue;
return true;
}
}//method
protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details)
{
//remember start - it may be different from source.TokenStart, we may have skipped prefix
int start = source.PreviewPosition;
char current = source.PreviewChar;
if (IsSet(NumberOptions.AllowSign) && (current == '-' || current == '+'))
{
details.Sign = current.ToString();
source.PreviewPosition++;
}
//Figure out digits set
string digits = GetDigits(details);
bool isDecimal = !details.IsSet((short)(NumberOptions.Binary | NumberOptions.Octal | NumberOptions.Hex));
bool allowFloat = !IsSet(NumberOptions.IntOnly);
bool foundDigits = false;
while (!source.EOF())
{
current = source.PreviewChar;
//1. If it is a digit, just continue going; the same for '_' if it is allowed
if (digits.IndexOf(current) >= 0 || IsSet(NumberOptions.AllowUnderscore) && current == '_')
{
source.PreviewPosition++;
foundDigits = true;
continue;
}
//2. Check if it is a dot in float number
bool isDot = current == DecimalSeparator;
if (allowFloat && isDot)
{
//If we had seen already a dot or exponent, don't accept this one;
bool hasDotOrExp = details.IsSet((short)(NumberFlagsInternal.HasDot | NumberFlagsInternal.HasExp));
if (hasDotOrExp)
{
break; //from while loop
}
//In python number literals (NumberAllowPointFloat) a point can be the first and last character,
//We accept dot only if it is followed by a digit
if (digits.IndexOf(source.NextPreviewChar) < 0 && !IsSet(NumberOptions.AllowStartEndDot))
{
break; //from while loop
}
details.Flags |= (int)NumberFlagsInternal.HasDot;
source.PreviewPosition++;
continue;
}
//3. Check if it is int number followed by dot or exp symbol
bool isExpSymbol = (details.ExponentSymbol == null) && _exponentsTable.ContainsKey(current);
if (!allowFloat && foundDigits && (isDot || isExpSymbol))
{
//If no partial float allowed then return false - it is not integer, let float terminal recognize it as float
if (IsSet(NumberOptions.NoDotAfterInt))
{
return(false);
}
//otherwise break, it is integer and we're done reading digits
break;
}
//4. Only for decimals - check if it is (the first) exponent symbol
if (allowFloat && isDecimal && isExpSymbol)
{
char next = source.NextPreviewChar;
bool nextIsSign = next == '-' || next == '+';
bool nextIsDigit = digits.IndexOf(next) >= 0;
if (!nextIsSign && !nextIsDigit)
{
break; //Exponent should be followed by either sign or digit
}
//ok, we've got real exponent
details.ExponentSymbol = current.ToString(); //remember the exp char
details.Flags |= (int)NumberFlagsInternal.HasExp;
source.PreviewPosition++;
if (nextIsSign)
{
source.PreviewPosition++; //skip +/- explicitly so we don't have to deal with them on the next iteration
}
continue;
}
//4. It is something else (not digit, not dot or exponent) - we're done
break; //from while loop
}//while
int end = source.PreviewPosition;
if (!foundDigits)
{
return(false);
}
details.Body = source.Text.Substring(start, end - start);
return(true);
}
private bool ConvertToFloat(TypeCode typeCode, CompoundTokenDetails details) {
//only decimal numbers can be fractions
if (details.IsSet((short)(NumberFlags.Binary | NumberFlags.Octal | NumberFlags.Hex))) {
details.Error = "Invalid number.";
return false;
}
string body = details.Body;
//Some languages allow exp symbols other than E. Check if it is the case, and change it to E
// - otherwise .NET conversion methods may fail
if (details.IsSet((short)NumberFlags.HasExp) && details.ExponentSymbol.ToUpper() != "E")
body = body.Replace(details.ExponentSymbol, "E");
//'.' decimal seperator required by invariant culture
if (details.IsSet((short)NumberFlags.HasDot) && DecimalSeparator != '.')
body = body.Replace(DecimalSeparator, '.');
switch (typeCode) {
case TypeCode.Double:
case TypeCodeImaginary:
double dValue;
if (!Double.TryParse(body, NumberStyles.Float, CultureInfo.InvariantCulture, out dValue)) return false;
if (typeCode == TypeCodeImaginary)
details.Value = new Complex64(0, dValue);
else
details.Value = dValue;
return true;
case TypeCode.Single:
float fValue;
if (!Single.TryParse(body, NumberStyles.Float, CultureInfo.InvariantCulture, out fValue)) return false;
details.Value = fValue;
return true;
case TypeCode.Decimal:
decimal decValue;
if (!Decimal.TryParse(body, NumberStyles.Float, CultureInfo.InvariantCulture, out decValue)) return false;
details.Value = decValue;
return true;
}//switch
return false;
}
private int GetRadix(CompoundTokenDetails details) {
if (details.IsSet((short)NumberFlags.Hex))
return 16;
if (details.IsSet((short)NumberFlags.Octal))
return 8;
if (details.IsSet((short)NumberFlags.Binary))
return 2;
return 10;
}
} //method
//Extract the string content from lexeme, adjusts the escaped and double-end symbols
protected override bool ConvertValue(CompoundTokenDetails details)
{
string value = details.Body;
bool escapeEnabled = !details.IsSet((short)StringOptions.NoEscapes);
//Fix all escapes
if (escapeEnabled && value.IndexOf(EscapeChar) >= 0)
{
details.Flags |= (int)StringFlagsInternal.HasEscapes;
string[] arr = value.Split(EscapeChar);
bool ignoreNext = false;
//we skip the 0 element as it is not preceeded by "\"
for (int i = 1; i < arr.Length; i++)
{
if (ignoreNext)
{
ignoreNext = false;
continue;
}
string s = arr[i];
if (string.IsNullOrEmpty(s))
{
//it is "\\" - escaped escape symbol.
arr[i] = @"\";
ignoreNext = true;
continue;
}
//The char is being escaped is the first one; replace it with char in Escapes table
char first = s[0];
char newFirst;
if (Escapes.TryGetValue(first, out newFirst))
{
arr[i] = newFirst + s.Substring(1);
}
else
{
arr[i] = HandleSpecialEscape(arr[i], details);
} //else
} //for i
value = string.Join(string.Empty, arr);
} // if EscapeEnabled
//Check for doubled end symbol
string endSymbol = details.EndSymbol;
if (details.IsSet((short)StringOptions.AllowsDoubledQuote) && value.IndexOf(endSymbol) >= 0)
{
value = value.Replace(endSymbol + endSymbol, endSymbol);
}
if (details.IsSet((short)StringOptions.IsChar))
{
if (value.Length != 1)
{
details.Error = Resources.ErrBadChar; //"Invalid length of char literal - should be a single character.";
return(false);
}
details.Value = value[0];
}
else
{
details.TypeCodes = new TypeCode[] { TypeCode.String };
details.Value = value;
}
return(true);
}
private bool CompleteReadBody(ISourceStream source, CompoundTokenDetails details)
{
bool escapeEnabled = !details.IsSet((short)StringOptions.NoEscapes);
int start = source.PreviewPosition;
string endQuoteSymbol = details.EndSymbol;
string endQuoteDoubled = endQuoteSymbol + endQuoteSymbol; //doubled quote symbol
bool lineBreakAllowed = details.IsSet((short)StringOptions.AllowsLineBreak);
//1. Find the string end
// first get the position of the next line break; we are interested in it to detect malformed string,
// therefore do it only if linebreak is NOT allowed; if linebreak is allowed, set it to -1 (we don't care).
int nlPos = lineBreakAllowed ? -1 : source.Text.IndexOf('\n', source.PreviewPosition);
//fix by ashmind for EOF right after opening symbol
while (true)
{
int endPos = source.Text.IndexOf(endQuoteSymbol, source.PreviewPosition);
//Check for partial token in line-scanning mode
if (endPos < 0 && details.PartialOk && lineBreakAllowed)
{
ProcessPartialBody(source, details);
return(true);
}
//Check for malformed string: either EndSymbol not found, or LineBreak is found before EndSymbol
bool malformed = endPos < 0 || nlPos >= 0 && nlPos < endPos;
if (malformed)
{
//Set source position for recovery: move to the next line if linebreak is not allowed.
if (nlPos > 0)
{
endPos = nlPos;
}
if (endPos > 0)
{
source.PreviewPosition = endPos + 1;
}
details.Error = Resources.ErrBadStrLiteral; // "Mal-formed string literal - cannot find termination symbol.";
return(true); //we did find start symbol, so it is definitely string, only malformed
}//if malformed
if (source.EOF())
{
return(true);
}
//We found EndSymbol - check if it is escaped; if yes, skip it and continue search
if (escapeEnabled && IsEndQuoteEscaped(source.Text, endPos))
{
source.PreviewPosition = endPos + endQuoteSymbol.Length;
continue; //searching for end symbol
}
//Check if it is doubled end symbol
source.PreviewPosition = endPos;
if (details.IsSet((short)StringOptions.AllowsDoubledQuote) && source.MatchSymbol(endQuoteDoubled, !CaseSensitive))
{
source.PreviewPosition = endPos + endQuoteDoubled.Length;
continue;
}//checking for doubled end symbol
//Ok, this is normal endSymbol that terminates the string.
// Advance source position and get out from the loop
details.Body = source.Text.Substring(start, endPos - start);
source.PreviewPosition = endPos + endQuoteSymbol.Length;
return(true); //if we come here it means we're done - we found string end.
} //end of loop to find string end;
}
private int GetSafeWordLength(CompoundTokenDetails details) {
if (details.IsSet((short)NumberFlags.Hex))
return 15;
if (details.IsSet((short)NumberFlags.Octal))
return 21; //maxWordLength 22
if (details.IsSet((short)NumberFlags.Binary))
return 63;
return 19; //maxWordLength 20
}
//Should support: \Udddddddd, \udddd, \xdddd, \N{name}, \0, \ddd (octal),
protected virtual string HandleSpecialEscape(string segment, CompoundTokenDetails details)
{
if (string.IsNullOrEmpty(segment))
{
return(string.Empty);
}
int len, p; string digits; char ch; string result;
char first = segment[0];
switch (first)
{
case 'u':
case 'U':
if (details.IsSet((short)StringOptions.AllowsUEscapes))
{
len = (first == 'u' ? 4 : 8);
if (segment.Length < len + 1)
{
details.Error = string.Format(Resources.ErrBadUnEscape, segment.Substring(len + 1), len);// "Invalid unicode escape ({0}), expected {1} hex digits."
return(segment);
}
digits = segment.Substring(1, len);
ch = (char)Convert.ToUInt32(digits, 16);
result = ch + segment.Substring(len + 1);
return(result);
}//if
break;
case 'x':
if (details.IsSet((short)StringOptions.AllowsXEscapes))
{
//x-escape allows variable number of digits, from one to 4; let's count them
p = 1; //current position
while (p < 5 && p < segment.Length)
{
if (Strings.HexDigits.IndexOf(segment[p]) < 0)
{
break;
}
p++;
}
//p now point to char right after the last digit
if (p <= 1)
{
details.Error = Resources.ErrBadXEscape; // @"Invalid \x escape, at least one digit expected.";
return(segment);
}
digits = segment.Substring(1, p - 1);
ch = (char)Convert.ToUInt32(digits, 16);
result = ch + segment.Substring(p);
return(result);
}//if
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
if (details.IsSet((short)StringOptions.AllowsOctalEscapes))
{
//octal escape allows variable number of digits, from one to 3; let's count them
p = 0; //current position
while (p < 3 && p < segment.Length)
{
if (Strings.OctalDigits.IndexOf(segment[p]) < 0)
{
break;
}
p++;
}
//p now point to char right after the last digit
digits = segment.Substring(0, p);
ch = (char)Convert.ToUInt32(digits, 8);
result = ch + segment.Substring(p);
return(result);
} //if
break;
} //switch
details.Error = string.Format(Resources.ErrInvEscape, segment); //"Invalid escape sequence: \{0}"
return(segment);
}//method
//radix^safeWordLength
private ulong GetSafeWordRadix(CompoundTokenDetails details) {
if (details.IsSet((short)NumberFlags.Hex))
return 1152921504606846976;
if (details.IsSet((short)NumberFlags.Octal))
return 9223372036854775808;
if (details.IsSet((short) NumberFlags.Binary))
return 9223372036854775808;
return 10000000000000000000;
}