/// <summary>
/// Scans for the longest valid EcmaScript identifier
/// </summary>
/// <returns>identifier</returns>
/// <remarks>
/// http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-262.pdf
///
/// IdentifierName =
/// IdentifierStart | IdentifierName IdentifierPart
/// IdentifierStart =
/// Letter | '$' | '_'
/// IdentifierPart =
/// IdentifierStart | Digit
/// </remarks>
private static string ScanIdentifier(ITextStream scanner)
{
bool identPart = false;
scanner.BeginChunk();
while (true)
{
char ch = scanner.Peek();
// digits are only allowed after first char
// rest can be in head or tail
if ((identPart && CharUtility.IsDigit(ch)) ||
CharUtility.IsLetter(ch) || (ch == '_') || (ch == '$'))
{
identPart = true;
scanner.Pop();
ch = scanner.Peek();
continue;
}
// get ident string
return(scanner.EndChunk());
}
}
private static string ScanString(ITextStream scanner)
{
// store for unterminated cases
long strPos = scanner.Index + 1;
int strLine = scanner.Line;
int strCol = scanner.Column;
char stringDelim = scanner.Peek();
scanner.Pop();
char ch = scanner.Peek();
// start chunking
scanner.BeginChunk();
StringBuilder buffer = new StringBuilder(JsonTokenizer.DefaultBufferSize);
while (true)
{
// look ahead
if (scanner.IsCompleted ||
CharUtility.IsControl(ch) && ch != '\t')
{
// reached end or line break before string delim
throw new DeserializationException(JsonTokenizer.ErrorUnterminatedString, strPos, strLine, strCol);
}
// check each character for ending delim
if (ch == stringDelim)
{
// end chunking
scanner.EndChunk(buffer);
// flush closing delim
scanner.Pop();
// output string
return(buffer.ToString());
}
if (ch != JsonGrammar.OperatorCharEscape)
{
// accumulate
scanner.Pop();
ch = scanner.Peek();
continue;
}
// pause chunking to replace escape char
scanner.EndChunk(buffer);
// flush escape char
scanner.Pop();
ch = scanner.Peek();
if (scanner.IsCompleted ||
CharUtility.IsControl(ch) && ch != '\t')
{
// reached end or line break before string delim
throw new DeserializationException(JsonTokenizer.ErrorUnterminatedString, strPos, strLine, strCol);
}
// begin decode
switch (ch)
{
case '0':
{
// consume and do not allow NULL char '\0'
// causes CStrings to terminate
scanner.Pop();
ch = scanner.Peek();
break;
}
case 'b':
{
// backspace
buffer.Append('\b');
scanner.Pop();
ch = scanner.Peek();
break;
}
case 'f':
{
// formfeed
buffer.Append('\f');
scanner.Pop();
ch = scanner.Peek();
break;
}
case 'n':
{
// newline
buffer.Append('\n');
scanner.Pop();
ch = scanner.Peek();
break;
}
case 'r':
{
// carriage return
buffer.Append('\r');
scanner.Pop();
ch = scanner.Peek();
break;
}
case 't':
{
// tab
buffer.Append('\t');
scanner.Pop();
ch = scanner.Peek();
break;
}
case 'u':
{
// Unicode escape sequence
// e.g. (c) => "\u00A9"
const int UnicodeEscapeLength = 4;
scanner.Pop();
ch = scanner.Peek();
string escapeSeq = String.Empty;
for (int i = UnicodeEscapeLength; !scanner.IsCompleted && CharUtility.IsHexDigit(ch) && (i > 0); i--)
{
escapeSeq += ch;
scanner.Pop();
ch = scanner.Peek();
}
// unicode ordinal
int utf16 = 0;
bool parsed = true;
try{
utf16 = Int32.Parse(
escapeSeq,
NumberStyles.AllowHexSpecifier,
NumberFormatInfo.InvariantInfo);
}catch (Exception) { parsed = false; };
if (escapeSeq.Length == UnicodeEscapeLength && parsed)
{
buffer.Append(CharUtility.ConvertFromUtf32(utf16));
}
else
{
// using FireFox-style recovery, if not a valid hex
// escape sequence then treat as single escaped 'u'
// followed by rest of string
buffer.Append('u');
buffer.Append(escapeSeq);
}
break;
}
default:
{
// all unrecognized sequences are interpreted as plain chars
buffer.Append(ch);
scanner.Pop();
ch = scanner.Peek();
break;
}
}
// resume chunking
scanner.BeginChunk();
}
}
private static Token <ModelTokenType> ScanNumber(ITextStream scanner)
{
// store for error cases
long numPos = scanner.Index + 1;
int numLine = scanner.Line;
int numCol = scanner.Column;
scanner.BeginChunk();
char ch = scanner.Peek();
bool isNeg = false;
if (ch == JsonGrammar.OperatorUnaryPlus)
{
// consume positive signing (as is extraneous)
scanner.Pop();
ch = scanner.Peek();
// reset buffering
scanner.BeginChunk();
}
else if (ch == JsonGrammar.OperatorUnaryMinus)
{
// optional minus part
scanner.Pop();
ch = scanner.Peek();
isNeg = true;
}
if (!CharUtility.IsDigit(ch) &&
ch != JsonGrammar.OperatorDecimalPoint)
{
// possibly "-Infinity"
scanner.EndChunk();
return(null);
}
// integer part
while (!scanner.IsCompleted && CharUtility.IsDigit(ch))
{
// consume digit
scanner.Pop();
ch = scanner.Peek();
}
bool hasDecimal = false;
if (!scanner.IsCompleted && (ch == JsonGrammar.OperatorDecimalPoint))
{
// consume decimal
scanner.Pop();
ch = scanner.Peek();
// fraction part
while (!scanner.IsCompleted && CharUtility.IsDigit(ch))
{
// consume digit
scanner.Pop();
ch = scanner.Peek();
hasDecimal = true;
}
if (!hasDecimal)
{
// fractional digits required when '.' present
throw new DeserializationException(JsonTokenizer.ErrorIllegalNumber, numPos, numLine, numCol);
}
}
// note the number of significant digits
int precision = scanner.ChunkSize;
if (hasDecimal)
{
precision--;
}
if (isNeg)
{
precision--;
}
if (precision < 1)
{
// missing digits all together
throw new DeserializationException(JsonTokenizer.ErrorIllegalNumber, numPos, numLine, numCol);
}
bool hasExponent = false;
// optional exponent part
if (!scanner.IsCompleted && (ch == 'e' || ch == 'E'))
{
// consume 'e'
scanner.Pop();
ch = scanner.Peek();
// optional minus/plus part
if (!scanner.IsCompleted &&
ch == JsonGrammar.OperatorUnaryMinus ||
ch == JsonGrammar.OperatorUnaryPlus)
{
// consume sign
scanner.Pop();
ch = scanner.Peek();
}
// exp part
while (!scanner.IsCompleted && CharUtility.IsDigit(ch))
{
// consume digit
scanner.Pop();
ch = scanner.Peek();
hasExponent = true;
}
if (!hasExponent)
{
// exponent digits required when 'e' present
throw new DeserializationException(JsonTokenizer.ErrorIllegalNumber, numPos, numLine, numCol);
}
}
// specifically check for 0x-style hex numbers
if (!scanner.IsCompleted && CharUtility.IsLetter(ch))
{
throw new DeserializationException(JsonTokenizer.ErrorIllegalNumber, numPos, numLine, numCol);
}
// by this point, we have the full number string and know its characteristics
string buffer = scanner.EndChunk();
if (!hasDecimal && !hasExponent && precision < 19)
{
// Integer value
decimal number = 0;
try{
number = Decimal.Parse(
buffer,
NumberStyles.Integer,
NumberFormatInfo.InvariantInfo);
}catch (Exception) {
throw new DeserializationException(JsonTokenizer.ErrorIllegalNumber, numPos, numLine, numCol);
}
if (number >= Int32.MinValue && number <= Int32.MaxValue)
{
// int most common
return(ModelGrammar.TokenPrimitive((int)number));
}
if (number >= Int64.MinValue && number <= Int64.MaxValue)
{
// long more flexible
return(ModelGrammar.TokenPrimitive((long)number));
}
// decimal most flexible
return(ModelGrammar.TokenPrimitive(number));
}
else
{
// Floating Point value
double number;
try{
number = Double.Parse(
buffer,
NumberStyles.Float,
NumberFormatInfo.InvariantInfo);
}catch (Exception) {
throw new DeserializationException(JsonTokenizer.ErrorIllegalNumber, numPos, numLine, numCol);
}
// native EcmaScript number (IEEE-754)
return(ModelGrammar.TokenPrimitive(number));
}
}
