private void ExtractLiteral(ReadCharBag charBag = null)
{
ReadCharBag currentBag = charBag ?? _readCharBag;
History <char> previousCharCircularArray = new History <char>(3);
while (true)
{
_currentChar = (char)_sourceStream.ReadByte();
if (_currentChar == '"')
{
if (previousCharCircularArray.Current == '\\')
{
if (previousCharCircularArray.Preceding(1) == '\\')
{
break;
}
}
else
{
break;
}
}
else if (_currentChar == '\uffff')
{
throw new EndOfFileExtractLiteralException();
}
currentBag.Add(_currentChar);
previousCharCircularArray.Add(_currentChar);
}
}
private void ExtractDigit(ReadCharBag charBag = null)
{
ReadCharBag currentBag = charBag ?? _readCharBag;
while (true)
{
_currentChar = (char)_sourceStream.ReadByte();
if (!char.IsDigit(_currentChar))
{
break;
}
currentBag.Add(_currentChar);
}
}
private void ExtractWhisteSpace(ReadCharBag charBag = null)
{
ReadCharBag currentBag = charBag ?? _readCharBag;
while (true)
{
_currentChar = (char)_sourceStream.ReadByte();
if (!char.IsWhiteSpace(_currentChar))
{
if (_currentChar != '\uffff')
{
_sourceStream.Position--;
}
break;
}
currentBag.Add(_currentChar);
}
}
private void ExtractKeyword(ReadCharBag charBag = null)
{
ReadCharBag currentBag = charBag ?? _readCharBag;
while (true)
{
_currentChar = (char)_sourceStream.ReadByte();
if (_currentChar == ':' || _currentChar == ',' || char.IsWhiteSpace(_currentChar))
{
break;
}
if (_currentChar == '\uffff')
{
throw new EndOfFileExtractLiteralException();
}
currentBag.Add(_currentChar);
}
}
/// <inheritdoc />
public IEnumerable <SyntaxToken> Lex()
{
SyntaxKind kind;
do
{
object value = null;
string text = null;
kind = SyntaxKind.Unknown;
long startPosition = _sourceStream.Position;
using (_readCharBag)
{
int readByte = _sourceStream.ReadByte();
if (readByte == -1)
{
kind = SyntaxKind.EndOfFile;
}
else
{
_currentChar = (char)readByte;
_readCharBag.Add(_currentChar);
switch (_currentChar)
{
case '\uffff':
case '\0':
kind = SyntaxKind.EndOfFile;
break;
case '{':
kind = SyntaxKind.OpenObjectCurlyBracket;
break;
case '}':
kind = SyntaxKind.CloseObjectCurlyBracket;
break;
case '[':
kind = SyntaxKind.OpenArrayBracket;
break;
case ']':
kind = SyntaxKind.CloseArrayBracket;
break;
case '\'':
case '"':
_readCharBag.EmptyBag();
try
{
ExtractLiteral();
kind = SyntaxKind.Literal;
}
catch (EndOfFileExtractLiteralException e)
{
text = _readCharBag.ToString();
Diagnostics.AddDiagnostic(new Diagnostic(new TextSpan(startPosition, text?.Length ?? 0), $"The string is open but never closed. {text}", e));
continue;
}
break;
case '-':
kind = SyntaxKind.Minus;
break;
case ',':
kind = SyntaxKind.Coma;
break;
case '/':
kind = SyntaxKind.Slash;
break;
case '\\':
kind = SyntaxKind.BackSlash;
break;
case ':':
kind = SyntaxKind.Colon;
break;
case ' ':
case '\t':
case '\n':
case '\r':
ExtractWhisteSpace();
kind = SyntaxKind.Whitespace;
break;
default:
if (char.IsLetter(_currentChar))
{
try
{
ExtractKeyword();
}
catch (EndOfFileExtractLiteralException e)
{
text = _readCharBag.ToString();
Diagnostics.AddDiagnostic(new Diagnostic(new TextSpan(startPosition, text?.Length ?? 0), $"the allowed keywords are boolean values, null and object", e));
continue;
}
string tmpValue = _readCharBag.ToString();
StringComparer currentCultureIgnoreCase = StringComparer.CurrentCultureIgnoreCase;
if (currentCultureIgnoreCase.Compare(tmpValue, "true") == 0 ||
currentCultureIgnoreCase.Compare(tmpValue, "false") == 0)
{
value = currentCultureIgnoreCase.Compare(tmpValue, "true") == 0;
kind = SyntaxKind.Boolean;
}
else if (currentCultureIgnoreCase.Compare(tmpValue, "null") == 0)
{
kind = SyntaxKind.Null;
}
}
else if (char.IsDigit(_currentChar))
{
ExtractDigit();
value = BigInteger.Parse(_readCharBag.ToString());
kind = SyntaxKind.Digit;
}
else
{
kind = SyntaxKind.Unknown;
}
break;
}
text = _readCharBag.ToString();
if (kind == SyntaxKind.Unknown || kind == SyntaxKind.Count)
{
Diagnostics.AddDiagnostic(new Diagnostic(new TextSpan(startPosition, text?.Length ?? 0),
$"Unknown character(s) {text}", new UnknownCharacterException()));
}
}
}
yield return(new SyntaxToken(kind, text, startPosition, value));
} while (kind != SyntaxKind.EndOfFile);
}