/// <summary>
/// Main tokenization method. Responsible for adding next token
/// to the list, if any. Returns if it is at the end of the
/// character stream. It is up to base class to terminate tokenization.
/// </summary>
public override void AddNextToken()
{
SkipWhitespace();
if (_cs.IsEndOfStream())
{
return;
}
// First look at the numbers. Note that it is hard to tell
// 12 +1 if it is a sum of numbers or a sequence. Note that
// R also supports complex numbers like 1.e+01+-37.5i
if (IsPossibleNumber())
{
var start = _cs.Position;
var length = NumberTokenizer.HandleNumber(_cs);
if (length > 0)
{
// HandleNumber will stop at 'i' as it will stop
// at any other non-digit (decimal or hex). Also,
// it doesn't know if + or - is an operator or part
// of the complex number.
HandleNumber(start, length);
return;
}
}
HandleCharacter();
}
public void NumberScannerInteger()
{
var source = "12345678";
var scanner = new StringScanner(source);
Assert.IsTrue(scanner.MoveNext());
var tokenizer = new NumberTokenizer();
var result = tokenizer.Next(scanner);
Assert.IsInstanceOf <NumberToken>(result);
Assert.AreEqual(12345678.0, ((NumberToken)result).Value);
}
public void NumberScannerScientificDotShouldStop()
{
var source = "1e1.2";
var scanner = new StringScanner(source);
Assert.IsTrue(scanner.MoveNext());
var tokenizer = new NumberTokenizer();
var result = tokenizer.Next(scanner);
Assert.IsInstanceOf <NumberToken>(result);
Assert.AreEqual(10.0, ((NumberToken)result).Value);
}
public void NumberScannerScientificMinus()
{
var source = "1e-1";
var scanner = new StringScanner(source);
Assert.IsTrue(scanner.MoveNext());
var tokenizer = new NumberTokenizer();
var result = tokenizer.Next(scanner);
Assert.IsInstanceOf <NumberToken>(result);
Assert.AreEqual(0.1, ((NumberToken)result).Value);
}
public void NumberScannerBinary()
{
var source = "0b01100011";
var scanner = new StringScanner(source);
Assert.IsTrue(scanner.MoveNext());
var tokenizer = new NumberTokenizer();
var result = tokenizer.Next(scanner);
Assert.IsInstanceOf <NumberToken>(result);
Assert.AreEqual(99, ((NumberToken)result).Value);
}
public void NumberScannerZeroFloat()
{
var source = "0.012340";
var scanner = new StringScanner(source);
Assert.IsTrue(scanner.MoveNext());
var tokenizer = new NumberTokenizer();
var result = tokenizer.Next(scanner);
Assert.IsInstanceOf <NumberToken>(result);
Assert.AreEqual(0.012340, ((NumberToken)result).Value);
}
public void NumberScannerLargeValue()
{
var source = "9223372036854775807";
var scanner = new StringScanner(source);
Assert.IsTrue(scanner.MoveNext());
var tokenizer = new NumberTokenizer();
var result = tokenizer.Next(scanner);
Assert.IsInstanceOf <NumberToken>(result);
Assert.AreEqual(9223372036854776000.0, ((NumberToken)result).Value);
}
private void HandleNumber(int numberStart, int length)
{
if (_cs.CurrentChar == 'i')
{
_cs.MoveToNextChar();
AddToken(RTokenType.Complex, numberStart, _cs.Position - numberStart);
return;
}
// Check if this is actually complex number
var imaginaryStart = _cs.Position;
var imaginaryLength = NumberTokenizer.HandleImaginaryPart(_cs);
if (imaginaryLength > 0)
{
AddToken(RTokenType.Complex, numberStart, length + imaginaryLength);
return;
}
_cs.Position = imaginaryStart;
AddToken(RTokenType.Number, numberStart, length);
}
