public void GetNameWithDigits()
{
Lexer lexer = new Lexer("name123");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("name123", result.Value);
Assert.AreEqual(TokenType.Name, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetKey()
{
Lexer lexer = new Lexer("name:");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("name", result.Value);
Assert.AreEqual(TokenType.Key, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetNameWithInitialUnderscore()
{
Lexer lexer = new Lexer("_foo");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("_foo", result.Value);
Assert.AreEqual(TokenType.Name, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void RaiseIfUnexpectedCharacter()
{
Lexer lexer = new Lexer("\\");
try
{
lexer.NextToken();
Assert.Fail();
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(SyntaxError));
Assert.AreEqual("unexpected '\\'", ex.Message);
}
}
public void RaiseWhenClassVarNameStartsWithADigit()
{
Lexer lexer = new Lexer("@@123");
try
{
lexer.NextToken();
Assert.Fail();
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(SyntaxError));
Assert.AreEqual("invalid class variable name", ex.Message);
}
}
public void GetSymbolInDoubleQuotes()
{
Lexer lexer = new Lexer(" :\"foo bar\"");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("foo bar", result.Value);
Assert.AreEqual(TokenType.Symbol, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetRealNumberWithSpacesUsingReader()
{
Lexer lexer = new Lexer(new StringReader(" 123.45 "));
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("123.45", result.Value);
Assert.AreEqual(TokenType.Real, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetComparisonOperators()
{
Lexer lexer = new Lexer("== != < > <= >=");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("==", result.Value);
Assert.AreEqual(TokenType.Operator, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("!=", result.Value);
Assert.AreEqual(TokenType.Operator, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("<", result.Value);
Assert.AreEqual(TokenType.Operator, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual(">", result.Value);
Assert.AreEqual(TokenType.Operator, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("<=", result.Value);
Assert.AreEqual(TokenType.Operator, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual(">=", result.Value);
Assert.AreEqual(TokenType.Operator, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetDoubleColonAsSeparator()
{
Lexer lexer = new Lexer("::");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("::", result.Value);
Assert.AreEqual(TokenType.Separator, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetStringWithEscapedChars()
{
Lexer lexer = new Lexer("\"foo\\t\\n\\r\"");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("foo\t\n\r", result.Value);
Assert.AreEqual(TokenType.String, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void RaiseIfDoubleQuoteStringIsNotClosed()
{
Lexer lexer = new Lexer("\"foo");
lexer.NextToken();
}
public void GetCharacter()
{
Lexer lexer = new Lexer("'f'");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("f", result.Value);
Assert.AreEqual(TokenType.Character, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetEscapedCharactersWithOctal()
{
Lexer lexer = new Lexer("'\\101' '\\123' '\\12' '\\1'");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("A", result.Value);
Assert.AreEqual(TokenType.Character, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("S", result.Value);
Assert.AreEqual(TokenType.Character, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("\n", result.Value);
Assert.AreEqual(TokenType.Character, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual(1, result.Value.Length);
Assert.AreEqual(1, (int)result.Value[0]);
Assert.AreEqual(TokenType.Character, result.Type);
result = lexer.NextToken();
Assert.IsNull(lexer.NextToken());
}
public void RaiseWhenInvalidInstanceVarName()
{
Lexer lexer = new Lexer("@");
try
{
lexer.NextToken();
Assert.Fail();
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(SyntaxError));
Assert.AreEqual("invalid instance variable name", ex.Message);
}
}
public void GetClassVarNameWithDigitsAndUnderscore()
{
Lexer lexer = new Lexer("@@a_123");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("a_123", result.Value);
Assert.AreEqual(TokenType.ClassVarName, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetOperatorAsSymbol()
{
Lexer lexer = new Lexer(":+");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("+", result.Value);
Assert.AreEqual(TokenType.Symbol, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetClassVarName()
{
Lexer lexer = new Lexer("@@a");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("a", result.Value);
Assert.AreEqual(TokenType.ClassVarName, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void RaiseIfSymbolStartsWithADigit()
{
Lexer lexer = new Lexer(":123");
try
{
lexer.NextToken();
Assert.Fail();
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(SyntaxError));
Assert.AreEqual("unexpected integer", ex.Message);
}
}
public void GetEscapedCharacters()
{
Lexer lexer = new Lexer("'\\t' '\\n' '\\r' '\\v' '\\f' '\\\\'");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("\t", result.Value);
Assert.AreEqual(TokenType.Character, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("\n", result.Value);
Assert.AreEqual(TokenType.Character, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("\r", result.Value);
Assert.AreEqual(TokenType.Character, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("\v", result.Value);
Assert.AreEqual(TokenType.Character, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("\f", result.Value);
Assert.AreEqual(TokenType.Character, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("\\", result.Value);
Assert.AreEqual(TokenType.Character, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetIntegerWithUnderscore()
{
Lexer lexer = new Lexer("123_000_000");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("123000000", result.Value);
Assert.AreEqual(TokenType.Integer, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetStringWithCommentChar()
{
Lexer lexer = new Lexer("\"#foo\"");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("#foo", result.Value);
Assert.AreEqual(TokenType.String, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetIntegerWithDotName()
{
Lexer lexer = new Lexer("123.foo");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("123", result.Value);
Assert.AreEqual(TokenType.Integer, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual(".", result.Value);
Assert.AreEqual(TokenType.Separator, result.Type);
result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("foo", result.Value);
Assert.AreEqual(TokenType.Name, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetOctalInteger()
{
Lexer lexer = new Lexer("0o123");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("0o123", result.Value);
Assert.AreEqual(TokenType.Integer, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetBinaryInteger()
{
Lexer lexer = new Lexer("0b11011");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("0b11011", result.Value);
Assert.AreEqual(TokenType.Integer, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void RaiseIfCharacterIsNotClosed()
{
Lexer lexer = new Lexer("'f");
try
{
lexer.NextToken();
Assert.Fail();
}
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(SyntaxError));
Assert.AreEqual("unclosed character", ex.Message);
}
}
public void GetHexadecimalInteger()
{
Lexer lexer = new Lexer("0x0123456789abcdef");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("0x0123456789abcdef", result.Value);
Assert.AreEqual(TokenType.Integer, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetComparisonAsOperator()
{
Lexer lexer = new Lexer("<=>");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("<=>", result.Value);
Assert.AreEqual(TokenType.Operator, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetHexadecimalIntegerWithUpperCaseLetters()
{
Lexer lexer = new Lexer("0x0123456789ABCDEF");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("0x0123456789ABCDEF", result.Value);
Assert.AreEqual(TokenType.Integer, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetBinaryOrAsOperator()
{
Lexer lexer = new Lexer("|");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("|", result.Value);
Assert.AreEqual(TokenType.Operator, result.Type);
Assert.IsNull(lexer.NextToken());
}
public void GetSymbolEndingWithExclamationMark()
{
Lexer lexer = new Lexer(" :foo!");
var result = lexer.NextToken();
Assert.IsNotNull(result);
Assert.AreEqual("foo!", result.Value);
Assert.AreEqual(TokenType.Symbol, result.Type);
Assert.IsNull(lexer.NextToken());
}
