private static void TestString(string code)
{
var tokenizer = new Tokenizer($"\"{code}\"", new TokenizerOptions());
Assert.IsTrue(
TokenizationTestUtils.Match(
tokenizer.GetNextToken(),
new Token(
0, code, 0, 0,
Tokenizer.State.Context.Global,
TokenType.String, TokenSubType.String
)
),
$"Should classify '{code}' as 'String'"
);
TokenizationTestUtils.TestEOF(tokenizer);
}
private static void TestCharacter(char code)
{
var tokenizer = new Tokenizer($"'{code}", new TokenizerOptions());
Assert.IsTrue(
TokenizationTestUtils.Match(
tokenizer.GetNextToken(),
new Token(
0, code.ToString(), 0, 0,
Tokenizer.State.Context.Global,
TokenType.Character, TokenSubType.Character
)
),
$"Should classify '{code}' as 'Character'"
);
TokenizationTestUtils.TestEOF(tokenizer);
}
private void TestOperator(string op, TokenSubType st, TokenType t)
{
var tokenizer = new Tokenizer(op, new TokenizerOptions());
Assert.IsTrue(
TokenizationTestUtils.Match(
tokenizer.GetNextToken(),
new Token(
0, op, 0, 0,
Tokenizer.State.Context.Global,
t, st
)
),
$"Should classify '{op}' as '{st}'"
);
TokenizationTestUtils.TestEOF(tokenizer);
}
private static void TestNumber(string code, TokenSubType targetType)
{
var tokenizer = new Tokenizer(code, new TokenizerOptions());
Assert.IsTrue(
TokenizationTestUtils.Match(
tokenizer.GetNextToken(),
new Token(
0, code, 0, 0,
Tokenizer.State.Context.Global,
TokenType.Number, targetType
)
),
$"Should tokenize number '{code}' and classify it as '{targetType.ToString()}'"
);
TokenizationTestUtils.TestEOF(tokenizer);
}
private static void TestDirective(string directive)
{
var tokenizer = new Tokenizer(directive, new TokenizerOptions());
Assert.IsTrue(
TokenizationTestUtils.Match(
tokenizer.GetNextToken(),
new Token(
0, directive, 0, 0,
Tokenizer.State.Context.Global,
TokenType.CompilerDirective,
TokenSubType.CompilerDirective
)
),
$"Should classify '{directive}' as 'Compiler directive'"
);
TokenizationTestUtils.TestEOF(tokenizer);
}
public void TestIdentifier_0_Basic()
{
foreach (string code in new[] { "test", "$_Test", "te5t" })
{
var tokenizer = new Tokenizer(code, new TokenizerOptions());
Assert.IsTrue(
TokenizationTestUtils.Match(
tokenizer.GetNextToken(),
new Token(
0, code, 0, 0,
Tokenizer.State.Context.Global,
TokenType.Identifier, TokenSubType.Identifier
)
),
$"Should classify '{code}' as 'Identifer'"
);
TokenizationTestUtils.TestEOF(tokenizer);
}
}
