public void ShouldReadWithTags()
{
ICharReader reader;
ILexer lexer;
Rule number, word;
TokenMatch tokenMatch;
reader = new MockedCharReader("12345abc");
number = new Rule("Number", Parse.Character('0').OrCharacter('1').OrCharacter('2').OrCharacter('3').OrCharacter('4').OrCharacter('5').OrCharacter('6').OrCharacter('7').OrCharacter('8').OrCharacter('9').OneOrMoreTimes());
number.Tags.Add(new Tag("Number", "Value"));
word = new Rule("Word", Parse.Character('a').OrCharacter('b').OrCharacter('c').OrCharacter('d').OneOrMoreTimes());
word.Tags.Add(new Tag("Word", "Value"));
lexer = new Lexer(word, number);
tokenMatch = lexer.Read(reader);
Assert.IsTrue(tokenMatch.Success);
Assert.AreEqual("Number", tokenMatch.Token.Class);
Assert.AreEqual("12345", tokenMatch.Token.Value);
Assert.IsNotNull(tokenMatch.Tags);
Assert.AreEqual(1, tokenMatch.Tags.Length);
Assert.AreEqual("Number", tokenMatch.Tags[0].Name);
tokenMatch = lexer.Read(reader);
Assert.AreEqual("Word", tokenMatch.Token.Class);
Assert.AreEqual("abc", tokenMatch.Token.Value);
Assert.IsNotNull(tokenMatch.Tags);
Assert.AreEqual(1, tokenMatch.Tags.Length);
Assert.AreEqual("Word", tokenMatch.Tags[0].Name);
}
public void ShouldFailToTryReadWhenInvalidCharAtFirstPosition()
{
ICharReader reader;
ILexer lexer;
Rule word;
TokenMatch tokenMatch;
word = new Rule("Word", Parse.Characters("token"));
reader = new MockedCharReader("token");
lexer = new Lexer(word);
tokenMatch = lexer.TryRead(reader);
Assert.IsTrue(tokenMatch.Success);
Assert.AreEqual("Word", tokenMatch.Token.Class);
Assert.AreEqual("token", tokenMatch.Token.Value);
reader = new MockedCharReader(",oken");
lexer = new Lexer(word);
tokenMatch = lexer.TryRead(reader);
Assert.IsFalse(tokenMatch.Success);
Assert.IsNull(tokenMatch.Token.Class);
Assert.AreEqual(",", tokenMatch.Token.Value);
}
public void ShouldReadAndSeek()
{
ICharReader reader;
ILexer lexer;
Rule letter, word;
TokenMatch tokenMatch;
reader = new MockedCharReader("abcde");
word = new Rule("Word", Parse.Characters("abcdef"));
letter = new Rule("Letter", Parse.CharacterRange('a', 'z'));
lexer = new Lexer(word, letter);
tokenMatch = lexer.Read(reader);
Assert.AreEqual("Letter", tokenMatch.Token.Class);
Assert.AreEqual("a", tokenMatch.Token.Value);
tokenMatch = lexer.Read(reader);
Assert.AreEqual("Letter", tokenMatch.Token.Class);
Assert.AreEqual("b", tokenMatch.Token.Value);
tokenMatch = lexer.Read(reader);
Assert.AreEqual("Letter", tokenMatch.Token.Class);
Assert.AreEqual("c", tokenMatch.Token.Value);
tokenMatch = lexer.Read(reader);
Assert.AreEqual("Letter", tokenMatch.Token.Class);
Assert.AreEqual("d", tokenMatch.Token.Value);
tokenMatch = lexer.Read(reader);
Assert.AreEqual("Letter", tokenMatch.Token.Class);
Assert.AreEqual("e", tokenMatch.Token.Value);
}
public void ShouldFailToReadAndThrowEndOfStreamException()
{
ICharReader reader;
ILexer lexer;
Rule word;
TokenMatch tokenMatch;
word = new Rule("Word", Parse.Characters("token"));
reader = new MockedCharReader("token");
lexer = new Lexer(word);
tokenMatch = lexer.Read(reader);
Assert.IsTrue(tokenMatch.Success);
Assert.AreEqual("Word", tokenMatch.Token.Class);
Assert.AreEqual("token", tokenMatch.Token.Value);
reader = new MockedCharReader("toke");
lexer = new Lexer(word);
Assert.ThrowsException <EndOfStreamException>(() => tokenMatch = lexer.Read(reader));
}
