public void ReadTokens_FromEmptyString_IsEmpty() { var lexer = new CharacterLexer(String.Empty); lexer.ReadTokens().ToList().Should().ContainSingle() .Which.IsEndOfStream.Should().BeTrue(); }
public void ReadTokens_WithTextContainingNewlines_NormalizesNewlines() { var word = "A\nB\r\nC\rD"; var lexer = new CharacterLexer(word); var newlines = lexer.ReadTokens().Where(t => t.Type == nameof(Environment.NewLine)).ToList(); newlines.Count.Should().Be(3); newlines.All(t => t.Value == "\n").Should().BeTrue(); }
public void ReadTokens_FromWord_ReturnsTokenForEachCharacter() { var word = "Hello World!"; var lexer = new CharacterLexer(word); var tokens = lexer.ReadTokens().ToList(); tokens.Count.Should().Be(word.Length + 1); tokens.Select(t => t.Value).StrJoin(String.Empty).Should().Be(word); }