public void Should_Identify_All_Delimiters_When_Multiple_Delimiters_Given()
{
_tokenizerMock.TryParseDelimiter(Arg.Is("1"), out Arg.Any <string>(), out Arg.Any <string>()).Returns(false);
_tokenizerMock.TryParseDelimiter(Arg.Is("1,"), out Arg.Any <string>(), out Arg.Any <string>()).Returns(arg => { arg[1] = ","; arg[2] = "1"; return(true); });
_tokenizerMock.Identify(Arg.Is("1")).Returns(new Token(TokenType.Number, "1", 1));
_tokenizerMock.Identify(Arg.Is(",")).Returns(new Token(TokenType.PlusOperator, ","));
_delimiterParserMock.TryParse(Arg.Any <string>(), out Arg.Any <int>(), out Arg.Any <Dictionary <string, TokenType> >()).Returns(false);
var actual = _lexer.Scan("1,1,1");
Assert.Equal(5, actual.Count());
var tokens = actual.ToArray();
Assert.Equal(TokenType.Number, tokens[0].Type);
Assert.Equal(TokenType.PlusOperator, tokens[1].Type);
Assert.Equal(TokenType.Number, tokens[2].Type);
Assert.Equal(TokenType.PlusOperator, tokens[3].Type);
Assert.Equal(TokenType.Number, tokens[4].Type);
_tokenizerMock.Received(2).TryParseDelimiter(Arg.Is("1,"), out Arg.Any <string>(), out Arg.Any <string>());
_tokenizerMock.Received(3).TryParseDelimiter(Arg.Is("1"), out Arg.Any <string>(), out Arg.Any <string>());
_tokenizerMock.DidNotReceive().TryParseDelimiter(Arg.Is((string s) => s.Length > 2 || !s.StartsWith("1")), out Arg.Any <string>(), out Arg.Any <string>());
_tokenizerMock.Received(3).Identify(Arg.Is("1"));
_tokenizerMock.Received(2).Identify(Arg.Is(","));
_tokenizerMock.DidNotReceive().Identify(Arg.Is((string s) => s.Length > 1 || s != "1" && s != ","));
}
void Should_Throw_OverlappingDelimitersException_When_Try_Add_Overlapping_Delimiters()
{
Assert.Throws <OverlappingDelimitersException>(() =>
{
_parser.TryParse("//[a][aa]\\n1a1", out var offset, out Dictionary <string, TokenType> delimiters);
});
}
public IEnumerable <Token> Scan(string source)
{
if (source == null) // Empty or whitespcae is valid
{
throw new ArgumentNullException(nameof(source));
}
// Try parse custom delimiters
if (_parser.TryParse(source, out var offset, out var delimiters))
{
_tokenizer.ApplyCustomRule(delimiters);
source = source.Remove(0, offset);
}