public async Task TokenizeAsyncCancellation()
{
using (var stream = LoadResourceStream("RobotsFile/InvalidField-Example.txt"))
using (var reader = new StreamReader(stream))
{
var tokenizer = new RobotsFileTokenizer();
await Assert.ThrowsExceptionAsync <OperationCanceledException>(
async() => await tokenizer.TokenizeAsync(reader, new CancellationToken(true))
);
}
}
public void RFCValidPatterns()
{
var robots = LoadRobotsRfcFileExample();
var tokenizer = new RobotsFileTokenizer();
var tokens = tokenizer.Tokenize(robots);
var validator = new RobotsFileTokenPatternValidator();
var result = validator.Validate(tokens);
Assert.IsTrue(result.IsValid);
}
public void InvalidFields()
{
var robots = LoadResource("RobotsFile/InvalidField-Example.txt");
var tokenizer = new RobotsFileTokenizer();
var tokens = tokenizer.Tokenize(robots);
var fieldTokens = tokens.Where(t => t.TokenType == TokenType.Field);
var valueTokens = tokens.Where(t => t.TokenType == TokenType.Value);
Assert.AreEqual(0, fieldTokens.Count());
Assert.AreEqual(19, valueTokens.Count());
}
public void RFCCommentTokenization()
{
var robots = LoadRobotsRfcFileExample();
var tokenizer = new RobotsFileTokenizer();
var tokens = tokenizer.Tokenize(robots);
var commentTokens = tokens.Where(t => t.TokenType == TokenType.Comment);
Assert.AreEqual(2, commentTokens.Count());
Assert.AreEqual("# /robots.txt for http://www.fict.org/", commentTokens.First().Value);
Assert.AreEqual("# comments to [email protected]", commentTokens.Last().Value);
}
public void RFCValueTokenization()
{
var robots = LoadRobotsRfcFileExample();
var tokenizer = new RobotsFileTokenizer();
var tokens = tokenizer.Tokenize(robots);
var valueTokens = tokens.Where(t => t.TokenType == TokenType.Value);
Assert.AreEqual(10, valueTokens.Count());
Assert.AreEqual(1, valueTokens.Count(t => t.Value == "unhipbot"));
Assert.AreEqual(2, valueTokens.Count(t => t.Value == "/"));
}
public void RFCFieldTokenization()
{
var robots = LoadRobotsRfcFileExample();
var tokenizer = new RobotsFileTokenizer();
var tokens = tokenizer.Tokenize(robots);
var fieldTokens = tokens.Where(t => t.TokenType == TokenType.Field);
Assert.AreEqual(11, fieldTokens.Count());
Assert.AreEqual(4, fieldTokens.Count(t => t.Value == "User-agent"));
Assert.AreEqual(4, fieldTokens.Count(t => t.Value == "Disallow"));
Assert.AreEqual(3, fieldTokens.Count(t => t.Value == "Allow"));
}
public void InvalidFields()
{
var robots = LoadResource("RobotsFile/InvalidField-Example.txt");
var tokenizer = new RobotsFileTokenizer();
var tokens = tokenizer.Tokenize(robots);
Assert.AreEqual(19, tokens.Count(t => t.TokenType == TokenType.NotDefined));
Assert.AreEqual(2, tokens.Count(t => t.TokenType == TokenType.Blank));
Assert.AreEqual(19, tokens.Count(t => t.TokenType == TokenType.Field));
Assert.AreEqual(19, tokens.Count(t => t.TokenType == TokenType.FieldValueDelimiter));
Assert.AreEqual(19, tokens.Count(t => t.TokenType == TokenType.Value));
Assert.AreEqual(18, tokens.Count(t => t.TokenType == TokenType.NewLine));
}
public void MalformedFieldPatterns()
{
var robots = LoadResource("RobotsFile/InvalidField-Example.txt");
var tokenizer = new RobotsFileTokenizer();
var tokens = tokenizer.Tokenize(robots);
var validator = new RobotsFileTokenPatternValidator();
var result = validator.Validate(tokens);
Assert.IsFalse(result.IsValid);
var firstErrorExpectedTokens = result.Errors.First().Expected;
Assert.AreEqual(TokenType.NewLine, firstErrorExpectedTokens.ElementAt(0));
Assert.AreEqual(1, firstErrorExpectedTokens.Count());
Assert.AreEqual(19, result.Errors.Count());
}
