C# (CSharp) Tests Reader.URLIsAllowed Examples

Example #1

        public void UrlIsAvailableToCrawl()
        {
            Reader _reader = new Reader(logger.Object);

            _reader.ParseRobotsTxt(defaultQueryAddress);

            var isAvailableToCrawl = _reader.URLIsAllowed(availableToCrawl);

            Assert.True(isAvailableToCrawl);
        }