public async Task ShouldCrawl()
{
await CrawlHandler.ProcessAsync(
new Configuration("http://google.com/", Path.Combine(Path.GetTempPath(), "TestFileWrite"))
{
Depth = 1,
Mode = TraversalMode.SameHost
}).ConfigureAwait(false);
}
/// <summary>
/// An example usage of crawler.
/// </summary>
private static async Task Main()
{
var result = await CrawlHandler.ProcessAsync(
new Configuration("http://google.com/", "${TempPath}\\TestFileWrite")
{
Depth = 2,
Mode = TraversalMode.AnyHost
}).ConfigureAwait(false);
foreach (var(uri, exception) in result.FailedUris)
{
Console.WriteLine($"{uri}: {exception}");
}
}