public async Task ShouldCrawl() { await CrawlHandler.ProcessAsync( new Configuration("http://google.com/", Path.Combine(Path.GetTempPath(), "TestFileWrite")) { Depth = 1, Mode = TraversalMode.SameHost }).ConfigureAwait(false); }
/// <summary> /// An example usage of crawler. /// </summary> private static async Task Main() { var result = await CrawlHandler.ProcessAsync( new Configuration("http://google.com/", "${TempPath}\\TestFileWrite") { Depth = 2, Mode = TraversalMode.AnyHost }).ConfigureAwait(false); foreach (var(uri, exception) in result.FailedUris) { Console.WriteLine($"{uri}: {exception}"); } }