static void Main(string[] args)
{
var p = new FluentCommandLineParser <Arguments>();
string[] lines = System.IO.File.ReadAllLines(@"C:\Users\g\Downloads\" + args[0]);
//string[] lines = System.IO.File.ReadAllLines(@"C:\Users\g\Downloads\myurls816.txt");
// Display the file contents by using a foreach loop.
foreach (string line in lines)
{
// Use a tab to indent each line of the file.
System.Console.WriteLine("Contents of text file = " + line);
Arguments arguments = new Arguments();
//arguments.RootUri = line;
arguments.RootUri = line.Replace('"', ' ').Replace(',', ' ').Trim();
arguments.ServiceName = "localhost";
arguments.IndexName = "IndexName";
//arguments.AdminApiKey = "ed83fde2a56041ab83d75fe2437adb19";
arguments.AdminApiKey = "1ccabb26e2134039ba6680434bd1a3ed";
//ed83fde2a56041ab83d75fe2437adb19
var indexer = new AzureSearchIndexer(arguments.ServiceName, arguments.IndexName, arguments.AdminApiKey, new TextExtractor());
var crawler = new Crawler(indexer);
crawler.Crawl(arguments.RootUri, maxPages: 20).Wait();
}
// Console.Read(); // keep console open until a button is pressed so we see the output
}
static void Main(string[] args)
{
var p = new FluentCommandLineParser <Arguments>();
p.Setup(arg => arg.RootUri)
.As('r', "rootUri")
.Required()
.WithDescription("Start crawling from this web page");
p.Setup(arg => arg.MaxPagesToIndex)
.As('m', "maxPages")
.SetDefault(DefaultMaxPagesToIndex)
.WithDescription("Stop after having indexed this many pages. Default is " + DefaultMaxPagesToIndex + "; 0 means no limit.");
p.Setup(arg => arg.ServiceName)
.As('s', "ServiceName")
.Required()
.WithDescription("The name of your Azure Search service");
p.Setup(arg => arg.IndexName)
.As('i', "IndexName")
.Required()
.WithDescription("The name of the index in your Azure Search service");
p.Setup(arg => arg.AdminApiKey)
.As('a', "AdminApiKey")
.Required();
p.SetupHelp("?", "h", "help").Callback(text => Console.Error.WriteLine(text));
ICommandLineParserResult result = p.Parse(args);
if (result.HasErrors)
{
Console.Error.WriteLine(result.ErrorText);
Console.Error.WriteLine("Usage: ");
p.HelpOption.ShowHelp(p.Options);
return;
}
if (result.HelpCalled)
{
return;
}
Arguments arguments = p.Object;
var indexer = new AzureSearchIndexer(arguments.ServiceName, arguments.IndexName, arguments.AdminApiKey, new TextExtractor());
var crawler = new Crawler(indexer);
crawler.Crawl(arguments.RootUri, maxPages: arguments.MaxPagesToIndex).Wait();
Console.Read(); // keep console open until a button is pressed so we see the output
}
