void init(string name, DataScraper boss, IDatacleaner datacleaner, int max_requests)
{
this.boss = boss;
this.datacleaner = datacleaner;
Name = name;
this.max_requests = max_requests;
requestHandler = new DSTRequestHandler(false);
my_data = new List <string>();
requests_handled = new List <TableMetadata>();
raw_data = new Dictionary <TableMetadata, string>();
}
static void Main(string[] args)
{
Console.WriteLine("Start scraping");
var ds = new DataScraper();
var exitcode = ds.Setup();
Console.WriteLine($"Setup exitcode: {exitcode}");
exitcode = ds.Run();
Console.WriteLine($"Run exitcode: {exitcode}");
Console.ReadLine();
}
public Worker(string name, DataScraper boss, IDatacleaner datacleaner, int max_requests)
{
init(name, boss, datacleaner, max_requests);
}
public Worker(string name, DataScraper boss, IDatacleaner datacleaner)
{
init(name, boss, datacleaner, int.MaxValue);
}
public Worker(string name, DataScraper boss)
{
init(name, boss, new NoCleaningStrategy(), int.MaxValue);
}
