public static IList <IWebScraper> GetScrapers(Type type, IServiceCollection services)
{
var webScrapers = new List <IWebScraper>();
var allWebScrapersTypes = Assembly.GetAssembly(type).GetTypes()
.Where(t => type.IsAssignableFrom(t) && t.IsAbstract == false);
foreach (var scraperType in allWebScrapersTypes)
{
BaseScraper scaper = Activator.CreateInstance(scraperType) as BaseScraper;
webScrapers.Add(scaper);
}
return(webScrapers);
}
public static void BuildScrapers()
{
var allScrapers = Assembly.GetAssembly(typeof(BaseScraper)).GetTypes()
.Where(t => typeof(BaseScraper).IsAssignableFrom(t) && t.IsAbstract == false);
var timers = new List <ScraperHolder>();
foreach (var scraperType in allScrapers)
{
ScraperAttribute attribute = (ScraperAttribute)Attribute.GetCustomAttribute(scraperType, typeof(ScraperAttribute));
if (attribute != null)
{
BaseScraper scaper = Activator.CreateInstance(scraperType) as BaseScraper;
string expression = @"*/5 * * * * *";
if (attribute.IsProductionReady)
{
expression = CronExpressionHelpers.CronMappings.GetValueOrDefault(attribute.Timer);
}
if (!timers.Any(t => t.CronExpression == expression))
{
timers.Add(new ScraperHolder {
CronExpression = expression, Scrapers = new List <BaseScraper> {
scaper
}
});
}
else
{
var holder = timers.First(t => t.CronExpression == expression);
holder.Scrapers.Add(scaper);
}
}
}
}
private async void button1_Click(object sender, EventArgs e)
{
this.button1.Enabled = false;
this.button1.Text = "実行中";
this.textBox2.Text = "";
lbl_resultCount.Text = "0件";
try
{
string src = "";
int analyzeMode = 1;
if (this.rdo_url.Checked)
{
analyzeMode = 1;
src = this.textBox1.Text;
}
else
{
analyzeMode = 2;
src = this.textBox3.Text;
}
if (src == string.Empty)
{
MessageBox.Show("URL/HTMLを入力してください。",
"エラー",
MessageBoxButtons.OK,
MessageBoxIcon.Error);
return;
}
var scraper = BaseScraper.factory(src, analyzeMode);
if (scraper == null)
{
MessageBox.Show("規定サイト以外のURLが入力されました。",
"エラー",
MessageBoxButtons.OK,
MessageBoxIcon.Error);
return;
}
// パラメタ初期化
var param = new Dictionary <string, object>();
param.Add("city_name", this.txt_cityName.Text);
param.Add("city_count", this.txt_count.Text);
var resultList = await scraper.execute(param);
var errMsg = scraper.getErrMessage();
var infMsg = scraper.getInfoMessage();
if (errMsg.Length > 0)
{
MessageBox.Show(errMsg,
"エラー",
MessageBoxButtons.OK,
MessageBoxIcon.Error);
}
if (infMsg.Length > 0)
{
MessageBox.Show(infMsg,
"情報",
MessageBoxButtons.OK,
MessageBoxIcon.Information);
}
lbl_resultCount.Text = resultList.Count.ToString() + "件";
textBox2.Text = string.Join("\r\n", resultList);
}
catch (Exception ex)
{
MessageBox.Show(ex.Message,
"エラー",
MessageBoxButtons.OK,
MessageBoxIcon.Error);
}
finally
{
this.button1.Enabled = true;
this.button1.Text = "実行";
}
}
private async Task <List <Item> > StartSearch(RequestModel model)
{
List <string> sites = model.CheckedSites;
string itemName = model.ItemName;
int itemCount = model.ItemCount;
int minPrice = model.MinPrice == null ? 0 : (int)model.MinPrice;
int maxPrice = model.MaxPrice == null ? 0 : (int)model.MaxPrice;
List <Item> AllItems = new List <Item>();
//AutoResetEvent auto = new AutoResetEvent(false);
List <Task> Tasks = new List <Task>();
foreach (string s in sites)
{
Task task = Task.Run(async() =>
{
Assembly asm = null;
try
{
string asmPath = HttpRuntime.AppDomainAppPath + $"Plugins/{s}Library.dll";
asm = Assembly.LoadFrom(asmPath);
}
catch (FileNotFoundException e)
{
Console.WriteLine(e.Message);
return;
//auto.Set();
}
catch (ArgumentException e)
{
Console.WriteLine(e.Message);
return;
//auto.Set();
}
var pluginTypes = from t in asm.GetTypes()
where t.IsClass && (t.BaseType == typeof(BaseScraper))
select t;
foreach (Type t in pluginTypes)
{
BaseScraper obj = (BaseScraper)Activator.CreateInstance(t);
int min = 0, max = 0;
if (minPrice != 0 || maxPrice != 0)
{
AvailableCurrencies currencyFrom = (AvailableCurrencies)model.Currency;
CurrencyAttribute curr_attr = t.GetCustomAttribute(typeof(CurrencyAttribute)) as CurrencyAttribute;
AvailableCurrencies currencyTo = curr_attr.Currency;
CurrencyScraper cs = new CurrencyScraper();
if (currencyFrom != currencyTo)
{
double rate = await cs.StartScraping(currencyFrom, currencyTo);
min = Convert.ToInt32(rate * minPrice);
max = Convert.ToInt32(rate * maxPrice);
}
else
{
min = minPrice;
max = maxPrice;
}
}
List <Item> items = await obj?.StartScraping(itemName, itemCount, min, max);
AllItems.AddRange(items);
//auto.Set();
}
});
Tasks.Add(task);
}
//foreach (Task t in Tasks)
// auto.WaitOne();
await Task.WhenAll(Tasks);
return(AllItems);
}