public static IList <IWebScraper> GetScrapers(Type type, IServiceCollection services) { var webScrapers = new List <IWebScraper>(); var allWebScrapersTypes = Assembly.GetAssembly(type).GetTypes() .Where(t => type.IsAssignableFrom(t) && t.IsAbstract == false); foreach (var scraperType in allWebScrapersTypes) { BaseScraper scaper = Activator.CreateInstance(scraperType) as BaseScraper; webScrapers.Add(scaper); } return(webScrapers); }
public static void BuildScrapers() { var allScrapers = Assembly.GetAssembly(typeof(BaseScraper)).GetTypes() .Where(t => typeof(BaseScraper).IsAssignableFrom(t) && t.IsAbstract == false); var timers = new List <ScraperHolder>(); foreach (var scraperType in allScrapers) { ScraperAttribute attribute = (ScraperAttribute)Attribute.GetCustomAttribute(scraperType, typeof(ScraperAttribute)); if (attribute != null) { BaseScraper scaper = Activator.CreateInstance(scraperType) as BaseScraper; string expression = @"*/5 * * * * *"; if (attribute.IsProductionReady) { expression = CronExpressionHelpers.CronMappings.GetValueOrDefault(attribute.Timer); } if (!timers.Any(t => t.CronExpression == expression)) { timers.Add(new ScraperHolder { CronExpression = expression, Scrapers = new List <BaseScraper> { scaper } }); } else { var holder = timers.First(t => t.CronExpression == expression); holder.Scrapers.Add(scaper); } } } }
private async void button1_Click(object sender, EventArgs e) { this.button1.Enabled = false; this.button1.Text = "実行中"; this.textBox2.Text = ""; lbl_resultCount.Text = "0件"; try { string src = ""; int analyzeMode = 1; if (this.rdo_url.Checked) { analyzeMode = 1; src = this.textBox1.Text; } else { analyzeMode = 2; src = this.textBox3.Text; } if (src == string.Empty) { MessageBox.Show("URL/HTMLを入力してください。", "エラー", MessageBoxButtons.OK, MessageBoxIcon.Error); return; } var scraper = BaseScraper.factory(src, analyzeMode); if (scraper == null) { MessageBox.Show("規定サイト以外のURLが入力されました。", "エラー", MessageBoxButtons.OK, MessageBoxIcon.Error); return; } // パラメタ初期化 var param = new Dictionary <string, object>(); param.Add("city_name", this.txt_cityName.Text); param.Add("city_count", this.txt_count.Text); var resultList = await scraper.execute(param); var errMsg = scraper.getErrMessage(); var infMsg = scraper.getInfoMessage(); if (errMsg.Length > 0) { MessageBox.Show(errMsg, "エラー", MessageBoxButtons.OK, MessageBoxIcon.Error); } if (infMsg.Length > 0) { MessageBox.Show(infMsg, "情報", MessageBoxButtons.OK, MessageBoxIcon.Information); } lbl_resultCount.Text = resultList.Count.ToString() + "件"; textBox2.Text = string.Join("\r\n", resultList); } catch (Exception ex) { MessageBox.Show(ex.Message, "エラー", MessageBoxButtons.OK, MessageBoxIcon.Error); } finally { this.button1.Enabled = true; this.button1.Text = "実行"; } }
private async Task <List <Item> > StartSearch(RequestModel model) { List <string> sites = model.CheckedSites; string itemName = model.ItemName; int itemCount = model.ItemCount; int minPrice = model.MinPrice == null ? 0 : (int)model.MinPrice; int maxPrice = model.MaxPrice == null ? 0 : (int)model.MaxPrice; List <Item> AllItems = new List <Item>(); //AutoResetEvent auto = new AutoResetEvent(false); List <Task> Tasks = new List <Task>(); foreach (string s in sites) { Task task = Task.Run(async() => { Assembly asm = null; try { string asmPath = HttpRuntime.AppDomainAppPath + $"Plugins/{s}Library.dll"; asm = Assembly.LoadFrom(asmPath); } catch (FileNotFoundException e) { Console.WriteLine(e.Message); return; //auto.Set(); } catch (ArgumentException e) { Console.WriteLine(e.Message); return; //auto.Set(); } var pluginTypes = from t in asm.GetTypes() where t.IsClass && (t.BaseType == typeof(BaseScraper)) select t; foreach (Type t in pluginTypes) { BaseScraper obj = (BaseScraper)Activator.CreateInstance(t); int min = 0, max = 0; if (minPrice != 0 || maxPrice != 0) { AvailableCurrencies currencyFrom = (AvailableCurrencies)model.Currency; CurrencyAttribute curr_attr = t.GetCustomAttribute(typeof(CurrencyAttribute)) as CurrencyAttribute; AvailableCurrencies currencyTo = curr_attr.Currency; CurrencyScraper cs = new CurrencyScraper(); if (currencyFrom != currencyTo) { double rate = await cs.StartScraping(currencyFrom, currencyTo); min = Convert.ToInt32(rate * minPrice); max = Convert.ToInt32(rate * maxPrice); } else { min = minPrice; max = maxPrice; } } List <Item> items = await obj?.StartScraping(itemName, itemCount, min, max); AllItems.AddRange(items); //auto.Set(); } }); Tasks.Add(task); } //foreach (Task t in Tasks) // auto.WaitOne(); await Task.WhenAll(Tasks); return(AllItems); }