public TffMatchInfoScraper() { _matchDataDriver = new MatchDataDriver(); _dataSourceDriver = new DataSourceDriver(); _dispatcher = new WebScraperDispatcher(); }
public void Run() { List<object> models = new List<object>(); var standardData = new { type = "tff", status = "0" }; Stopwatch watch = Stopwatch.StartNew(); StringBuilder result = new StringBuilder(); TaskCompletionSource<bool> complete = new TaskCompletionSource<bool>(); WebScraperBrowser browser = _dispatcher.AddBrowser(); _dispatcher.Navigate(browser.ID, "http://www.tff.org/default.aspx?pageID=322"); IDomDocument document = browser.Document; document.Find("#ctl00_MPane_m_322_1480_ctnr_m_322_1480_txtKulupAdi").Value = "%"; document.Find("#ctl00_MPane_m_322_1480_ctnr_m_322_1480_btnSave").Trigger("click"); browser.LoadCompletedAction = (b) => { complete.SetResult(true); }; complete.Task.Wait(); IEnumerable<object> allList = null; bool isContinue = true; do { document = browser.Document; WebUrl url = _dispatcher.GetUrl( browser.ID); _urlBinder.SetDocument(document); _urlBinder.Execute(); var secList = (_urlBinder.Model as IContentBindingModel)["urlList"] as IContentBindingCollection; foreach(var sec in secList) { var sourceModel = (sec as IContentBindingModel)["source"]; (sec as IContentBindingModel)["source"] = url.GetUrl(sourceModel.ToString()); } if (allList == null) { allList = secList; } else { allList = allList.Concat(secList); } IDomElement nextPage = document.Find("a[title='Next page']"); isContinue = nextPage.Length > 0; if (isContinue) { complete = new TaskCompletionSource<bool>(); nextPage.Trigger("click"); browser.LoadCompletedAction = (b) => { complete.SetResult(true); }; complete.Task.Wait(); } } while (isContinue); DataSourceDriver driver = new DataSourceDriver(); driver.AddDataSource(standardData,allList.ToArray()); Console.WriteLine(watch.ElapsedMilliseconds); }