public static Collection <CrawlerDetail> GetCrawlerDetails(string websiteUrl) { StringBuilder query = new StringBuilder(); query.AppendLine("SELECT RowID, CrawlDate FROM CrawlerDetail WHERE WebsiteUrl=@WebsiteUrl"); DataTable dt = new DataTable(); using (MySqlConnection conn = new MySqlConnection(ConnectionString)) { MySqlDataAdapter dap = new MySqlDataAdapter(query.ToString(), conn); dap.SelectCommand.Parameters.AddWithValue("@WebsiteUrl", websiteUrl); dap.Fill(dt); } Collection <CrawlerDetail> details = new Collection <CrawlerDetail>(); foreach (DataRow row in dt.Rows) { CrawlerDetail item = new CrawlerDetail(); System.DateTime dat = default(System.DateTime); if (System.DateTime.TryParse(row["CrawlDate"] + "", out dat) == true) { item.CrawlDate = dat; } item.CrawlID = int.Parse(row["RowID"] + ""); } return(details); }
public static Collection <CrawlerDetail> GetCrawlerDetails() { StringBuilder query = new StringBuilder(); query.AppendLine("SELECT * FROM CrawlerDetail ORDER BY CrawlDate DESC"); DataTable dt = new DataTable(); using (MySqlConnection conn = new MySqlConnection(ConnectionString)) { MySqlDataAdapter dap = new MySqlDataAdapter(query.ToString(), conn); dap.Fill(dt); } Collection <CrawlerDetail> details = new Collection <CrawlerDetail>(); foreach (DataRow row in dt.Rows) { CrawlerDetail item = new CrawlerDetail(); System.DateTime dat = default(System.DateTime); if (System.DateTime.TryParse(row["CrawlDate"] + "", out dat) == true) { item.CrawlDate = dat; } item.CrawlID = int.Parse(row["RowID"] + ""); int d = 0; if (int.TryParse(row["TotalCrawled"] + "", out d) == true) { item.TotalCrawled = d; } if (int.TryParse(row["BrokenLinks"] + "", out d) == true) { item.BrokenLinks = d; } item.WebsiteUrl = row["WebsiteUrl"] + ""; details.Add(item); } return(details); }
public static void SaveCrawlerDetail(CrawlerDetail detail) { StringBuilder query = new StringBuilder(); query.AppendLine("INSERT INTO CrawlerDetail (WebsiteUrl, BrokenLinks, TotalCrawled, CrawlDate) VALUES (@WebsiteUrl, @BrokenLinks, @TotalCrawled, @CrawlDate)"); using (MySqlConnection conn = new MySqlConnection(ConnectionString)) { conn.Open(); using (MySqlCommand cmd = new MySqlCommand(query.ToString(), conn)) { cmd.Parameters.AddWithValue("@WebsiteUrl", detail.WebsiteUrl); cmd.Parameters.AddWithValue("@BrokenLinks", detail.BrokenLinks); cmd.Parameters.AddWithValue("@TotalCrawled", detail.TotalCrawled); cmd.Parameters.AddWithValue("@CrawlDate", detail.CrawlDate); cmd.ExecuteNonQuery(); } } }
private void btnStartSearch_Click(object sender, RoutedEventArgs e) { try { mCrawlDetail = new CrawlDetail(); this.DataContext = mCrawlDetail; string url = txtSearchUrl.Text; MyBrowser.Navigate(url); tbkCurrentStatus.Text = "Running"; Thread th = new Thread(new ThreadStart(() => { WebCrawler spider = new WebCrawler(url, url, 100); //spider.WebPageManager.WebPageContentHandler = [Delegate].Combine(spider.WebPageManager.WebPageContentHandler, New WebPageContentDelegate(AddressOf spider_WebPageContentHandler)) spider.WebPageManager.WebPageTaskCompleted = new WebPageContentDelegate(spider_WebPageContentHandler); spider.Execute(); var lst = spider.WebPages.Values; this.Dispatcher.Invoke(new Action(() => { tbkCurrentStatus.Text = "Completed"; CrawlerDetail result = new CrawlerDetail(); result.ProjectName = ProjectName; result.CrawlDate = DateAndTime.Now; result.WebsiteUrl = url; result.TotalCrawled = mCrawlDetail.TotalCrawled; result.BrokenLinks = mCrawlDetail.TotalBrokenLink; CrawlerQueries.SaveCrawlerDetail(result); })); })); th.Start(); } catch (Exception ex) { MessageBox.Show("Error"); } }