public static Collection <CrawlerDetail> GetCrawlerDetails(string websiteUrl)
{
StringBuilder query = new StringBuilder();
query.AppendLine("SELECT RowID, CrawlDate FROM CrawlerDetail WHERE WebsiteUrl=@WebsiteUrl");
DataTable dt = new DataTable();
using (MySqlConnection conn = new MySqlConnection(ConnectionString))
{
MySqlDataAdapter dap = new MySqlDataAdapter(query.ToString(), conn);
dap.SelectCommand.Parameters.AddWithValue("@WebsiteUrl", websiteUrl);
dap.Fill(dt);
}
Collection <CrawlerDetail> details = new Collection <CrawlerDetail>();
foreach (DataRow row in dt.Rows)
{
CrawlerDetail item = new CrawlerDetail();
System.DateTime dat = default(System.DateTime);
if (System.DateTime.TryParse(row["CrawlDate"] + "", out dat) == true)
{
item.CrawlDate = dat;
}
item.CrawlID = int.Parse(row["RowID"] + "");
}
return(details);
}
public static Collection <CrawlerDetail> GetCrawlerDetails()
{
StringBuilder query = new StringBuilder();
query.AppendLine("SELECT * FROM CrawlerDetail ORDER BY CrawlDate DESC");
DataTable dt = new DataTable();
using (MySqlConnection conn = new MySqlConnection(ConnectionString))
{
MySqlDataAdapter dap = new MySqlDataAdapter(query.ToString(), conn);
dap.Fill(dt);
}
Collection <CrawlerDetail> details = new Collection <CrawlerDetail>();
foreach (DataRow row in dt.Rows)
{
CrawlerDetail item = new CrawlerDetail();
System.DateTime dat = default(System.DateTime);
if (System.DateTime.TryParse(row["CrawlDate"] + "", out dat) == true)
{
item.CrawlDate = dat;
}
item.CrawlID = int.Parse(row["RowID"] + "");
int d = 0;
if (int.TryParse(row["TotalCrawled"] + "", out d) == true)
{
item.TotalCrawled = d;
}
if (int.TryParse(row["BrokenLinks"] + "", out d) == true)
{
item.BrokenLinks = d;
}
item.WebsiteUrl = row["WebsiteUrl"] + "";
details.Add(item);
}
return(details);
}
public static void SaveCrawlerDetail(CrawlerDetail detail)
{
StringBuilder query = new StringBuilder();
query.AppendLine("INSERT INTO CrawlerDetail (WebsiteUrl, BrokenLinks, TotalCrawled, CrawlDate) VALUES (@WebsiteUrl, @BrokenLinks, @TotalCrawled, @CrawlDate)");
using (MySqlConnection conn = new MySqlConnection(ConnectionString))
{
conn.Open();
using (MySqlCommand cmd = new MySqlCommand(query.ToString(), conn))
{
cmd.Parameters.AddWithValue("@WebsiteUrl", detail.WebsiteUrl);
cmd.Parameters.AddWithValue("@BrokenLinks", detail.BrokenLinks);
cmd.Parameters.AddWithValue("@TotalCrawled", detail.TotalCrawled);
cmd.Parameters.AddWithValue("@CrawlDate", detail.CrawlDate);
cmd.ExecuteNonQuery();
}
}
}
private void btnStartSearch_Click(object sender, RoutedEventArgs e)
{
try
{
mCrawlDetail = new CrawlDetail();
this.DataContext = mCrawlDetail;
string url = txtSearchUrl.Text;
MyBrowser.Navigate(url);
tbkCurrentStatus.Text = "Running";
Thread th = new Thread(new ThreadStart(() =>
{
WebCrawler spider = new WebCrawler(url, url, 100);
//spider.WebPageManager.WebPageContentHandler = [Delegate].Combine(spider.WebPageManager.WebPageContentHandler, New WebPageContentDelegate(AddressOf spider_WebPageContentHandler))
spider.WebPageManager.WebPageTaskCompleted = new WebPageContentDelegate(spider_WebPageContentHandler);
spider.Execute();
var lst = spider.WebPages.Values;
this.Dispatcher.Invoke(new Action(() =>
{
tbkCurrentStatus.Text = "Completed";
CrawlerDetail result = new CrawlerDetail();
result.ProjectName = ProjectName;
result.CrawlDate = DateAndTime.Now;
result.WebsiteUrl = url;
result.TotalCrawled = mCrawlDetail.TotalCrawled;
result.BrokenLinks = mCrawlDetail.TotalBrokenLink;
CrawlerQueries.SaveCrawlerDetail(result);
}));
}));
th.Start();
}
catch (Exception ex)
{
MessageBox.Show("Error");
}
}
