private void btnSpider_Click(object sender, EventArgs e)
{
string Dname;
string line;
int cntr = 0;
double timeAvg = 0;
double timeTotal = 0;
ProxyShuttler PrxShtlr = new ProxyShuttler();
while ((Dname = GetDN()) != null)
{
DateTime startTime = DateTime.Now;
M4Spider Spidey = new M4Spider();
Spidey.MaxOutboundLinks = 3;
Spidey.ProxyDomain = PrxShtlr.CurrentProxy.ToString();
Spidey.AddUnspidered("http://www.alexa.com/site/linksin/" + Dname);//"http://www.databasejobs.com/");//
Spidey.BackLinkDN = Dname;
ArrayList BackLinksList;
if (AV_links > 0)
{
bool lb_rtn = Spidey.CrawlNext();
if (!lb_rtn)
{
continue;
}
if (Spidey.SR1 != null)
{
BackLinksList = Spidey.SR1.HyperLinks;
}
else
{
BackLinksList = new ArrayList();
BackLinksList.Add(Dname);
}
}
else
{
BackLinksList = new ArrayList();
BackLinksList.Add(Dname);
}
LinkRaker LR = new LinkRaker();
LR.lb_Proxy = true; // While at work . . .
LR.RunAlexa(Dname);
LR.RunCompete(Dname);
//Run against Quantcast
LR.RunQuantcast(Dname);
DateTime StopTime = DateTime.Now;
TimeSpan timeDif = StopTime - startTime;
double procTime = timeDif.TotalSeconds;
bool blrtn = CycleTime("Parent Domain", Dname, (float)procTime);
using (SqlConnection conn = new SqlConnection())
{
conn.ConnectionString = Properties.Settings.Default.ConnStr;
conn.Open();
SqlCommand cmd = new SqlCommand();
cmd.CommandText = "INSERT INTO TrafficRoutine4"
+ " (traf4_traf3_seqno, traf4_domname, traf4_bl_domainname, traf4_compete_esttraffic, traf4_compete_perdiff,"
+ " traf4_quantcast_esttraffic, traf4_alexa_tr, traf4_alexa_reach, traf4_alexa_search, traf4_alexa_country1,"
+ " traf4_alexa_country1_per, traf4_alexa_country2, traf4_alexa_country2_per, traf4_alexa_country3, traf4_alexa_country3_per, traf4_procTime)"
+ " VALUES (@seqno, @domname, @bl_domname, @compete_esttraf, @compete_perdiff, @quantcast_esttraf, @alexa_tr,"
+ " @alexa_reach, @alexa_search, @alexa_ctry1, @alexa_ctry1_per, @alexa_ctry2, @alexa_ctry2_per, @alexa_ctry3, @alexa_ctry3_per, @timedif)";
cmd.Parameters.AddWithValue("@seqno", traf3seqno);
cmd.Parameters.AddWithValue("@domname", Dname);
cmd.Parameters.AddWithValue("@bl_domname", "");
cmd.Parameters.AddWithValue("@compete_esttraf", LR.CmptRec.CmptMonth1);
cmd.Parameters.AddWithValue("@compete_perdiff", 0);
cmd.Parameters.AddWithValue("@quantcast_esttraf", LR.dnRec.QuantcastCtr);
cmd.Parameters.AddWithValue("@alexa_tr", LR.AlxRec.AlxHighRank);
cmd.Parameters.AddWithValue("@alexa_reach", LR.AlxRec.AlxHighReach);
cmd.Parameters.AddWithValue("@alexa_search", LR.AlxRec.AlxHighSearch);
cmd.Parameters.AddWithValue("@alexa_ctry1", (LR.AlxRec.AlxCountry1.ToString() == null ? "" : LR.AlxRec.AlxCountry1.ToString()));
cmd.Parameters.AddWithValue("@alexa_ctry1_per", LR.AlxRec.AlxCntry1_per);
cmd.Parameters.AddWithValue("@alexa_ctry2", (LR.AlxRec.AlxCountry2.ToString() == null ? "" : LR.AlxRec.AlxCountry2.ToString()));
cmd.Parameters.AddWithValue("@alexa_ctry2_per", LR.AlxRec.AlxCntry2_per);
cmd.Parameters.AddWithValue("@alexa_ctry3", (LR.AlxRec.AlxCountry3.ToString() == null ? "" : LR.AlxRec.AlxCountry3.ToString()));
cmd.Parameters.AddWithValue("@alexa_ctry3_per", LR.AlxRec.AlxCntry3_per);
cmd.Parameters.AddWithValue("@timedif", procTime);
cmd.Connection = conn;
int rtn = cmd.ExecuteNonQuery();
}
DateTime bckLnkStartTime = DateTime.Now;
int count = 0;
foreach (object ALobj in BackLinksList)
{
if (ALobj.ToString().ToUpper().Contains(Dname.ToUpper()))
{
continue;
}
count++;
LinkRaker LR1 = new LinkRaker();
LR1.lb_Proxy = true; // While at work . . .
LR1.RunAlexa(ALobj.ToString());
LR1.RunCompete(ALobj.ToString());
LR.RunQuantcast(ALobj.ToString());
StopTime = DateTime.Now;
TimeSpan timeDifA = StopTime - bckLnkStartTime;
double subProcTime = timeDifA.TotalSeconds;
blrtn = CycleTime("Backlink" + count.ToString() + " Domain", Dname, (float)subProcTime);
using (SqlConnection conn = new SqlConnection())
{
conn.ConnectionString = Properties.Settings.Default.ConnStr;
conn.Open();
SqlCommand cmd = new SqlCommand();
cmd.CommandText = "INSERT INTO TrafficRoutine4"
+ " (traf4_traf3_seqno, traf4_domname, traf4_bl_domainname, traf4_compete_esttraffic, traf4_compete_perdiff,"
+ " traf4_quantcast_esttraffic, traf4_alexa_tr, traf4_alexa_reach, traf4_alexa_search, traf4_alexa_country1,"
+ " traf4_alexa_country1_per, traf4_alexa_country2, traf4_alexa_country2_per, traf4_alexa_country3, traf4_alexa_country3_per, traf4_procTime)"
+ " VALUES (@seqno, @domname, @bl_domname, @compete_esttraf, @compete_perdiff, @quantcast_esttraf, @alexa_tr,"
+ " @alexa_reach, @alexa_search, @alexa_ctry1, @alexa_ctry1_per, @alexa_ctry2, @alexa_ctry2_per, @alexa_ctry3, @alexa_ctry3_per, @timedif)";
cmd.Parameters.AddWithValue("@seqno", traf3seqno);
cmd.Parameters.AddWithValue("@domname", Dname);
cmd.Parameters.AddWithValue("@bl_domname", webAddr(ALobj.ToString()));
cmd.Parameters.AddWithValue("@compete_esttraf", LR1.CmptRec.CmptMonth1);
cmd.Parameters.AddWithValue("@compete_perdiff", 0);
cmd.Parameters.AddWithValue("@quantcast_esttraf", LR1.dnRec.QuantcastCtr);
cmd.Parameters.AddWithValue("@alexa_tr", LR1.AlxRec.AlxHighRank);
cmd.Parameters.AddWithValue("@alexa_reach", LR1.AlxRec.AlxHighReach);
cmd.Parameters.AddWithValue("@alexa_search", LR1.AlxRec.AlxHighSearch);
cmd.Parameters.AddWithValue("@alexa_ctry1", (LR1.AlxRec.AlxCountry1.ToString() == null ? "" : LR1.AlxRec.AlxCountry1.ToString()));
cmd.Parameters.AddWithValue("@alexa_ctry1_per", LR1.AlxRec.AlxCntry1_per);
cmd.Parameters.AddWithValue("@alexa_ctry2", (LR1.AlxRec.AlxCountry2.ToString() == null ? "" : LR1.AlxRec.AlxCountry2.ToString()));
cmd.Parameters.AddWithValue("@alexa_ctry2_per", LR1.AlxRec.AlxCntry2_per);
cmd.Parameters.AddWithValue("@alexa_ctry3", (LR1.AlxRec.AlxCountry3.ToString() == null ? "" : LR1.AlxRec.AlxCountry3.ToString()));
cmd.Parameters.AddWithValue("@alexa_ctry3_per", LR1.AlxRec.AlxCntry3_per);
cmd.Parameters.AddWithValue("@timedif", subProcTime);
cmd.Connection = conn;
bckLnkStartTime = DateTime.Now;
int rtn = cmd.ExecuteNonQuery();
StopTime = DateTime.Now;
timeDifA = StopTime - bckLnkStartTime;
subProcTime = timeDifA.TotalSeconds;
blrtn = CycleTime("Backlink" + count.ToString() + " UpdateTime", Dname, (float)subProcTime);
}
LR1 = null;
}
LR = null;
Spidey = null;
txtBxExecTime.Text = timeDif.TotalSeconds.ToString();
UpdateDNprocessTime(Dname, timeDif);
cntr++;
label2.Text = cntr.ToString();
StopTime = DateTime.Now;
timeDif = StopTime - startTime;
line = cntr.ToString() + " - " + Dname + " -- Processing time: " + timeDif.TotalSeconds.ToString();
timeTotal = timeTotal + timeDif.TotalSeconds;
timeAvg = timeTotal / cntr;
label3.Text = timeAvg.ToString();
//listBox1.SelectedItem = listBox1.Items.Add(line.ToString());
Application.DoEvents();
}
MessageBox.Show("It is finished! Total: " + cntr.ToString() + " Domains cycled.");
}
private void btnProcess_Click(object sender, EventArgs e)
{
//LinkRaker LR = new LinkRaker();
LR.lb_Proxy = checkBox1.Checked;
foreach (object itemChecked in checkedListBoxResources.CheckedItems)
{
if (checkedListBoxResources.GetItemChecked(checkedListBoxResources.Items.IndexOf(itemChecked)))
{
switch (itemChecked.ToString())
{
case "Alexa":
LR.lb_Alexa = true;
break;
case "Compete":
LR.lb_Compete = true;
break;
case "Quantcast":
LR.lb_Quant = true;
break;
case "Archive":
LR.lb_Archive = true;
break;
case "PageRank":
LR.lb_PageRank = true;
break;
case "WhoIs":
LR.lb_WhoIs = true;
break;
case "Estibot":
LR.lb_Estibot = true;
break;
case "DMOZ":
LR.lb_DMOZ = true;
break;
case "IndexPages":
LR.lb_IndexPages = true;
break;
case "BackLinks":
LR.lb_BackLinks = true;
break;
default:
break;
}
}
}
//LR.RunResources();
string DDName;
bool lb_rtn = false;
if (LR.lb_Proxy)
{
LR.proxAddr = LR.GetProxy();
}
DDName = LR.GetDDN();
while (DDName != null)
{
if (LR.lb_PageRank)
{
LR.RunPageRank(DDName);
}
if (LR.lb_Archive)
{
LR.RunArchive(DDName);
}
if (LR.lb_Quant)
{
LR.RunQuantcast(DDName);
}
if (LR.lb_DMOZ)
{
LR.RunDMOZ(DDName);
}
if (LR.lb_PageRank || LR.lb_Archive || LR.lb_Quant || LR.lb_DMOZ)
{
lb_rtn = LR.dnRec.Update();
}
if (LR.lb_Alexa)
{
LR.RunAlexa(DDName);
}
if (LR.lb_Compete)
{
LR.RunCompete(DDName);
}
if (LR.lb_WhoIs)
{
LR.RunWhoIs(DDName);
}
if (LR.lb_Estibot)
{
LR.RunEstibot(DDName);
}
if (LR.lb_IndexPages)
{
LR.RunIndexPages(DDName);
}
if (LR.lb_BackLinks)
{
LR.RunBackLinks(DDName);
}
DDName = LR.GetDDN();
}
}