Example #1
0
        private void btnSpider_Click(object sender, EventArgs e)
        {
            string Dname;
            string line;
            int    cntr      = 0;
            double timeAvg   = 0;
            double timeTotal = 0;

            ProxyShuttler PrxShtlr = new ProxyShuttler();

            while ((Dname = GetDN()) != null)
            {
                DateTime startTime = DateTime.Now;
                M4Spider Spidey    = new M4Spider();
                Spidey.MaxOutboundLinks = 3;
                Spidey.ProxyDomain      = PrxShtlr.CurrentProxy.ToString();
                Spidey.AddUnspidered("http://www.alexa.com/site/linksin/" + Dname);//"http://www.databasejobs.com/");//
                Spidey.BackLinkDN = Dname;
                ArrayList BackLinksList;
                if (AV_links > 0)
                {
                    bool lb_rtn = Spidey.CrawlNext();
                    if (!lb_rtn)
                    {
                        continue;
                    }

                    if (Spidey.SR1 != null)
                    {
                        BackLinksList = Spidey.SR1.HyperLinks;
                    }
                    else
                    {
                        BackLinksList = new ArrayList();
                        BackLinksList.Add(Dname);
                    }
                }
                else
                {
                    BackLinksList = new ArrayList();
                    BackLinksList.Add(Dname);
                }



                LinkRaker LR = new LinkRaker();
                LR.lb_Proxy = true; // While at work . . .
                LR.RunAlexa(Dname);

                LR.RunCompete(Dname);

                //Run against Quantcast
                LR.RunQuantcast(Dname);

                DateTime StopTime = DateTime.Now;
                TimeSpan timeDif  = StopTime - startTime;
                double   procTime = timeDif.TotalSeconds;

                bool blrtn = CycleTime("Parent Domain", Dname, (float)procTime);

                using (SqlConnection conn = new SqlConnection())
                {
                    conn.ConnectionString = Properties.Settings.Default.ConnStr;
                    conn.Open();
                    SqlCommand cmd = new SqlCommand();
                    cmd.CommandText = "INSERT INTO TrafficRoutine4"
                                      + " (traf4_traf3_seqno, traf4_domname, traf4_bl_domainname, traf4_compete_esttraffic, traf4_compete_perdiff,"
                                      + " traf4_quantcast_esttraffic, traf4_alexa_tr, traf4_alexa_reach, traf4_alexa_search, traf4_alexa_country1,"
                                      + " traf4_alexa_country1_per, traf4_alexa_country2, traf4_alexa_country2_per, traf4_alexa_country3, traf4_alexa_country3_per, traf4_procTime)"
                                      + " VALUES (@seqno, @domname, @bl_domname, @compete_esttraf, @compete_perdiff, @quantcast_esttraf, @alexa_tr,"
                                      + " @alexa_reach, @alexa_search, @alexa_ctry1, @alexa_ctry1_per, @alexa_ctry2, @alexa_ctry2_per, @alexa_ctry3, @alexa_ctry3_per, @timedif)";
                    cmd.Parameters.AddWithValue("@seqno", traf3seqno);
                    cmd.Parameters.AddWithValue("@domname", Dname);
                    cmd.Parameters.AddWithValue("@bl_domname", "");
                    cmd.Parameters.AddWithValue("@compete_esttraf", LR.CmptRec.CmptMonth1);
                    cmd.Parameters.AddWithValue("@compete_perdiff", 0);
                    cmd.Parameters.AddWithValue("@quantcast_esttraf", LR.dnRec.QuantcastCtr);
                    cmd.Parameters.AddWithValue("@alexa_tr", LR.AlxRec.AlxHighRank);
                    cmd.Parameters.AddWithValue("@alexa_reach", LR.AlxRec.AlxHighReach);
                    cmd.Parameters.AddWithValue("@alexa_search", LR.AlxRec.AlxHighSearch);
                    cmd.Parameters.AddWithValue("@alexa_ctry1", (LR.AlxRec.AlxCountry1.ToString() == null ? "" : LR.AlxRec.AlxCountry1.ToString()));
                    cmd.Parameters.AddWithValue("@alexa_ctry1_per", LR.AlxRec.AlxCntry1_per);
                    cmd.Parameters.AddWithValue("@alexa_ctry2", (LR.AlxRec.AlxCountry2.ToString() == null ? "" : LR.AlxRec.AlxCountry2.ToString()));
                    cmd.Parameters.AddWithValue("@alexa_ctry2_per", LR.AlxRec.AlxCntry2_per);
                    cmd.Parameters.AddWithValue("@alexa_ctry3", (LR.AlxRec.AlxCountry3.ToString() == null ? "" : LR.AlxRec.AlxCountry3.ToString()));
                    cmd.Parameters.AddWithValue("@alexa_ctry3_per", LR.AlxRec.AlxCntry3_per);
                    cmd.Parameters.AddWithValue("@timedif", procTime);
                    cmd.Connection = conn;

                    int rtn = cmd.ExecuteNonQuery();
                }

                DateTime bckLnkStartTime = DateTime.Now;
                int      count           = 0;
                foreach (object ALobj in BackLinksList)
                {
                    if (ALobj.ToString().ToUpper().Contains(Dname.ToUpper()))
                    {
                        continue;
                    }
                    count++;
                    LinkRaker LR1 = new LinkRaker();
                    LR1.lb_Proxy = true; // While at work . . .
                    LR1.RunAlexa(ALobj.ToString());

                    LR1.RunCompete(ALobj.ToString());

                    LR.RunQuantcast(ALobj.ToString());

                    StopTime = DateTime.Now;
                    TimeSpan timeDifA    = StopTime - bckLnkStartTime;
                    double   subProcTime = timeDifA.TotalSeconds;
                    blrtn = CycleTime("Backlink" + count.ToString() + " Domain", Dname, (float)subProcTime);

                    using (SqlConnection conn = new SqlConnection())
                    {
                        conn.ConnectionString = Properties.Settings.Default.ConnStr;
                        conn.Open();
                        SqlCommand cmd = new SqlCommand();
                        cmd.CommandText = "INSERT INTO TrafficRoutine4"
                                          + " (traf4_traf3_seqno, traf4_domname, traf4_bl_domainname, traf4_compete_esttraffic, traf4_compete_perdiff,"
                                          + " traf4_quantcast_esttraffic, traf4_alexa_tr, traf4_alexa_reach, traf4_alexa_search, traf4_alexa_country1,"
                                          + " traf4_alexa_country1_per, traf4_alexa_country2, traf4_alexa_country2_per, traf4_alexa_country3, traf4_alexa_country3_per, traf4_procTime)"
                                          + " VALUES (@seqno, @domname, @bl_domname, @compete_esttraf, @compete_perdiff, @quantcast_esttraf, @alexa_tr,"
                                          + " @alexa_reach, @alexa_search, @alexa_ctry1, @alexa_ctry1_per, @alexa_ctry2, @alexa_ctry2_per, @alexa_ctry3, @alexa_ctry3_per, @timedif)";
                        cmd.Parameters.AddWithValue("@seqno", traf3seqno);
                        cmd.Parameters.AddWithValue("@domname", Dname);
                        cmd.Parameters.AddWithValue("@bl_domname", webAddr(ALobj.ToString()));
                        cmd.Parameters.AddWithValue("@compete_esttraf", LR1.CmptRec.CmptMonth1);
                        cmd.Parameters.AddWithValue("@compete_perdiff", 0);
                        cmd.Parameters.AddWithValue("@quantcast_esttraf", LR1.dnRec.QuantcastCtr);
                        cmd.Parameters.AddWithValue("@alexa_tr", LR1.AlxRec.AlxHighRank);
                        cmd.Parameters.AddWithValue("@alexa_reach", LR1.AlxRec.AlxHighReach);
                        cmd.Parameters.AddWithValue("@alexa_search", LR1.AlxRec.AlxHighSearch);
                        cmd.Parameters.AddWithValue("@alexa_ctry1", (LR1.AlxRec.AlxCountry1.ToString() == null ? "" : LR1.AlxRec.AlxCountry1.ToString()));
                        cmd.Parameters.AddWithValue("@alexa_ctry1_per", LR1.AlxRec.AlxCntry1_per);
                        cmd.Parameters.AddWithValue("@alexa_ctry2", (LR1.AlxRec.AlxCountry2.ToString() == null ? "" : LR1.AlxRec.AlxCountry2.ToString()));
                        cmd.Parameters.AddWithValue("@alexa_ctry2_per", LR1.AlxRec.AlxCntry2_per);
                        cmd.Parameters.AddWithValue("@alexa_ctry3", (LR1.AlxRec.AlxCountry3.ToString() == null ? "" : LR1.AlxRec.AlxCountry3.ToString()));
                        cmd.Parameters.AddWithValue("@alexa_ctry3_per", LR1.AlxRec.AlxCntry3_per);
                        cmd.Parameters.AddWithValue("@timedif", subProcTime);
                        cmd.Connection = conn;

                        bckLnkStartTime = DateTime.Now;

                        int rtn = cmd.ExecuteNonQuery();

                        StopTime    = DateTime.Now;
                        timeDifA    = StopTime - bckLnkStartTime;
                        subProcTime = timeDifA.TotalSeconds;
                        blrtn       = CycleTime("Backlink" + count.ToString() + " UpdateTime", Dname, (float)subProcTime);
                    }

                    LR1 = null;
                }

                LR                 = null;
                Spidey             = null;
                txtBxExecTime.Text = timeDif.TotalSeconds.ToString();
                UpdateDNprocessTime(Dname, timeDif);
                cntr++;
                label2.Text = cntr.ToString();

                StopTime = DateTime.Now;
                timeDif  = StopTime - startTime;

                line      = cntr.ToString() + " - " + Dname + " -- Processing time: " + timeDif.TotalSeconds.ToString();
                timeTotal = timeTotal + timeDif.TotalSeconds;
                timeAvg   = timeTotal / cntr;

                label3.Text = timeAvg.ToString();
                //listBox1.SelectedItem = listBox1.Items.Add(line.ToString());

                Application.DoEvents();
            }

            MessageBox.Show("It is finished! Total: " + cntr.ToString() + " Domains cycled.");
        }
Example #2
0
        private void btnProcess_Click(object sender, EventArgs e)
        {
            //LinkRaker LR = new LinkRaker();
            LR.lb_Proxy = checkBox1.Checked;

            foreach (object itemChecked in checkedListBoxResources.CheckedItems)
            {
                if (checkedListBoxResources.GetItemChecked(checkedListBoxResources.Items.IndexOf(itemChecked)))
                {
                    switch (itemChecked.ToString())
                    {
                    case "Alexa":
                        LR.lb_Alexa = true;
                        break;

                    case "Compete":
                        LR.lb_Compete = true;
                        break;

                    case "Quantcast":
                        LR.lb_Quant = true;
                        break;

                    case "Archive":
                        LR.lb_Archive = true;
                        break;

                    case "PageRank":
                        LR.lb_PageRank = true;
                        break;

                    case "WhoIs":
                        LR.lb_WhoIs = true;
                        break;

                    case "Estibot":
                        LR.lb_Estibot = true;
                        break;

                    case "DMOZ":
                        LR.lb_DMOZ = true;
                        break;

                    case "IndexPages":
                        LR.lb_IndexPages = true;
                        break;

                    case "BackLinks":
                        LR.lb_BackLinks = true;
                        break;

                    default:

                        break;
                    }
                }
            }

            //LR.RunResources();

            string DDName;
            bool   lb_rtn = false;

            if (LR.lb_Proxy)
            {
                LR.proxAddr = LR.GetProxy();
            }

            DDName = LR.GetDDN();

            while (DDName != null)
            {
                if (LR.lb_PageRank)
                {
                    LR.RunPageRank(DDName);
                }
                if (LR.lb_Archive)
                {
                    LR.RunArchive(DDName);
                }
                if (LR.lb_Quant)
                {
                    LR.RunQuantcast(DDName);
                }
                if (LR.lb_DMOZ)
                {
                    LR.RunDMOZ(DDName);
                }
                if (LR.lb_PageRank || LR.lb_Archive || LR.lb_Quant || LR.lb_DMOZ)
                {
                    lb_rtn = LR.dnRec.Update();
                }

                if (LR.lb_Alexa)
                {
                    LR.RunAlexa(DDName);
                }

                if (LR.lb_Compete)
                {
                    LR.RunCompete(DDName);
                }
                if (LR.lb_WhoIs)
                {
                    LR.RunWhoIs(DDName);
                }
                if (LR.lb_Estibot)
                {
                    LR.RunEstibot(DDName);
                }
                if (LR.lb_IndexPages)
                {
                    LR.RunIndexPages(DDName);
                }
                if (LR.lb_BackLinks)
                {
                    LR.RunBackLinks(DDName);
                }


                DDName = LR.GetDDN();
            }
        }