Beispiel #1
0
        public Form1()
        {
            InitializeComponent();

            List<Html> stat = new List<Html>();

            #region StartUrl
            WebLink.LinkToVisit.Add("http://vortex.infosth.com");
            WebLink.LinkToVisit.Add("http://tva.ca");
            #endregion

            Random random = new Random();

            for (int i = 0; i < WebLink.LinkToVisit.Count; i++)
            {
                int r = random.Next(0, WebLink.LinkToVisit.Count);

                Html webSite = new Html(WebLink.LinkToVisit[r]);

                #region TestCurrentWebSite
                //------------Test----------------------
                Console.WriteLine("#########");
                Console.ForegroundColor = ConsoleColor.Green;
                Console.WriteLine(WebLink.LinkToVisit[r]);
                Console.ResetColor();
                //--------------------------------------
                #endregion

                webSite.ReadHtml();
                stat.Add(webSite);

                WebLink.LinkVisited.Add(WebLink.LinkToVisit[r]);
                WebLink.LinkToVisit.RemoveAt(r);
                WebLink.CompteurLinkToVisit--;
            }
        }
Beispiel #2
0
 private void ConsoleTitle(ref Html webSite)
 {
     Console.WriteLine(webSite.Title);
 }
Beispiel #3
0
 private void ConsoleMeta(ref Html webSite)
 {
     for (int i = 0; i < webSite.ListMeta.Count; i++)
     {
         Console.WriteLine(webSite.ListMeta[i]);
     }
 }
Beispiel #4
0
 private void ConsoleLinks(ref Html webSite)
 {
     for (int i = 0; i < webSite.ListLink.Count; i++)
     {
         Console.WriteLine(webSite.ListLink[i]);
     }
 }
Beispiel #5
0
 private void ConsoleH2(ref Html webSite)
 {
     for (int i = 0; i < webSite.ListH2.Count; i++)
     {
         Console.WriteLine(webSite.ListH2[i]);
     }
 }
Beispiel #6
0
 private void ConsoleCurrentLink(ref Html webSite)
 {
     //------------Test----------------------
     Console.WriteLine("#########");
     Console.ForegroundColor = ConsoleColor.Green;
     Console.WriteLine(webSite.Link + "    " + m_stat.Count.ToString());
     Console.ResetColor();
     Console.WriteLine("--------------------------------");
     //--------------------------------------
 }
Beispiel #7
0
        public void Search()
        {
            m_stopwatchRobots.Start();

            for (int i = 0; i < WebLink.LinkToVisit.Count; i++)
            {
                if (m_stopwatchRobots.ElapsedMilliseconds > 500000)
                {
                    m_stopwatchRobots.Stop();

                    if (m_dataBase.RobotTable.Count(r => r.IsWorking) != 0)
                        m_aloneOnWork = false;

                    m_stopwatchRobots.Reset();
                    m_stopwatchRobots.Start();
                }

                if (WebLink.LinkToVisit.Count == 0)
                {
                    WaitForTableUnvisited();

                    m_htmlDataBase.DropUnvisitedSiteInDataBase();
                }

                int ra = m_random.Next(0, WebLink.LinkToVisit.Count);
                Html webSite = new Html(WebLink.LinkToVisit[ra]);

                if (CurrentLinkOnConsole)
                    ConsoleCurrentLink(ref webSite);

                m_errorIndex = WebLink.LinkCauseError.Count;

                if (!m_htmlDataBase.IsLinkAlreadyVisited(webSite.Link))
                    webSite.ReadHtml();

                #region WriteInConsole

                if (LinksOnConsole)
                    ConsoleLinks(ref webSite);

                if (H1OnConsole)
                    ConsoleH1(ref webSite);

                if (H2OnConsole)
                    ConsoleH2(ref webSite);

                if (MetaOnConsole)
                    ConsoleMeta(ref webSite);

                if (TitleOnConsole)
                    ConsoleTitle(ref webSite);

                if (m_errorIndex != WebLink.LinkCauseError.Count && ErrorOnConsole)
                    ConsoleError();
                #endregion

                m_stat.Add(webSite);

                WebLink.LinkVisited.Add(WebLink.LinkToVisit[ra]);
                WebLink.LinkToVisit.RemoveAt(ra);
                WebLink.CompteurLinkToVisit--;

                if (m_stat.Count >= 500)
                {
                    if (!m_aloneOnWork)
                        WaitForDropData();

                    m_tbot.IsDroppingDataInTables = true;
                    m_tbot.TotalLinkVisited += 500;
                    m_dataBase.SubmitChanges();

                    m_htmlDataBase.DropHtmlObjectInDataBase(ref m_stat);
                    m_htmlDataBase.DropErrorInDataBase();
                    m_htmlDataBase.DropUnvisitedSiteInDataBase();
                    WebLink.LinkVisited.Clear();

                    m_tbot.IsDroppingDataInTables = false;
                    m_dataBase.SubmitChanges();
                }

            }
        }