示例#1
0
        public void Dispose()
        {
            MONITOR.objects_disposed++;

            MONITOR.LogDiagnostic("object has been disposed." + MONITOR.objects_disposed);
        }
示例#2
0
        public List <string> spawn(string url)
        //public void spawn(string url)
        {
            HtmlWeb       hw                   = new HtmlWeb();
            HtmlDocument  doc                  = new HtmlDocument();
            List <string> hrefTags             = new List <string>();
            List <string> text                 = new List <string>();
            List <string> links_from_this_page = new List <string>();

            Random rnd       = new Random();
            int    wait_time = 5000 + rnd.Next(1000);

            //10000 + rnd.Next(1000);

            //MONITOR.Log( "Elapsed: " + (DateTime.Now.Subtract(MONITOR.START_TIME).TotalMinutes.ToString() +"  wt@"+wait_time + "mS " + url));
            MONITOR.LogMulti("Elapsed: " + (DateTime.Now.Subtract(MONITOR.START_TIME).TotalMinutes.ToString() + ",  wt@" + wait_time + "mS, " + url));

            try
            {
                System.Threading.Thread.Sleep(wait_time);
                doc = hw.Load(url);
            }
            catch (Exception e)
            {
                MONITOR.LogError(e.Message);
            }

            try
            {
                foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//text()"))
                {
                    text.Add(node.InnerText);
                }
            }
            catch (Exception e)
            {
                MONITOR.LogError(e.Message);
            }
            try
            {
                foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]"))
                {
                    HtmlAttribute att = link.Attributes["href"];
                    hrefTags.Add(att.Value);
                }
            }
            catch (Exception e)
            {
                MONITOR.LogError(e.Message);
            }

            doc = null;
            GC.Collect();

            foreach (var tag in hrefTags)
            {
                if (Regex.IsMatch(tag, @"profile/"))
                {
                    var user = tag.Split(("/").ToCharArray()).Last();
                    if (!MONITOR.listOfUniqueUsers.Contains(user))
                    {
                        using (SQLiteConnection qdbconnection = new SQLiteConnection("Data Source=D:/cache/quora.sqlite;Version=3"))
                        {
                            qdbconnection.Open();
                            SQLiteCommand command = new SQLiteCommand(@"INSERT INTO quora (name, link) values ('" + user + "', '" + url + "')", qdbconnection);
                            command.ExecuteNonQuery();
                        }
                        MONITOR.listOfUniqueUsers.Add(user);
                    }
                }
            }

            //using (StreamWriter sw2 = File.AppendText(@"D:/cache/cache-quora-text.txt"))
            //{
            //    sw2.WriteLine(String.Join(String.Empty, text.ToArray()));
            //}
            //using (StreamWriter sw = new StreamWriter(@"D:/cache/cache-quora-usernames.txt"))
            //{
            //    sw.WriteLine(String.Join(Environment.NewLine, MONITOR.listOfUniqueUsers.ToArray()));
            //}
            //using (StreamWriter sw3 = new StreamWriter(@"D:/cache/cache-quora-visited-sites.txt"))
            //{
            //    sw3.WriteLine(String.Join(Environment.NewLine, MONITOR.visitedpages.ToArray()));
            //}


            Parallel.ForEach(hrefTags, hrefTag =>
            {
                if (!hrefTag.Contains("topic") && !hrefTag.Contains("profile") && !hrefTag.Contains("answer") && !hrefTag.Contains("careers/"))
                {
                    //if (!MONITOR.visitedpages.Contains(hrefTag))
                    //{
                    if (hrefTag[0].Equals('/') && hrefTag.Contains("-"))
                    {
                        try
                        {
                            links_from_this_page.Add(@"https://www.quora.com" + hrefTag.Trim(Environment.NewLine.ToCharArray()));
                            MONITOR.listOfUnvisitedPages.Add(@"https://www.quora.com" + hrefTag.Trim(Environment.NewLine.ToCharArray()));
                        }
                        catch (Exception e)
                        {
                            MONITOR.LogError(e.Message + e.StackTrace);
                        }

                        //using (spider n = new spider())
                        //{
                        //    n.spawn(@"https://www.quora.com" + hrefTag);
                        //    Dispose();
                        //}
                    }
                    //}
                }
            });
            //MONITOR.visitedpages.Add(url);
            return(links_from_this_page);
        }
示例#3
0
        //Server=localhost\SQLEXPRESS;Database=master;Trusted_Connection=True;

        static void Main(string[] args)
        {
            Console.SetBufferSize(480, 160);
            string url  = @"https://www.quora.com/Can-social-networks-be-scraped-for-data";
            string url2 = @"https://www.instagram.com/angiesuco/";

            MONITOR.START_TIME       = DateTime.Now;
            MONITOR.pages_visited    = 0;
            MONITOR.objects_disposed = 0;

            //using (spider s = new spider())
            //{
            //    s.spawn(url);
            //}

            //foreach(var child in MONITOR.listOfUnvisitedPages)
            //{
            //    using (spider s = new spider())
            //    {
            //        s.spawn(child);
            //    }
            //}

            var stack = new Stack <List <string> >();

            using (spider n = new spider())
            {
                stack.Push(n.spawn(url));
                while (stack.Any())
                {
                    try
                    {
                        var current = stack.Pop();
                        foreach (var child in current)
                        {
                            try
                            {
                                using (spider n2 = new spider())
                                {
                                    stack.Push(n2.spawn(child));
                                }
                            }
                            catch (Exception e)
                            {
                                MONITOR.LogError(e.Message);
                            }
                        }
                    }
                    catch (Exception e)
                    {
                        MONITOR.LogError(e.Message);
                    }
                }
            }



            Console.WriteLine("PAGES VISITED: " + MONITOR.pages_visited + ". OBJECTS DISPOSED: " + MONITOR.objects_disposed);

            //using (spider n = new spider())
            //{
            //    Parallel.ForEach( n.spawn(url), page =>{
            //        using (spider n2 = new spider())
            //        {
            //            n2.spawn(page);
            //        }

            //    });
            //}
        }
示例#4
0
        public void R(string url)
        {
            HtmlWeb       hw       = new HtmlWeb();
            HtmlDocument  doc      = hw.Load(url);
            List <string> hrefTags = new List <string>();
            List <string> text     = new List <string>();

            foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//text()"))
            {
                if (node.InnerText.Length > 10)
                {
                    text.Add(node.InnerText);
                }
            }
            foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]"))
            {
                HtmlAttribute att = link.Attributes["href"];
                hrefTags.Add(att.Value);
            }
            List <string> uniqueUsers = new List <string>();
            List <string> userTags    = new List <string>();

            foreach (var tag in hrefTags)
            {
                if (Regex.IsMatch(tag, @"users/\d+"))
                {
                    var user = tag.Split(("/").ToCharArray()).Last();
                    userTags.Add(user);
                    if (!uniqueUsers.Contains(user) && !MONITOR.listOfUniqueUsers.Contains(user))
                    {
                        uniqueUsers.Add(user);
                        MONITOR.listOfUniqueUsers.Add(user);

                        using (StreamWriter sw = File.AppendText(@"D:/cache/cache-so-usernames.txt"))
                        {
                            sw.WriteLine(user);
                            //MONITOR.Log(user);
                        }
                    }
                }
            }


            foreach (var tag in hrefTags)
            {
                if (Regex.IsMatch(tag, @"http.*?questions/\d+"))
                {
                    if (!tag.Contains("promotion"))
                    {
                        if (!MONITOR.visitedpages.Contains(tag))
                        {
                            MONITOR.Log(tag);
                            System.Threading.Thread.Sleep(60000);
                            MONITOR.visitedpages.Add(tag);
                            using (N n = new N())
                            {
                                n.R(tag);
                            }
                        }
                    }
                }
            }
        }
示例#5
0
 public void Dispose()
 {
     MONITOR.Log("object has been disposed.");
 }
示例#6
0
 public static void LogError(string data)
 {
     Console.ForegroundColor = ConsoleColor.Red;
     MONITOR.Log(data);
     Console.ResetColor();
 }
示例#7
0
 public static void LogDiagnostic(string data)
 {
     Console.ForegroundColor = ConsoleColor.Cyan;
     MONITOR.Log(data);
     Console.ResetColor();
 }