示例#1
0
        private void ThreadProcessData(ImgLink img)
        {
            Console.WriteLine("Start Check: " + img.linkimg);
            var _bolService = new BOLService.BOLService();

            try
            {
                var req = (HttpWebRequest)HttpWebRequest.Create(img.linkimg);
                req.Method = "HEAD";
                using (var resp = req.GetResponse())
                {
                    if (!resp.ContentType.ToLower(CultureInfo.InvariantCulture).StartsWith("image/"))
                    {
                        Console.WriteLine("Invalid: " + img.linkimg);
                        _bolService.UpdateBadURL(img.ID, true);
                    }
                    else
                    {
                        _bolService.UpdateBadURL(img.ID, false);
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error: " + ex.Message);
                _bolService.UpdateBadURL(img.ID);
            }
            Console.WriteLine("End Check: " + img.linkimg);
        }
示例#2
0
        public static void TestImageOnePage(string strPage, string strCounter, string category, string strTitle)
        {
            try
            {
                var bolService = new BOLService.BOLService();
                if (bolService.CheckExistLinkByDomain(strCounter, "xemlasuong.org"))
                {
                    return;
                }

                var lst = new List <ImgLink>();
                var web = new HtmlWeb();
                var doc = web.Load(strPage);
                //TODO: Check valid
                var divContainer = doc.DocumentNode.SelectSingleNode("//div[@class = 'blog-single-para']");
                if (divContainer == null)
                {
                    return;
                }

                HtmlNodeCollection nodeImagesCollection = divContainer.SelectNodes("//img");
                foreach (var node in nodeImagesCollection)
                {
                    try
                    {
                        string strLink  = node.Attributes["src"].Value;
                        string strClass = node.Attributes["class"] != null
                            ? node.Attributes["class"].Value
                            : string.Empty;
                        if (strClass.Contains("size-full"))
                        {
                            var item = new ImgLink()
                            {
                                Category   = category,
                                Counter    = strCounter,
                                CreateDate = DateTime.Now,
                                Domain     = "xemlasuong.org",
                                GroupName  = strTitle,
                                linkimg    = strLink
                            };
                            lst.Add(item);
                        }
                    }
                    catch (Exception ex)
                    {
                        //TODO: show error;
                        Console.WriteLine(ex.ToString());
                    }
                }
                if (lst.Count > 0)
                {
                    bolService.SaveImg(lst);
                    Console.WriteLine(strPage);
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
        }
示例#3
0
 public static void TestImageOnePage(string strPage, string strCounter, string category, string strTitle)
 {
     try
     {
         var bolService = new BOLService.BOLService();
         var lst        = new List <ImgLink>();
         var web        = new HtmlWeb();
         var doc        = web.Load(strPage);
         //TODO: Check valid
         var divContainer = doc.DocumentNode.SelectSingleNode("//div[@class = 'photo-container']");
         if (divContainer != null)
         {
             //var imgNodes = divContainer.SelectNodes("//img[@onerror = 'imgerror(this)']");
             foreach (HtmlNode node in divContainer.ChildNodes)
             {
                 try
                 {
                     if (node.Name.Equals("a"))
                     {
                         foreach (HtmlNode imgNode in node.ChildNodes)
                         {
                             if (imgNode.Name.Equals("img"))
                             {
                                 string strLink = imgNode.Attributes["src"].Value;
                                 if (!strLink.Contains("Content/themes/noimg_big.jpg") &&
                                     !strLink.Contains("http://api.xinhvl.tekreds.com"))
                                 {
                                     ImgLink obj = new ImgLink()
                                     {
                                         CreateDate = DateTime.UtcNow,
                                         Domain     = "xinh.hay.la",
                                         Counter    = strCounter,
                                         linkimg    = strLink,
                                         Category   = category,
                                         GroupName  = strTitle
                                     };
                                     lst.Add(obj);
                                     break;
                                 }
                             }
                         }
                     }
                 }
                 catch (Exception exception)
                 {
                     Console.WriteLine(exception.ToString());
                 }
             }
         }
         if (lst.Count > 0)
         {
             bolService.SaveImg(lst);
             Console.WriteLine(strPage);
         }
     }
     catch (Exception ex)
     {
         Console.WriteLine(ex.ToString());
     }
 }
示例#4
0
        public static void TestImageOnePage(string strPage, string strCounter, string category, string strTitle)
        {
            try
            {
                var bolService = new BOLService.BOLService();
                if (bolService.CheckExistLinkByDomain(strCounter, "phunuvn.net"))
                {
                    return;
                }

                var lst = new List <ImgLink>();
                var web = new HtmlWeb();
                var doc = web.Load(strPage);
                //TODO: Check valid
                var divContainer = doc.DocumentNode.SelectSingleNode("//blockquote[@class = 'messageText ugc baseHtml']");
                if (divContainer == null)
                {
                    return;
                }

                HtmlNodeCollection nodeImagesCollection = divContainer.SelectNodes("//img[@class = 'bbCodeImage LbImage']");
                foreach (var node in nodeImagesCollection)
                {
                    try
                    {
                        string strLink = node.Attributes["src"].Value;
                        var    item    = new ImgLink()
                        {
                            Category   = category,
                            Counter    = strCounter,
                            CreateDate = DateTime.Now,
                            Domain     = "phunuvn.net",
                            GroupName  = strTitle,
                            linkimg    = strLink
                        };
                        lst.Add(item);
                    }
                    catch (Exception ex)
                    {
                        //TODO: show error;
                        Console.WriteLine(ex.ToString());
                    }
                }
                if (lst.Count > 0)
                {
                    bolService.SaveImg(lst);
                    Console.WriteLine(strPage);
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
        }
示例#5
0
        private static void GetChanDaiPhoto()
        {
            var            bolService = new BOLService.BOLService();
            List <ImgLink> lst        = new List <ImgLink>();
            var            web        = new HtmlWeb();
            int            counter    = 0;
            int            error      = 0;
            int            start      = 1;
            int            end        = 100;

            var vStartObj = bolService.GetLastestChanDaiImage();

            if (vStartObj != null)
            {
                if (!int.TryParse(vStartObj.Counter, out start))
                {
                    Console.WriteLine("Not Start ^_^");
                    return;
                }
            }

            end = start * 1000;

            for (int i = start; i < end; i++)
            {
                try
                {
                    var doc  = web.Load("http://chandai.tv/photo/" + i);
                    var vImg = doc.DocumentNode.SelectSingleNode("//img[@class = 'img-responsive']");
                    if (vImg != null && vImg.Attributes.Count > 1)
                    {
                        counter = i;
                        var vResult = vImg.Attributes[1];
                        if (vResult.Value.Equals("/Content/images/notfound.jpg"))
                        {
                            error += 1;
                            if (error == 300)
                            {
                                break;
                            }
                        }
                        else
                        {
                            error = 0; //Only end when 10 sequence times no image
                            string strResult = string.Format("http://chandai.tv{0}", vResult.Value);
                            var    img       = new ImgLink()
                            {
                                linkimg    = strResult,
                                Counter    = i.ToString(),
                                CreateDate = DateTime.Now,
                                Domain     = "chandai.tv"
                            };

                            lst.Add(img);
                            if (lst.Count == 10)
                            {
                                bolService.SaveImg(lst);
                                lst.Clear();
                                Console.WriteLine("Save link " + i);
                            }
                        }
                    }
                }
                catch (Exception exception)
                {
                    log4net.Config.XmlConfigurator.Configure();
                    log.Info("Error: " + exception);
                    Console.WriteLine(exception.ToString());
                }
            }
            if (lst.Count > 0)
            {
                bolService.SaveImg(lst);
                lst.Clear();
            }
            Console.WriteLine("Finish ^_^");
        }
示例#6
0
        public static void TestImageOnePage(string strPage)
        {
            try
            {
                var bolService = new BOLService.BOLService();
                var lst        = new List <ImgLink>();
                var web        = new HtmlWeb();
                var doc        = web.Load(strPage);
                //TODO: Check valid
                var divContainer = doc.DocumentNode.SelectNodes("//a[@imageanchor = '1']");
                if (divContainer == null)
                {
                    var v = doc.DocumentNode.SelectSingleNode("//div[@class = 'post-content']");
                    if (v != null && v.HasChildNodes)
                    {
                        foreach (HtmlNode htmlNode in v.ChildNodes)
                        {
                            if (htmlNode.Name.Equals("div") && htmlNode.Attributes["style"].Value == "padding:10px")
                            {
                                var vTagP = htmlNode.SelectSingleNode("//p");
                                if (!vTagP.HasAttributes)
                                {
                                    divContainer = vTagP.SelectNodes("//img");
                                    foreach (HtmlNode node in divContainer)
                                    {
                                        try
                                        {
                                            string str = node.Attributes["src"].Value;
                                            if (str.Contains("photo.hoanvu.net/wp-content/uploads/thumbnail/") ||
                                                str.Contains("photo.hoanvu.net/wp-content/themes/iphoto/images/logo.png"))
                                            {
                                                continue;
                                            }
                                            if (str.Contains("blogspot.com") || str.Contains("photo.hoanvu.net/wp-content/") ||
                                                str.Contains("data.hoanvu.net/images"))
                                            {
                                                ImgLink obj = new ImgLink()
                                                {
                                                    CreateDate = DateTime.UtcNow,
                                                    Domain     = "photo.hoanvu.net",
                                                    //Counter = strCounter,
                                                    linkimg = str,
                                                    //Category = strCategory,
                                                    //GroupName = strTitle
                                                };
                                                lst.Add(obj);
                                                Console.WriteLine(str);
                                            }
                                        }
                                        catch (Exception exception)
                                        {
                                            Console.WriteLine(exception.ToString());
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
                else
                {
                    foreach (HtmlNode node in divContainer)
                    {
                        try
                        {
                            ImgLink obj = new ImgLink()
                            {
                                CreateDate = DateTime.UtcNow,
                                Domain     = "photo.hoanvu.net",
                                //Counter = strCounter,
                                linkimg = node.Attributes["href"].Value,
                                //Category = strCategory,
                                //GroupName = strTitle
                            };
                            lst.Add(obj);
                        }
                        catch (Exception exception)
                        {
                            Console.WriteLine(exception.ToString());
                        }
                    }
                }

                bolService.SaveImg(lst);
                Console.WriteLine(strPage);
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
        }
示例#7
0
        public static void TestImageOnePage(string strPage, string strCounter, string category, string strTitle)
        {
            try
            {
                var bolService = new BOLService.BOLService();
                if (bolService.CheckExistLinkByDomain(strCounter, "xemanh.net"))
                {
                    return;
                }

                var lst = new List <ImgLink>();
                var web = new HtmlWeb();
                var doc = web.Load(strPage);
                //TODO: Check valid
                var divContainer = doc.DocumentNode.SelectSingleNode("//div[@id = 'main']");
                if (divContainer == null)
                {
                    return;
                }

                HtmlNode articleNode = divContainer.SelectSingleNode("//article");
                HtmlNode sectionNode = articleNode.SelectSingleNode("//section");
                foreach (var node in sectionNode.ChildNodes)
                {
                    try
                    {
                        if (node.Name.Equals("p") && node.HasAttributes && node.HasChildNodes)
                        {
                            HtmlNode nodeA = node.FirstChild;
                            if (node.ChildNodes.Count > 1)
                            {
                                foreach (HtmlNode f1Node in node.ChildNodes)
                                {
                                    if (f1Node.Name.Equals("a") || f1Node.Name.Equals("img"))
                                    {
                                        nodeA = f1Node;
                                        break;
                                    }
                                }
                            }

                            HtmlNode nodeImg = nodeA;
                            if (nodeA.Name.Equals("a") && nodeA.HasChildNodes)
                            {
                                foreach (HtmlNode fNode in nodeA.ChildNodes)
                                {
                                    if (fNode.Name.Equals("img"))
                                    {
                                        nodeImg = fNode;
                                        break;
                                    }
                                }
                            }
                            if (nodeImg != null && nodeImg.Name.Equals("img") && nodeImg.Attributes["src"] != null)
                            {
                                string strLink = nodeImg.Attributes["src"].Value;
                                var    item    = new ImgLink()
                                {
                                    Category   = category,
                                    Counter    = strCounter,
                                    CreateDate = DateTime.Now,
                                    Domain     = "xemanh.net",
                                    GroupName  = strTitle,
                                    linkimg    = strLink
                                };
                                lst.Add(item);
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        //TODO: show error;
                        Console.WriteLine(ex.ToString());
                    }
                }
                if (lst.Count > 0)
                {
                    bolService.SaveImg(lst);
                    Console.WriteLine(strPage);
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
        }
示例#8
0
        public static void TestImageOnePage(string strPage, string strCounter, string category, string strTitle)
        {
            try
            {
                var bolService = new BOLService.BOLService();
                if (bolService.CheckExistLinkByDomain(strCounter, "tructiepso.com"))
                {
                    return;
                }

                var lst = new List <ImgLink>();
                var web = new HtmlWeb();
                var doc = web.Load(strPage);
                //TODO: Check valid
                var divContainer = doc.DocumentNode.SelectSingleNode("//div[@class = 'entry-content rich-content']");
                if (divContainer == null)
                {
                    divContainer = doc.DocumentNode.SelectSingleNode("//div[@id = 'content_document']");
                    if (divContainer == null)
                    {
                        return;
                    }
                }

                HtmlNodeCollection nodeImagesCollection = divContainer.SelectNodes("//img");
                foreach (var node in nodeImagesCollection)
                {
                    try
                    {
                        string strLink = node.Attributes["src"].Value;
                        if (!strLink.Contains("tructiepso.com/wp-content/uploads") &&
                            !strLink.Contains("gamer.gif"))
                        {
                            var item = new ImgLink()
                            {
                                Category   = category,
                                Counter    = strCounter,
                                CreateDate = DateTime.Now,
                                Domain     = "tructiepso.com",
                                GroupName  = strTitle,
                                linkimg    = strLink
                            };
                            lst.Add(item);
                        }
                    }
                    catch (Exception ex)
                    {
                        //TODO: show error;
                        Console.WriteLine(ex.ToString());
                    }
                }
                if (lst.Count > 0)
                {
                    bolService.SaveImg(lst);
                    Console.WriteLine(strPage);
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
        }