public static void UgirlsSexyGirl()
{
try
{
var bolService = new BOLService.BOLService();
List <ImgLink> lst = new List <ImgLink>();
var web = new HtmlWeb();
int start = 1;
int end = 300; //TODO: current max value 75
for (int i = start; i <= end; i++)
{
string strURL = string.Format("http://www.ugirls.com/Content/List/Magazine-{0}.html", i);
try
{
string strTitle = string.Format("Magazine-" + i);
string strPage = strURL;
string strCategory = "Sexy Girl";
string strCounter = i.ToString();
TestImageOnePage(strPage, strCounter, strCategory, strTitle);
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
Console.WriteLine("Finish page ^_^: " + strURL);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
public static void GetHome()
{
var bolService = new BOLService.BOLService();
List <ImgLink> lst = new List <ImgLink>();
var web = new HtmlWeb();
int start = 1;
int end = 100;
//track counter: id="post-5740"
for (int i = start; i < end; i++)
{
try
{
var doc = web.Load("http://photo.hoanvu.net");
//TODO: Check valid URL
var divContainer = doc.DocumentNode.SelectSingleNode("//div[@id = 'container']");
if (divContainer == null)
{
continue;
}
foreach (HtmlNode node in divContainer.ChildNodes)
{
if (node != null && node.Name == "div")
{
try
{
string strCounter = node.Attributes["id"].Value;
if (strCounter != "post-tags")
{
bool bDownloaded = bolService.CheckLinkDownloaded("photo.hoanvu.net", strCounter);
if (!bDownloaded && node.HasChildNodes)
{
HtmlNode nodeDiv = node.SelectSingleNode("//div[@class = 'post-thumbnail']");
HtmlNode nodea = nodeDiv.SelectSingleNode("//a[@class = 'img']");
string strTitle = nodea.Attributes["title"].Value;
string strPage = nodea.Attributes["href"].Value;
//TODO: call function Get Images
SaveImageOnePage(strPage, strCounter, "bikini-girl", strTitle);
}
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
}
}
}
catch (Exception exception)
{
//log4net.Config.XmlConfigurator.Configure();
//log.Info("Error: " + exception);
Console.WriteLine(exception.ToString());
}
}
Console.WriteLine("Finish GetBikiniGirl ^_^");
}
public static void CheckValidImageURL()
{
var bolService = new BOLService.BOLService();
int totalRecord = bolService.TotalImages();
int iCounter = totalRecord / 1000;
for (int i = 0; i <= iCounter; i++)
{
Console.WriteLine("Group: " + i);
var lst = bolService.GetImgLinks(i, 1000);
foreach (ImgLink link in lst)
{
Console.WriteLine(link.linkimg);
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(link.linkimg);
request.Method = "HEAD";
try
{
request.GetResponse();
continue;
}
catch
{
bolService.UpdateBadURL(link.ID);
}
}
}
}
public ActionResult MakeBadURL(int idImage)
{
var bolService = new BOLService.BOLService();
bolService.UpdateBadURL(idImage);
return(Json(true));
}
public static void TestImageOnePage(string strPage, string strCounter, string category, string strTitle)
{
try
{
var bolService = new BOLService.BOLService();
var lst = new List <ImgLink>();
var web = new HtmlWeb();
var doc = web.Load(strPage);
//TODO: Check valid
var divContainer = doc.DocumentNode.SelectSingleNode("//div[@class = 'photo-container']");
if (divContainer != null)
{
//var imgNodes = divContainer.SelectNodes("//img[@onerror = 'imgerror(this)']");
foreach (HtmlNode node in divContainer.ChildNodes)
{
try
{
if (node.Name.Equals("a"))
{
foreach (HtmlNode imgNode in node.ChildNodes)
{
if (imgNode.Name.Equals("img"))
{
string strLink = imgNode.Attributes["src"].Value;
if (!strLink.Contains("Content/themes/noimg_big.jpg") &&
!strLink.Contains("http://api.xinhvl.tekreds.com"))
{
ImgLink obj = new ImgLink()
{
CreateDate = DateTime.UtcNow,
Domain = "xinh.hay.la",
Counter = strCounter,
linkimg = strLink,
Category = category,
GroupName = strTitle
};
lst.Add(obj);
break;
}
}
}
}
}
catch (Exception exception)
{
Console.WriteLine(exception.ToString());
}
}
}
if (lst.Count > 0)
{
bolService.SaveImg(lst);
Console.WriteLine(strPage);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
private static void ClearInvalidURLImage()
{
var _bolService = new BOLService.BOLService();
var lstImgs = _bolService.GetListImgForCheck(50);
if (lstImgs.Count > 0)
{
foreach (ImgLink img in lstImgs)
{
try
{
var req = (HttpWebRequest)HttpWebRequest.Create(img.linkimg);
req.Method = "HEAD";
using (var resp = req.GetResponse())
{
if (!resp.ContentType.ToLower(CultureInfo.InvariantCulture).StartsWith("image/"))
{
Console.WriteLine("Invalid: " + img.linkimg);
}
_bolService.UpdateStatus(img.ID);
}
}
catch (Exception ex)
{
Console.WriteLine("Error: " + ex.Message);
_bolService.UpdateBadURL(img.ID);
}
}
}
}
public static void TestImageOnePage(string strPage, string strCounter, string category, string strTitle)
{
try
{
var bolService = new BOLService.BOLService();
if (bolService.CheckExistLinkByDomain(strCounter, "xemlasuong.org"))
{
return;
}
var lst = new List <ImgLink>();
var web = new HtmlWeb();
var doc = web.Load(strPage);
//TODO: Check valid
var divContainer = doc.DocumentNode.SelectSingleNode("//div[@class = 'blog-single-para']");
if (divContainer == null)
{
return;
}
HtmlNodeCollection nodeImagesCollection = divContainer.SelectNodes("//img");
foreach (var node in nodeImagesCollection)
{
try
{
string strLink = node.Attributes["src"].Value;
string strClass = node.Attributes["class"] != null
? node.Attributes["class"].Value
: string.Empty;
if (strClass.Contains("size-full"))
{
var item = new ImgLink()
{
Category = category,
Counter = strCounter,
CreateDate = DateTime.Now,
Domain = "xemlasuong.org",
GroupName = strTitle,
linkimg = strLink
};
lst.Add(item);
}
}
catch (Exception ex)
{
//TODO: show error;
Console.WriteLine(ex.ToString());
}
}
if (lst.Count > 0)
{
bolService.SaveImg(lst);
Console.WriteLine(strPage);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
private void ThreadProcessData(ImgLink img)
{
Console.WriteLine("Start Check: " + img.linkimg);
var _bolService = new BOLService.BOLService();
try
{
var req = (HttpWebRequest)HttpWebRequest.Create(img.linkimg);
req.Method = "HEAD";
using (var resp = req.GetResponse())
{
if (!resp.ContentType.ToLower(CultureInfo.InvariantCulture).StartsWith("image/"))
{
Console.WriteLine("Invalid: " + img.linkimg);
_bolService.UpdateBadURL(img.ID, true);
}
else
{
_bolService.UpdateBadURL(img.ID, false);
}
}
}
catch (Exception ex)
{
Console.WriteLine("Error: " + ex.Message);
_bolService.UpdateBadURL(img.ID);
}
Console.WriteLine("End Check: " + img.linkimg);
}
public static void HotGirl()
{
var bolService = new BOLService.BOLService();
List <ImgLink> lst = new List <ImgLink>();
var web = new HtmlWeb();
int start = 1;
int end = 2; //TODO: max value 5;
//track counter: id="post-5740"
for (int i = start; i < end; i++)
{
try
{
var doc = web.Load("http://tructiepso.com/category/hot-girl/page/" + i);
//TODO: Check valid URL
var divContainer = doc.DocumentNode.SelectSingleNode("//div[@class = 'nag cf']");
if (divContainer == null)
{
continue;
}
HtmlNodeCollection linkPage = divContainer.SelectNodes("//a[@class = 'clip-link']");
if (linkPage == null)
{
continue;
}
foreach (HtmlNode node in linkPage)
{
try
{
string strCounter = node.Attributes["data-id"].Value;
bool bDownloaded = bolService.CheckLinkDownloaded("tructiepso.com", strCounter);
if (!bDownloaded)
{
string strTitle = node.Attributes["title"].Value;
string strPage = node.Attributes["href"].Value;
//TODO: call function Get Images
TestImageOnePage(strPage, strCounter, "Hot-Girl", strTitle);
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
}
}
catch (Exception exception)
{
//log4net.Config.XmlConfigurator.Configure();
//log.Info("Error: " + exception);
Console.WriteLine(exception.ToString());
}
}
Console.WriteLine("Finish GetBikiniGirl ^_^");
}
public void ProcessCheckURL()
{
var _bolService = new BOLService.BOLService();
var lstImgs = _bolService.GetListImgForCheck(150);
var parallelOptions = new ParallelOptions();
parallelOptions.MaxDegreeOfParallelism = 100;
Parallel.ForEach(lstImgs, ThreadProcessData);
}
//
// GET: /Lastest/
public ActionResult Index()
{
var bolService = new BOLService.BOLService();
var obj = new ImageDisplay();
obj.PageIndex = 0;
obj.PageSize = 1000;
obj.ListImg = bolService.GetImgLinks(obj.PageIndex, obj.PageSize);
return(View(obj));
}
public static void Bikini()
{
var bolService = new BOLService.BOLService();
List <ImgLink> lst = new List <ImgLink>();
var web = new HtmlWeb();
int start = 1;
int end = 3; //TODO: Current max 9;
//track counter: id="post-5740"
for (int i = start; i < end; i++)
{
try
{
var doc = web.Load("http://xinhvl.com/channels/bikini-xinh/?page=" + i);
//TODO: Check valid URL
var divContainer = doc.DocumentNode.SelectSingleNode("//div[@id = 'xinhVLListBit']");
if (divContainer == null)
{
continue;
}
foreach (HtmlNode htmlNode in divContainer.ChildNodes)
{
if (htmlNode.Name.Equals("div"))
{
foreach (HtmlNode childNode in htmlNode.ChildNodes)
{
if (childNode.Name.Equals("a"))
{
string strPage = childNode.Attributes["href"].Value;
string strCounter = BuildCounter(strPage);
string strTitle = childNode.NextSibling.NextSibling.InnerText;
bool bDownloaded = bolService.CheckLinkDownloaded("xinhvl.com", strCounter);
if (!bDownloaded)
{
//TODO: call function Get Images
TestImageOnePage(strPage, strCounter, "bikini", strTitle);
}
break;
}
}
}
}
}
catch (Exception exception)
{
//log4net.Config.XmlConfigurator.Configure();
//log.Info("Error: " + exception);
Console.WriteLine(exception.ToString());
}
}
Console.WriteLine("Finish Bikini ^_^");
}
public static void TestImageOnePage(string strPage, string strCounter, string category, string strTitle)
{
try
{
var bolService = new BOLService.BOLService();
if (bolService.CheckExistLinkByDomain(strCounter, "phunuvn.net"))
{
return;
}
var lst = new List <ImgLink>();
var web = new HtmlWeb();
var doc = web.Load(strPage);
//TODO: Check valid
var divContainer = doc.DocumentNode.SelectSingleNode("//blockquote[@class = 'messageText ugc baseHtml']");
if (divContainer == null)
{
return;
}
HtmlNodeCollection nodeImagesCollection = divContainer.SelectNodes("//img[@class = 'bbCodeImage LbImage']");
foreach (var node in nodeImagesCollection)
{
try
{
string strLink = node.Attributes["src"].Value;
var item = new ImgLink()
{
Category = category,
Counter = strCounter,
CreateDate = DateTime.Now,
Domain = "phunuvn.net",
GroupName = strTitle,
linkimg = strLink
};
lst.Add(item);
}
catch (Exception ex)
{
//TODO: show error;
Console.WriteLine(ex.ToString());
}
}
if (lst.Count > 0)
{
bolService.SaveImg(lst);
Console.WriteLine(strPage);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
public ActionResult PagingImage(int?domaind, int pageSize, int pageIndex)
{
var bolService = new BOLService.BOLService();
var obj = new ImageDisplay();
int domainId = domaind ?? 12;
string domainName = GetDomainName(domainId);
int? totals = 0;
obj.ListImg = bolService.GetTopLastest(domainName, pageSize, (pageIndex - 1) * pageSize, ref totals);
obj.TotalRecords = totals.GetValueOrDefault();
return(Json(new { success = true, data = obj.ListImg }, JsonRequestBehavior.AllowGet));
}
public static void InitTGOD()
{
try
{
var bolService = new BOLService.BOLService();
List <ImgLink> lst = new List <ImgLink>();
var web = new HtmlWeb();
int start = 0;
int end = 6; //1
for (int i = end; i >= start; i--)
{
string strURL = string.Format("http://www.tgod.cn/travelpic.aspx?page={0}", i);
var doc = web.Load(strURL);
//TODO: Check valid
var divContainer = doc.DocumentNode.SelectSingleNode("//ul[@id = 'ulnvshen']");
if (divContainer == null)
{
return;
}
foreach (var node in divContainer.ChildNodes)
{
if (node.Name.Equals("li") && node.HasChildNodes)
{
try
{
foreach (HtmlNode nodeLink in node.ChildNodes)
{
if (nodeLink.Name.Equals("a"))
{
string strPage = string.Format("http://www.tgod.cn{0}", nodeLink.Attributes["href"].Value);
ProcessDetailPage(strPage);
}
}
}
catch (Exception ex)
{
//TODO: show error;
Console.WriteLine(ex.ToString());
}
}
}
Console.WriteLine("Finish page ^_^: " + strURL);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
public static void XinhHayLaLoop(string category, string strPage)
{
try
{
var bolService = new BOLService.BOLService();
var lst = new List <ImgLink>();
var web = new HtmlWeb();
var doc = web.Load(strPage);
//TODO: Check valid URL
string strCounter = strPage.Substring(strPage.LastIndexOf('/') + 1);
HtmlNode nodeTitle = doc.DocumentNode.SelectSingleNode("//h1[@class = 'title']");
string strTitle = nodeTitle != null ? nodeTitle.InnerText : string.Empty;
//TODO: Check exist Link
bool existCouter = bolService.CheckExistLinkByDomain(strCounter, "xinh.hay.la");
if (!existCouter)
{
TestImageOnePage(strPage, strCounter, category, strTitle);
}
var divContainer = doc.DocumentNode.SelectSingleNode("//div[@class = 'navButtons']");
if (divContainer == null)
{
return;
}
foreach (HtmlNode node in divContainer.ChildNodes)
{
try
{
if (node.Name == "a" && node.Attributes["class"].Value == "prev")
{
string strLoopPage = string.Format("http://xinh.hay.la{0}", node.Attributes["href"].Value);
XinhHayLaLoop(category, strLoopPage);
break;
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
}
}
catch (Exception exception)
{
//log4net.Config.XmlConfigurator.Configure();
//log.Info("Error: " + exception);
Console.WriteLine(exception.ToString());
}
}
public ActionResult TestDomain(string id)
{
int i = 0;
int.TryParse(id, out i);
string strDomain = GetDomainName(i);
var bolService = new BOLService.BOLService();
var obj = new ImageDisplay();
obj.PageIndex = 0;
obj.PageSize = 1000;
obj.ListImg = bolService.GetImgLinksByDomain(obj.PageIndex, obj.PageSize, strDomain);
obj.ListImg.Shuffle();
return(View(obj));
}
public ActionResult FastCheckImage(int?id)
{
int domainId = id ?? 12;
var bolService = new BOLService.BOLService();
var obj = new ImageDisplay();
int pageSize = 10;
int pageIndex = 1;
int? totals = 0;
string domainName = GetDomainName(domainId);
obj.PageSize = pageSize;
obj.PageIndex = pageIndex;
obj.CurrentPage = 1;
obj.ListImg = bolService.GetTopLastest(domainName, pageSize, (pageIndex - 1) * pageSize, ref totals);
obj.TotalRecords = totals.GetValueOrDefault();
return(View(obj));
}
public ActionResult Review(int?id)
{
var bolService = new BOLService.BOLService();
var obj = new ImageDisplay();
int domainId = id.HasValue ? id.Value : 0;
if (domainId == 0)
{
obj.ListImg = bolService.GetBadURL(string.Empty);
}
else
{
string domainName = GetDomainName(domainId);
obj.ListImg = bolService.GetBadURL(domainName);
}
return(View(obj));
}
static void Main(string[] args)
{
var bolService = new BOLService.BOLService();
var lstUpdate = new List <ImgLink>();
for (int i = 0; i < 2000; i++)
{
var lstImg = bolService.GetImgToDownLoad(100);
if (lstImg.Count == 0)
{
break;
}
foreach (var link in lstImg)
{
try
{
string str = link.linkimg;
int index = str.LastIndexOf('.');
string strExtension = str.Substring(index + 1);
var webClient = new WebClient();
webClient.DownloadFile(str, "D:\\Public\\Project_Tools_\\DemoDepVDWeb\\Demo1\\Download_\\" + link.ID + "." + strExtension);
Console.WriteLine(link.ID);
lstUpdate.Add(link);
}
catch (Exception exception)
{
log4net.Config.XmlConfigurator.Configure();
log.Info("Error: " + exception);
Console.WriteLine(exception.ToString());
}
}
bolService.UpdateDownloaded(lstUpdate);
lstUpdate.Clear();
Console.WriteLine("Group : " + i);
}
Console.WriteLine("Finish! ^_^");
}
public static void ProcessDetailPage(string strPage)
{
try
{
var bolService = new BOLService.BOLService();
var lst = new List <BOLService.ImgLink>();
var web = new HtmlWeb();
var doc = web.Load(strPage);
//TODO: Check valid
var divContainer = doc.DocumentNode.SelectSingleNode("//ul[@class = 'pic clearfix']");
if (divContainer == null)
{
return;
}
foreach (HtmlNode childNode in divContainer.ChildNodes)
{
if (childNode.HasChildNodes && childNode.Name.Equals("li"))
{
foreach (var node in childNode.ChildNodes)
{
if (node.Name.Equals("img") && node.Attributes["src"] != null)
{
try
{
string strLink = node.Attributes["src"].Value;
strLink = strLink.Replace("/upload_x/", "/upload_big/");
var item = new BOLService.ImgLink()
{
Category = "Girl",
Counter = strLink,
CreateDate = DateTime.Now,
Domain = "tgod.cn",
GroupName = "TGOD",
linkimg = strLink
};
if (!bolService.CheckExistLinkByDomain(strLink, "tgod.cn"))
{
lst.Add(item);
Console.WriteLine(strLink);
}
}
catch (Exception ex)
{
//TODO: show error;
Console.WriteLine(ex.ToString());
}
}
}
}
}
if (lst.Count > 0)
{
bolService.SaveImgDepVD(lst);
Console.WriteLine(strPage);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
public static void TestImageOnePage(string strPage, string strCounter, string category, string strTitle)
{
try
{
var bolService = new BOLService.BOLService();
if (bolService.CheckExistLinkByDomain(strCounter, "xemanh.net"))
{
return;
}
var lst = new List <ImgLink>();
var web = new HtmlWeb();
var doc = web.Load(strPage);
//TODO: Check valid
var divContainer = doc.DocumentNode.SelectSingleNode("//div[@id = 'main']");
if (divContainer == null)
{
return;
}
HtmlNode articleNode = divContainer.SelectSingleNode("//article");
HtmlNode sectionNode = articleNode.SelectSingleNode("//section");
foreach (var node in sectionNode.ChildNodes)
{
try
{
if (node.Name.Equals("p") && node.HasAttributes && node.HasChildNodes)
{
HtmlNode nodeA = node.FirstChild;
if (node.ChildNodes.Count > 1)
{
foreach (HtmlNode f1Node in node.ChildNodes)
{
if (f1Node.Name.Equals("a") || f1Node.Name.Equals("img"))
{
nodeA = f1Node;
break;
}
}
}
HtmlNode nodeImg = nodeA;
if (nodeA.Name.Equals("a") && nodeA.HasChildNodes)
{
foreach (HtmlNode fNode in nodeA.ChildNodes)
{
if (fNode.Name.Equals("img"))
{
nodeImg = fNode;
break;
}
}
}
if (nodeImg != null && nodeImg.Name.Equals("img") && nodeImg.Attributes["src"] != null)
{
string strLink = nodeImg.Attributes["src"].Value;
var item = new ImgLink()
{
Category = category,
Counter = strCounter,
CreateDate = DateTime.Now,
Domain = "xemanh.net",
GroupName = strTitle,
linkimg = strLink
};
lst.Add(item);
}
}
}
catch (Exception ex)
{
//TODO: show error;
Console.WriteLine(ex.ToString());
}
}
if (lst.Count > 0)
{
bolService.SaveImg(lst);
Console.WriteLine(strPage);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
public static void SexyGirl()
{
var bolService = new BOLService.BOLService();
List <ImgLink> lst = new List <ImgLink>();
var web = new HtmlWeb();
int start = 1;
int end = 3; //TODO: current max value 36
//track counter: id="post-5740"
for (int i = start; i < end; i++)
{
try
{
var doc = web.Load("http://xemlasuong.org/anh/page/" + i);
//TODO: Check valid URL
var divContainer = doc.DocumentNode.SelectSingleNode("//div[@class = 'items-container items-container-blog']");
if (divContainer == null)
{
continue;
}
foreach (HtmlNode htmlNode in divContainer.ChildNodes)
{
if (htmlNode.Name.Equals("div") && htmlNode.Attributes["class"] != null)
{
string strClass = htmlNode.Attributes["class"].Value;
if (strClass.Equals("items-container-responsive"))
{
foreach (HtmlNode node in htmlNode.ChildNodes)
{
if (node.Name.Equals("div"))
{
string strID = node.Attributes["id"].Value;
foreach (HtmlNode childNode in node.ChildNodes)
{
if (childNode.Name.Equals("a"))
{
string strPage = childNode.Attributes["href"].Value;
string strCounter = strID;
string strTitle = BuildCounter(strPage);
bool bDownloaded = bolService.CheckLinkDownloaded("xemlasuong.org", strCounter);
if (!bDownloaded)
{
//TODO: call function Get Images
TestImageOnePage(strPage, strCounter, "sexy-girl", strTitle);
}
break;
}
}
}
}
break;
}
}
}
}
catch (Exception exception)
{
//log4net.Config.XmlConfigurator.Configure();
//log.Info("Error: " + exception);
Console.WriteLine(exception.ToString());
}
}
Console.WriteLine("Finish sexy-girl ^_^");
}
public static void ImportOnePageVietlottVN(int pageId)
{
try
{
var bolService = new BOLService.BOLService();
List <VietlottVNDto> lst = new List <VietlottVNDto>();
var web = new HtmlWeb();
string strURL = string.Format("http://www.vietlott.vn/vi/trung-thuong/ket-qua-trung-thuong/mega-6-45/winning-numbers/?p={0}", pageId);
var doc = web.Load(strURL);
//TODO: Check valid
var divContainer = doc.DocumentNode.SelectSingleNode("//table[@class = 'table table-striped']");
if (divContainer == null)
{
Console.WriteLine("Nothing to import ^_^");
return;
}
foreach (HtmlNode childNode in divContainer.ChildNodes)
{
if (childNode.Name.Equals("tbody"))
{
if (childNode.HasChildNodes)
{
foreach (HtmlNode mychildNode in childNode.ChildNodes)
{
if (mychildNode.Name.Equals("tr"))
{
Console.WriteLine(mychildNode.InnerText);
var obj = new VietlottVNDto();
foreach (var tdChildNodes in mychildNode.ChildNodes)
{
if (tdChildNodes.Name.Equals("td"))
{
if (tdChildNodes.Attributes["style"] != null)
{
obj.StrNumber = tdChildNodes.InnerText;
if (tdChildNodes.HasChildNodes)
{
foreach (HtmlNode spanChildNode in tdChildNodes.ChildNodes)
{
if (spanChildNode.Name.Equals("span"))
{
string strNumber = spanChildNode.InnerText;
obj.ListNumbers.Add(strNumber);
}
}
}
}
else
{
obj.DatePize = tdChildNodes.InnerText;
}
}
}
lst.Add(obj);
}
}
}
}
}
if (lst.Count > 0)
{
bolService.ImportVietLottPage(lst);
}
Console.WriteLine("Finish page ^_^: " + strURL);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
public static void Daily_AnhGirlXinh(string strHost)
{
var bolService = new BOLService.BOLService();
List <ImgLink> lst = new List <ImgLink>();
var web = new HtmlWeb();
int start = 1;
int end = 5; //TODO: Current max 9;
//track counter: id="post-5740"
for (int i = start; i < end; i++)
{
try
{
//strHost = http://www.xemanh.net/category/anh-girl-xinh/page/
string strURL = strHost + "/" + i;
Console.WriteLine(i + " : " + strURL);
var doc = web.Load(strURL);
//TODO: Check valid URL
var divContainer = doc.DocumentNode.SelectSingleNode("//div[@id = 'main']");
if (divContainer == null)
{
continue;
}
foreach (HtmlNode htmlNode in divContainer.ChildNodes)
{
try
{
if (htmlNode.Name.Equals("article"))
{
foreach (HtmlNode childNode in htmlNode.ChildNodes)
{
if (childNode.Name.Equals("a") && childNode.Attributes["class"] != null && childNode.Attributes["class"].Value == "home-thumb")
{
string strPage = childNode.Attributes["href"].Value;
string strCounter = htmlNode.Id;
string strTitle = childNode.Attributes["title"].Value;
bool bDownloaded = bolService.CheckLinkDownloaded("xemanh.net", strCounter);
if (!bDownloaded)
{
//TODO: call function Get Images
TestImageOnePage(strPage, strCounter, "anh-girl-xinh", strTitle);
}
break;
}
}
}
}
catch (Exception exceptionArticle)
{
Console.WriteLine(exceptionArticle.ToString());
}
}
}
catch (Exception exception)
{
//log4net.Config.XmlConfigurator.Configure();
//log.Info("Error: " + exception);
Console.WriteLine(exception.ToString());
}
}
Console.WriteLine("Finish AnhGirlXinh ^_^");
}
public static void XinhHayLaDaily(string catagory, string strURL)
{
try
{
var bolService = new BOLService.BOLService();
var lst = new List <ImgLink>();
var web = new HtmlWeb();
var doc = web.Load(strURL);
//TODO: Check valid URL
var divContainer = doc.DocumentNode.SelectSingleNode("//div[@class = 'items post']");
if (divContainer == null)
{
return;
}
foreach (HtmlNode node in divContainer.ChildNodes)
{
if (node != null && node.Name == "div")
{
try
{
foreach (HtmlNode childNode in node.ChildNodes)
{
if (childNode.Name.Equals("a"))
{
HtmlNode nodeDivImage = childNode.SelectSingleNode("//div[@class = 'contest-item-image']");
HtmlNode nodeImage = nodeDivImage.SelectSingleNode("//img");
string strImg = nodeImage.Attributes["src"].Value;
if (strImg.Contains("Content/themes/noimg_big.jpg"))
{
break;
}
string strPage = string.Format("http://xinh.hay.la{0}", childNode.Attributes["href"].Value);
string strCounter = strPage.Substring(strPage.LastIndexOf('/') + 1);
HtmlNode nodeTitle = childNode.SelectSingleNode("//div[@class = 'ext-info title']");
string strTitle = nodeTitle != null ? nodeTitle.InnerText : string.Empty;
//TODO: Check exist Link
if (!bolService.CheckExistLinkByDomain(strCounter, "xinh.hay.la"))
{
TestImageOnePage(strPage, strCounter, catagory, strTitle);
}
break;
}
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
}
}
}
catch (Exception exception)
{
//log4net.Config.XmlConfigurator.Configure();
//log.Info("Error: " + exception);
Console.WriteLine(exception.ToString());
}
Console.WriteLine("Finish " + catagory + " ^_^");
}
public static void Bikini_Chau_A()
{
var bolService = new BOLService.BOLService();
List <ImgLink> lst = new List <ImgLink>();
var web = new HtmlWeb();
int start = 0;
int end = 44; //TODO: current max value 39
//track counter: id="post-5740"
for (int i = end; i > start; i--)
{
try
{
var doc = web.Load("http://phunuvn.net/forums/anh-girl-xinh-bikini-chau-a.9/page-" + i);
//TODO: Check valid URL
var divContainer = doc.DocumentNode.SelectSingleNode("//ol[@class = 'discussionListItems']");
if (divContainer == null)
{
continue;
}
foreach (HtmlNode htmlNode in divContainer.ChildNodes)
{
if (htmlNode.Name.Equals("li") && htmlNode.Attributes["id"] != null)
{
string strClass = htmlNode.Attributes["id"].Value;
string strCounter = strClass.Replace("thread-", "");
if (strClass.Contains("thread-"))
{
bool bDownloaded = bolService.CheckLinkDownloaded("phunuvn.net", strCounter);
if (!bDownloaded)
{
foreach (HtmlNode node in htmlNode.ChildNodes)
{
if (node.Name.Equals("div") && node.Attributes["class"] != null &&
node.Attributes["class"].Value == "listBlock main")
{
HtmlNode nodeTitle = node.ChildNodes.FirstOrDefault(o => o.Name == "div" && o.HasAttributes && o.Attributes["class"].Value == "titleText");
if (nodeTitle != null && nodeTitle.HasChildNodes)
{
HtmlNode nodeh3 = nodeTitle.ChildNodes.FirstOrDefault(o => o.Name == "h3" && o.HasAttributes && o.Attributes["class"].Value == "title");
if (nodeh3 != null && nodeh3.HasChildNodes)
{
HtmlNode nodeLink = nodeh3.ChildNodes.FirstOrDefault(o => o.Name == "a" && o.HasAttributes && o.Attributes["class"].Value == "PreviewTooltip");
string strTitle = nodeLink.InnerText;
string strPage = string.Format("http://phunuvn.net/{0}", nodeLink.Attributes["href"].Value);
//TODO: call function Get Images
TestImageOnePage(strPage, strCounter, "Bikini-Chau-A", strTitle);
break;
}
}
}
}
}
}
}
}
}
catch (Exception exception)
{
//log4net.Config.XmlConfigurator.Configure();
//log.Info("Error: " + exception);
Console.WriteLine(exception.ToString());
}
}
Console.WriteLine("Finish Bikini_VietNam ^_^");
}
public static void TestImageOnePage(string strPage)
{
try
{
var bolService = new BOLService.BOLService();
var lst = new List <ImgLink>();
var web = new HtmlWeb();
var doc = web.Load(strPage);
//TODO: Check valid
var divContainer = doc.DocumentNode.SelectNodes("//a[@imageanchor = '1']");
if (divContainer == null)
{
var v = doc.DocumentNode.SelectSingleNode("//div[@class = 'post-content']");
if (v != null && v.HasChildNodes)
{
foreach (HtmlNode htmlNode in v.ChildNodes)
{
if (htmlNode.Name.Equals("div") && htmlNode.Attributes["style"].Value == "padding:10px")
{
var vTagP = htmlNode.SelectSingleNode("//p");
if (!vTagP.HasAttributes)
{
divContainer = vTagP.SelectNodes("//img");
foreach (HtmlNode node in divContainer)
{
try
{
string str = node.Attributes["src"].Value;
if (str.Contains("photo.hoanvu.net/wp-content/uploads/thumbnail/") ||
str.Contains("photo.hoanvu.net/wp-content/themes/iphoto/images/logo.png"))
{
continue;
}
if (str.Contains("blogspot.com") || str.Contains("photo.hoanvu.net/wp-content/") ||
str.Contains("data.hoanvu.net/images"))
{
ImgLink obj = new ImgLink()
{
CreateDate = DateTime.UtcNow,
Domain = "photo.hoanvu.net",
//Counter = strCounter,
linkimg = str,
//Category = strCategory,
//GroupName = strTitle
};
lst.Add(obj);
Console.WriteLine(str);
}
}
catch (Exception exception)
{
Console.WriteLine(exception.ToString());
}
}
}
}
}
}
}
else
{
foreach (HtmlNode node in divContainer)
{
try
{
ImgLink obj = new ImgLink()
{
CreateDate = DateTime.UtcNow,
Domain = "photo.hoanvu.net",
//Counter = strCounter,
linkimg = node.Attributes["href"].Value,
//Category = strCategory,
//GroupName = strTitle
};
lst.Add(obj);
}
catch (Exception exception)
{
Console.WriteLine(exception.ToString());
}
}
}
bolService.SaveImg(lst);
Console.WriteLine(strPage);
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
private static void GetChanDaiPhoto()
{
var bolService = new BOLService.BOLService();
List <ImgLink> lst = new List <ImgLink>();
var web = new HtmlWeb();
int counter = 0;
int error = 0;
int start = 1;
int end = 100;
var vStartObj = bolService.GetLastestChanDaiImage();
if (vStartObj != null)
{
if (!int.TryParse(vStartObj.Counter, out start))
{
Console.WriteLine("Not Start ^_^");
return;
}
}
end = start * 1000;
for (int i = start; i < end; i++)
{
try
{
var doc = web.Load("http://chandai.tv/photo/" + i);
var vImg = doc.DocumentNode.SelectSingleNode("//img[@class = 'img-responsive']");
if (vImg != null && vImg.Attributes.Count > 1)
{
counter = i;
var vResult = vImg.Attributes[1];
if (vResult.Value.Equals("/Content/images/notfound.jpg"))
{
error += 1;
if (error == 300)
{
break;
}
}
else
{
error = 0; //Only end when 10 sequence times no image
string strResult = string.Format("http://chandai.tv{0}", vResult.Value);
var img = new ImgLink()
{
linkimg = strResult,
Counter = i.ToString(),
CreateDate = DateTime.Now,
Domain = "chandai.tv"
};
lst.Add(img);
if (lst.Count == 10)
{
bolService.SaveImg(lst);
lst.Clear();
Console.WriteLine("Save link " + i);
}
}
}
}
catch (Exception exception)
{
log4net.Config.XmlConfigurator.Configure();
log.Info("Error: " + exception);
Console.WriteLine(exception.ToString());
}
}
if (lst.Count > 0)
{
bolService.SaveImg(lst);
lst.Clear();
}
Console.WriteLine("Finish ^_^");
}
public static void TestImageOnePage(string strPage, string strCounter, string category, string strTitle)
{
try
{
var bolService = new BOLService.BOLService();
if (bolService.CheckExistLinkByDomain(strCounter, "xiuren.org"))
{
Console.WriteLine("Exist :" + strCounter);
return;
}
var lst = new List <BOLService.ImgLink>();
var web = new HtmlWeb();
var doc = web.Load(strPage);
//TODO: Check valid
var divContainer = doc.DocumentNode.SelectSingleNode("//div[@id = 'main']");
if (divContainer == null)
{
return;
}
HtmlNodeCollection nodeImagesCollection = divContainer.SelectNodes("//span[@class = 'photoThum']");
foreach (HtmlNode childNode in nodeImagesCollection)
{
if (childNode.HasChildNodes)
{
foreach (var node in childNode.ChildNodes)
{
if (node.Name.Equals("a") && node.Attributes["title"] != null)
{
try
{
string strLink = node.Attributes["href"].Value;
var item = new BOLService.ImgLink()
{
Category = category,
Counter = strCounter,
CreateDate = DateTime.Now,
Domain = "xiuren.org",
GroupName = strTitle,
linkimg = strLink
};
lst.Add(item);
}
catch (Exception ex)
{
//TODO: show error;
Console.WriteLine(ex.ToString());
}
}
}
}
}
if (lst.Count > 0)
{
bolService.SaveImgDepVD(lst);
Console.WriteLine(strPage);
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
}
