private static List <string> GetProxyList() { var proxyList = new List <string>(); var proxyPage = PageEncoding.GetPage("http://www.checker.freeproxy.ru/checker/last_checked_proxies.php", "", "", "", 0); if (proxyPage != null) { foreach (var proxyText in proxyPage.Split(new[] { "Don't try to parse this code! This is second notification!" }, StringSplitOptions.None)) { var proxy = ParceProxy(proxyText); if (proxy != "") { proxyList.Add(proxy); } } } // убрать из списка неподходящие прокси if (new FileInfo("BannedProxies.txt").Exists) { var fr = new StreamReader("BannedProxies.txt"); while (!fr.EndOfStream) { proxyList.Remove(fr.ReadLine()); } fr.Close(); } return(proxyList); }
private static bool IsGoodProxy2(string url, string ip, int port) { WebResponse response = null; Stream stream = null; try { HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); request.Timeout = 25000; request.UserAgent = "MSIE 6.0"; if (ip != "" && port != 0) { request.Proxy = new WebProxy(ip, port); } response = request.GetResponse(); stream = response.GetResponseStream(); var b = PageEncoding.ReadFully(stream); Encoding enc = PageEncoding.def_code(b, 100); string content = PageEncoding.DecodeContent(b, enc); if (!content.Contains("Bot or Human") && !content.Contains("checkcaptcha")) { return(true); } } catch (WebException e) { return(false); } catch (Exception e) { return(false); } return(false); }
public bool GetDescriptions(Product currentProduct, string searchURL) { var findResult = PageEncoding.GetUTFPage(searchURL, ip, port); if (findResult == null) { GetProxy(); return(false); } //if (findResult.Contains("margin-top: 10px; font-size: 15px")) if (findResult.Contains("<div class=\"result-item *sr\">")) { findResult = findResult.Split(new[] { "<div class=\"result-item *sr\">" }, 2, StringSplitOptions.None)[1]; findResult = findResult.Split(new[] { "href=\"" }, 2, StringSplitOptions.None)[1]; if (findResult.Contains("\" class=\"ux-thumb-wrap")) { var firstModelURL = findResult.Split(new[] { "\" class=\"ux-thumb-wrap" }, 2, StringSplitOptions.None)[0]; if (firstModelURL.Contains("&")) { firstModelURL = firstModelURL.Split(new[] { "&" }, 2, StringSplitOptions.None)[0]; } firstModelURL = "http://youtube.com/v/" + firstModelURL.Replace("/watch?v=", "") + "?fs=1&hl=ru_RU"; d.ProductsVideos.InsertOnSubmit(new ProductsVideo { VideoChecked = false, Link = firstModelURL }); d.SubmitChanges(); currentProduct.VideoId = d.ProductsVideos.Max(t => t.VideoId); d.SubmitChanges(); Console.WriteLine(currentProduct.ProductFullName); return(true); } else { currentProduct.VideoId = -1; d.SubmitChanges(); return(false); } } else { //var color = d.ProductsColors.ToList().Where(t => searchURL.Contains(t.ColorName)).FirstOrDefault(); //if (color != null) //{ // searchURL = searchURL.Replace(color.ColorName, ""); // return GetDescriptions(currentProduct, searchURL); //} currentProduct.VideoId = -1; d.SubmitChanges(); return(false); } }
public bool FindNewImages(Product product) { d = new DataClasses1DataContext(); bannedHosts = d.BannedImagesSites.Select(t => t.BannedHost).ToList(); bannedImages = d.BannedImages.Select(t => t.BannedImage1).ToList(); potentialImages.Clear(); var productName = product.ProductFullName; foreach (var c in chars) { productName = productName.Replace(c, " "); } var searchURL = "http://ajax.googleapis.com/ajax/services/search/images?v=1.0&q=" + productName; var findResult = PageEncoding.GetPage(searchURL, null, null, ip, port); if (findResult == null || findResult.Contains("Bot or Human")) { return(false); } // выбираем самую большую GetImagesList(findResult); foreach (var potentialImage in potentialImages.OrderByDescending(t => t.ContentLength).ToList()) { var result = GetImage(potentialImage); if (result) { //imageHost = potentialImage.ResponseUri.Host; imageHost = potentialImage.ResponseUri.AbsoluteUri.Replace("www.", ""); return(true); } } return(false); }
private int GetPhoto(string searchURL, int count) { var findResult = PageEncoding.GetPage(searchURL, null, null, ip, port); if (findResult == null || findResult.Contains("checkcaptcha")) { GetNewProxy(); return(1); } var x = new string[1]; x[0] = "<img id=\"i-main-pic\" src=\""; if (!findResult.Contains(x[0])) { return(1); } var href = findResult.Split(x, StringSplitOptions.None)[1]; x[0] = "\" onload="; href = href.Split(x, StringSplitOptions.None)[0].Replace("www.", ""); try { var host = (new Uri(href)).Host; if (bannedHosts.Contains(host) || bannedImages.Contains(href)) { return(FindAgain(searchURL, count, findResult, ref x, ref href)); } var _filePath = homeFolder + "temp.jpeg"; // запрос по ссылке WebResponse response; var request = (HttpWebRequest)WebRequest.Create(href); request.Timeout = 16000; response = request.GetResponse(); href = response.ResponseUri.AbsoluteUri; host = (new Uri(href)).Host.Replace("www.", ""); if (response.ContentType.Contains("image/jpeg") && !bannedHosts.Contains(host) && !bannedImages.Contains(href.Replace("www.", ""))) { if (response.ContentLength > minImageSize || response.ContentLength < 0) { var remoteStream = response.GetResponseStream(); // Create the local file var localStream = File.Create(_filePath); var buffer = new byte[2048]; int bytesRead; var counter = 0; remoteStream.ReadTimeout = 5000; do { bytesRead = remoteStream.Read(buffer, 0, buffer.Length); localStream.Write(buffer, 0, bytesRead); counter++; } while (bytesRead > 0 && counter < 800); localStream.Close(); remoteStream.Close(); if (counter == 800 || counter == 1) { return(FindAgain(searchURL, count, findResult, ref x, ref href)); } //imageHost = (new Uri(href)).Host; imageHost = href; } else { potentialImages.Add(response); return(FindAgain(searchURL, count, findResult, ref x, ref href)); } } else { return(FindAgain(searchURL, count, findResult, ref x, ref href)); } } catch (WebException) { return(FindAgain(searchURL, count, findResult, ref x, ref href)); } catch (Exception) { return(1); } return(0); }