private static List <string> GetProxyList()
{
var proxyList = new List <string>();
var proxyPage = PageEncoding.GetPage("http://www.checker.freeproxy.ru/checker/last_checked_proxies.php", "", "", "", 0);
if (proxyPage != null)
{
foreach (var proxyText in proxyPage.Split(new[] { "Don't try to parse this code! This is second notification!" }, StringSplitOptions.None))
{
var proxy = ParceProxy(proxyText);
if (proxy != "")
{
proxyList.Add(proxy);
}
}
}
// убрать из списка неподходящие прокси
if (new FileInfo("BannedProxies.txt").Exists)
{
var fr = new StreamReader("BannedProxies.txt");
while (!fr.EndOfStream)
{
proxyList.Remove(fr.ReadLine());
}
fr.Close();
}
return(proxyList);
}
private static bool IsGoodProxy2(string url, string ip, int port)
{
WebResponse response = null;
Stream stream = null;
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 25000;
request.UserAgent = "MSIE 6.0";
if (ip != "" && port != 0)
{
request.Proxy = new WebProxy(ip, port);
}
response = request.GetResponse();
stream = response.GetResponseStream();
var b = PageEncoding.ReadFully(stream);
Encoding enc = PageEncoding.def_code(b, 100);
string content = PageEncoding.DecodeContent(b, enc);
if (!content.Contains("Bot or Human") && !content.Contains("checkcaptcha"))
{
return(true);
}
}
catch (WebException e)
{
return(false);
}
catch (Exception e)
{
return(false);
}
return(false);
}
public bool GetDescriptions(Product currentProduct, string searchURL)
{
var findResult = PageEncoding.GetUTFPage(searchURL, ip, port);
if (findResult == null)
{
GetProxy();
return(false);
}
//if (findResult.Contains("margin-top: 10px; font-size: 15px"))
if (findResult.Contains("<div class=\"result-item *sr\">"))
{
findResult = findResult.Split(new[] { "<div class=\"result-item *sr\">" }, 2, StringSplitOptions.None)[1];
findResult = findResult.Split(new[] { "href=\"" }, 2, StringSplitOptions.None)[1];
if (findResult.Contains("\" class=\"ux-thumb-wrap"))
{
var firstModelURL = findResult.Split(new[] { "\" class=\"ux-thumb-wrap" }, 2, StringSplitOptions.None)[0];
if (firstModelURL.Contains("&"))
{
firstModelURL = firstModelURL.Split(new[] { "&" }, 2, StringSplitOptions.None)[0];
}
firstModelURL = "http://youtube.com/v/" + firstModelURL.Replace("/watch?v=", "") + "?fs=1&hl=ru_RU";
d.ProductsVideos.InsertOnSubmit(new ProductsVideo {
VideoChecked = false, Link = firstModelURL
});
d.SubmitChanges();
currentProduct.VideoId = d.ProductsVideos.Max(t => t.VideoId);
d.SubmitChanges();
Console.WriteLine(currentProduct.ProductFullName);
return(true);
}
else
{
currentProduct.VideoId = -1;
d.SubmitChanges();
return(false);
}
}
else
{
//var color = d.ProductsColors.ToList().Where(t => searchURL.Contains(t.ColorName)).FirstOrDefault();
//if (color != null)
//{
// searchURL = searchURL.Replace(color.ColorName, "");
// return GetDescriptions(currentProduct, searchURL);
//}
currentProduct.VideoId = -1;
d.SubmitChanges();
return(false);
}
}
public bool FindNewImages(Product product)
{
d = new DataClasses1DataContext();
bannedHosts = d.BannedImagesSites.Select(t => t.BannedHost).ToList();
bannedImages = d.BannedImages.Select(t => t.BannedImage1).ToList();
potentialImages.Clear();
var productName = product.ProductFullName;
foreach (var c in chars)
{
productName = productName.Replace(c, " ");
}
var searchURL = "http://ajax.googleapis.com/ajax/services/search/images?v=1.0&q=" + productName;
var findResult = PageEncoding.GetPage(searchURL, null, null, ip, port);
if (findResult == null || findResult.Contains("Bot or Human"))
{
return(false);
}
// выбираем самую большую
GetImagesList(findResult);
foreach (var potentialImage in potentialImages.OrderByDescending(t => t.ContentLength).ToList())
{
var result = GetImage(potentialImage);
if (result)
{
//imageHost = potentialImage.ResponseUri.Host;
imageHost = potentialImage.ResponseUri.AbsoluteUri.Replace("www.", "");
return(true);
}
}
return(false);
}
private int GetPhoto(string searchURL, int count)
{
var findResult = PageEncoding.GetPage(searchURL, null, null, ip, port);
if (findResult == null || findResult.Contains("checkcaptcha"))
{
GetNewProxy();
return(1);
}
var x = new string[1];
x[0] = "<img id=\"i-main-pic\" src=\"";
if (!findResult.Contains(x[0]))
{
return(1);
}
var href = findResult.Split(x, StringSplitOptions.None)[1];
x[0] = "\" onload=";
href = href.Split(x, StringSplitOptions.None)[0].Replace("www.", "");
try
{
var host = (new Uri(href)).Host;
if (bannedHosts.Contains(host) || bannedImages.Contains(href))
{
return(FindAgain(searchURL, count, findResult, ref x, ref href));
}
var _filePath = homeFolder + "temp.jpeg";
// запрос по ссылке
WebResponse response;
var request = (HttpWebRequest)WebRequest.Create(href);
request.Timeout = 16000;
response = request.GetResponse();
href = response.ResponseUri.AbsoluteUri;
host = (new Uri(href)).Host.Replace("www.", "");
if (response.ContentType.Contains("image/jpeg") && !bannedHosts.Contains(host) &&
!bannedImages.Contains(href.Replace("www.", "")))
{
if (response.ContentLength > minImageSize || response.ContentLength < 0)
{
var remoteStream = response.GetResponseStream();
// Create the local file
var localStream = File.Create(_filePath);
var buffer = new byte[2048];
int bytesRead;
var counter = 0;
remoteStream.ReadTimeout = 5000;
do
{
bytesRead = remoteStream.Read(buffer, 0, buffer.Length);
localStream.Write(buffer, 0, bytesRead);
counter++;
} while (bytesRead > 0 && counter < 800);
localStream.Close();
remoteStream.Close();
if (counter == 800 || counter == 1)
{
return(FindAgain(searchURL, count, findResult, ref x, ref href));
}
//imageHost = (new Uri(href)).Host;
imageHost = href;
}
else
{
potentialImages.Add(response);
return(FindAgain(searchURL, count, findResult, ref x, ref href));
}
}
else
{
return(FindAgain(searchURL, count, findResult, ref x, ref href));
}
}
catch (WebException)
{
return(FindAgain(searchURL, count, findResult, ref x, ref href));
}
catch (Exception)
{
return(1);
}
return(0);
}
