public List <Proxy> Scrape(string data)
{
List <Proxy> scraped = new List <Proxy>();
Generic g = new Generic();
var url = data.RegexMatch(@"(http:\/\/|https:\/\/)(.*?)\/favicon.ico").Groups[2].Value;
var searchPageURL = "http://" + url + "/search?max-results=10";
var searchPage = HTTP.DoWebRequest(searchPageURL);
string[] pages = GetPages(searchPage, url);
if (pages == null)
{
return(scraped);
}
var options = new ParallelOptions {
MaxDegreeOfParallelism = 10
};
Task t = new Task(() =>
{
Parallel.ForEach(pages, options, (item) =>
{
try
{
string html = HTTP.DoWebRequest(item);
if (string.IsNullOrEmpty(html))
{
return;
}
lock (scraped)
{
scraped.AddRange(g.Scrape(html));
}
}
catch
{
}
});
});
t.Start();
Task.WaitAll(t);
return(scraped);
}
public List <Proxy> Scrape(string data)
{
List <string> pages = new List <string>();
List <Proxy> scraped = new List <Proxy>();
Generic g = new Generic();
var options = new ParallelOptions {
MaxDegreeOfParallelism = 10
};
for (int i = 0; i < 23; i++)
{
pages.Add("https://proxyrox.com/?p=" + i + "&sortdir=desc&sort=reliability");
}
Task t = new Task(() =>
{
Parallel.ForEach(pages, options, (item) =>
{
try
{
string html = HTTP.DoWebRequest(item);
if (string.IsNullOrEmpty(html))
{
return;
}
lock (scraped)
{
scraped.AddRange(g.Scrape(html));
}
}
catch
{
}
});
});
t.Start();
Task.WaitAll(t);
return(scraped);
}
public List <Proxy> Scrape(string data)
{
List <Proxy> scraped = new List <Proxy>();
List <string> pages = new List <string>();
List <string> ippages = new List <string>();
Generic g = new Generic();
var options = new ParallelOptions {
MaxDegreeOfParallelism = 10
};
pages.Add("http://www.samair.ru/proxy/proxy-1.htm");
for (int i = 2; i < 30; i++)
{
if (i <= 9 && i > 1)
{
pages.Add("http://www.samair.ru/proxy/proxy-0" + i + ".htm");
}
else
{
pages.Add("http://www.samair.ru/proxy/proxy-" + i + ".htm");
}
//<a href="/proxy/ip-port/977482367.html">You can do it there</a>
}
Task t = new Task(() =>
{
Parallel.ForEach(pages, options, (item) =>
{
try
{
string html = HTTP.DoWebRequest(item);
if (string.IsNullOrEmpty(html))
{
return;
}
var page = html.GetBetween("<a href=\"/proxy/ip-port/", ".html");
var linkToPage = "http://www.samair.ru/proxy/ip-port/" + page + ".html";
ippages.Add(linkToPage);
var page2 = HTTP.DoWebRequest(linkToPage);
if (string.IsNullOrEmpty(page2))
{
return;
}
lock (scraped)
{
scraped.AddRange(g.Scrape(page2));
}
}
catch
{
}
});
});
t.Start();
Task.WaitAll(t);
return(scraped);
}