//public void QueueGooglePages(int numPage, string key, Regex rx, WaitObj waiter)
//{
// Uri uri = new Uri("http://www.google.ru/search?q=" + key + "&sourceid=opera&num=0&ie=utf-8&oe=utf-8&start=" + numPage);
// DownloaderObj obj = new DownloaderObj(uri, EndGetPage, true, null, false, 10, rx);
// Downloader.Queue(obj);
//}
void EndGetPage(DownloaderObj obj)
{
object[] args = obj.Arg as object[];
if (obj.DataStr != null)
{
List<string> urls = new List<string>();
Regex rx = args[0] as Regex;
WaitObj waiter = args[1] as WaitObj;
MatchCollection urlsMatches = rx.Matches(obj.DataStr);
foreach (Match urlMatch in urlsMatches)
{
urls.Add(urlMatch.Groups[1].Value);
}
if (OnParsed!=null) OnParsed(urls);
if (Interlocked.Decrement(ref waiter.Count) == 0 && OnCompleted != null) OnCompleted(this, EventArgs.Empty);
}
else
{
//ProxyRotator proxyGiver = args[2] as ProxyRotator;
//proxyGiver.TryGetProxy(ref obj.PrxContainer);
//obj.Proxies = proxyGiver;
obj.Attempts = 10;
obj.CallBack = EndGetPageWithProxy;
}
}
public async Task Wait(TimeSpan delay)
{
WaitObj w = new WaitObj();
w.Expiration = _Now + delay;
w.CTS = new TaskCompletionSource <bool>();
lock (waits)
{
waits.Add(w);
}
await w.CTS.Task;
}
void EndDownloadAndParse(DownloaderObj obj)
{
object[] args = obj.Arg as object[];
IProxySiteProvider proxySiteProvider = args[0] as IProxySiteProvider;
WaitObj waiter = args[1] as WaitObj;
List <RatedProxy> proxies = null;
proxies = proxySiteProvider.ParsePage(obj.DataStr);
NotifyAboutProgress(waiter, proxies);
}
public void BeginDownloadPages(int count, string uriStr, string replaseSubstr, IProxySiteProvider proxySiteProvider)
{
if (count == 0 || string.IsNullOrEmpty(uriStr) || string.IsNullOrEmpty(replaseSubstr) || proxySiteProvider == null)
throw new ArgumentException("Bad argumenst");
WaitObj waiter = new WaitObj(count);
for (int i = 0; i < count; i++)
{
Uri uri = new Uri(uriStr.Replace(replaseSubstr, i.ToString()));
DownloaderObj obj = new DownloaderObj(uri, EndDownloadAndParse, true, null, CookieOptions.NoCookies, 10, new object[] { proxySiteProvider, waiter });
Downloader.Queue(obj);
}
}
private void NotifyAboutProgress(WaitObj waiter, List <RatedProxy> proxies)
{
if (OnPageGrabbed != null)
{
OnPageGrabbed(proxies);
}
if (Interlocked.Decrement(ref waiter.Count) == 0)
{
if (OnParseCompleted != null)
{
OnParseCompleted();
}
}
}
public void BeginDownloadPages(int count, string uriStr, string replaseSubstr, IProxySiteProvider proxySiteProvider)
{
if (count == 0 || string.IsNullOrEmpty(uriStr) || string.IsNullOrEmpty(replaseSubstr) || proxySiteProvider == null)
{
throw new ArgumentException("Bad argumenst");
}
WaitObj waiter = new WaitObj(count);
for (int i = 0; i < count; i++)
{
Uri uri = new Uri(uriStr.Replace(replaseSubstr, i.ToString()));
DownloaderObj obj = new DownloaderObj(uri, EndDownloadAndParse, true, null, CookieOptions.NoCookies, 10, new object[] { proxySiteProvider, waiter });
Downloader.Queue(obj);
}
}
private void QueueGooglePages(string pattern, List<string> keys)
{
Regex rx = new Regex(pattern);
WaitObj waiter = new WaitObj(keys.Count * 50);
///ProxyRotator proxyGiver = new ProxyRotator(ProxyManager.LoadProxies());
foreach (var key in keys)
{
for (int i = 0; i < 50; i++)
{
Uri uri = new Uri("http://www.google.ru/search?q=" + key + "&sourceid=opera&num=0&ie=utf-8&oe=utf-8&start=" + i);
//DownloaderObj obj = new DownloaderObj(uri, EndGetPage, true, null, CookieOptions.NoCookies, 10, new object[] { rx, waiter, proxyGiver }, null, false, 3000);
//Downloader.Queue(obj);
}
}
}
void ParseSerpAsync(string uriPattern, List<string> keys, Action<int, string, Regex, WaitObj> requestgiver)
{
List<Uri> urls = new List<Uri>();
string pattern = uriPattern;
Regex rx = new Regex(pattern);
List<string> keyPages = new List<string>();
WaitObj waiter = new WaitObj(keys.Count * 50);
for (int i = 0; i < keys.Count; i++)
{
for (int p = 0; p < 50; p++)
{
requestgiver.Invoke(p, keys[i], rx, waiter);
}
}
}
public async Task Wait(TimeSpan delay, CancellationToken cancellation)
{
WaitObj w = new WaitObj();
w.Expiration = _Now + delay;
w.CTS = new TaskCompletionSource <bool>();
using (cancellation.Register(() =>
{
w.CTS.TrySetCanceled();
}))
{
lock (waits)
{
waits.Add(w);
}
await w.CTS.Task;
}
}
private void QueueGooglePages(string pattern, List<string> keys)
{
Regex rx = new Regex(pattern);
WaitObj waiter = new WaitObj(keys.Count * 50);
///ProxyRotator proxyGiver = new ProxyRotator(ProxyManager.LoadProxies());
foreach (var key in keys)
{
for (int i = 0; i < 50; i++)
{
Uri uri = new Uri("http://www.google.ru/search?q=" + key + "&sourceid=opera&num=0&ie=utf-8&oe=utf-8&start=" + i);
//DownloaderObj obj = new DownloaderObj(uri, EndGetPage, true, null, CookieOptions.NoCookies, 10, new object[] { rx, waiter, proxyGiver }, null, false, 3000);
//Downloader.Queue(obj);
}
}
}
void ParseSerpAsync(string uriPattern, List<string> keys, Action<int, string, Regex, WaitObj> requestgiver)
{
List<Uri> urls = new List<Uri>();
string pattern = uriPattern;
Regex rx = new Regex(pattern);
List<string> keyPages = new List<string>();
WaitObj waiter = new WaitObj(keys.Count * 50);
for (int i = 0; i < keys.Count; i++)
{
for (int p = 0; p < 50; p++)
{
requestgiver.Invoke(p, keys[i], rx, waiter);
}
}
}
void GetYandexPage(int numPage, string key, Regex rx, WaitObj waiter)
{
Uri pageUri = new Uri("http://yandex.ru/yandsearch?p=" + numPage + "&text=" + Uri.EscapeDataString(key));
DownloaderObj obj = new DownloaderObj(pageUri, null);
Downloader.DownloadSync(obj);
}
private void NotifyAboutProgress(WaitObj waiter, List<RatedProxy> proxies)
{
if (OnPageGrabbed != null) OnPageGrabbed(proxies);
if (Interlocked.Decrement(ref waiter.Count) == 0)
{
if (OnParseCompleted != null) OnParseCompleted();
}
}
void GetYandexPage(int numPage, string key, Regex rx, WaitObj waiter)
{
Uri pageUri = new Uri("http://yandex.ru/yandsearch?p=" + numPage + "&text=" + Uri.EscapeDataString(key));
DownloaderObj obj = new DownloaderObj(pageUri, null);
Downloader.DownloadSync(obj);
}
