//ASYNC(NOT TESTED)
#region Async RBL
//void BeginRBLCheck(RatedProxy proxy)
//{
// List<string[]> rblList = GlobalResourceCache.RBLList;
// int indx = 0;
// double rblBanRate = 0;
// int rblChecks = rblList.Count;
// BeginRBLCheck(indx, rblBanRate, rblChecks, proxy);
//}
//void EndRBLCheck(DownloaderObj obj)
//{
// object[] args = obj.Arg as object[];
// int indx = (int)args[0];
// double rblBanRate = (double)args[1];
// int rblChecks = (int)args[2];
// List<string[]> rblList = GlobalResourceCache.RBLList;
// string[] rblSet = rblList[indx];
// Regex positiveRx = new Regex(rblSet[1]);
// Regex negativeRx = new Regex(rblSet[2]);
// double rate = GetRblBanRate(obj.DataStr, positiveRx, negativeRx);
// if (rate > -1)
// {
// rblBanRate += rate;
// }
// else
// rblChecks--;
// indx++;
// if (indx < rblList.Count)
// {
// BeginRBLCheck(indx, rblBanRate, rblChecks, obj.PrxContainer);
// }
// else if (rblChecks > 0)
// {
// obj.PrxContainer.RBLBanRate = rblBanRate / rblChecks;
// }
//}
//double GetRblBanRate(string data, Regex positiveRx, Regex negativeRx)
//{
// if (data == null)
// return -1;
// double blocked = negativeRx.Matches(data).Count;
// double ok = positiveRx.Matches(data).Count;
// bool noResults = blocked == 0 && ok == 0;
// if (noResults)
// return -1;
// if (blocked == 0)
// return 0;
// if (ok == 0)
// return 1;
// return blocked / ok;
//}
//void BeginRBLCheck(int indx, double rblBanRate, int rblChecks, RatedProxy proxy)
//{
// List<string[]> rblList = GlobalResourceCache.RBLList;
// if (indx < rblList.Count)
// {
// string[] rblSet = rblList[indx];
// DownloaderObj obj = new DownloaderObj(new Uri(rblSet[0]), EndRBLCheck, true, proxy, CookieOptions.NoCookies, 3, new object[] { indx, rblBanRate, rblChecks });
// Downloader.Queue(obj);
// }
//}
#endregion
#endregion
#region DownloadsCheck
void BeginDownloads(RatedProxy proxy)
{
//Console.WriteLine("Downloads START {0}", DateTime.Now.ToShortTimeString());
List <PatternsContainer> sitePatterns = PagePatternGrabber.LoadPatterns(PATH.TagClassPatterns);
SyncWaitObj waiter = new SyncWaitObj(0);
foreach (var patt in sitePatterns)
{
object[] args = { patt, waiter, sitePatterns.Count };
DownloaderObj obj = new DownloaderObj(patt.Uri, EndDownload, true, proxy, CookieOptions.NoCookies, DownloadsAttempts, args);
Downloader.Queue(obj);
}
}
public bool Validate(string page)
{
int score = 0;
List <string> tagSheme = PagePatternGrabber.GrabClassStructure(page);
foreach (var tagClass in tagSheme)
{
if (_patterns.Contains(tagClass))
{
score++;
}
}
return((score / _patterns.Count) >= _passScore ? true : false);
}
void VerifySites(RatedProxy proxy)
{
//DEBUG
DateTime start = DateTime.Now;
int downloadCheked = 0;
int latencyCheked = 0;
double totalSiteRate = 0;
double totalSpeed = 0;
long totalLatency = 0;
List <PatternsContainer> siteChecks = PagePatternGrabber.LoadPatterns(PATH.TagClassPatterns);
foreach (var siteCheck in siteChecks)
{
double siteRate = 0;
double downloadSpeed = 0;
if (TryCheckDownloadSpeed(proxy, siteCheck.Uri, siteCheck.Validation, ref siteRate, ref downloadSpeed))
{
totalSiteRate += siteRate;
totalSpeed += downloadSpeed;
downloadCheked++;
}
int?latency = GetAvgLatency(siteCheck.Uri, proxy, 4);
if (latency.HasValue)
{
latencyCheked++;
totalLatency += latency.Value;
}
}
if (latencyCheked > 0)
{
proxy.AvgLatency = (int)(totalLatency / latencyCheked);
}
if (downloadCheked > 0)
{
proxy.SitesRate = totalSiteRate / downloadCheked;
proxy.AvgSpeed = totalSpeed / downloadCheked;
}
//DEBUG
Console.WriteLine("VerifySites - elapsed {0}s, latency {1}, sites rate {2}, speed {3} ", (DateTime.Now - start).TotalSeconds, (int)(totalLatency / latencyCheked), totalSiteRate / downloadCheked, totalSpeed / downloadCheked); //DEBUG
proxy.CheckTimes++;
}
