void EndDownload(DownloaderObj obj) { object[] args = obj.Arg as object[]; PatternsContainer sitePatt = args[0] as PatternsContainer; SyncWaitObj waiter = args[1] as SyncWaitObj; int siteChecksCount = (int)args[2]; if (obj.DataStr != null) { bool originalPage = sitePatt.Validation.Validate(obj.DataStr); if (originalPage) { waiter.MultidownloadRate += (double)obj.Attempts / (DownloadsAttempts - 1); } } if (Interlocked.Increment(ref waiter.Count) == siteChecksCount) { obj.Proxy.MultidownloadRate = waiter.MultidownloadRate / siteChecksCount; EndTest(obj.Proxy); ////DEBUG //Console.WriteLine("Downloads END {0}, rate ", DateTime.Now.ToShortTimeString(), obj.Proxy.MultidownloadRate); } }
//ASYNC(NOT TESTED) #region Async RBL //void BeginRBLCheck(RatedProxy proxy) //{ // List<string[]> rblList = GlobalResourceCache.RBLList; // int indx = 0; // double rblBanRate = 0; // int rblChecks = rblList.Count; // BeginRBLCheck(indx, rblBanRate, rblChecks, proxy); //} //void EndRBLCheck(DownloaderObj obj) //{ // object[] args = obj.Arg as object[]; // int indx = (int)args[0]; // double rblBanRate = (double)args[1]; // int rblChecks = (int)args[2]; // List<string[]> rblList = GlobalResourceCache.RBLList; // string[] rblSet = rblList[indx]; // Regex positiveRx = new Regex(rblSet[1]); // Regex negativeRx = new Regex(rblSet[2]); // double rate = GetRblBanRate(obj.DataStr, positiveRx, negativeRx); // if (rate > -1) // { // rblBanRate += rate; // } // else // rblChecks--; // indx++; // if (indx < rblList.Count) // { // BeginRBLCheck(indx, rblBanRate, rblChecks, obj.PrxContainer); // } // else if (rblChecks > 0) // { // obj.PrxContainer.RBLBanRate = rblBanRate / rblChecks; // } //} //double GetRblBanRate(string data, Regex positiveRx, Regex negativeRx) //{ // if (data == null) // return -1; // double blocked = negativeRx.Matches(data).Count; // double ok = positiveRx.Matches(data).Count; // bool noResults = blocked == 0 && ok == 0; // if (noResults) // return -1; // if (blocked == 0) // return 0; // if (ok == 0) // return 1; // return blocked / ok; //} //void BeginRBLCheck(int indx, double rblBanRate, int rblChecks, RatedProxy proxy) //{ // List<string[]> rblList = GlobalResourceCache.RBLList; // if (indx < rblList.Count) // { // string[] rblSet = rblList[indx]; // DownloaderObj obj = new DownloaderObj(new Uri(rblSet[0]), EndRBLCheck, true, proxy, CookieOptions.NoCookies, 3, new object[] { indx, rblBanRate, rblChecks }); // Downloader.Queue(obj); // } //} #endregion #endregion #region DownloadsCheck void BeginDownloads(RatedProxy proxy) { //Console.WriteLine("Downloads START {0}", DateTime.Now.ToShortTimeString()); List <PatternsContainer> sitePatterns = PagePatternGrabber.LoadPatterns(PATH.TagClassPatterns); SyncWaitObj waiter = new SyncWaitObj(0); foreach (var patt in sitePatterns) { object[] args = { patt, waiter, sitePatterns.Count }; DownloaderObj obj = new DownloaderObj(patt.Uri, EndDownload, true, proxy, CookieOptions.NoCookies, DownloadsAttempts, args); Downloader.Queue(obj); } }
void BeginDownloads(RatedProxy proxy) { //Console.WriteLine("Downloads START {0}", DateTime.Now.ToShortTimeString()); List<PatternsContainer> sitePatterns = PagePatternGrabber.LoadPatterns(PATH.TagClassPatterns); SyncWaitObj waiter = new SyncWaitObj(0); foreach (var patt in sitePatterns) { object[] args = { patt, waiter, sitePatterns.Count }; DownloaderObj obj = new DownloaderObj(patt.Uri, EndDownload, true, proxy, CookieOptions.NoCookies, DownloadsAttempts, args); Downloader.Queue(obj); } }