Exemple #1
0
        //ASYNC(NOT TESTED)
        #region Async RBL
        //void BeginRBLCheck(RatedProxy proxy)
        //{
        //    List<string[]> rblList = GlobalResourceCache.RBLList;
        //    int indx = 0;
        //    double rblBanRate = 0;
        //    int rblChecks = rblList.Count;

        //    BeginRBLCheck(indx, rblBanRate, rblChecks, proxy);
        //}
        //void EndRBLCheck(DownloaderObj obj)
        //{
        //    object[] args = obj.Arg as object[];
        //    int indx = (int)args[0];
        //    double rblBanRate = (double)args[1];
        //    int rblChecks = (int)args[2];

        //    List<string[]> rblList = GlobalResourceCache.RBLList;
        //    string[] rblSet = rblList[indx];
        //    Regex positiveRx = new Regex(rblSet[1]);
        //    Regex negativeRx = new Regex(rblSet[2]);

        //    double rate = GetRblBanRate(obj.DataStr, positiveRx, negativeRx);
        //    if (rate > -1)
        //    {
        //        rblBanRate += rate;
        //    }
        //    else
        //        rblChecks--;

        //    indx++;
        //    if (indx < rblList.Count)
        //    {
        //        BeginRBLCheck(indx, rblBanRate, rblChecks, obj.PrxContainer);
        //    }
        //    else if (rblChecks > 0)
        //    {
        //        obj.PrxContainer.RBLBanRate = rblBanRate / rblChecks;
        //    }
        //}
        //double GetRblBanRate(string data, Regex positiveRx, Regex negativeRx)
        //{
        //    if (data == null)
        //        return -1;
        //    double blocked = negativeRx.Matches(data).Count;
        //    double ok = positiveRx.Matches(data).Count;

        //    bool noResults = blocked == 0 && ok == 0;

        //    if (noResults)
        //        return -1;
        //    if (blocked == 0)
        //        return 0;
        //    if (ok == 0)
        //        return 1;

        //    return blocked / ok;
        //}
        //void BeginRBLCheck(int indx, double rblBanRate, int rblChecks, RatedProxy proxy)
        //{
        //    List<string[]> rblList = GlobalResourceCache.RBLList;

        //    if (indx < rblList.Count)
        //    {
        //        string[] rblSet = rblList[indx];
        //        DownloaderObj obj = new DownloaderObj(new Uri(rblSet[0]), EndRBLCheck, true, proxy, CookieOptions.NoCookies, 3, new object[] { indx, rblBanRate, rblChecks });
        //        Downloader.Queue(obj);
        //    }
        //}
        #endregion
        #endregion

        #region DownloadsCheck
        void BeginDownloads(RatedProxy proxy)
        {
            //Console.WriteLine("Downloads START {0}", DateTime.Now.ToShortTimeString());
            List <PatternsContainer> sitePatterns = PagePatternGrabber.LoadPatterns(PATH.TagClassPatterns);
            SyncWaitObj waiter = new SyncWaitObj(0);

            foreach (var patt in sitePatterns)
            {
                object[]      args = { patt, waiter, sitePatterns.Count };
                DownloaderObj obj  = new DownloaderObj(patt.Uri, EndDownload, true, proxy, CookieOptions.NoCookies, DownloadsAttempts, args);
                Downloader.Queue(obj);
            }
        }
Exemple #2
0
        public bool Validate(string page)
        {
            int score = 0;

            List <string> tagSheme = PagePatternGrabber.GrabClassStructure(page);

            foreach (var tagClass in tagSheme)
            {
                if (_patterns.Contains(tagClass))
                {
                    score++;
                }
            }
            return((score / _patterns.Count) >= _passScore ? true : false);
        }
Exemple #3
0
        void VerifySites(RatedProxy proxy)
        {
            //DEBUG
            DateTime start = DateTime.Now;

            int    downloadCheked = 0;
            int    latencyCheked  = 0;
            double totalSiteRate  = 0;
            double totalSpeed     = 0;
            long   totalLatency   = 0;

            List <PatternsContainer> siteChecks = PagePatternGrabber.LoadPatterns(PATH.TagClassPatterns);

            foreach (var siteCheck in siteChecks)
            {
                double siteRate      = 0;
                double downloadSpeed = 0;
                if (TryCheckDownloadSpeed(proxy, siteCheck.Uri, siteCheck.Validation, ref siteRate, ref downloadSpeed))
                {
                    totalSiteRate += siteRate;
                    totalSpeed    += downloadSpeed;
                    downloadCheked++;
                }

                int?latency = GetAvgLatency(siteCheck.Uri, proxy, 4);
                if (latency.HasValue)
                {
                    latencyCheked++;
                    totalLatency += latency.Value;
                }
            }
            if (latencyCheked > 0)
            {
                proxy.AvgLatency = (int)(totalLatency / latencyCheked);
            }
            if (downloadCheked > 0)
            {
                proxy.SitesRate = totalSiteRate / downloadCheked;
                proxy.AvgSpeed  = totalSpeed / downloadCheked;
            }
            //DEBUG
            Console.WriteLine("VerifySites - elapsed {0}s, latency {1}, sites rate {2}, speed {3} ", (DateTime.Now - start).TotalSeconds, (int)(totalLatency / latencyCheked), totalSiteRate / downloadCheked, totalSpeed / downloadCheked); //DEBUG
            proxy.CheckTimes++;
        }