Example #1
0
        internal bool Insert(ad_network obj)
        {
            var connectionSettings = new ConnectionSettings(MainVariables.conElasticSearch);
            connectionSettings.SetDefaultIndex("ad_network");

            var elasticClient = new ElasticClient(connectionSettings);
            //var indexaaa = elasticClient.DeleteIndex("advertisement");
            var index = elasticClient.Index(obj);
            if (index.ServerError == null) return true;
            else return false;
        }
Example #2
0
        internal bool Update(ad_network obj)
        {
            var connectionSettings = new ConnectionSettings(MainVariables.conElasticSearch);
            connectionSettings.SetDefaultIndex("ad_network");

            var elasticClient = new ElasticClient(connectionSettings);

            var index = elasticClient.Index(obj);
            if (index.ServerError == null && !index.Created && index.ConnectionStatus.HttpStatusCode.Value < 203) return true;
            else return false;
        }
Example #3
0
        public void YahooGemini(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
        {
            Stopwatch watch = new Stopwatch();
            watch.Start();
            List<string> adIds = new List<string>();
            List<string> adUrls = new List<string>();
            List<string> adImages = new List<string>();
            List<string> adTitle = new List<string>();
            List<string> imgwidth = new List<string>();
            List<string> imgheight = new List<string>();
            try
            {
            ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath("//li[@class='content has-image voh-parent cf']/div/div/div/h3/a"));
            //NetworkFunctions ns = new NetworkFunctions();
            List<NetworkFunctions> List = new List<NetworkFunctions>();

            foreach (var v1 in getAdds)
            {

                string s2 = v1.GetAttribute("href");

                adUrls.Add(s2);

                //ns.adUrl = s2;
                //List.Add(ns);
                try
                {
                    string s4 = v1.GetAttribute("outerText");
                    adTitle.Add(s4);
                    //ns.adTitle = s4;
                    //List.Add(ns);
                }
                catch
                { }

            }
            ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath("//li[@class='content has-image voh-parent cf']/div/div/img"));
            foreach (var v2 in getImages)
            {
                string s3 = v2.GetAttribute("src");
                if (s3 != null)
                {
                    adImages.Add(s3);
                    //ns.adImage = s3;
                    //List.Add(ns);

                }

                try
                {
                    string s5 = v2.GetAttribute("width");
                    if (s5 != null)
                    {
                        imgwidth.Add(s5);
                        //ns.imgHeight = s5;
                        //List.Add(ns);
                    }
                }
                catch
                { }
                try
                {
                    string s6 = v2.GetAttribute("height");
                    if (s6 != null)
                    {
                        imgheight.Add(s6);
                       // ns.imgWidth = s6;
                       // List.Add(ns);
                    }
                }
                catch
                { }

            }
            ReadOnlyCollection<IWebElement> getID = driver.FindElements(By.XPath("//li[@class='content has-image voh-parent cf']"));
            foreach (var v3 in getID)
            {
                try
                {
                    string v7 = v3.GetAttribute("data-uuid");
                    if (v7 != null)
                    {
                        adIds.Add(v7);
                        //ns.adId = v7.Trim();
                        //List.Add(ns);
                    }
                }
                catch { }

            }

                watch.Stop();
                string time = watch.Elapsed.ToString();
                if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
                {
                    _form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent, time);

                }
            }

            catch (Exception)
            { }
            driver.Quit();
            driver.Dispose();
            adIds = null;
            adUrls = null;
            adImages = null;
            adTitle = null;
        }
Example #4
0
        public void ZergNetDotcom(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
        {
            Stopwatch watch = new Stopwatch();
            watch.Start();
            List<string> adIds = new List<string>();
            List<string> adUrls = new List<string>();
            List<string> adImages = new List<string>();
            List<string> adTitle = new List<string>();
            List<string> imgwidth = new List<string>();
            List<string> imgheight = new List<string>();
            try
            {
                ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath(path1));
                //NetworkFunctions ns = new NetworkFunctions();////"//div[@class='zergentity']/div/a"
                List<NetworkFunctions> List = new List<NetworkFunctions>();

                foreach (var v1 in getAdds)
                {
                    string s2 = v1.GetAttribute("href");
                    adUrls.Add(s2);
                    try
                    {
                        string s1 = s2.Split('/')[4];
                        // ns.adId = s1.Trim();
                        // List.Add(ns);
                        adIds.Add(s1.Trim());

                    }
                    catch { }
                    try
                    {
                        string s4 = v1.GetAttribute("outerText");
                        adTitle.Add(s4);
                        // ns.adTitle = s4;
                        // List.Add(ns);
                    }
                    catch
                    { }

                }
                ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath(path2));//"//div[@class='zergentity']/a/img"
                foreach (var v2 in getImages)
                {
                    string s3 = v2.GetAttribute("src");
                    adImages.Add(s3);
                    //ns.adImage = s3;
                    //List.Add(ns);

                    //  s3 = s3.Split('_','/')[2];
                    //  adIds.Add(s3);
                    try
                    {
                        string s5 = v2.GetAttribute("style");
                        string width = s5.Split(';', ':')[5];
                        string height = s5.Split(';', ':')[7];
                        imgwidth.Add(width);
                        imgheight.Add(height);
                    }
                    catch
                    { }

                }
                watch.Stop();
                string time = watch.Elapsed.ToString();
                if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
                {
                    _form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent, time);

                }
            }

            catch(Exception)
            { }
            driver.Quit();
            driver.Dispose();
            adIds = null;
            adUrls = null;
            adImages = null;
            adTitle = null;
        }
Example #5
0
        public void Outbrain(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
        {
            Stopwatch watch = new Stopwatch();
            watch.Start();
            List<string> adIds = new List<string>();
            List<string> adUrls = new List<string>();
            List<string> adImages = new List<string>();
            List<string> adTitle = new List<string>();
            List<string> imgwidth = new List<string>();
            List<string> imgheight = new List<string>();
            try
            {
            ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath(path1));
               // NetworkFunctions ns = new NetworkFunctions();////a[contains(@onmousedown,'paid.outbrain.com/')]
            List<NetworkFunctions> List = new List<NetworkFunctions>();

            foreach (var v1 in getAdds)
            {

                string s2 = v1.GetAttribute("href");
                adUrls.Add(s2);
                //ns.adUrl = s2;
                //List.Add(ns);

            }
            ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath(path2));
            foreach (var v2 in getImages)//"//img[@class='ob-rec-image ob-show']"
            {
                string s3 = v2.GetAttribute("src");
                adImages.Add(s3);
                string id = s3.Split('/')[8];
                adIds.Add(id);
                //ns.adImage = s3;
                //List.Add(ns);

            }
            ReadOnlyCollection<IWebElement> getID = driver.FindElements(By.XPath(path3));
            foreach (var v3 in getID)//"//span[@class='ob-unit ob-rec-text']"
            {
                string v7 = v3.GetAttribute("title");
                if (v7 != null)
                {
                    adTitle.Add(v7);
                    // List.Add(ns);
                }
            }
                watch.Stop();
                string time = watch.Elapsed.ToString();
                if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
                {
                    _form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent, time);

                }
            }

            catch (Exception)
            { }
            driver.Quit();
            driver.Dispose();
            adIds = null;
            adUrls = null;
            adImages = null;
            adTitle = null;
        }
Example #6
0
        public void Taboola(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
        {
            Stopwatch watch = new Stopwatch();
            watch.Start();
            List<string> adIds = new List<string>();
            List<string> adUrls = new List<string>();
            List<string> adImages = new List<string>();
            List<string> adTitle = new List<string>();
            List<string> imgwidth = new List<string>();
            List<string> imgheight = new List<string>();
            try
            {
                ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath(path1));
               // NetworkFunctions ns = new NetworkFunctions();
                List<NetworkFunctions> List = new List<NetworkFunctions>();

                foreach (var v1 in getAdds)
                {

                    try
                    {
                        string s1 = v1.GetAttribute("data-item-id");
                        if (s1.Contains("~"))
                        {
                            s1 = s1.Split('~')[4];
                            adIds.Add(s1.Trim().Replace("-", ""));
                            //ns.adId = s1;
                            //List.Add(ns);
                        }
                        else
                        {
                            adIds.Add(s1.Trim().Replace("-", ""));
                            //ns.adId = s1.Trim().Replace("-", "");
                            // List.Add(ns);
                        }
                    }
                    catch { }

                    try
                    {
                        string s4 = v1.GetAttribute("data-item-title").Replace("&nbsp;", "");
                        if (s4 != "")
                        {
                            adTitle.Add(s4);
                            //ns.adTitle = s4;
                            //List.Add(ns);
                        }

                    }

                    catch
                    { }

                    try
                    {
                        string s3 = v1.GetAttribute("data-item-thumb");
                        if (s3 != null)
                        {
                            adImages.Add(s3);
                            //ns.adImage = s3;
                            //List.Add(ns);
                        }
                    }
                    catch
                    { }

                }
                ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath(path2));
                foreach (var v2 in getImages)
                {
                    string s2 = v2.GetAttribute("href");
                    adUrls.Add(s2);
                    //ns.adUrl = s2;
                    //List.Add(ns);
                }
                watch.Stop();
                string time = watch.Elapsed.ToString();
                if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
                {
                    _form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent, time);

                }
            }

            catch (Exception)
            { }
            driver.Quit();
                driver.Dispose();
                adIds = null;
                adUrls = null;
                adImages = null;
                adTitle = null;
        }
Example #7
0
        public void Adblade(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
        {
            Stopwatch watch = new Stopwatch();
            watch.Start();
            var service1 = PhantomJSDriverService.CreateDefaultService();
            service1.HideCommandPromptWindow = true;
            var driver1 = new PhantomJSDriver(service1);
            List<string> adIds = new List<string>();
            List<string> adUrls = new List<string>();
            List<string> adImages = new List<string>();
            List<string> adTitle = new List<string>();
            List<string> imgwidth = new List<string>();
            List<string> imgheight = new List<string>();
            try
            {
            ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath(path1));
            //NetworkFunctions ns = new NetworkFunctions();
            List<NetworkFunctions> List = new List<NetworkFunctions>();

            foreach (var v1 in getAdds)
            {
                try
                {
                    string s2 = v1.GetAttribute("href");

                    try
                    {

                        driver1.Navigate().GoToUrl(s2);
                        //driver.GetScreenshot().SaveAsFile("image3.png", ImageFormat.Png);
                        IWebElement geturl = driver1.FindElement(By.XPath("//meta[@property='og:url']"));
                        string content = geturl.GetAttribute("content");
                        adUrls.Add(content);
                    }
                    catch (Exception ex)
                    {
                        adUrls.Add(s2);
                    }
                }
                catch
                { }
                // ns.adUrl = s2;
                //List.Add(ns);
                try
                    {
                        string s4 = v1.GetAttribute("outerText");
                        adTitle.Add(s4);
                        // ns.adTitle = s4;
                        //List.Add(ns);
                    }
                    catch
                    { }
                }
            ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath(path2));
                foreach (var v2 in getImages)
                {//http://staticd.cdn.adblade.com/banners/images/298x224/4365_548f126ed8024.jpg
                    string s3 = v2.GetAttribute("src");
                    if (s3 != null)
                    {
                        adImages.Add(s3);
                        //ns.adImage = s3;
                        //List.Add(ns);
                        s3 = s3.Split('/')[5];
                        string width = s3.Split('x')[0];
                        string height = s3.Split('x')[1];
                        imgwidth.Add(width);
                        imgheight.Add(height);

                    }

                }
                ReadOnlyCollection<IWebElement> getID = driver.FindElements(By.XPath(path3));
                foreach (var v3 in getID)
                {

                    try
                    {
                        string v7 = v3.GetAttribute("data-adid");
                        if (v7 != null)
                        {
                            adIds.Add(v7);
                            //ns.adId = v7.Trim();
                            // List.Add(ns);
                        }
                    }
                    catch { }

                }

                watch.Stop();
                string time = watch.Elapsed.ToString();
                if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
                {
                    _form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent, time);

                }
            }

            catch (Exception)
            { }
            driver.Quit();
            driver.Dispose();
            driver1.Quit();
            driver1.Dispose();
            adIds = null;
            adUrls = null;
            adImages = null;
            adTitle = null;
        }
Example #8
0
        public void Mgid(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
        {
            Stopwatch watch = new Stopwatch();
            watch.Start();
            List<string> adIds = new List<string>();
            List<string> adUrls = new List<string>();////div[@class='text-elements']/div/a  //div[@class='mcimg']/a/img
            List<string> adImages = new List<string>();////div[contains(@class,'mgline teaser')]
            List<string> adTitle = new List<string>();
            List<string> imgwidth = new List<string>();
            List<string> imgheight = new List<string>();
            try
            {
            ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath("//div[@class='text-elements']/div/a"));
            //NetworkFunctions ns = new NetworkFunctions();
            List<NetworkFunctions> List = new List<NetworkFunctions>();

            foreach (var v1 in getAdds)
            {

                string s2 = v1.GetAttribute("href");

                adUrls.Add(s2);

                //ns.adUrl = s2;
                ///List.Add(ns);
                try
                {
                    string s4 = v1.GetAttribute("outerText");
                    adTitle.Add(s4);
                    //ns.adTitle = s4;
                    //List.Add(ns);
                }
                catch
                { }

            }
            ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath("//div[@class='mcimg']/a/img"));

            foreach (var v2 in getImages)
            {
                string s3 = v2.GetAttribute("src");
                if (s3 != null)
                {
                    adImages.Add(s3);
                    //ns.adImage = s3;
                    //List.Add(ns);
                    s3 = s3.Split('_')[1].Replace(".jpg", "");
                    string width = s3.Split('x')[0];
                    string height = s3.Split('x')[1];
                    imgwidth.Add(width);
                    imgheight.Add(height);

                }

                //try
                //{
                //    string s5 = v2.GetAttribute("width");
                //    if (s5 != null)
                //    {
                //        imgwidth.Add(s5);
                //        //ns.imgHeight = s5;
                //        //List.Add(ns);
                //    }
                //}
                //catch
                //{ }
                //try
                //{
                //    string s6 = v2.GetAttribute("height");
                //    if (s6 != null)
                //    {
                //        imgheight.Add(s6);
                //       // ns.imgWidth = s6;
                //        //List.Add(ns);
                //    }
                //}
                //catch
                //{ }

            }
            ReadOnlyCollection<IWebElement> getID = driver.FindElements(By.XPath(path3));
            foreach (var v3 in getID)
            {
                string v7 = v3.GetAttribute("class");
                if (v7 != null)
                {
                    adIds.Add(v7.Split('-')[1].Trim());
                   // List.Add(ns);
                }
            }
                watch.Stop();
                string time = watch.Elapsed.ToString();
                if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
                {
                    _form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent, time);

                }
            }

            catch (Exception)
            { }
            driver.Quit();
            driver.Dispose();
            adIds = null;
            adUrls = null;
            adImages = null;
            adTitle = null;
        }
Example #9
0
        private void startCrawler()
        {
            updateUI(new System.Action(() => grid("Title","crawler started............", Color.Green)));

            websites dt = new websites();

            /*
            dt.website_url = "http://viralkeen.com/";
            dt.website_networks = new List<string> { "Contentad" };
            dt.last_run = "";
            var ad = dt;
            */

            var list1 = dt.SelectAll();
            updateUI(new System.Action(() => grid("Websites", "We have " + list1.Count+" websites.", Color.Green)));
            ad_network an = new ad_network();
            ANlist = new List<ad_network>();
            ANlist = an.SelectAll();
            updateUI(new System.Action(() => grid("Networks", "We have  " + ANlist.Count + " Networks.", Color.Green)));

            foreach (var ad in list1)
            {

                if (ad.website_networks.Count != 0)
                {

                    if (checkBox1.Checked == true)
                    {

                        CrawlerPool.QueueWorkItem(dummy => ScrapeAdds(ad, false, country, device, resltn, agent), new object());
                        //ScrapeAdds(ad, true, country, device, resltn,agent);
                    }
                    else//ajsfdhgas
                    {

                        CrawlerPool.QueueWorkItem(dummy => ScrapeAdds(ad, false, country, device, resltn, agent), new object());
                        //ScrapeAdds(ad, false, country, device, resltn,agent);
                    }
                }
            }

            //try { CrawlerPool.WaitForIdle(); CrawlerPool.Shutdown(); }
            //catch { }
        }
Example #10
0
        public void ContentDotAd(string url, PhantomJSDriver driver, string path1, string path2, string path3, ad_network adN, string agent)
        {
            Stopwatch watch = new Stopwatch();
            watch.Start();
            List<string> adIds = new List<string>();
            List<string> adUrls = new List<string>();
            List<string> adImages = new List<string>();
            List<string> adTitle = new List<string>();
            List<string> imgwidth = new List<string>();
            List<string> imgheight = new List<string>();
            // List<NetworkFunctions> List = new List<NetworkFunctions>();
            try
            {
                ReadOnlyCollection<IWebElement> getAdds = driver.FindElements(By.XPath(path1));

                foreach (var v1 in getAdds)
                {
                    string s1 = v1.GetAttribute("onclick");
                    s1 = s1.Split(',')[2].Replace("'", "");
                    adIds.Add(s1.Trim());
                    string s2 = v1.GetAttribute("href");
                    adUrls.Add(s2);

                }//div[@class='ac_container']/a/img
                ReadOnlyCollection<IWebElement> getImages = driver.FindElements(By.XPath("//div[@class='ac_container']/a/img"));
                foreach (var v2 in getImages)
                {

                    string s3 = v2.GetAttribute("src");
                    adImages.Add(s3);
                    string s4 = v2.GetAttribute("title");
                    adTitle.Add(s4);
                    string height = v2.GetAttribute("height");
                    string width = v2.GetAttribute("width");
                    imgwidth.Add(height);
                    imgheight.Add(width);

                }
                watch.Stop();
                string time = watch.Elapsed.ToString();
                if (adIds.Count == adImages.Count && adIds.Count == adUrls.Count)
                {
                    _form1.combineAdd(url, adIds, adUrls, adImages, adTitle, imgwidth, imgheight, adN, agent,time);

                }
            }
            catch(Exception)
            { }
            driver.Quit();
            driver.Dispose();
            adIds = null;
            adUrls = null;
            adImages = null;
            adTitle = null;
        }
Example #11
0
 private void button4_Click(object sender, EventArgs e)
 {
     ad_network an = new ad_network();
     string id = textBox6.Text;
     an.Delete(id);
     //advertisement ad = new advertisement();
     //ad.DeleteAll();
 }
Example #12
0
        private void button3_Click(object sender, EventArgs e)
        {
            ad_network an = new ad_network();

            an.id = int.Parse(textBox1.Text);
            an.name = textBox3.Text;

            an.xpath = textBox2.Text + "|" + textBox4.Text + "|" + textBox5.Text;
            an.Update(an);
        }
Example #13
0
        private void button2_Click(object sender, EventArgs e)
        {
            ad_network an = new ad_network();

            an.id = int.Parse(textBox1.Text);
            an.name = textBox3.Text;

            an.xpath = textBox2.Text+"|"+ textBox4.Text+"|"+ textBox5.Text;
            an.Insert(an);
            //websites dt = new websites();
            //var list1 = dt.SelectAll();
            //foreach(var s in list1)
            //{

            //}
        }
Example #14
0
        internal void combineAdd(string url, List<string> adIds, List<string> adUrls, List<string> adImages, List<string> adTitle, List<string> imgwidth, List<string> imgheight, ad_network adN,string agent,string time)
        {
            updateUI(new System.Action(() => grid("Combining Add", "Total: "+adIds.Count+" From "+url, Color.Green)));
            for (int i = 0; i < adUrls.Count; i++)
            {

                //file.Write(adUrls[i] +"|" + adIds + "|" + adImages[i] + "|" + adTitle + "|" + FirstSeen[i] + "|" + LastSeen[i]);
                advertisement ad = new advertisement();
                ad.Id = adIds[i] + "_" + adN.name + "_" + Form1.country + "_" + Form1.device;
                ad.ad_networkId = adN.id;
                ad.ad_networkName = adN.name;
                ad.AddId = adIds[i];
                ad.Time = time;
                ad.title = adTitle[i];// "19 Things That Probably Only Happen in Dubai";
                                      //ad.image = adImages[i];// "http://d3dytsf4vrjn5x.cloudfront.net/2724/300x250/6afe33ad3159d67dd9b77b210a1a4335.jpg";
                                      //ad.imageheight = 120;
                                      //ad.imageWidth = 150;

                ad.firstSceen = System.DateTime.Now;
                ad.lastSceen = System.DateTime.Now;
                ad.advertiserUrl = new List<string> { url }; //url;
                                                             // ad.publisherUrl[0] = adUrls[i];
                ad.publisherUrl = new List<string> { adUrls[i] };

                //ad.publisherUrl = new List<string> { "http://firsttoknow.com/only-in-dubai/?utm_source=contentad_backfill&utm_campaign=only-in-dubai-102596&pp=1", "http://firsttoknow.com/make-a-ring-out-of-a-quarter/?utm_source=contentad_backfill&utm_campaign=make-a-ring-out-of-a-qua-108675&pp=1" };
                ad.Country = Form1.country;
                ad.deviceName = Form1.device;

                ad.Useragent = agent;//"Opera/9.80 (Android; Opera Mini/5.1.22460/23.334; U; en) Presto/2.5.25 Version/10.54";//"Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0)";//"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36";
                ad.Hits = new List<ad_hits>();
                ad_hits hit = new ad_hits();
                hit.date = System.DateTime.Now.Date;
                hit.count = 1;
                ad.Hits.Add(hit);
                ad.ad_details = new List<ad_Detail>();
                ad_Detail adeatail = new ad_Detail();
                adeatail.advertiserUrl = url;
                adeatail.publisherUrl = new List<string> { adUrls[i] };
                adeatail.firstSceen = System.DateTime.Now;
                adeatail.lastSceen = System.DateTime.Now;
                adeatail.Hits = new List<ad_hits> { hit };
                //adeatail.Hits.Add(hit);
                ad_Images aImage = new ad_Images();
                aImage.ImageUrl = adImages[i];
                try
                {
                    aImage.imageheight = imgheight[i];
                    aImage.imageWidth = imgwidth[i];
                }
                catch
                {
                    aImage.imageheight = "300";
                    aImage.imageWidth = "250";
                }
                aImage.IsServerUploaded = false;
                aImage.IsAmazonUploaded = false;
                aImage.ServerUrl = "";
                aImage.AmazonUrl = "";

                adeatail.Images = new List<ad_Images> { aImage };
                ad.ad_details.Add(adeatail);
                saveAdvertisement(ad);
            }
            adIds = new List<string>();
            adUrls = new List<string>();
            adImages = new List<string>();
            adTitle = new List<string>();
        }
Example #15
0
        public void ScrapeAdds(websites web, bool proxyCondtition, string country, string device, string resolution,string agent)
        {
            Stopwatch watch = new Stopwatch();
            watch.Start();
            updateUI(new System.Action(() => grid("Crawler","Starting crawler............",Color.Green)));

            List<advertisement> adList = new List<advertisement>();
            var service = PhantomJSDriverService.CreateDefaultService();
            service.HideCommandPromptWindow = true;
            var driver = new PhantomJSDriver(service);

            updateUI(new System.Action(() =>  grid("Loading","loading website...............", Color.Green)));

            if (proxyCondtition == true)
            {
                List<string> proxy = Proxy();

                Random rand = new Random();
                int index = rand.Next(proxy.Count);
                string selectProxy = proxy[index];
                string _proxy = selectProxy.Split('&')[0];
                string _credentials = selectProxy.Split('&')[1];
                Proxy prox = new Proxy();
                prox.HttpProxy = string.Format(_proxy);
                service.ProxyType = "http";
                service.Proxy = prox.HttpProxy;
            }

            driver.Navigate().GoToUrl(web.website_url);
            try
            {
                //List<string> resolution = ScreenResolution();
                //Random ran = new Random();
                //int index = ran.Next(resolution.Count);
                //string ScrnRes = resolution[index];
                string ScrnRes = resolution;

                int x = int.Parse(ScrnRes.Split('x')[0]);
                int y = int.Parse(ScrnRes.Split('x')[1]);

                driver.Manage().Window.Size = new Size(x, y);

            }
            catch (Exception ex)
            { }
            //  driver.GetScreenshot().SaveAsFile("image2.png", ImageFormat.Png);

            foreach (var networkName in web.website_networks)
            {
                ad_network adN = new ad_network();
                //adN = adN.SearchByName(networkName);

                adN = ANlist.Find(a => a.name.Equals(networkName));
                if (adN != null)
                {
                    updateUI(new System.Action(() => grid("Network","Network Found:"+ adN.name, Color.Green)));
                    string path1 = null;
                    string path2 = null;
                    string path3 = null;
                    string xpaths = adN.xpath;
                    string[] path = xpaths.Split('|');
                    if (path.Length == 3)
                    {
                        path1 = path[0];
                        path2 = path[1];
                        path3 = path[2];
                    }
                    if (path.Length == 2)
                    {
                        path1 = path[0];
                        path2 = path[1];
                        path3 = "";
                    }

                    NetworkFunctions nf = new NetworkFunctions(this);

                    if (adN.name == "Mgid")
                    {
                        updateUI(new System.Action(() => grid("Scraping:", "Migid Ads", Color.Green)));
                        nf.Mgid(web.website_url, driver, path1, path2, path3, adN, agent);

                    }
                    if (adN.name == "Adblade")
                    {
                        updateUI(new System.Action(() => grid("Scraping:", "Adblade Ads", Color.Green)));
                        nf.Adblade(web.website_url, driver, path1, path2, path3, adN, agent);

                    }
                    if (adN.name == "Gravity")
                    {
                        updateUI(new System.Action(() => grid("Scraping:", "Gravity Ads", Color.Green)));
                        nf.Gravity(web.website_url, driver, path1, path2, path3, adN, agent);
                    }
                    if (adN.name == "Taboola")
                    {
                        updateUI(new System.Action(() => grid("Scraping:", "Taboola Ads", Color.Green)));
                        nf.Taboola(web.website_url, driver, path1, path2, path3, adN, agent);
                    }
                    if (adN.name == "Outbrain")
                    {
                        updateUI(new System.Action(() => grid("Scraping:", "Outbrain Ads", Color.Green)));
                        nf.Outbrain(web.website_url, driver, path1, path2, path3, adN, agent);
                    }
                    if (adN.name == "Zergnet")
                    {
                        updateUI(new System.Action(() => grid("Scraping:", "Zergnet Ads", Color.Green)));
                        nf.ZergNetDotcom(web.website_url, driver, path1, path2, path3, adN, agent);
                    }
                    if (adN.name == "YahooGemini")
                    {
                        updateUI(new System.Action(() => grid("Scraping:", "YahooGemini Ads", Color.Green)));
                        nf.YahooGemini(web.website_url, driver, path1, path2, path3, adN, agent);
                    }
                    if (adN.name == "Hexagram")
                    {
                        updateUI(new System.Action(() => grid("Scraping:", "Hexagram Ads", Color.Green)));
                        nf.Hexagram(web.website_url, driver, path1, path2, path3, adN, agent);
                    }

                    if (adN.name == "Contentad" || adN.name == "Content.ad")
                    {
                        updateUI(new System.Action(() => grid("Scraping:", "Content.ad Ads", Color.Green)));
                        nf.ContentDotAd(web.website_url, driver, path1, path2, path3, adN, agent);
                    }
                    //if (adN.name == "Mgid")
                    //{
                    //    ls = nf.Mgid(driver, path1, path2, path3);
                    //}

                }
            }
            watch.Stop();
            List<websites> list = new List<websites>();
            list = web.SelectAll();
            string ScrapingTime = watch.Elapsed.ToString();
               foreach (var v in list)
            {
                if (v.website_url == web.website_url)
                {
                    web.Update(v._id, ScrapingTime);
                }
            }
            updateUI(new System.Action(() => grid("Time:", web.website_url+":" + ScrapingTime, Color.Green)));
            driver.Quit();
            driver.Dispose();
        }