Exemplo n.º 1
0
        static void Main(string[] args)
        {
            //credit to jimevans over at #selenium! :D
            IWebDriver driver = new OpenQA.Selenium.PhantomJS.PhantomJSDriver();
            Dictionary<string, KeyValuePair<int, double>> cards = new Dictionary<string, KeyValuePair<int, double>>();
            driver.Url = "http://steamcommunity.com/market/search?q=appid%3A753+trading+card";
            var nextPage = driver.FindElement(By.Id("searchResults_btn_next"));
            string classAttribute = nextPage.GetAttribute("class");
            int i = 1;
            while (classAttribute != null && !classAttribute.Contains("disabled"))
            {
                File.WriteAllText("f" + i.ToString() + ".html", driver.PageSource);
                var elementList = driver.FindElements(By.CssSelector("div.market_listing_row.market_recent_listing_row"));
                if (elementList.Count == 0)
                {
                    int h = 6;
                    break;
                }
                Console.WriteLine("Done page {0} with {1} elements", i, elementList.Count());
                ++i;
                foreach (var element in elementList)
                {
                    var itemNameElement = element.FindElement(By.CssSelector("span.market_listing_item_name"));
                    var gameNameElement = element.FindElement(By.CssSelector("span.market_listing_game_name"));
                    if (gameNameElement.Text.Contains("Emoticon"))
                        continue;
                    var gameName = gameNameElement.Text.Replace(" Trading Card", "");

                    var priceElement = element.FindElement(By.CssSelector("div.market_listing_right_cell.market_listing_num_listings"));
                    string priceWithJunk = priceElement.Text;
                    priceWithJunk = priceWithJunk.Substring(priceWithJunk.IndexOf('$') + 1);
                    double price = Convert.ToDouble(priceWithJunk.Remove(priceWithJunk.IndexOf('U')).Trim());
                    //Console.WriteLine("Item: {0}, Game:{1}, Price: {2}", itemNameElement.Text, gameName, price);
                    if (cards.Keys.Contains(gameName)) //add 1
                    {
                        cards[gameName] = new KeyValuePair<int, double>(cards[gameName].Key + 1, cards[gameName].Value + price);
                    }
                    else
                    {
                        cards[gameName] = new KeyValuePair<int, double>(1, price);
                    }
                }

                // Note: you might be able to get away with not clicking on this
                // element and using a direct navigation.
                nextPage.Click();
                WebDriverWait wait = new WebDriverWait(driver, TimeSpan.FromSeconds(60));
                try
                {
                    wait.Until<bool>((d) =>
                    {
                        try
                        {
                            // This will wait until the listing item name element goes
                            // "stale", meaning it's been removed from the DOM and replaced
                            // by a similar element on the next page.
                            bool dummy = elementList[0].FindElement(By.CssSelector("span.market_listing_item_name")).Displayed;
                            return false;
                        }
                        catch (StaleElementReferenceException)
                        {
                            return true;
                        }
                    });
                }
                catch (WebDriverTimeoutException)
                {
                    break;
                }
            }

            driver.Quit();

            /*var driver = new OpenQA.Selenium.PhantomJS.PhantomJSDriver();
            //var driver = new OpenQA.Selenium.Firefox.FirefoxDriver();
            string lastFirstElement = "novalue", currFirstElement="novalue";
            for (int i = 1; i < 111; ++i)
            {
                //load the page
                string url = "http://steamcommunity.com/market/search?q=appid%3A753+trading+card#p" + i.ToString();
                driver.Url = url;
                driver.Manage().Timeouts().ImplicitlyWait(new TimeSpan(0, 0, 15));
                HtmlDocument doc = new HtmlDocument();
                WebDriverWait wait = new WebDriverWait(driver, new TimeSpan(0, 1, 0));

                wait.Until(d => d.FindElement(By.Id("searchResultsTable")));
                //try to find the cards
                IEnumerable<HtmlNode> findThings = null;
                do
                {

                    if (i == 1)
                    {
                        System.Threading.Thread.Sleep(5000);
                    }
                    else
                    {
                        System.Threading.Thread.Sleep(1000);
                    }
                    string s = driver.PageSource;
                    doc.LoadHtml(s);
                    findThings = doc.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("class") &&
                        d.Attributes["class"].Value == "market_listing_row market_recent_listing_row");
                    currFirstElement = findThings.First().InnerText;

                } while (lastFirstElement.Equals(currFirstElement) && i != 1);

                //reset
                //TODO timeout
                lastFirstElement = currFirstElement;
                 Console.WriteLine("Done page" + i);

                foreach (var thing in findThings)
                {
                    string name = thing.Descendants("span").Where(d => d.Attributes.Contains("class") &&
                        d.Attributes["class"].Value == "market_listing_item_name").First().InnerHtml;
                    string type = thing.Descendants("span").Where(d => d.Attributes.Contains("class") &&
                        d.Attributes["class"].Value == "market_listing_game_name").First().InnerHtml.Replace(" Trading Card", "");
                    string priceHtml = thing.Descendants("div").Where(d => d.Attributes.Contains("class") &&
                        d.Attributes["class"].Value == "market_listing_right_cell market_listing_num_listings").First().InnerHtml;
                    string priceWithJunk = priceHtml.Substring(priceHtml.IndexOf('$') + 1);
                    double price = Convert.ToDouble(priceWithJunk.Remove(priceWithJunk.IndexOf('U')).Trim());

                    //if(type.Contains("Walking Dead"))
                    Console.WriteLine("Page {0} adding {1} from {2}", i, name, type);

                }
            }

            driver.Quit();*/

            List<string> print = new List<string>();
            foreach(KeyValuePair<string, KeyValuePair<int, double>> cardset in cards)
            {
                print.Add(string.Join(",", cardset.Key, cardset.Value.Key, cardset.Value.Value));
            }
            File.WriteAllLines("print.csv", print);
            int j = 4;
        }
Exemplo n.º 2
0
        public Location GetLocation(string urlRe, string province, string district, string pacelNo)
        {
            var result = new Location();
            var parcel = 0;

            if (string.IsNullOrEmpty(province) || string.IsNullOrEmpty(district) || string.IsNullOrEmpty(pacelNo))
            {
                return result;
            }

            if (pacelNo.Contains("-"))
            {
                var stat = int.TryParse(pacelNo.Split('-')[0], out parcel);
                if (!stat)
                    return result;
            }
            else
            {
                var stat = int.TryParse(pacelNo, out parcel);
                if (!stat)
                    return result;
            }

            var loc = DataHelper.GetLocation(province, district, parcel);
            if (loc != null)
            {
                return loc;
            }

            string driverPath = @"D:\MyProjects\FRES\src\FRES.Source.Map\Drivers\";
            IWebDriver driver = new OpenQA.Selenium.PhantomJS.PhantomJSDriver(driverPath);
            //IWebDriver driver = new OpenQA.Selenium.Chrome.ChromeDriver(driverPath);
            try
            {
                var url = "http://dolwms.dol.go.th/tvwebp/";
                driver.Navigate().GoToUrl(url);
                var wait = new WebDriverWait(driver, TimeSpan.FromSeconds(TIMEOUT));

                Thread.Sleep(DELAY);

                var ddlProvince = driver.FindSelectElementWhenPopulated(By.Name("ddlProvince"), TIMEOUT);
                ddlProvince.SelectBySubText(province);

                Thread.Sleep(DELAY);

                var ddlAmphur = driver.FindSelectElementWhenPopulated(By.Name("ddlAmphur"), TIMEOUT);
                ddlAmphur.SelectBySubText(district);

                Thread.Sleep(DELAY);

                var txtPacelNo = new WebDriverWait(driver, TimeSpan.FromSeconds(TIMEOUT)).Until(ExpectedConditions.ElementExists(By.Name("txtPacelNo")));
                txtPacelNo.SendKeys(pacelNo);

                Thread.Sleep(DELAY);

                var btnFind = driver.FindElement(By.Name("btnFind"));
                IJavaScriptExecutor js = driver as IJavaScriptExecutor;
                js.ExecuteScript("arguments[0].click();", btnFind);

                Thread.Sleep(DELAY);
                //var element = new WebDriverWait(driver, TimeSpan.FromSeconds(3)).Until(ExpectedConditions.TextToBePresentInElement(driver.FindElement(By.Id("ddlAmphur")), "01"));
                //wait.Until(ExpectedConditions.ElementExists(By.CssSelector("div[style=\"transform: translateZ(0px); position: absolute; left: 0px; top: 0px; z-index: 107; width: 100%;\"]")));

                var isExist = wait.Until((d) => { return driver.PageSource.Contains("createMarker( new Array("); });

                if (isExist)
                {
                    var html = driver.PageSource;
                    html = GetStrBtw(html, "createMarker( new Array(", "));");//.Replace("'", string.Empty);
                    var dtls = html.Split(',').Select(x => x.Replace("'", "")).ToArray();
                    result = new Location
                    {
                        Amphur = district,
                        Province = province,
                        ParcelCode = parcel,
                        Lat = double.Parse(dtls[7]),
                        Lon = double.Parse(dtls[8])
                    };

                    DataHelper.InsertLocation(result);
                }
                else
                {
                    throw new Exception("Can't find location");
                }
            }
            catch (Exception ex)
            {
                lock (sync)
                    File.AppendAllText("D:/RE/M.log", DateTime.Now.ToString("yyyyMMdd HH:mm") + "," + province + "," + district + "," + parcel + "," + urlRe + "," + ex.GetBaseException().Message + "\r\n");
            }
            finally
            {
                //Thread.Sleep(3000);
                driver.Close();
                driver.Quit();
                driver.Dispose();
            }
            return result;
        }