static void Main(string[] args)
{
//credit to jimevans over at #selenium! :D
IWebDriver driver = new OpenQA.Selenium.PhantomJS.PhantomJSDriver();
Dictionary<string, KeyValuePair<int, double>> cards = new Dictionary<string, KeyValuePair<int, double>>();
driver.Url = "http://steamcommunity.com/market/search?q=appid%3A753+trading+card";
var nextPage = driver.FindElement(By.Id("searchResults_btn_next"));
string classAttribute = nextPage.GetAttribute("class");
int i = 1;
while (classAttribute != null && !classAttribute.Contains("disabled"))
{
File.WriteAllText("f" + i.ToString() + ".html", driver.PageSource);
var elementList = driver.FindElements(By.CssSelector("div.market_listing_row.market_recent_listing_row"));
if (elementList.Count == 0)
{
int h = 6;
break;
}
Console.WriteLine("Done page {0} with {1} elements", i, elementList.Count());
++i;
foreach (var element in elementList)
{
var itemNameElement = element.FindElement(By.CssSelector("span.market_listing_item_name"));
var gameNameElement = element.FindElement(By.CssSelector("span.market_listing_game_name"));
if (gameNameElement.Text.Contains("Emoticon"))
continue;
var gameName = gameNameElement.Text.Replace(" Trading Card", "");
var priceElement = element.FindElement(By.CssSelector("div.market_listing_right_cell.market_listing_num_listings"));
string priceWithJunk = priceElement.Text;
priceWithJunk = priceWithJunk.Substring(priceWithJunk.IndexOf('$') + 1);
double price = Convert.ToDouble(priceWithJunk.Remove(priceWithJunk.IndexOf('U')).Trim());
//Console.WriteLine("Item: {0}, Game:{1}, Price: {2}", itemNameElement.Text, gameName, price);
if (cards.Keys.Contains(gameName)) //add 1
{
cards[gameName] = new KeyValuePair<int, double>(cards[gameName].Key + 1, cards[gameName].Value + price);
}
else
{
cards[gameName] = new KeyValuePair<int, double>(1, price);
}
}
// Note: you might be able to get away with not clicking on this
// element and using a direct navigation.
nextPage.Click();
WebDriverWait wait = new WebDriverWait(driver, TimeSpan.FromSeconds(60));
try
{
wait.Until<bool>((d) =>
{
try
{
// This will wait until the listing item name element goes
// "stale", meaning it's been removed from the DOM and replaced
// by a similar element on the next page.
bool dummy = elementList[0].FindElement(By.CssSelector("span.market_listing_item_name")).Displayed;
return false;
}
catch (StaleElementReferenceException)
{
return true;
}
});
}
catch (WebDriverTimeoutException)
{
break;
}
}
driver.Quit();
/*var driver = new OpenQA.Selenium.PhantomJS.PhantomJSDriver();
//var driver = new OpenQA.Selenium.Firefox.FirefoxDriver();
string lastFirstElement = "novalue", currFirstElement="novalue";
for (int i = 1; i < 111; ++i)
{
//load the page
string url = "http://steamcommunity.com/market/search?q=appid%3A753+trading+card#p" + i.ToString();
driver.Url = url;
driver.Manage().Timeouts().ImplicitlyWait(new TimeSpan(0, 0, 15));
HtmlDocument doc = new HtmlDocument();
WebDriverWait wait = new WebDriverWait(driver, new TimeSpan(0, 1, 0));
wait.Until(d => d.FindElement(By.Id("searchResultsTable")));
//try to find the cards
IEnumerable<HtmlNode> findThings = null;
do
{
if (i == 1)
{
System.Threading.Thread.Sleep(5000);
}
else
{
System.Threading.Thread.Sleep(1000);
}
string s = driver.PageSource;
doc.LoadHtml(s);
findThings = doc.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("class") &&
d.Attributes["class"].Value == "market_listing_row market_recent_listing_row");
currFirstElement = findThings.First().InnerText;
} while (lastFirstElement.Equals(currFirstElement) && i != 1);
//reset
//TODO timeout
lastFirstElement = currFirstElement;
Console.WriteLine("Done page" + i);
foreach (var thing in findThings)
{
string name = thing.Descendants("span").Where(d => d.Attributes.Contains("class") &&
d.Attributes["class"].Value == "market_listing_item_name").First().InnerHtml;
string type = thing.Descendants("span").Where(d => d.Attributes.Contains("class") &&
d.Attributes["class"].Value == "market_listing_game_name").First().InnerHtml.Replace(" Trading Card", "");
string priceHtml = thing.Descendants("div").Where(d => d.Attributes.Contains("class") &&
d.Attributes["class"].Value == "market_listing_right_cell market_listing_num_listings").First().InnerHtml;
string priceWithJunk = priceHtml.Substring(priceHtml.IndexOf('$') + 1);
double price = Convert.ToDouble(priceWithJunk.Remove(priceWithJunk.IndexOf('U')).Trim());
//if(type.Contains("Walking Dead"))
Console.WriteLine("Page {0} adding {1} from {2}", i, name, type);
}
}
driver.Quit();*/
List<string> print = new List<string>();
foreach(KeyValuePair<string, KeyValuePair<int, double>> cardset in cards)
{
print.Add(string.Join(",", cardset.Key, cardset.Value.Key, cardset.Value.Value));
}
File.WriteAllLines("print.csv", print);
int j = 4;
}
