private List <ItemForSale> GetLocalPartData(string partId, string colourId)
{
Stream inputStream;
StreamReader reader;
inputStream = new FileStream(_filepath + partId + "-" + colourId + ".txt", FileMode.Open, FileAccess.Read);
reader = new StreamReader(inputStream);
List <ItemForSale> items = new List <ItemForSale>();
while (!reader.EndOfStream)
{
string[] partData = reader.ReadLine().Split('|');
ItemForSale item = new ItemForSale();
item.SellerName = partData[0];
item.Quantity = Int32.Parse(partData[1]);
item.Price = Decimal.Parse(partData[2]);
item.Category = partData[3];
item.PartId = partId;
item.ColorId = colourId;
items.Add(item);
}
reader.Close();
inputStream.Close();
return(items);
}
public ItemForSale Copy()
{
ItemForSale newItem = new ItemForSale();
newItem.SellerName = this.SellerName;
newItem.Quantity = this.Quantity;
newItem.Price = this.Price;
newItem.Category = this.Category;
newItem.PartId = this.PartId;
newItem.ColorId = this.ColorId;
newItem.Notes = this.Notes;
newItem.QuantityRequired = this.QuantityRequired;
return(newItem);
}
/// <summary>
/// Scrape the price guide data and return the list of wanted items updated with availability data
/// </summary>
public List <WantedListItem> ScrapePriceGuideData()
{
foreach (var wantedItem in _wantedItems)
{
List <ItemForSale> itemsForSale = null;
if (this.LocalPartsDataExists(wantedItem.Id, wantedItem.ColorId))
{
Console.WriteLine(wantedItem.Name + " being retrieved from local cache");
itemsForSale = this.GetLocalPartData(wantedItem.Id, wantedItem.ColorId);
Console.WriteLine(itemsForSale.Count + " available items found");
}
else
{
Console.WriteLine("Sleeping...");
System.Threading.Thread.Sleep(SleepTime);
try {
Console.WriteLine(wantedItem.Name + " being retrieved from Bricklink...");
itemsForSale = new List <ItemForSale>();
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create("http://www.bricklink.com/catalogPG.asp?P=" + wantedItem.Id + "&colorID=" + wantedItem.ColorId);
req.Proxy = _proxySettings;
req.CookieContainer = _cookieJar;
HttpWebResponse response = (HttpWebResponse)req.GetResponse();
StreamReader reader = new StreamReader(response.GetResponseStream());
string pgdata = reader.ReadToEnd();
reader.Close();
response.Close();
string[] pageparts = pgdata.Split(new string[] { "<B>Currently Available</B>" }, StringSplitOptions.None);
// extract the image url from the first part of the page
Regex imageUrlRegex = new Regex(@"SRC=\'(http\:\/\/img\.bricklink\.com\/.*?)\'");
Match matchImg = imageUrlRegex.Match(pageparts[0]);
if (matchImg.Success)
{
string imageUrl = matchImg.Groups[1].ToString();
if (!_imageUrls.ContainsKey(wantedItem.Id + "-" + wantedItem.ColorId))
{
_imageUrls.Add(wantedItem.Id + "-" + wantedItem.ColorId, imageUrl);
}
}
string newdata = "";
string useddata = "";
if (pageparts.Length == 2)
{
// either new or used is missing
// if used is missing then pageparts[1] will contain <TD WIDTH="25%" BGCOLOR="DDDDDD"> </TD>
Regex emptyTestRegex = new Regex(@"<TD WIDTH=\""25%\"" BGCOLOR=\""DDDDDD\""> </TD>");
if (emptyTestRegex.IsMatch(pageparts[1]))
{
newdata = pageparts[1];
}
else
{
useddata = pageparts[1];
}
}
else
{
newdata = pageparts[1];
useddata = pageparts[2];
}
Regex itemListingRegex = new Regex(@"<A HREF=\""\/store\.asp\?sID=(\d*)\&.*?<IMG SRC=\""\/images\/box16(.)\.png\"".*?TITLE=\""Store\: (.*?)\"" ALIGN=\""ABSMIDDLE\""\>.*?<\/TD><TD>(\d*)<\/TD><TD>.*?\ \;\D*([\d,]*)\.(\d+)");
if (!string.IsNullOrEmpty(newdata))
{
string[] newitems = newdata.Split(new string[] { "</TD></TR>" }, StringSplitOptions.None);
foreach (var item in newitems)
{
Match match = itemListingRegex.Match(item);
if (match.Success)
{
ItemForSale saleItem = new ItemForSale();
saleItem.SellerName = match.Groups[3].ToString().Replace("|", "");
saleItem.Category = "N";
saleItem.Quantity = Int32.Parse(match.Groups[4].ToString());
saleItem.Price = Decimal.Parse(match.Groups[5].ToString() + "." + match.Groups[6].ToString());
saleItem.PartId = wantedItem.Id;
saleItem.ColorId = wantedItem.ColorId;
itemsForSale.Add(saleItem);
}
}
}
if (!string.IsNullOrEmpty(useddata))
{
string[] useditems = useddata.Split(new string[] { "</TD></TR>" }, StringSplitOptions.None);
foreach (var item in useditems)
{
Match match = itemListingRegex.Match(item);
if (match.Success)
{
ItemForSale saleItem = new ItemForSale();
saleItem.SellerName = match.Groups[3].ToString().Replace("|", "");
saleItem.Category = "U";
saleItem.Quantity = Int32.Parse(match.Groups[4].ToString());
saleItem.Price = Decimal.Parse(match.Groups[5].ToString() + "." + match.Groups[6].ToString());
saleItem.PartId = wantedItem.Id;
saleItem.ColorId = wantedItem.ColorId;
itemsForSale.Add(saleItem);
}
}
}
this.SavePartData(wantedItem.Id, wantedItem.ColorId, itemsForSale);
Console.WriteLine(itemsForSale.Count + " available items found and saved to cache");
} catch (Exception ex) {
Console.WriteLine("Failed: " + ex.Message);
}
}
wantedItem.AvailableItems = itemsForSale;
}
Console.WriteLine("Saving image URLs");
this.SaveImageUrlData();
return(_wantedItems);
}
