private static void GrabProfile(string symbol)
{
Console.Write("Grabbing Profile for " + symbol);
string url = @"http://finance.yahoo.com/q/pr?s=" + symbol;
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
AgentDocument document;
try
{
document = AgentHandler.Instance.PerformAction(session, action);
}
catch
{
return;
}
string doc = document.ResponseString.ToLower();
var extractor = new ProgressiveDataExtractor(doc, ">details<");
if (extractor.NotFound)
{
Console.WriteLine(" - not found");
return;
}
Console.WriteLine(" - found");
ModelMadness madness = new ModelMadness();
string sector = extractor.ExtractString("sector");
Sector dbSector = null;
Industry dbIndustry = null;
if (sector != null && sector != "n/a")
{
sector = sector.Replace("&", "&");
string industry = extractor.ExtractString("industry").Replace("&", "&");
var obj = (from q in madness.Sectors where q.Sector1 == sector select q).ToArray();
if (obj.Length == 1)
{
dbSector = obj[0];
}
else
{
dbSector = new Sector {
Sector1 = sector
};
madness.AddToSectors(dbSector);
}
var obj2 = (from q in madness.Industries
where q.Sector.Sector1 == sector && q.Iindustry == industry
select q).ToArray();
if (obj2.Length == 1)
{
dbIndustry = obj2[0];
}
else
{
dbIndustry = new Industry {
Iindustry = industry, Sector = dbSector
};
madness.AddToIndustries(dbIndustry);
}
}
StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First();
CompanyProfile profile = new CompanyProfile
{
Sector = dbSector,
Industry = dbIndustry,
num_employees = extractor.ExtractInt("full time employees"),
StockName = stock,
scrape_day = DateTime.Now.Date
};
profile.summary = extractor.ExtractString("business summary");
profile.cgq = extractor.ExtractDecimal("corporate governance", "<b");
extractor.ExtractString("key executives");
int totalAge = 0;
int numAges = 0;
long totalPay = 0;
int numPays = 0;
int? curAge;
long?curPay;
do
{
curAge = extractor.ExtractInt("yfnc_tabledata1", "</b");
if (curAge > 111)
{
curAge = null;
}
if (curAge != null)
{
numAges++;
totalAge += (int)curAge;
curPay = extractor.ExtractLong("yfnc_tabledata1", "nowrap");
if (curPay != null)
{
numPays++;
totalPay += (long)curPay;
}
}
} while (curAge != null);
profile.avg_executive_age = totalAge == 0 ? null : (int?)totalAge / numAges;
profile.avg_executive_pay = totalPay == 0 ? null : (long?)totalPay / numPays;
madness.AddToCompanyProfiles(profile);
madness.SaveChanges();
}
