private static void GetWeatherForDay(DateTime day, ModelMadness db)
{
string url =
"http://www.wunderground.com/history/airport/KNYC/`/DailyHistory.html?req_city=New+York&req_state=NY&req_statename=New+York&MR=1";
url = url.Replace("`", day.Year + "/" + day.Month + "/" + day.Day);
AgentAction action = new AgentAction(url, false);
AgentDocument document = AgentHandler.Instance.PerformAction(new AgentSession(), action);
int startIdx = document.ResponseString.IndexOf("Mean Temperature");
startIdx = document.ResponseString.IndexOf("<span class=\"b\">", startIdx) + "<span class=\"b\">".Length;
int endidx = document.ResponseString.IndexOf("<", startIdx);
int actual = int.Parse(document.ResponseString.Substring(startIdx, endidx - startIdx));
startIdx = document.ResponseString.IndexOf("<span class=\"b\">", startIdx) + "<span class=\"b\">".Length;
endidx = document.ResponseString.IndexOf("<", startIdx);
int average = int.Parse(document.ResponseString.Substring(startIdx, endidx - startIdx));
Weather weather = new Weather
{
day = day,
from_avg = actual - average
};
db.AddToWeathers(weather);
db.SaveChanges();
Console.WriteLine(day.ToString("d") + " was off by " + (actual - average) + " degrees on wall street");
}
public void Debug(string message)
{
ModelMadness madness = new ModelMadness();
Log log = new Log
{
message = message,
transaction_date = DateTime.Now
};
madness.AddToLogs(log);
madness.SaveChanges();
}
public static void PullAll(DateTime start, DateTime stop)
{
ModelMadness madness = new ModelMadness();
StockName[] stocks = (from q in madness.StockNames orderby q.Stock select q).ToArray();
foreach (StockName stock in stocks)
{
PullQuotes(start, stop, stock.Stock);
}
Console.WriteLine("Done");
}
public static void Start()
{
ModelMadness madness = new ModelMadness();
string[] stockNames = (from q in madness.StockNames where q.HasData != null && ((bool)q.HasData) orderby q.Stock select q.Stock).ToArray();
foreach (string stock in stockNames)
{
GrabProfile(stock);
GrabStats(stock);
}
Console.WriteLine("Done");
}
public void Error(string message, Exception ex)
{
ModelMadness madness = new ModelMadness();
Log log = new Log
{
message = message,
exception_message = ex.Message,
exception_stacktrace = ex.StackTrace,
transaction_date = DateTime.Now
};
madness.AddToLogs(log);
madness.SaveChanges();
}
public void Init(bool forTraining)
{
Console.WriteLine("Retrieving madness from database...");
ModelMadness madness = new ModelMadness();
madness.CommandTimeout = 0;
simplestockquote[] stats;
if (forTraining)
{
stats = (madness.simplestockquotes.Where(q => q.day > new DateTime(2006, 1, 1))
.OrderBy(q => q.day))
.ToArray();
}
else
{
stats = (madness.simplestockquotes.Where(q => q.day > new DateTime(2010, 1, 1) &&
q.day <= new DateTime(2010, 12, 09))
.OrderBy(q => q.day))
.ToArray();
}
long lastDay = 0;
DataForDay currentDay = null;
// these are in order, from oldest to newest
foreach (simplestockquote stat in stats)
{
if (lastDay != stat.day.Ticks)
{
lastDay = stat.day.Ticks;
currentDay = new DataForDay();
Console.WriteLine("Analyzing Day " + stat.day.ToString("D"));
DayCount++;
SnapshotByDay[DayCount] = currentDay;
}
currentDay.AddStat(stat);
}
Console.WriteLine("Done retrieving");
}
public static void Go()
{
var encog = new EncogPersistedCollection("market-network.dat", FileMode.Open);
Console.WriteLine(@"Loading network");
var network = (BasicNetwork)encog.Find("market-network");
Console.WriteLine(@"Reading current data from db");
var market = new StockMarket();
market.Init(false);
var data = market.GetCurrentData();
Console.WriteLine(@"Running network on data");
var madness = new ModelMadness();
foreach (StockMarket.WorkableStockInfo info in data)
{
var input = InputOutputMadness.CreateInputs(info);
var neuralInput = new BasicMLData(input);
var output = network.Compute(neuralInput);
Console.WriteLine(@"Stock " + info.ViewToday.stock + @" will change " + output[0] + @"% in the next 20 trading days");
var future = new prediction
{
day = DateTime.Now.Date,
C20_Days_Close = 100 * (decimal)output[0],
stock = info.ViewToday.stock
};
madness.AddTopredictions(future);
}
madness.SaveChanges();
Console.WriteLine(@"Done - begin making $millions");
}
public static void Dance()
{
Console.WriteLine("Checking the weather");
ModelMadness db = new ModelMadness();
// from 2006-01-01 through present
HashSet <DateTime> existingDays = new HashSet <DateTime>(db.Weathers.Select(x => x.day).ToArray());
DateTime current = new DateTime(2006, 1, 1);
while (current < DateTime.Now)
{
if (!existingDays.Contains(current))
{
GetWeatherForDay(current, db);
}
current = current.AddDays(1);
}
Console.WriteLine("Done");
}
public static void PullQuotes(DateTime start, DateTime stop, string stockName)
{
ModelMadness madness = new ModelMadness();
StockName stock = (from q in madness.StockNames where q.Stock == stockName select q).First();
Console.WriteLine("Working on " + stock.Stock);
//http://ichart.finance.yahoo.com/table.csv?s=TKR&a=00&b=1&c=2000&d=01&e=18&f=2010&g=d&ignore=.csv
string url = @"http://ichart.finance.yahoo.com/table.csv?s=" +
stock.Stock + "&a=" + TwoDig(start.Month - 1) + "&b=" + start.Day + "&c=" + start.Year +
"&d=" + TwoDig(stop.Month - 1) + "&e=" + stop.Day + "&f=" + stop.Year +
"&g=d&ignore=.csv";
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
bool found = false;
AgentDocument document = null;
try
{
document = AgentHandler.Instance.PerformAction(session, action);
found = true;
}
catch (Exception ex)
{
Console.WriteLine("ERROR: " + url + " - " + ex.Message);
}
if (!found)
{
return;
}
//Date,Open,High,Low,Close,Volume,Adj Close
string[] rows = document.ResponseString.Split('\n');
if (rows.Length < 2)
{
return;
}
for (int i = 1; i < rows.Length; i++)
{
string row = rows[i];
string[] fields = row.Split(',');
if (fields.Length < 7)
{
Console.WriteLine((i - 2) + " records stored");
continue;
}
quote q = new quote
{
day = (DateTime) new DateTimeConverter().ConvertFrom(fields[0]),
open = decimal.Parse(fields[1]),
high = decimal.Parse(fields[2]),
low = decimal.Parse(fields[3]),
close = decimal.Parse(fields[4]),
volume = long.Parse(fields[5]),
adjusted_close = decimal.Parse(fields[6]),
StockName = stock
};
madness.AddToquotes(q);
}
madness.SaveChanges();
}
public static DateTime GetMaxDate()
{
ModelMadness madness = new ModelMadness();
return((from q in madness.quotes orderby q.day descending select q.day).First());
}
private static void GrabProfile(string symbol)
{
Console.Write("Grabbing Profile for " + symbol);
string url = @"http://finance.yahoo.com/q/pr?s=" + symbol;
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
AgentDocument document;
try
{
document = AgentHandler.Instance.PerformAction(session, action);
}
catch
{
return;
}
string doc = document.ResponseString.ToLower();
var extractor = new ProgressiveDataExtractor(doc, ">details<");
if (extractor.NotFound)
{
Console.WriteLine(" - not found");
return;
}
Console.WriteLine(" - found");
ModelMadness madness = new ModelMadness();
string sector = extractor.ExtractString("sector");
Sector dbSector = null;
Industry dbIndustry = null;
if (sector != null && sector != "n/a")
{
sector = sector.Replace("&", "&");
string industry = extractor.ExtractString("industry").Replace("&", "&");
var obj = (from q in madness.Sectors where q.Sector1 == sector select q).ToArray();
if (obj.Length == 1)
{
dbSector = obj[0];
}
else
{
dbSector = new Sector {
Sector1 = sector
};
madness.AddToSectors(dbSector);
}
var obj2 = (from q in madness.Industries
where q.Sector.Sector1 == sector && q.Iindustry == industry
select q).ToArray();
if (obj2.Length == 1)
{
dbIndustry = obj2[0];
}
else
{
dbIndustry = new Industry {
Iindustry = industry, Sector = dbSector
};
madness.AddToIndustries(dbIndustry);
}
}
StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First();
CompanyProfile profile = new CompanyProfile
{
Sector = dbSector,
Industry = dbIndustry,
num_employees = extractor.ExtractInt("full time employees"),
StockName = stock,
scrape_day = DateTime.Now.Date
};
profile.summary = extractor.ExtractString("business summary");
profile.cgq = extractor.ExtractDecimal("corporate governance", "<b");
extractor.ExtractString("key executives");
int totalAge = 0;
int numAges = 0;
long totalPay = 0;
int numPays = 0;
int? curAge;
long?curPay;
do
{
curAge = extractor.ExtractInt("yfnc_tabledata1", "</b");
if (curAge > 111)
{
curAge = null;
}
if (curAge != null)
{
numAges++;
totalAge += (int)curAge;
curPay = extractor.ExtractLong("yfnc_tabledata1", "nowrap");
if (curPay != null)
{
numPays++;
totalPay += (long)curPay;
}
}
} while (curAge != null);
profile.avg_executive_age = totalAge == 0 ? null : (int?)totalAge / numAges;
profile.avg_executive_pay = totalPay == 0 ? null : (long?)totalPay / numPays;
madness.AddToCompanyProfiles(profile);
madness.SaveChanges();
}
private static void GrabStats(string symbol)
{
Console.Write("Grabbing Key Stats for " + symbol);
string url = @"http://finance.yahoo.com/q/ks?s=" + symbol;
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
AgentDocument document;
try
{
document = AgentHandler.Instance.PerformAction(session, action);
}
catch
{
return;
}
var doc = document.ResponseString;
var extractor = new ProgressiveDataExtractor(doc, "Valuation Measures");
if (extractor.NotFound)
{
Console.WriteLine(" - Not found");
return;
}
Console.WriteLine(" - found");
ModelMadness madness = new ModelMadness();
StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First();
CompanyStat stats = new CompanyStat
{
market_cap = extractor.ExtractLong("Market Cap"),
enterprise_value = extractor.ExtractLong("Enterprise Value"),
trailing_pe = extractor.ExtractDecimal("Trailing P/E"),
forward_pe = extractor.ExtractDecimal("Forward P/E"),
peg_ratio = extractor.ExtractDecimal("PEG Ratio"),
price_sales = extractor.ExtractDecimal("Price/Sales"),
price_book = extractor.ExtractDecimal("Price/Book"),
ent_value_rev = extractor.ExtractLong("Enterprise Value/Revenue"),
ent_value_ebitda = extractor.ExtractDecimal("Enterprise Value/EBITDA"),
fiscal_year_ends = extractor.ExtractString("Fiscal Year Ends"),
most_recent_quarter = extractor.ExtractString("Most Recent Quarter"),
profit_margin = extractor.ExtractDecimal("Profit Margin"),
operating_margin = extractor.ExtractDecimal("Operating Margin"),
return_on_assets = extractor.ExtractDecimal("Return on Assets"),
return_on_equity = extractor.ExtractDecimal("Return on Equity"),
revenue = extractor.ExtractLong("Revenue"),
revenue_per_share = extractor.ExtractDecimal("Revenue Per Share"),
qrt_rev_growth = extractor.ExtractDecimal("Qtrly Revenue Growth"),
gross_profit = extractor.ExtractLong("Gross Profit"),
ebitda = extractor.ExtractLong("EBITDA"),
net_income_a_c = extractor.ExtractLong("Net Income Avl to Common"),
diluted_eps = extractor.ExtractDecimal("Diluted EPS"),
qrt_earnings_growth = extractor.ExtractDecimal("Qtrly Earnings Growth"),
total_cash = extractor.ExtractLong("Total Cash"),
cash_per_share = extractor.ExtractDecimal("Total Cash Per Share"),
total_debt = extractor.ExtractLong("Total Debt"),
total_debt_equit = extractor.ExtractDecimal("Total Debt/Equity"),
current_ratio = extractor.ExtractDecimal("Current Ratio"),
book_value_p_share = extractor.ExtractDecimal("Book Value Per Share"),
operating_cash_flow = extractor.ExtractLong("Operating Cash Flow"),
levered_free_cash_flow = extractor.ExtractLong("Levered Free Cash Flow"),
StockName = stock,
scrape_day = DateTime.Now.Date
};
madness.AddToCompanyStats(stats);
madness.SaveChanges();
}
