private static void GetWeatherForDay(DateTime day, ModelMadness db) { string url = "http://www.wunderground.com/history/airport/KNYC/`/DailyHistory.html?req_city=New+York&req_state=NY&req_statename=New+York&MR=1"; url = url.Replace("`", day.Year + "/" + day.Month + "/" + day.Day); AgentAction action = new AgentAction(url, false); AgentDocument document = AgentHandler.Instance.PerformAction(new AgentSession(), action); int startIdx = document.ResponseString.IndexOf("Mean Temperature"); startIdx = document.ResponseString.IndexOf("<span class=\"b\">", startIdx) + "<span class=\"b\">".Length; int endidx = document.ResponseString.IndexOf("<", startIdx); int actual = int.Parse(document.ResponseString.Substring(startIdx, endidx - startIdx)); startIdx = document.ResponseString.IndexOf("<span class=\"b\">", startIdx) + "<span class=\"b\">".Length; endidx = document.ResponseString.IndexOf("<", startIdx); int average = int.Parse(document.ResponseString.Substring(startIdx, endidx - startIdx)); Weather weather = new Weather { day = day, from_avg = actual - average }; db.AddToWeathers(weather); db.SaveChanges(); Console.WriteLine(day.ToString("d") + " was off by " + (actual - average) + " degrees on wall street"); }
public void Debug(string message) { ModelMadness madness = new ModelMadness(); Log log = new Log { message = message, transaction_date = DateTime.Now }; madness.AddToLogs(log); madness.SaveChanges(); }
public static void PullAll(DateTime start, DateTime stop) { ModelMadness madness = new ModelMadness(); StockName[] stocks = (from q in madness.StockNames orderby q.Stock select q).ToArray(); foreach (StockName stock in stocks) { PullQuotes(start, stop, stock.Stock); } Console.WriteLine("Done"); }
public static void Start() { ModelMadness madness = new ModelMadness(); string[] stockNames = (from q in madness.StockNames where q.HasData != null && ((bool)q.HasData) orderby q.Stock select q.Stock).ToArray(); foreach (string stock in stockNames) { GrabProfile(stock); GrabStats(stock); } Console.WriteLine("Done"); }
public void Error(string message, Exception ex) { ModelMadness madness = new ModelMadness(); Log log = new Log { message = message, exception_message = ex.Message, exception_stacktrace = ex.StackTrace, transaction_date = DateTime.Now }; madness.AddToLogs(log); madness.SaveChanges(); }
public void Init(bool forTraining) { Console.WriteLine("Retrieving madness from database..."); ModelMadness madness = new ModelMadness(); madness.CommandTimeout = 0; simplestockquote[] stats; if (forTraining) { stats = (madness.simplestockquotes.Where(q => q.day > new DateTime(2006, 1, 1)) .OrderBy(q => q.day)) .ToArray(); } else { stats = (madness.simplestockquotes.Where(q => q.day > new DateTime(2010, 1, 1) && q.day <= new DateTime(2010, 12, 09)) .OrderBy(q => q.day)) .ToArray(); } long lastDay = 0; DataForDay currentDay = null; // these are in order, from oldest to newest foreach (simplestockquote stat in stats) { if (lastDay != stat.day.Ticks) { lastDay = stat.day.Ticks; currentDay = new DataForDay(); Console.WriteLine("Analyzing Day " + stat.day.ToString("D")); DayCount++; SnapshotByDay[DayCount] = currentDay; } currentDay.AddStat(stat); } Console.WriteLine("Done retrieving"); }
public static void Go() { var encog = new EncogPersistedCollection("market-network.dat", FileMode.Open); Console.WriteLine(@"Loading network"); var network = (BasicNetwork)encog.Find("market-network"); Console.WriteLine(@"Reading current data from db"); var market = new StockMarket(); market.Init(false); var data = market.GetCurrentData(); Console.WriteLine(@"Running network on data"); var madness = new ModelMadness(); foreach (StockMarket.WorkableStockInfo info in data) { var input = InputOutputMadness.CreateInputs(info); var neuralInput = new BasicMLData(input); var output = network.Compute(neuralInput); Console.WriteLine(@"Stock " + info.ViewToday.stock + @" will change " + output[0] + @"% in the next 20 trading days"); var future = new prediction { day = DateTime.Now.Date, C20_Days_Close = 100 * (decimal)output[0], stock = info.ViewToday.stock }; madness.AddTopredictions(future); } madness.SaveChanges(); Console.WriteLine(@"Done - begin making $millions"); }
public static void Dance() { Console.WriteLine("Checking the weather"); ModelMadness db = new ModelMadness(); // from 2006-01-01 through present HashSet <DateTime> existingDays = new HashSet <DateTime>(db.Weathers.Select(x => x.day).ToArray()); DateTime current = new DateTime(2006, 1, 1); while (current < DateTime.Now) { if (!existingDays.Contains(current)) { GetWeatherForDay(current, db); } current = current.AddDays(1); } Console.WriteLine("Done"); }
public static void PullQuotes(DateTime start, DateTime stop, string stockName) { ModelMadness madness = new ModelMadness(); StockName stock = (from q in madness.StockNames where q.Stock == stockName select q).First(); Console.WriteLine("Working on " + stock.Stock); //http://ichart.finance.yahoo.com/table.csv?s=TKR&a=00&b=1&c=2000&d=01&e=18&f=2010&g=d&ignore=.csv string url = @"http://ichart.finance.yahoo.com/table.csv?s=" + stock.Stock + "&a=" + TwoDig(start.Month - 1) + "&b=" + start.Day + "&c=" + start.Year + "&d=" + TwoDig(stop.Month - 1) + "&e=" + stop.Day + "&f=" + stop.Year + "&g=d&ignore=.csv"; AgentSession session = new AgentSession(); AgentAction action = new AgentAction(url, false); bool found = false; AgentDocument document = null; try { document = AgentHandler.Instance.PerformAction(session, action); found = true; } catch (Exception ex) { Console.WriteLine("ERROR: " + url + " - " + ex.Message); } if (!found) { return; } //Date,Open,High,Low,Close,Volume,Adj Close string[] rows = document.ResponseString.Split('\n'); if (rows.Length < 2) { return; } for (int i = 1; i < rows.Length; i++) { string row = rows[i]; string[] fields = row.Split(','); if (fields.Length < 7) { Console.WriteLine((i - 2) + " records stored"); continue; } quote q = new quote { day = (DateTime) new DateTimeConverter().ConvertFrom(fields[0]), open = decimal.Parse(fields[1]), high = decimal.Parse(fields[2]), low = decimal.Parse(fields[3]), close = decimal.Parse(fields[4]), volume = long.Parse(fields[5]), adjusted_close = decimal.Parse(fields[6]), StockName = stock }; madness.AddToquotes(q); } madness.SaveChanges(); }
public static DateTime GetMaxDate() { ModelMadness madness = new ModelMadness(); return((from q in madness.quotes orderby q.day descending select q.day).First()); }
private static void GrabProfile(string symbol) { Console.Write("Grabbing Profile for " + symbol); string url = @"http://finance.yahoo.com/q/pr?s=" + symbol; AgentSession session = new AgentSession(); AgentAction action = new AgentAction(url, false); AgentDocument document; try { document = AgentHandler.Instance.PerformAction(session, action); } catch { return; } string doc = document.ResponseString.ToLower(); var extractor = new ProgressiveDataExtractor(doc, ">details<"); if (extractor.NotFound) { Console.WriteLine(" - not found"); return; } Console.WriteLine(" - found"); ModelMadness madness = new ModelMadness(); string sector = extractor.ExtractString("sector"); Sector dbSector = null; Industry dbIndustry = null; if (sector != null && sector != "n/a") { sector = sector.Replace("&", "&"); string industry = extractor.ExtractString("industry").Replace("&", "&"); var obj = (from q in madness.Sectors where q.Sector1 == sector select q).ToArray(); if (obj.Length == 1) { dbSector = obj[0]; } else { dbSector = new Sector { Sector1 = sector }; madness.AddToSectors(dbSector); } var obj2 = (from q in madness.Industries where q.Sector.Sector1 == sector && q.Iindustry == industry select q).ToArray(); if (obj2.Length == 1) { dbIndustry = obj2[0]; } else { dbIndustry = new Industry { Iindustry = industry, Sector = dbSector }; madness.AddToIndustries(dbIndustry); } } StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First(); CompanyProfile profile = new CompanyProfile { Sector = dbSector, Industry = dbIndustry, num_employees = extractor.ExtractInt("full time employees"), StockName = stock, scrape_day = DateTime.Now.Date }; profile.summary = extractor.ExtractString("business summary"); profile.cgq = extractor.ExtractDecimal("corporate governance", "<b"); extractor.ExtractString("key executives"); int totalAge = 0; int numAges = 0; long totalPay = 0; int numPays = 0; int? curAge; long?curPay; do { curAge = extractor.ExtractInt("yfnc_tabledata1", "</b"); if (curAge > 111) { curAge = null; } if (curAge != null) { numAges++; totalAge += (int)curAge; curPay = extractor.ExtractLong("yfnc_tabledata1", "nowrap"); if (curPay != null) { numPays++; totalPay += (long)curPay; } } } while (curAge != null); profile.avg_executive_age = totalAge == 0 ? null : (int?)totalAge / numAges; profile.avg_executive_pay = totalPay == 0 ? null : (long?)totalPay / numPays; madness.AddToCompanyProfiles(profile); madness.SaveChanges(); }
private static void GrabStats(string symbol) { Console.Write("Grabbing Key Stats for " + symbol); string url = @"http://finance.yahoo.com/q/ks?s=" + symbol; AgentSession session = new AgentSession(); AgentAction action = new AgentAction(url, false); AgentDocument document; try { document = AgentHandler.Instance.PerformAction(session, action); } catch { return; } var doc = document.ResponseString; var extractor = new ProgressiveDataExtractor(doc, "Valuation Measures"); if (extractor.NotFound) { Console.WriteLine(" - Not found"); return; } Console.WriteLine(" - found"); ModelMadness madness = new ModelMadness(); StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First(); CompanyStat stats = new CompanyStat { market_cap = extractor.ExtractLong("Market Cap"), enterprise_value = extractor.ExtractLong("Enterprise Value"), trailing_pe = extractor.ExtractDecimal("Trailing P/E"), forward_pe = extractor.ExtractDecimal("Forward P/E"), peg_ratio = extractor.ExtractDecimal("PEG Ratio"), price_sales = extractor.ExtractDecimal("Price/Sales"), price_book = extractor.ExtractDecimal("Price/Book"), ent_value_rev = extractor.ExtractLong("Enterprise Value/Revenue"), ent_value_ebitda = extractor.ExtractDecimal("Enterprise Value/EBITDA"), fiscal_year_ends = extractor.ExtractString("Fiscal Year Ends"), most_recent_quarter = extractor.ExtractString("Most Recent Quarter"), profit_margin = extractor.ExtractDecimal("Profit Margin"), operating_margin = extractor.ExtractDecimal("Operating Margin"), return_on_assets = extractor.ExtractDecimal("Return on Assets"), return_on_equity = extractor.ExtractDecimal("Return on Equity"), revenue = extractor.ExtractLong("Revenue"), revenue_per_share = extractor.ExtractDecimal("Revenue Per Share"), qrt_rev_growth = extractor.ExtractDecimal("Qtrly Revenue Growth"), gross_profit = extractor.ExtractLong("Gross Profit"), ebitda = extractor.ExtractLong("EBITDA"), net_income_a_c = extractor.ExtractLong("Net Income Avl to Common"), diluted_eps = extractor.ExtractDecimal("Diluted EPS"), qrt_earnings_growth = extractor.ExtractDecimal("Qtrly Earnings Growth"), total_cash = extractor.ExtractLong("Total Cash"), cash_per_share = extractor.ExtractDecimal("Total Cash Per Share"), total_debt = extractor.ExtractLong("Total Debt"), total_debt_equit = extractor.ExtractDecimal("Total Debt/Equity"), current_ratio = extractor.ExtractDecimal("Current Ratio"), book_value_p_share = extractor.ExtractDecimal("Book Value Per Share"), operating_cash_flow = extractor.ExtractLong("Operating Cash Flow"), levered_free_cash_flow = extractor.ExtractLong("Levered Free Cash Flow"), StockName = stock, scrape_day = DateTime.Now.Date }; madness.AddToCompanyStats(stats); madness.SaveChanges(); }