private static void GetWeatherForDay(DateTime day, ModelMadness db) { string url = "http://www.wunderground.com/history/airport/KNYC/`/DailyHistory.html?req_city=New+York&req_state=NY&req_statename=New+York&MR=1"; url = url.Replace("`", day.Year + "/" + day.Month + "/" + day.Day); AgentAction action = new AgentAction(url, false); AgentDocument document = AgentHandler.Instance.PerformAction(new AgentSession(), action); int startIdx = document.ResponseString.IndexOf("Mean Temperature"); startIdx = document.ResponseString.IndexOf("<span class=\"b\">", startIdx) + "<span class=\"b\">".Length; int endidx = document.ResponseString.IndexOf("<", startIdx); int actual = int.Parse(document.ResponseString.Substring(startIdx, endidx - startIdx)); startIdx = document.ResponseString.IndexOf("<span class=\"b\">", startIdx) + "<span class=\"b\">".Length; endidx = document.ResponseString.IndexOf("<", startIdx); int average = int.Parse(document.ResponseString.Substring(startIdx, endidx - startIdx)); Weather weather = new Weather { day = day, from_avg = actual - average }; db.AddToWeathers(weather); db.SaveChanges(); Console.WriteLine(day.ToString("d") + " was off by " + (actual - average) + " degrees on wall street"); }
public void Debug(string message) { ModelMadness madness = new ModelMadness(); Log log = new Log { message = message, transaction_date = DateTime.Now }; madness.AddToLogs(log); madness.SaveChanges(); }
public void Error(string message, Exception ex) { ModelMadness madness = new ModelMadness(); Log log = new Log { message = message, exception_message = ex.Message, exception_stacktrace = ex.StackTrace, transaction_date = DateTime.Now }; madness.AddToLogs(log); madness.SaveChanges(); }
public static void Go() { var encog = new EncogPersistedCollection("market-network.dat", FileMode.Open); Console.WriteLine(@"Loading network"); var network = (BasicNetwork)encog.Find("market-network"); Console.WriteLine(@"Reading current data from db"); var market = new StockMarket(); market.Init(false); var data = market.GetCurrentData(); Console.WriteLine(@"Running network on data"); var madness = new ModelMadness(); foreach (StockMarket.WorkableStockInfo info in data) { var input = InputOutputMadness.CreateInputs(info); var neuralInput = new BasicMLData(input); var output = network.Compute(neuralInput); Console.WriteLine(@"Stock " + info.ViewToday.stock + @" will change " + output[0] + @"% in the next 20 trading days"); var future = new prediction { day = DateTime.Now.Date, C20_Days_Close = 100 * (decimal)output[0], stock = info.ViewToday.stock }; madness.AddTopredictions(future); } madness.SaveChanges(); Console.WriteLine(@"Done - begin making $millions"); }
public static void PerformOilSlick() { Console.WriteLine("Reading weekly oil prices"); var document = AgentHandler.Instance.PerformAction(new AgentSession(), new AgentAction("http://www.nyse.tv/crude-oil-price-history.htm", false)); var match = OilSlicker.Match(document.ResponseString); while (match.Success) { var month = match.Result("$1"); var day = match.Result("$2"); var year = match.Result("$3"); var price = decimal.Parse(match.Result("$4")); var date = (DateTime) new DateTimeConverter().ConvertFromString(month + " " + day + ", " + year); if (DB.CrudeOils.Any(x => x.Day == date)) { Console.WriteLine(date.ToString("MM/dd/yyyy") + " already in database"); } else { Console.WriteLine("Saving price for " + date.ToString("MM/dd/yyyy") + " - " + price.ToString("c")); DB.AddToCrudeOils(new CrudeOil { Day = date, price_at_close = price }); } match = match.NextMatch(); } DB.SaveChanges(true); Console.WriteLine("Done"); }
public static void PullQuotes(DateTime start, DateTime stop, string stockName) { ModelMadness madness = new ModelMadness(); StockName stock = (from q in madness.StockNames where q.Stock == stockName select q).First(); Console.WriteLine("Working on " + stock.Stock); //http://ichart.finance.yahoo.com/table.csv?s=TKR&a=00&b=1&c=2000&d=01&e=18&f=2010&g=d&ignore=.csv string url = @"http://ichart.finance.yahoo.com/table.csv?s=" + stock.Stock + "&a=" + TwoDig(start.Month - 1) + "&b=" + start.Day + "&c=" + start.Year + "&d=" + TwoDig(stop.Month - 1) + "&e=" + stop.Day + "&f=" + stop.Year + "&g=d&ignore=.csv"; AgentSession session = new AgentSession(); AgentAction action = new AgentAction(url, false); bool found = false; AgentDocument document = null; try { document = AgentHandler.Instance.PerformAction(session, action); found = true; } catch (Exception ex) { Console.WriteLine("ERROR: " + url + " - " + ex.Message); } if (!found) { return; } //Date,Open,High,Low,Close,Volume,Adj Close string[] rows = document.ResponseString.Split('\n'); if (rows.Length < 2) { return; } for (int i = 1; i < rows.Length; i++) { string row = rows[i]; string[] fields = row.Split(','); if (fields.Length < 7) { Console.WriteLine((i - 2) + " records stored"); continue; } quote q = new quote { day = (DateTime) new DateTimeConverter().ConvertFrom(fields[0]), open = decimal.Parse(fields[1]), high = decimal.Parse(fields[2]), low = decimal.Parse(fields[3]), close = decimal.Parse(fields[4]), volume = long.Parse(fields[5]), adjusted_close = decimal.Parse(fields[6]), StockName = stock }; madness.AddToquotes(q); } madness.SaveChanges(); }
private static void SavePage(string raptureURL) { AgentAction action = new AgentAction(raptureURL, false); AgentDocument document = null; int errorCount = 0; while (errorCount < 5) { try { document = AgentHandler.Instance.PerformAction(new AgentSession(), action); break; } catch (Exception ex) { errorCount++; } } int startIdx = document.ResponseString.IndexOf("Rapture Index ") + "Rapture Index ".Length; int endIdx = document.ResponseString.IndexOf("<", startIdx); int raptureIdx = int.Parse(document.ResponseString.Substring(startIdx, endIdx - startIdx)); startIdx = document.ResponseString.IndexOf("Updated ") + "Updated ".Length; endIdx = document.ResponseString.IndexOf("<", startIdx); string dateString = document.ResponseString.Substring(startIdx, endIdx - startIdx); dateString = dateString.Replace("Sept", "Sep"); DateTime raptureDate = (DateTime) new DateTimeConverter().ConvertFrom(dateString); if (!DB.RaptureIndexes.Where(x => x.rapture_day.Equals(raptureDate)).Any()) { int oil = GetRaptureAmount(document.ResponseString, "Oil Supply", dateString); int debt = GetRaptureAmount(document.ResponseString, "Debt and Trade", dateString); int weather = GetRaptureAmount(document.ResponseString, "Wild Weather", dateString); int globalism = GetRaptureAmount(document.ResponseString, "Globalism", dateString); int economy = GetRaptureAmount(document.ResponseString, "Economy", dateString); Console.WriteLine("Rapture on " + raptureDate.ToString("d") + " was " + raptureIdx + " - oil: " + oil + ", debt: " + debt + ", weather: " + weather + ", globalism: " + globalism + ", economy: " + economy); RaptureIndex rapture = new RaptureIndex { rapture_day = raptureDate, rapture_index = raptureIdx, oil = oil, debt = debt, weather = weather, globalism = globalism, economy = economy }; DB.AddToRaptureIndexes(rapture); DB.SaveChanges(); } else { Console.WriteLine("Rapture already happened on " + raptureDate.ToString("d")); } }
private static void GrabProfile(string symbol) { Console.Write("Grabbing Profile for " + symbol); string url = @"http://finance.yahoo.com/q/pr?s=" + symbol; AgentSession session = new AgentSession(); AgentAction action = new AgentAction(url, false); AgentDocument document; try { document = AgentHandler.Instance.PerformAction(session, action); } catch { return; } string doc = document.ResponseString.ToLower(); var extractor = new ProgressiveDataExtractor(doc, ">details<"); if (extractor.NotFound) { Console.WriteLine(" - not found"); return; } Console.WriteLine(" - found"); ModelMadness madness = new ModelMadness(); string sector = extractor.ExtractString("sector"); Sector dbSector = null; Industry dbIndustry = null; if (sector != null && sector != "n/a") { sector = sector.Replace("&", "&"); string industry = extractor.ExtractString("industry").Replace("&", "&"); var obj = (from q in madness.Sectors where q.Sector1 == sector select q).ToArray(); if (obj.Length == 1) { dbSector = obj[0]; } else { dbSector = new Sector { Sector1 = sector }; madness.AddToSectors(dbSector); } var obj2 = (from q in madness.Industries where q.Sector.Sector1 == sector && q.Iindustry == industry select q).ToArray(); if (obj2.Length == 1) { dbIndustry = obj2[0]; } else { dbIndustry = new Industry { Iindustry = industry, Sector = dbSector }; madness.AddToIndustries(dbIndustry); } } StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First(); CompanyProfile profile = new CompanyProfile { Sector = dbSector, Industry = dbIndustry, num_employees = extractor.ExtractInt("full time employees"), StockName = stock, scrape_day = DateTime.Now.Date }; profile.summary = extractor.ExtractString("business summary"); profile.cgq = extractor.ExtractDecimal("corporate governance", "<b"); extractor.ExtractString("key executives"); int totalAge = 0; int numAges = 0; long totalPay = 0; int numPays = 0; int? curAge; long?curPay; do { curAge = extractor.ExtractInt("yfnc_tabledata1", "</b"); if (curAge > 111) { curAge = null; } if (curAge != null) { numAges++; totalAge += (int)curAge; curPay = extractor.ExtractLong("yfnc_tabledata1", "nowrap"); if (curPay != null) { numPays++; totalPay += (long)curPay; } } } while (curAge != null); profile.avg_executive_age = totalAge == 0 ? null : (int?)totalAge / numAges; profile.avg_executive_pay = totalPay == 0 ? null : (long?)totalPay / numPays; madness.AddToCompanyProfiles(profile); madness.SaveChanges(); }
private static void GrabStats(string symbol) { Console.Write("Grabbing Key Stats for " + symbol); string url = @"http://finance.yahoo.com/q/ks?s=" + symbol; AgentSession session = new AgentSession(); AgentAction action = new AgentAction(url, false); AgentDocument document; try { document = AgentHandler.Instance.PerformAction(session, action); } catch { return; } var doc = document.ResponseString; var extractor = new ProgressiveDataExtractor(doc, "Valuation Measures"); if (extractor.NotFound) { Console.WriteLine(" - Not found"); return; } Console.WriteLine(" - found"); ModelMadness madness = new ModelMadness(); StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First(); CompanyStat stats = new CompanyStat { market_cap = extractor.ExtractLong("Market Cap"), enterprise_value = extractor.ExtractLong("Enterprise Value"), trailing_pe = extractor.ExtractDecimal("Trailing P/E"), forward_pe = extractor.ExtractDecimal("Forward P/E"), peg_ratio = extractor.ExtractDecimal("PEG Ratio"), price_sales = extractor.ExtractDecimal("Price/Sales"), price_book = extractor.ExtractDecimal("Price/Book"), ent_value_rev = extractor.ExtractLong("Enterprise Value/Revenue"), ent_value_ebitda = extractor.ExtractDecimal("Enterprise Value/EBITDA"), fiscal_year_ends = extractor.ExtractString("Fiscal Year Ends"), most_recent_quarter = extractor.ExtractString("Most Recent Quarter"), profit_margin = extractor.ExtractDecimal("Profit Margin"), operating_margin = extractor.ExtractDecimal("Operating Margin"), return_on_assets = extractor.ExtractDecimal("Return on Assets"), return_on_equity = extractor.ExtractDecimal("Return on Equity"), revenue = extractor.ExtractLong("Revenue"), revenue_per_share = extractor.ExtractDecimal("Revenue Per Share"), qrt_rev_growth = extractor.ExtractDecimal("Qtrly Revenue Growth"), gross_profit = extractor.ExtractLong("Gross Profit"), ebitda = extractor.ExtractLong("EBITDA"), net_income_a_c = extractor.ExtractLong("Net Income Avl to Common"), diluted_eps = extractor.ExtractDecimal("Diluted EPS"), qrt_earnings_growth = extractor.ExtractDecimal("Qtrly Earnings Growth"), total_cash = extractor.ExtractLong("Total Cash"), cash_per_share = extractor.ExtractDecimal("Total Cash Per Share"), total_debt = extractor.ExtractLong("Total Debt"), total_debt_equit = extractor.ExtractDecimal("Total Debt/Equity"), current_ratio = extractor.ExtractDecimal("Current Ratio"), book_value_p_share = extractor.ExtractDecimal("Book Value Per Share"), operating_cash_flow = extractor.ExtractLong("Operating Cash Flow"), levered_free_cash_flow = extractor.ExtractLong("Levered Free Cash Flow"), StockName = stock, scrape_day = DateTime.Now.Date }; madness.AddToCompanyStats(stats); madness.SaveChanges(); }