private static void GetWeatherForDay(DateTime day, ModelMadness db)
{
string url =
"http://www.wunderground.com/history/airport/KNYC/`/DailyHistory.html?req_city=New+York&req_state=NY&req_statename=New+York&MR=1";
url = url.Replace("`", day.Year + "/" + day.Month + "/" + day.Day);
AgentAction action = new AgentAction(url, false);
AgentDocument document = AgentHandler.Instance.PerformAction(new AgentSession(), action);
int startIdx = document.ResponseString.IndexOf("Mean Temperature");
startIdx = document.ResponseString.IndexOf("<span class=\"b\">", startIdx) + "<span class=\"b\">".Length;
int endidx = document.ResponseString.IndexOf("<", startIdx);
int actual = int.Parse(document.ResponseString.Substring(startIdx, endidx - startIdx));
startIdx = document.ResponseString.IndexOf("<span class=\"b\">", startIdx) + "<span class=\"b\">".Length;
endidx = document.ResponseString.IndexOf("<", startIdx);
int average = int.Parse(document.ResponseString.Substring(startIdx, endidx - startIdx));
Weather weather = new Weather
{
day = day,
from_avg = actual - average
};
db.AddToWeathers(weather);
db.SaveChanges();
Console.WriteLine(day.ToString("d") + " was off by " + (actual - average) + " degrees on wall street");
}
private void ScrapePage(string pageName)
{
Console.WriteLine("Doing page " + pageName);
string url = @"http://www.nyse.com/about/listed/lc_ny_name_`.js".Replace("`", pageName);
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
AgentDocument document = AgentHandler.Instance.PerformAction(session, action);
string[] segments = document.ResponseString.Split(']');
foreach (string segment in segments)
{
int startIdx = segment.IndexOf("\"") + 1;
if (startIdx == 0)
{
continue;
}
int endIdx = segment.IndexOf("\"", startIdx);
string stockTicker = segment.Substring(startIdx, endIdx - startIdx);
startIdx = segment.IndexOf("\"", endIdx + 1) + 1;
endIdx = segment.IndexOf("\"", startIdx);
string companyName = segment.Substring(startIdx, endIdx - startIdx);
startIdx = segment.IndexOf("\"", endIdx + 1) + 1;
endIdx = segment.IndexOf("\"", startIdx);
string country = segment.Substring(startIdx, endIdx - startIdx);
;
StockName name = new StockName
{
Stock = stockTicker,
Company_Name = companyName,
Country = country,
Exchange = "NYSE"
};
if (!Madness.StockNames.Any(x => x.Stock == stockTicker))
{
Console.WriteLine("Found new stock: " + stockTicker);
Madness.AddToStockNames(name);
}
}
}
public static void Historical()
{
AgentAction action = new AgentAction("http://web.archive.org/web/*/http://raptureready.com/rap2.html", false);
AgentDocument document = AgentHandler.Instance.PerformAction(new AgentSession(), action);
string indexPage = document.ResponseString;
int startIdx = indexPage.IndexOf("Feb 13, 2004");
startIdx = indexPage.IndexOf("<a href=\"", startIdx) + 9;
int stopIdx = indexPage.IndexOf("TEXT FOLLOWING URL");
while (startIdx < stopIdx)
{
int endIdx = indexPage.IndexOf("\"", startIdx);
string dateURL = indexPage.Substring(startIdx, endIdx - startIdx);
SavePage(dateURL);
startIdx = indexPage.IndexOf("<a href=\"", startIdx) + 9;
}
}
public static void PullQuotes(DateTime start, DateTime stop, string stockName)
{
ModelMadness madness = new ModelMadness();
StockName stock = (from q in madness.StockNames where q.Stock == stockName select q).First();
Console.WriteLine("Working on " + stock.Stock);
//http://ichart.finance.yahoo.com/table.csv?s=TKR&a=00&b=1&c=2000&d=01&e=18&f=2010&g=d&ignore=.csv
string url = @"http://ichart.finance.yahoo.com/table.csv?s=" +
stock.Stock + "&a=" + TwoDig(start.Month - 1) + "&b=" + start.Day + "&c=" + start.Year +
"&d=" + TwoDig(stop.Month - 1) + "&e=" + stop.Day + "&f=" + stop.Year +
"&g=d&ignore=.csv";
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
bool found = false;
AgentDocument document = null;
try
{
document = AgentHandler.Instance.PerformAction(session, action);
found = true;
}
catch (Exception ex)
{
Console.WriteLine("ERROR: " + url + " - " + ex.Message);
}
if (!found)
{
return;
}
//Date,Open,High,Low,Close,Volume,Adj Close
string[] rows = document.ResponseString.Split('\n');
if (rows.Length < 2)
{
return;
}
for (int i = 1; i < rows.Length; i++)
{
string row = rows[i];
string[] fields = row.Split(',');
if (fields.Length < 7)
{
Console.WriteLine((i - 2) + " records stored");
continue;
}
quote q = new quote
{
day = (DateTime) new DateTimeConverter().ConvertFrom(fields[0]),
open = decimal.Parse(fields[1]),
high = decimal.Parse(fields[2]),
low = decimal.Parse(fields[3]),
close = decimal.Parse(fields[4]),
volume = long.Parse(fields[5]),
adjusted_close = decimal.Parse(fields[6]),
StockName = stock
};
madness.AddToquotes(q);
}
madness.SaveChanges();
}
private static void SavePage(string raptureURL)
{
AgentAction action = new AgentAction(raptureURL, false);
AgentDocument document = null;
int errorCount = 0;
while (errorCount < 5)
{
try
{
document = AgentHandler.Instance.PerformAction(new AgentSession(), action);
break;
}
catch (Exception ex)
{
errorCount++;
}
}
int startIdx = document.ResponseString.IndexOf("Rapture Index ") + "Rapture Index ".Length;
int endIdx = document.ResponseString.IndexOf("<", startIdx);
int raptureIdx = int.Parse(document.ResponseString.Substring(startIdx, endIdx - startIdx));
startIdx = document.ResponseString.IndexOf("Updated ") + "Updated ".Length;
endIdx = document.ResponseString.IndexOf("<", startIdx);
string dateString = document.ResponseString.Substring(startIdx, endIdx - startIdx);
dateString = dateString.Replace("Sept", "Sep");
DateTime raptureDate = (DateTime) new DateTimeConverter().ConvertFrom(dateString);
if (!DB.RaptureIndexes.Where(x => x.rapture_day.Equals(raptureDate)).Any())
{
int oil = GetRaptureAmount(document.ResponseString, "Oil Supply", dateString);
int debt = GetRaptureAmount(document.ResponseString, "Debt and Trade", dateString);
int weather = GetRaptureAmount(document.ResponseString, "Wild Weather", dateString);
int globalism = GetRaptureAmount(document.ResponseString, "Globalism", dateString);
int economy = GetRaptureAmount(document.ResponseString, "Economy", dateString);
Console.WriteLine("Rapture on " + raptureDate.ToString("d") + " was " + raptureIdx + " - oil: " + oil + ", debt: " + debt +
", weather: " + weather + ", globalism: " + globalism + ", economy: " + economy);
RaptureIndex rapture = new RaptureIndex
{
rapture_day = raptureDate,
rapture_index = raptureIdx,
oil = oil,
debt = debt,
weather = weather,
globalism = globalism,
economy = economy
};
DB.AddToRaptureIndexes(rapture);
DB.SaveChanges();
}
else
{
Console.WriteLine("Rapture already happened on " + raptureDate.ToString("d"));
}
}
