public void Init(bool forTraining)
{
Console.WriteLine("Retrieving madness from database...");
ModelMadness madness = new ModelMadness();
madness.CommandTimeout = 0;
simplestockquote[] stats;
if (forTraining)
{
stats = (madness.simplestockquotes.Where(q => q.day > new DateTime(2006, 1, 1))
.OrderBy(q => q.day))
.ToArray();
}
else
{
stats = (madness.simplestockquotes.Where(q => q.day > new DateTime(2010, 1, 1) &&
q.day <= new DateTime(2010, 12, 09))
.OrderBy(q => q.day))
.ToArray();
}
long lastDay = 0;
DataForDay currentDay = null;
// these are in order, from oldest to newest
foreach (simplestockquote stat in stats)
{
if (lastDay != stat.day.Ticks)
{
lastDay = stat.day.Ticks;
currentDay = new DataForDay();
Console.WriteLine("Analyzing Day " + stat.day.ToString("D"));
DayCount++;
SnapshotByDay[DayCount] = currentDay;
}
currentDay.AddStat(stat);
}
Console.WriteLine("Done retrieving");
}
public IList <WorkableStockInfo> GetCurrentData()
{
IList <WorkableStockInfo> toReturn = new List <WorkableStockInfo>();
DataForDay dataToday = SnapshotByDay[SnapshotByDay.Count - 1];
DataForDay data5DaysAgo = SnapshotByDay[SnapshotByDay.Count - 6];
DataForDay data15DaysAgo = SnapshotByDay[SnapshotByDay.Count - 16];
DataForDay data25DaysAgo = SnapshotByDay[SnapshotByDay.Count - 26];
DataForDay data50DaysAgo = SnapshotByDay[SnapshotByDay.Count - 51];
Console.WriteLine("Creating Prediction Data View");
foreach (simplestockquote stockInfo in dataToday.ByStockID.Values)
{
// don't add stocks to the training data if we don't have past info
if (!data5DaysAgo.ByStockID.ContainsKey(stockInfo.id))
{
continue;
}
if (!data15DaysAgo.ByStockID.ContainsKey(stockInfo.id))
{
continue;
}
if (!data25DaysAgo.ByStockID.ContainsKey(stockInfo.id))
{
continue;
}
if (!data50DaysAgo.ByStockID.ContainsKey(stockInfo.id))
{
continue;
}
var workableStockInfo = new WorkableStockInfo
{
ViewToday = stockInfo,
View5DaysAgo = data5DaysAgo.ByStockID[stockInfo.id],
View15DaysAgo = data15DaysAgo.ByStockID[stockInfo.id],
View25DaysAgo = data25DaysAgo.ByStockID[stockInfo.id],
View50DaysAgo = data50DaysAgo.ByStockID[stockInfo.id],
IndustryAverage = dataToday.IndustryAVG[(int)stockInfo.industry_id].GetAvg(),
IndustryAverage5DaysAgo = data5DaysAgo.IndustryAVG[(int)stockInfo.industry_id].GetAvg(),
IndustryAverage15DaysAgo = data15DaysAgo.IndustryAVG[(int)stockInfo.industry_id].GetAvg(),
IndustryAverage25DaysAgo = data25DaysAgo.IndustryAVG[(int)stockInfo.industry_id].GetAvg(),
IndustryAverage50DaysAgo = data50DaysAgo.IndustryAVG[(int)stockInfo.industry_id].GetAvg(),
SectorAverage = dataToday.SectorAVG[(int)stockInfo.sector_id].GetAvg(),
SectorAverage5DaysAgo = data5DaysAgo.SectorAVG[(int)stockInfo.sector_id].GetAvg(),
SectorAverage15DaysAgo = data15DaysAgo.SectorAVG[(int)stockInfo.sector_id].GetAvg(),
SectorAverage25DaysAgo = data25DaysAgo.SectorAVG[(int)stockInfo.sector_id].GetAvg(),
SectorAverage50DaysAgo = data50DaysAgo.SectorAVG[(int)stockInfo.sector_id].GetAvg(),
PEIndustryDevs = dataToday.IndustryPE[(int)stockInfo.industry_id].GetStandardDevs((double)stockInfo.forward_pe),
VolumeIndustryDevs = dataToday.IndustryVolume[(int)stockInfo.industry_id].GetStandardDevs(stockInfo.volume),
ExecAgeIndustryDevs = dataToday.IndustryAvgExecAge[(int)stockInfo.industry_id].GetStandardDevs((double)stockInfo.avg_executive_age),
EmployeesIndustryDevs = dataToday.IndustryEmployees[(int)stockInfo.industry_id].GetStandardDevs((double)stockInfo.num_employees),
TotalDebtIndustryDevs = dataToday.IndustryTotalDebt[(int)stockInfo.industry_id].GetStandardDevs((double)stockInfo.total_debt),
NetIncomeIndustryDevs = dataToday.IndustryNetIncome[(int)stockInfo.industry_id].GetStandardDevs((double)stockInfo.net_income_a_c),
EnterpriseValueIndDevs = dataToday.IndustryEntValue[(int)stockInfo.industry_id].GetStandardDevs((double)stockInfo.enterprise_value),
IsBasicMaterials = stockInfo.sector_id == 102,
IsConglomerates = stockInfo.sector_id == 109,
IsConsumerGoods = stockInfo.sector_id == 106,
IsFinancial = stockInfo.sector_id == 104,
IsHealthcare = stockInfo.sector_id == 107,
IsIndustrialGoods = stockInfo.sector_id == 105,
IsServices = stockInfo.sector_id == 103,
IsTechnology = stockInfo.sector_id == 101,
IsUtilities = stockInfo.sector_id == 108,
WeatherChange = (int)stockInfo.from_avg
};
toReturn.Add(workableStockInfo);
}
return(toReturn);
}
public IList <WorkableStockInfo> GetTrainingData()
{
IList <WorkableStockInfo> toReturn = new List <WorkableStockInfo>();
// don't use the first 50 or the last 20 - first ones won't have necessary historical view, and last ones won't have prediction
for (int i = 50; i < SnapshotByDay.Count - 20; i++)
{
Console.WriteLine("Creating Training Data View for " + i + " of " + (SnapshotByDay.Count - 20));
DataForDay dataToday = SnapshotByDay[i];
DataForDay data5DaysAgo = SnapshotByDay[i - 5];
DataForDay data15DaysAgo = SnapshotByDay[i - 15];
DataForDay data25DaysAgo = SnapshotByDay[i - 25];
DataForDay data50DaysAgo = SnapshotByDay[i - 50];
DataForDay data20DaysFuture = SnapshotByDay[i + 20];
foreach (simplestockquote stockInfo in dataToday.ByStockID.Values)
{
// don't add stocks to the training data if we don't have past info
if (!data5DaysAgo.ByStockID.ContainsKey(stockInfo.id))
{
continue;
}
if (!data15DaysAgo.ByStockID.ContainsKey(stockInfo.id))
{
continue;
}
if (!data25DaysAgo.ByStockID.ContainsKey(stockInfo.id))
{
continue;
}
if (!data50DaysAgo.ByStockID.ContainsKey(stockInfo.id))
{
continue;
}
// likewise, don't use stocks without future info
if (!data20DaysFuture.ByStockID.ContainsKey(stockInfo.id))
{
continue;
}
if (!(data5DaysAgo.IndustryAVG.ContainsKey((int)stockInfo.industry_id) &&
data15DaysAgo.IndustryAVG.ContainsKey((int)stockInfo.industry_id) &&
data25DaysAgo.IndustryAVG.ContainsKey((int)stockInfo.industry_id) &&
data50DaysAgo.IndustryAVG.ContainsKey((int)stockInfo.industry_id)))
{
continue;
}
var workableStockInfo = new WorkableStockInfo
{
ViewToday = stockInfo,
View5DaysAgo = data5DaysAgo.ByStockID[stockInfo.id],
View15DaysAgo = data15DaysAgo.ByStockID[stockInfo.id],
View25DaysAgo = data25DaysAgo.ByStockID[stockInfo.id],
View50DaysAgo = data50DaysAgo.ByStockID[stockInfo.id],
IndustryAverage = dataToday.IndustryAVG[(int)stockInfo.industry_id].GetAvg(),
IndustryAverage5DaysAgo = data5DaysAgo.IndustryAVG[(int)stockInfo.industry_id].GetAvg(),
IndustryAverage15DaysAgo = data15DaysAgo.IndustryAVG[(int)stockInfo.industry_id].GetAvg(),
IndustryAverage25DaysAgo = data25DaysAgo.IndustryAVG[(int)stockInfo.industry_id].GetAvg(),
IndustryAverage50DaysAgo = data50DaysAgo.IndustryAVG[(int)stockInfo.industry_id].GetAvg(),
SectorAverage = dataToday.SectorAVG[(int)stockInfo.sector_id].GetAvg(),
SectorAverage5DaysAgo = data5DaysAgo.SectorAVG[(int)stockInfo.sector_id].GetAvg(),
SectorAverage15DaysAgo = data15DaysAgo.SectorAVG[(int)stockInfo.sector_id].GetAvg(),
SectorAverage25DaysAgo = data25DaysAgo.SectorAVG[(int)stockInfo.sector_id].GetAvg(),
SectorAverage50DaysAgo = data50DaysAgo.SectorAVG[(int)stockInfo.sector_id].GetAvg(),
PEIndustryDevs = dataToday.IndustryPE[(int)stockInfo.industry_id].GetStandardDevs((double)stockInfo.forward_pe),
VolumeIndustryDevs = dataToday.IndustryVolume[(int)stockInfo.industry_id].GetStandardDevs(stockInfo.volume),
ExecAgeIndustryDevs = dataToday.IndustryAvgExecAge[(int)stockInfo.industry_id].GetStandardDevs((double)stockInfo.avg_executive_age),
EmployeesIndustryDevs = dataToday.IndustryEmployees[(int)stockInfo.industry_id].GetStandardDevs((double)stockInfo.num_employees),
TotalDebtIndustryDevs = dataToday.IndustryTotalDebt[(int)stockInfo.industry_id].GetStandardDevs((double)stockInfo.total_debt),
NetIncomeIndustryDevs = dataToday.IndustryNetIncome[(int)stockInfo.industry_id].GetStandardDevs((double)stockInfo.net_income_a_c),
EnterpriseValueIndDevs = dataToday.IndustryEntValue[(int)stockInfo.industry_id].GetStandardDevs((double)stockInfo.enterprise_value),
IsBasicMaterials = stockInfo.sector_id == 102,
IsConglomerates = stockInfo.sector_id == 109,
IsConsumerGoods = stockInfo.sector_id == 106,
IsFinancial = stockInfo.sector_id == 104,
IsHealthcare = stockInfo.sector_id == 107,
IsIndustrialGoods = stockInfo.sector_id == 105,
IsServices = stockInfo.sector_id == 103,
IsTechnology = stockInfo.sector_id == 101,
IsUtilities = stockInfo.sector_id == 108,
WeatherChange = (int)stockInfo.from_avg,
AdjCloseNext20 = data20DaysFuture.ByStockID[stockInfo.id].adjusted_close
};
toReturn.Add(workableStockInfo);
}
}
return(toReturn);
}
