private static void CalculateSampleStatsPerRegime(SampleStatsPerRegime p_statsPerRegime, double p_pctChgTotalAMean)
{
for (int i = 0; i <= 16; i++) // write only from T-17 to T+17
{
CalculateSampleStatsPerDayOffset(ref p_statsPerRegime.TotMForward[i], p_pctChgTotalAMean);
CalculateSampleStatsPerDayOffset(ref p_statsPerRegime.TotMBackward[i], p_pctChgTotalAMean);
CalculateSampleStatsPerDayOffset(ref p_statsPerRegime.TotMidMForward[i], p_pctChgTotalAMean);
CalculateSampleStatsPerDayOffset(ref p_statsPerRegime.TotMidMBackward[i], p_pctChgTotalAMean);
}
}
private void RenderStatsForDisplaying(string p_title, SampleStatsPerRegime p_stats, bool p_isHtml, StringBuilder p_sbStats)
{
RenderStatsForDisplaying(p_title + ", TotM", p_stats.TotMForward, p_stats.TotMBackward, p_isHtml, p_sbStats);
RenderStatsForDisplaying(p_title + ", TotMidM", p_stats.TotMidMForward, p_stats.TotMidMBackward, p_isHtml, p_sbStats);
}
private SampleStatsPerRegime CreateSampleStatsPerRegime(string p_name, int p_estimatedNsamplesPerDayOffset)
{
SampleStatsPerRegime stats = new SampleStatsPerRegime() { Name = p_name, TotMForward = new SampleStats[cMaxDayOffset], TotMBackward = new SampleStats[cMaxDayOffset], TotMidMForward = new SampleStats[cMaxDayOffset], TotMidMBackward = new SampleStats[cMaxDayOffset] };
for (int i = 0; i < cMaxDayOffset; i++)
{
stats.TotMForward[i].Name = "TotM+";
stats.TotMBackward[i].Name = "TotM-";
stats.TotMidMForward[i].Name = "TotMidM+";
stats.TotMidMBackward[i].Name = "TotMidM-";
stats.TotMForward[i].DayOffset = i + 1; // 1= T+1, 1 based
stats.TotMBackward[i].DayOffset = i + 1;
stats.TotMidMForward[i].DayOffset = i + 1;
stats.TotMidMBackward[i].DayOffset = i + 1;
stats.TotMForward[i].Samples = new List<Tuple<DateTime, double>>(p_estimatedNsamplesPerDayOffset);
stats.TotMBackward[i].Samples = new List<Tuple<DateTime, double>>(p_estimatedNsamplesPerDayOffset);
stats.TotMidMForward[i].Samples = new List<Tuple<DateTime, double>>(p_estimatedNsamplesPerDayOffset);
stats.TotMidMBackward[i].Samples = new List<Tuple<DateTime, double>>(p_estimatedNsamplesPerDayOffset);
}
// in BackTester, the mask as string went into this function, but it is not needed just yet.
// setting up stats.TotMForward[i].IsBullish in the QuickTester followed here. (based on Mask to be played)
// only needed in Backtests, not in Significance analysis of the day, so maybe create a separate variable for this later in QuickTester2, so that the code base can be the same.
return stats;
}
private bool PrepareHistoricalStatsForAllRegimes()
{
DateTime pvStartDate = m_spy[0].Date;
DateTime pvEndDate = m_spy[m_spy.Length - 1].Date;
// maybe (maybe not) put the Today date and tomorrow date too into m_spy. (but don't use them as samples for Training) That way we will easily know if tomorrow is TotM + ? or TotMidM + ?
// or calculate it in any other way. Create a Separate function: GetAllOffset(DateTime p_day), but that will be very inefficient to do it for 6000 items in the array. So, just use it for Tomorrow.
// or just put tomorrow into the sample temporarily.
// 1.1 calculate totMForwardDayOffset
DateTime iDate = new DateTime(pvStartDate.Year, pvStartDate.Month, 1); // Time is 00:00, which is OK, but strangely it is visualized as "12:00 AM", but yeah. Correct. it is not noon.
iDate = DbUtils.GetNextUsaMarketOpenDayLoc(iDate, true); // this is day T+1
int iDateOffset = 1; // T+1
while (iDate < pvStartDate) // marching forward until iDate = startDate
{
iDate = DbUtils.GetNextUsaMarketOpenDayLoc(iDate, false);
iDateOffset++;
}
m_spy[0].TotMForwardOffset = iDateOffset;
for (int i = 1; i < m_spy.Length; i++) // march over on p_quotes, not pv
{
if (m_spy[i].Date.Month != m_spy[i - 1].Date.Month)
iDateOffset = 1; // T+1
else
iDateOffset++;
m_spy[i].TotMForwardOffset = iDateOffset;
}
// 1.2 calculate totMBackwardDayOffset
iDate = new DateTime(pvEndDate.Year, pvEndDate.Month, 1);
iDate = iDate.AddMonths(1); // next month can be in the following year; this is the first calendar day of the next month
iDate = DbUtils.GetPreviousUsaMarketOpenDayLoc(iDate, false); // this is day T-1
iDateOffset = 1; // T-1
while (iDate > pvEndDate) // marching backward until iDate == endDate
{
iDate = DbUtils.GetPreviousUsaMarketOpenDayLoc(iDate, false);
iDateOffset++;
}
m_spy[m_spy.Length - 1].TotMBackwardOffset = iDateOffset; // last day (today) is set
for (int i = m_spy.Length - 2; i >= 0; i--) // march over on p_quotes, not pv
{
if (m_spy[i].Date.Month != m_spy[i + 1].Date.Month) // what if market closes for 3 months (or we don't have the data in DB)
iDateOffset = 1; // T-1
else
iDateOffset++;
m_spy[i].TotMBackwardOffset = iDateOffset;
}
// 1.3 calculate totMidMForwardDayOffset
iDate = new DateTime(pvStartDate.Year, pvStartDate.Month, 15);
if (iDate > pvStartDate)
iDate = iDate.AddMonths(-1);
iDate = DbUtils.GetNextUsaMarketOpenDayLoc(iDate, true); // // this is day T+1
iDateOffset = 1; // T+1
while (iDate < pvStartDate) // marching forward until iDate = startDate
{
iDate = DbUtils.GetNextUsaMarketOpenDayLoc(iDate, false);
iDateOffset++;
}
m_spy[0].TotMidMForwardOffset = iDateOffset;
for (int i = 1; i < m_spy.Length; i++) // march over on p_quotes, not pv
{
if (((m_spy[i].Date.Month == m_spy[i - 1].Date.Month) && m_spy[i].Date.Day >= 15 && m_spy[i - 1].Date.Day < 15) || // what if market closes for 3 months (or we don't have the data in DB)
(m_spy[i].Date.Month != m_spy[i - 1].Date.Month) && m_spy[i].Date.Day >= 15) // if some months are skipped from data
iDateOffset = 1; // T+1
else
iDateOffset++;
m_spy[i].TotMidMForwardOffset = iDateOffset;
}
// 1.4 calculate totMBackwardDayOffset
iDate = new DateTime(pvEndDate.Year, pvEndDate.Month, 15);
if (iDate <= pvEndDate)
iDate = iDate.AddMonths(1); // next month can be in the following year; better to use AddMonths();
iDate = DbUtils.GetPreviousUsaMarketOpenDayLoc(iDate, false); // this is day T-1
iDateOffset = 1; // T-1
while (iDate > pvEndDate) // marching backward until iDate == endDate
{
iDate = DbUtils.GetPreviousUsaMarketOpenDayLoc(iDate, false);
iDateOffset++;
}
m_spy[m_spy.Length - 1].TotMidMBackwardOffset = iDateOffset; // last day (today) is set
for (int i = m_spy.Length - 2; i >= 0; i--) // march over on p_quotes, not pv
{
if (((m_spy[i].Date.Month == m_spy[i + 1].Date.Month) && m_spy[i].Date.Day < 15 && m_spy[i + 1].Date.Day >= 15) || // what if market closes for 3 months (or we don't have the data in DB)
(m_spy[i].Date.Month != m_spy[i + 1].Date.Month) && m_spy[i].Date.Day < 15) // if some months are skipped from data
iDateOffset = 1; // T-1
else
iDateOffset++;
m_spy[i].TotMidMBackwardOffset = iDateOffset;
}
// in BackTester, the mask as string went into this function, but it is not needed just yet.
// setting up stats.TotMForward[i].IsBullish in the QuickTester followed here. (based on Mask to be played)
// only needed in Backtests, not in Significance analysis of the day, so maybe create a separate variable for this later in QuickTester2, so that the code base can be the same.
m_winterStats = CreateSampleStatsPerRegime("Winter", (int)(m_spy.Length / 260.0 * 7.0 * 1.1)); // give a little 10% overhead, so List<> will be not re-allocated many times
m_summerStats = CreateSampleStatsPerRegime("Summer", (int)(m_spy.Length / 260.0 * 6.0 * 1.1)); // give a little 10% overhead, so List<> will be not re-allocated many times
m_allYearStats = CreateSampleStatsPerRegime("AllYear", (int)(m_spy.Length/260.0*12.0*1.1)); // give a little 10% overhead, so List<> will be not re-allocated many times
double outlierBasicZscore_PctThreshold = Double.NaN;
int nNegativeOutliers = 0, nPositiveOutliers = 0;
if (uberVxxConfig.TotM_OutlierElimination != OutlierElimination.None)
{
//-calculate StDev.and http://www.itl.nist.gov/div898/handbook/eda/section3/eda35h.htm
// "Although it is common practice to use Z-scores to identify possible outliers,
// Iglewicz and Hoaglin recommend using the modified Z - score"
//- Balazs used +8% or lower than -8% percentages threshold for VXX TotM. So determine 8% is how many StDev away,
// and if it 2.2 times, than use that for the other samples (like QQQ, which will have less StDev)
//-(not likely, but) double check, that max. only 5 or 6% of the samples are eliminated. If 10% of the samples are eliminated
// as outliers. (10 out of 100), then those are not random outliers.
double n = (double)m_spy.Length;
List<double> samples = m_spy.Select(r => r.PctChg).ToList();
double aMean = samples.Average();
double correctedStDev = Math.Sqrt(samples.Sum(r => (r - aMean) * (r - aMean)) / (n - 1.0)); // it is about 1.1% for the SPX, http://www.investopedia.com/articles/04/021804.asp
outlierBasicZscore_PctThreshold = correctedStDev * uberVxxConfig.OutlierBasicZscore_Zscore;
}
// create 2 lists, a Forward list, a backward list (maybe later to test day T+12..T+16) Jay's "Monthly 10", which is 4 days in the middle month
double pctChgUsedSamplesTotal = 0.0;
for (int i = 0; i < m_spy.Length; i++) // march over on p_quotes, not pv
{
DateTime day = m_spy[i].Date;
double pctChg = m_spy[i].PctChg;
if (uberVxxConfig.TotM_OutlierElimination == OutlierElimination.BasicZscore)
{
if (Math.Abs(pctChg) > outlierBasicZscore_PctThreshold)
{
if (pctChg > 0)
nPositiveOutliers++;
else if (pctChg < 0)
nNegativeOutliers++;
continue;
}
}
pctChgUsedSamplesTotal += pctChg;
int totMForwardInd = m_spy[i].TotMForwardOffset, totMBackwardInd = m_spy[i].TotMBackwardOffset, totMidMForwardInd = m_spy[i].TotMidMForwardOffset, totMidMBackwardInd = m_spy[i].TotMidMBackwardOffset;
m_allYearStats.TotMForward[totMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
m_allYearStats.TotMBackward[totMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
m_allYearStats.TotMidMForward[totMidMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
m_allYearStats.TotMidMBackward[totMidMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
if (IsBullishWinterDay(day))
{
m_winterStats.TotMForward[totMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
m_winterStats.TotMBackward[totMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
m_winterStats.TotMidMForward[totMidMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
m_winterStats.TotMidMBackward[totMidMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
}
else
{
m_summerStats.TotMForward[totMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
m_summerStats.TotMBackward[totMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
m_summerStats.TotMidMForward[totMidMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
m_summerStats.TotMidMBackward[totMidMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
}
}
if (uberVxxConfig.TotM_OutlierElimination != OutlierElimination.None)
{
//Maybe AdvancedOutlier elimination is not even needed because with the BasicZscore, here are how many samples are eliminated:
//OutlierBasicZscore_Zscore = 2.7: SPY outliers skipped at 3.21 %.Pos:70,Neg: 57, 2.17 % of samples.
//OutlierBasicZscore_Zscore = 4.0: SPY outliers skipped at 4.76 %.Pos:17,Neg: 18, 0.6 % of samples.
//OutlierBasicZscore_Zscore = 5.0: SPY outliers skipped at 5.95 %.Pos:8,Neg: 9, 0.29 % of samples.
//It is very even.Because there are big panic days, but there are buy upside days too. e.g.: Oct 13, 2008: up + 14.5 %
double pctOutliers = ((double)(nPositiveOutliers + nNegativeOutliers) / m_spy.Length);
//Console.WriteLine($"SPY outliers skipped at {outlierBasicZscore_PctThreshold * 100:0.##}%. Pos:{nPositiveOutliers},Neg:{nNegativeOutliers}, {pctOutliers * 100.0:0.##}% of samples.");
Utils.Logger.Info($"SPY outliers skipped at {outlierBasicZscore_PctThreshold * 100:0.##}%. Pos:{nPositiveOutliers},Neg:{nNegativeOutliers}, {pctOutliers * 100.0:0.##}% of samples.");
StrongAssert.True(pctOutliers < 0.05, Severity.NoException, "If 5%+ of the samples are eliminated, that means they are not random outliers. This is unexpected.");
}
int nUsedSamples = m_spy.Length - nNegativeOutliers - nPositiveOutliers;
double pctChgTotalAMean = (nUsedSamples <= 0) ? 0.0 : pctChgUsedSamplesTotal / (double)(nUsedSamples);
CalculateSampleStatsPerRegime(m_winterStats, pctChgTotalAMean);
CalculateSampleStatsPerRegime(m_summerStats, pctChgTotalAMean);
CalculateSampleStatsPerRegime(m_allYearStats, pctChgTotalAMean);
StringBuilder sbStats = new StringBuilder(Environment.NewLine); // looks better in the log file if it starts in a blank new line
RenderStatsForDisplaying("Winter", m_winterStats, false, sbStats);
RenderStatsForDisplaying("Summer", m_summerStats, false, sbStats);
RenderStatsForDisplaying("AllYear", m_allYearStats, false, sbStats);
Utils.Logger.Info(sbStats.ToString());
return true;
}