private static void CalculateSampleStatsPerRegime(SampleStatsPerRegime p_statsPerRegime, double p_pctChgTotalAMean) { for (int i = 0; i <= 16; i++) // write only from T-17 to T+17 { CalculateSampleStatsPerDayOffset(ref p_statsPerRegime.TotMForward[i], p_pctChgTotalAMean); CalculateSampleStatsPerDayOffset(ref p_statsPerRegime.TotMBackward[i], p_pctChgTotalAMean); CalculateSampleStatsPerDayOffset(ref p_statsPerRegime.TotMidMForward[i], p_pctChgTotalAMean); CalculateSampleStatsPerDayOffset(ref p_statsPerRegime.TotMidMBackward[i], p_pctChgTotalAMean); } }
private void RenderStatsForDisplaying(string p_title, SampleStatsPerRegime p_stats, bool p_isHtml, StringBuilder p_sbStats) { RenderStatsForDisplaying(p_title + ", TotM", p_stats.TotMForward, p_stats.TotMBackward, p_isHtml, p_sbStats); RenderStatsForDisplaying(p_title + ", TotMidM", p_stats.TotMidMForward, p_stats.TotMidMBackward, p_isHtml, p_sbStats); }
private SampleStatsPerRegime CreateSampleStatsPerRegime(string p_name, int p_estimatedNsamplesPerDayOffset) { SampleStatsPerRegime stats = new SampleStatsPerRegime() { Name = p_name, TotMForward = new SampleStats[cMaxDayOffset], TotMBackward = new SampleStats[cMaxDayOffset], TotMidMForward = new SampleStats[cMaxDayOffset], TotMidMBackward = new SampleStats[cMaxDayOffset] }; for (int i = 0; i < cMaxDayOffset; i++) { stats.TotMForward[i].Name = "TotM+"; stats.TotMBackward[i].Name = "TotM-"; stats.TotMidMForward[i].Name = "TotMidM+"; stats.TotMidMBackward[i].Name = "TotMidM-"; stats.TotMForward[i].DayOffset = i + 1; // 1= T+1, 1 based stats.TotMBackward[i].DayOffset = i + 1; stats.TotMidMForward[i].DayOffset = i + 1; stats.TotMidMBackward[i].DayOffset = i + 1; stats.TotMForward[i].Samples = new List<Tuple<DateTime, double>>(p_estimatedNsamplesPerDayOffset); stats.TotMBackward[i].Samples = new List<Tuple<DateTime, double>>(p_estimatedNsamplesPerDayOffset); stats.TotMidMForward[i].Samples = new List<Tuple<DateTime, double>>(p_estimatedNsamplesPerDayOffset); stats.TotMidMBackward[i].Samples = new List<Tuple<DateTime, double>>(p_estimatedNsamplesPerDayOffset); } // in BackTester, the mask as string went into this function, but it is not needed just yet. // setting up stats.TotMForward[i].IsBullish in the QuickTester followed here. (based on Mask to be played) // only needed in Backtests, not in Significance analysis of the day, so maybe create a separate variable for this later in QuickTester2, so that the code base can be the same. return stats; }
private bool PrepareHistoricalStatsForAllRegimes() { DateTime pvStartDate = m_spy[0].Date; DateTime pvEndDate = m_spy[m_spy.Length - 1].Date; // maybe (maybe not) put the Today date and tomorrow date too into m_spy. (but don't use them as samples for Training) That way we will easily know if tomorrow is TotM + ? or TotMidM + ? // or calculate it in any other way. Create a Separate function: GetAllOffset(DateTime p_day), but that will be very inefficient to do it for 6000 items in the array. So, just use it for Tomorrow. // or just put tomorrow into the sample temporarily. // 1.1 calculate totMForwardDayOffset DateTime iDate = new DateTime(pvStartDate.Year, pvStartDate.Month, 1); // Time is 00:00, which is OK, but strangely it is visualized as "12:00 AM", but yeah. Correct. it is not noon. iDate = DbUtils.GetNextUsaMarketOpenDayLoc(iDate, true); // this is day T+1 int iDateOffset = 1; // T+1 while (iDate < pvStartDate) // marching forward until iDate = startDate { iDate = DbUtils.GetNextUsaMarketOpenDayLoc(iDate, false); iDateOffset++; } m_spy[0].TotMForwardOffset = iDateOffset; for (int i = 1; i < m_spy.Length; i++) // march over on p_quotes, not pv { if (m_spy[i].Date.Month != m_spy[i - 1].Date.Month) iDateOffset = 1; // T+1 else iDateOffset++; m_spy[i].TotMForwardOffset = iDateOffset; } // 1.2 calculate totMBackwardDayOffset iDate = new DateTime(pvEndDate.Year, pvEndDate.Month, 1); iDate = iDate.AddMonths(1); // next month can be in the following year; this is the first calendar day of the next month iDate = DbUtils.GetPreviousUsaMarketOpenDayLoc(iDate, false); // this is day T-1 iDateOffset = 1; // T-1 while (iDate > pvEndDate) // marching backward until iDate == endDate { iDate = DbUtils.GetPreviousUsaMarketOpenDayLoc(iDate, false); iDateOffset++; } m_spy[m_spy.Length - 1].TotMBackwardOffset = iDateOffset; // last day (today) is set for (int i = m_spy.Length - 2; i >= 0; i--) // march over on p_quotes, not pv { if (m_spy[i].Date.Month != m_spy[i + 1].Date.Month) // what if market closes for 3 months (or we don't have the data in DB) iDateOffset = 1; // T-1 else iDateOffset++; m_spy[i].TotMBackwardOffset = iDateOffset; } // 1.3 calculate totMidMForwardDayOffset iDate = new DateTime(pvStartDate.Year, pvStartDate.Month, 15); if (iDate > pvStartDate) iDate = iDate.AddMonths(-1); iDate = DbUtils.GetNextUsaMarketOpenDayLoc(iDate, true); // // this is day T+1 iDateOffset = 1; // T+1 while (iDate < pvStartDate) // marching forward until iDate = startDate { iDate = DbUtils.GetNextUsaMarketOpenDayLoc(iDate, false); iDateOffset++; } m_spy[0].TotMidMForwardOffset = iDateOffset; for (int i = 1; i < m_spy.Length; i++) // march over on p_quotes, not pv { if (((m_spy[i].Date.Month == m_spy[i - 1].Date.Month) && m_spy[i].Date.Day >= 15 && m_spy[i - 1].Date.Day < 15) || // what if market closes for 3 months (or we don't have the data in DB) (m_spy[i].Date.Month != m_spy[i - 1].Date.Month) && m_spy[i].Date.Day >= 15) // if some months are skipped from data iDateOffset = 1; // T+1 else iDateOffset++; m_spy[i].TotMidMForwardOffset = iDateOffset; } // 1.4 calculate totMBackwardDayOffset iDate = new DateTime(pvEndDate.Year, pvEndDate.Month, 15); if (iDate <= pvEndDate) iDate = iDate.AddMonths(1); // next month can be in the following year; better to use AddMonths(); iDate = DbUtils.GetPreviousUsaMarketOpenDayLoc(iDate, false); // this is day T-1 iDateOffset = 1; // T-1 while (iDate > pvEndDate) // marching backward until iDate == endDate { iDate = DbUtils.GetPreviousUsaMarketOpenDayLoc(iDate, false); iDateOffset++; } m_spy[m_spy.Length - 1].TotMidMBackwardOffset = iDateOffset; // last day (today) is set for (int i = m_spy.Length - 2; i >= 0; i--) // march over on p_quotes, not pv { if (((m_spy[i].Date.Month == m_spy[i + 1].Date.Month) && m_spy[i].Date.Day < 15 && m_spy[i + 1].Date.Day >= 15) || // what if market closes for 3 months (or we don't have the data in DB) (m_spy[i].Date.Month != m_spy[i + 1].Date.Month) && m_spy[i].Date.Day < 15) // if some months are skipped from data iDateOffset = 1; // T-1 else iDateOffset++; m_spy[i].TotMidMBackwardOffset = iDateOffset; } // in BackTester, the mask as string went into this function, but it is not needed just yet. // setting up stats.TotMForward[i].IsBullish in the QuickTester followed here. (based on Mask to be played) // only needed in Backtests, not in Significance analysis of the day, so maybe create a separate variable for this later in QuickTester2, so that the code base can be the same. m_winterStats = CreateSampleStatsPerRegime("Winter", (int)(m_spy.Length / 260.0 * 7.0 * 1.1)); // give a little 10% overhead, so List<> will be not re-allocated many times m_summerStats = CreateSampleStatsPerRegime("Summer", (int)(m_spy.Length / 260.0 * 6.0 * 1.1)); // give a little 10% overhead, so List<> will be not re-allocated many times m_allYearStats = CreateSampleStatsPerRegime("AllYear", (int)(m_spy.Length/260.0*12.0*1.1)); // give a little 10% overhead, so List<> will be not re-allocated many times double outlierBasicZscore_PctThreshold = Double.NaN; int nNegativeOutliers = 0, nPositiveOutliers = 0; if (uberVxxConfig.TotM_OutlierElimination != OutlierElimination.None) { //-calculate StDev.and http://www.itl.nist.gov/div898/handbook/eda/section3/eda35h.htm // "Although it is common practice to use Z-scores to identify possible outliers, // Iglewicz and Hoaglin recommend using the modified Z - score" //- Balazs used +8% or lower than -8% percentages threshold for VXX TotM. So determine 8% is how many StDev away, // and if it 2.2 times, than use that for the other samples (like QQQ, which will have less StDev) //-(not likely, but) double check, that max. only 5 or 6% of the samples are eliminated. If 10% of the samples are eliminated // as outliers. (10 out of 100), then those are not random outliers. double n = (double)m_spy.Length; List<double> samples = m_spy.Select(r => r.PctChg).ToList(); double aMean = samples.Average(); double correctedStDev = Math.Sqrt(samples.Sum(r => (r - aMean) * (r - aMean)) / (n - 1.0)); // it is about 1.1% for the SPX, http://www.investopedia.com/articles/04/021804.asp outlierBasicZscore_PctThreshold = correctedStDev * uberVxxConfig.OutlierBasicZscore_Zscore; } // create 2 lists, a Forward list, a backward list (maybe later to test day T+12..T+16) Jay's "Monthly 10", which is 4 days in the middle month double pctChgUsedSamplesTotal = 0.0; for (int i = 0; i < m_spy.Length; i++) // march over on p_quotes, not pv { DateTime day = m_spy[i].Date; double pctChg = m_spy[i].PctChg; if (uberVxxConfig.TotM_OutlierElimination == OutlierElimination.BasicZscore) { if (Math.Abs(pctChg) > outlierBasicZscore_PctThreshold) { if (pctChg > 0) nPositiveOutliers++; else if (pctChg < 0) nNegativeOutliers++; continue; } } pctChgUsedSamplesTotal += pctChg; int totMForwardInd = m_spy[i].TotMForwardOffset, totMBackwardInd = m_spy[i].TotMBackwardOffset, totMidMForwardInd = m_spy[i].TotMidMForwardOffset, totMidMBackwardInd = m_spy[i].TotMidMBackwardOffset; m_allYearStats.TotMForward[totMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg)); m_allYearStats.TotMBackward[totMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg)); m_allYearStats.TotMidMForward[totMidMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg)); m_allYearStats.TotMidMBackward[totMidMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg)); if (IsBullishWinterDay(day)) { m_winterStats.TotMForward[totMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg)); m_winterStats.TotMBackward[totMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg)); m_winterStats.TotMidMForward[totMidMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg)); m_winterStats.TotMidMBackward[totMidMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg)); } else { m_summerStats.TotMForward[totMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg)); m_summerStats.TotMBackward[totMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg)); m_summerStats.TotMidMForward[totMidMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg)); m_summerStats.TotMidMBackward[totMidMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg)); } } if (uberVxxConfig.TotM_OutlierElimination != OutlierElimination.None) { //Maybe AdvancedOutlier elimination is not even needed because with the BasicZscore, here are how many samples are eliminated: //OutlierBasicZscore_Zscore = 2.7: SPY outliers skipped at 3.21 %.Pos:70,Neg: 57, 2.17 % of samples. //OutlierBasicZscore_Zscore = 4.0: SPY outliers skipped at 4.76 %.Pos:17,Neg: 18, 0.6 % of samples. //OutlierBasicZscore_Zscore = 5.0: SPY outliers skipped at 5.95 %.Pos:8,Neg: 9, 0.29 % of samples. //It is very even.Because there are big panic days, but there are buy upside days too. e.g.: Oct 13, 2008: up + 14.5 % double pctOutliers = ((double)(nPositiveOutliers + nNegativeOutliers) / m_spy.Length); //Console.WriteLine($"SPY outliers skipped at {outlierBasicZscore_PctThreshold * 100:0.##}%. Pos:{nPositiveOutliers},Neg:{nNegativeOutliers}, {pctOutliers * 100.0:0.##}% of samples."); Utils.Logger.Info($"SPY outliers skipped at {outlierBasicZscore_PctThreshold * 100:0.##}%. Pos:{nPositiveOutliers},Neg:{nNegativeOutliers}, {pctOutliers * 100.0:0.##}% of samples."); StrongAssert.True(pctOutliers < 0.05, Severity.NoException, "If 5%+ of the samples are eliminated, that means they are not random outliers. This is unexpected."); } int nUsedSamples = m_spy.Length - nNegativeOutliers - nPositiveOutliers; double pctChgTotalAMean = (nUsedSamples <= 0) ? 0.0 : pctChgUsedSamplesTotal / (double)(nUsedSamples); CalculateSampleStatsPerRegime(m_winterStats, pctChgTotalAMean); CalculateSampleStatsPerRegime(m_summerStats, pctChgTotalAMean); CalculateSampleStatsPerRegime(m_allYearStats, pctChgTotalAMean); StringBuilder sbStats = new StringBuilder(Environment.NewLine); // looks better in the log file if it starts in a blank new line RenderStatsForDisplaying("Winter", m_winterStats, false, sbStats); RenderStatsForDisplaying("Summer", m_summerStats, false, sbStats); RenderStatsForDisplaying("AllYear", m_allYearStats, false, sbStats); Utils.Logger.Info(sbStats.ToString()); return true; }