Exemplo n.º 1
0
 private static void CalculateSampleStatsPerRegime(SampleStatsPerRegime p_statsPerRegime, double p_pctChgTotalAMean)
 {
     for (int i = 0; i <= 16; i++)   // write only from T-17 to T+17
     {
         CalculateSampleStatsPerDayOffset(ref p_statsPerRegime.TotMForward[i], p_pctChgTotalAMean);
         CalculateSampleStatsPerDayOffset(ref p_statsPerRegime.TotMBackward[i], p_pctChgTotalAMean);
         CalculateSampleStatsPerDayOffset(ref p_statsPerRegime.TotMidMForward[i], p_pctChgTotalAMean);
         CalculateSampleStatsPerDayOffset(ref p_statsPerRegime.TotMidMBackward[i], p_pctChgTotalAMean);
     }
 }
Exemplo n.º 2
0
 private void RenderStatsForDisplaying(string p_title, SampleStatsPerRegime p_stats, bool p_isHtml, StringBuilder p_sbStats)
 {
     RenderStatsForDisplaying(p_title + ", TotM", p_stats.TotMForward, p_stats.TotMBackward, p_isHtml, p_sbStats);
     RenderStatsForDisplaying(p_title + ", TotMidM", p_stats.TotMidMForward, p_stats.TotMidMBackward, p_isHtml, p_sbStats);
 }
Exemplo n.º 3
0
        private SampleStatsPerRegime CreateSampleStatsPerRegime(string p_name, int p_estimatedNsamplesPerDayOffset)
        {
            SampleStatsPerRegime stats = new SampleStatsPerRegime() { Name = p_name, TotMForward = new SampleStats[cMaxDayOffset], TotMBackward = new SampleStats[cMaxDayOffset], TotMidMForward = new SampleStats[cMaxDayOffset], TotMidMBackward = new SampleStats[cMaxDayOffset] };

            for (int i = 0; i < cMaxDayOffset; i++)
            {
                stats.TotMForward[i].Name = "TotM+";
                stats.TotMBackward[i].Name = "TotM-";
                stats.TotMidMForward[i].Name = "TotMidM+";
                stats.TotMidMBackward[i].Name = "TotMidM-";
                stats.TotMForward[i].DayOffset = i + 1;     // 1= T+1, 1 based
                stats.TotMBackward[i].DayOffset = i + 1;
                stats.TotMidMForward[i].DayOffset = i + 1;
                stats.TotMidMBackward[i].DayOffset = i + 1;
                stats.TotMForward[i].Samples = new List<Tuple<DateTime, double>>(p_estimatedNsamplesPerDayOffset);
                stats.TotMBackward[i].Samples = new List<Tuple<DateTime, double>>(p_estimatedNsamplesPerDayOffset);
                stats.TotMidMForward[i].Samples = new List<Tuple<DateTime, double>>(p_estimatedNsamplesPerDayOffset);
                stats.TotMidMBackward[i].Samples = new List<Tuple<DateTime, double>>(p_estimatedNsamplesPerDayOffset);
            }

            // in BackTester, the mask as string went into this function, but it is not needed just yet.
            // setting up stats.TotMForward[i].IsBullish in the QuickTester followed here. (based on Mask to be played)
            // only needed in Backtests, not in Significance analysis of the day, so maybe create a separate variable for this later in QuickTester2, so that the code base can be the same.

            return stats;
        }
Exemplo n.º 4
0
        private bool PrepareHistoricalStatsForAllRegimes()
        {
            DateTime pvStartDate = m_spy[0].Date;
            DateTime pvEndDate = m_spy[m_spy.Length - 1].Date;

            // maybe (maybe not) put the Today date and tomorrow date too into m_spy. (but don't use them as samples for Training) That way we will easily know if tomorrow is TotM + ? or TotMidM + ?
            // or calculate it in any other way. Create a Separate function: GetAllOffset(DateTime p_day), but that will be very inefficient to do it for 6000 items in the array. So, just use it for Tomorrow.
            // or just put tomorrow into the sample temporarily.

            // 1.1 calculate totMForwardDayOffset
            DateTime iDate = new DateTime(pvStartDate.Year, pvStartDate.Month, 1);  // Time is 00:00, which is OK, but strangely it is visualized as "12:00 AM", but yeah. Correct. it is not noon.
            iDate = DbUtils.GetNextUsaMarketOpenDayLoc(iDate, true);    // this is day T+1
            int iDateOffset = 1;    // T+1
            while (iDate < pvStartDate) // marching forward until iDate = startDate
            {
                iDate = DbUtils.GetNextUsaMarketOpenDayLoc(iDate, false);
                iDateOffset++;
            }
            m_spy[0].TotMForwardOffset = iDateOffset;
            for (int i = 1; i < m_spy.Length; i++)  // march over on p_quotes, not pv
            {
                if (m_spy[i].Date.Month != m_spy[i - 1].Date.Month)
                    iDateOffset = 1;    // T+1
                else
                    iDateOffset++;
                m_spy[i].TotMForwardOffset = iDateOffset;
            }

            // 1.2 calculate totMBackwardDayOffset
            iDate = new DateTime(pvEndDate.Year, pvEndDate.Month, 1);
            iDate = iDate.AddMonths(1);     // next month can be in the following year; this is the first calendar day of the next month
            iDate = DbUtils.GetPreviousUsaMarketOpenDayLoc(iDate, false);   // this is day T-1
            iDateOffset = 1;    // T-1
            while (iDate > pvEndDate)   // marching backward until iDate == endDate
            {
                iDate = DbUtils.GetPreviousUsaMarketOpenDayLoc(iDate, false);
                iDateOffset++;
            }
            m_spy[m_spy.Length - 1].TotMBackwardOffset = iDateOffset;  // last day (today) is set
            for (int i = m_spy.Length - 2; i >= 0; i--)  // march over on p_quotes, not pv
            {
                if (m_spy[i].Date.Month != m_spy[i + 1].Date.Month)   // what if market closes for 3 months (or we don't have the data in DB)
                    iDateOffset = 1;    // T-1
                else
                    iDateOffset++;
                m_spy[i].TotMBackwardOffset = iDateOffset;
            }

            // 1.3 calculate totMidMForwardDayOffset
            iDate = new DateTime(pvStartDate.Year, pvStartDate.Month, 15);
            if (iDate > pvStartDate)
                iDate = iDate.AddMonths(-1);
            iDate = DbUtils.GetNextUsaMarketOpenDayLoc(iDate, true); // // this is day T+1
            iDateOffset = 1;    // T+1
            while (iDate < pvStartDate) // marching forward until iDate = startDate
            {
                iDate = DbUtils.GetNextUsaMarketOpenDayLoc(iDate, false);
                iDateOffset++;
            }
            m_spy[0].TotMidMForwardOffset = iDateOffset;
            for (int i = 1; i < m_spy.Length; i++)  // march over on p_quotes, not pv
            {
                if (((m_spy[i].Date.Month == m_spy[i - 1].Date.Month) && m_spy[i].Date.Day >= 15 && m_spy[i - 1].Date.Day < 15) ||  // what if market closes for 3 months (or we don't have the data in DB)
                    (m_spy[i].Date.Month != m_spy[i - 1].Date.Month) && m_spy[i].Date.Day >= 15)   // if some months are skipped from data
                    iDateOffset = 1;    // T+1
                else
                    iDateOffset++;
                m_spy[i].TotMidMForwardOffset = iDateOffset;
            }

            // 1.4 calculate totMBackwardDayOffset
            iDate = new DateTime(pvEndDate.Year, pvEndDate.Month, 15);
            if (iDate <= pvEndDate)
                iDate = iDate.AddMonths(1); // next month can be in the following year; better to use AddMonths();
            iDate = DbUtils.GetPreviousUsaMarketOpenDayLoc(iDate, false);    // this is day T-1
            iDateOffset = 1;    // T-1
            while (iDate > pvEndDate)   // marching backward until iDate == endDate
            {
                iDate = DbUtils.GetPreviousUsaMarketOpenDayLoc(iDate, false);
                iDateOffset++;
            }
            m_spy[m_spy.Length - 1].TotMidMBackwardOffset = iDateOffset;  // last day (today) is set
            for (int i = m_spy.Length - 2; i >= 0; i--)  // march over on p_quotes, not pv
            {
                if (((m_spy[i].Date.Month == m_spy[i + 1].Date.Month) && m_spy[i].Date.Day < 15 && m_spy[i + 1].Date.Day >= 15) ||  // what if market closes for 3 months (or we don't have the data in DB)
                    (m_spy[i].Date.Month != m_spy[i + 1].Date.Month) && m_spy[i].Date.Day < 15)   // if some months are skipped from data
                    iDateOffset = 1;    // T-1
                else
                    iDateOffset++;
                m_spy[i].TotMidMBackwardOffset = iDateOffset;
            }

            // in BackTester, the mask as string went into this function, but it is not needed just yet.
            // setting up stats.TotMForward[i].IsBullish in the QuickTester followed here. (based on Mask to be played)
            // only needed in Backtests, not in Significance analysis of the day, so maybe create a separate variable for this later in QuickTester2, so that the code base can be the same.
            m_winterStats = CreateSampleStatsPerRegime("Winter", (int)(m_spy.Length / 260.0 * 7.0 * 1.1)); // give a little 10% overhead, so List<> will be not re-allocated many times
            m_summerStats = CreateSampleStatsPerRegime("Summer", (int)(m_spy.Length / 260.0 * 6.0 * 1.1)); // give a little 10% overhead, so List<> will be not re-allocated many times
            m_allYearStats = CreateSampleStatsPerRegime("AllYear", (int)(m_spy.Length/260.0*12.0*1.1));   // give a little 10% overhead, so List<> will be not re-allocated many times

            double outlierBasicZscore_PctThreshold = Double.NaN;
            int nNegativeOutliers = 0, nPositiveOutliers = 0;
            if (uberVxxConfig.TotM_OutlierElimination != OutlierElimination.None)
            {
                //-calculate StDev.and http://www.itl.nist.gov/div898/handbook/eda/section3/eda35h.htm
                //	"Although it is common practice to use Z-scores to identify possible outliers, 
                //    Iglewicz and Hoaglin recommend using the modified Z - score"
                //- Balazs used +8% or lower than -8% percentages threshold for VXX TotM. So determine 8% is how many StDev away, 
                //	and if it 2.2 times, than use that for the other samples (like QQQ, which will have less StDev)
                //-(not likely, but) double check, that max. only 5 or 6% of the samples are eliminated. If 10% of the samples are eliminated
                //	as outliers. (10 out of 100), then those are not random outliers.

                double n = (double)m_spy.Length;
                List<double> samples = m_spy.Select(r => r.PctChg).ToList();
                double aMean = samples.Average();
                double correctedStDev = Math.Sqrt(samples.Sum(r => (r - aMean) * (r - aMean)) / (n - 1.0));     // it is about 1.1% for the SPX, http://www.investopedia.com/articles/04/021804.asp
                outlierBasicZscore_PctThreshold = correctedStDev * uberVxxConfig.OutlierBasicZscore_Zscore;
            }

            // create 2 lists, a Forward list, a backward list (maybe later to test day T+12..T+16) Jay's "Monthly 10", which is 4 days in the middle month
            double pctChgUsedSamplesTotal = 0.0;
            for (int i = 0; i < m_spy.Length; i++)  // march over on p_quotes, not pv
            {
                DateTime day = m_spy[i].Date;
                double pctChg = m_spy[i].PctChg;

                if (uberVxxConfig.TotM_OutlierElimination == OutlierElimination.BasicZscore)
                {
                    if (Math.Abs(pctChg) > outlierBasicZscore_PctThreshold)
                    {
                        if (pctChg > 0)
                            nPositiveOutliers++;
                        else if (pctChg < 0)
                            nNegativeOutliers++;

                        continue;
                    }
                }

                pctChgUsedSamplesTotal += pctChg;
                int totMForwardInd = m_spy[i].TotMForwardOffset, totMBackwardInd = m_spy[i].TotMBackwardOffset, totMidMForwardInd = m_spy[i].TotMidMForwardOffset, totMidMBackwardInd = m_spy[i].TotMidMBackwardOffset;

                m_allYearStats.TotMForward[totMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
                m_allYearStats.TotMBackward[totMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
                m_allYearStats.TotMidMForward[totMidMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
                m_allYearStats.TotMidMBackward[totMidMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));

                if (IsBullishWinterDay(day))
                {
                    m_winterStats.TotMForward[totMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
                    m_winterStats.TotMBackward[totMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
                    m_winterStats.TotMidMForward[totMidMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
                    m_winterStats.TotMidMBackward[totMidMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
                }
                else
                {
                    m_summerStats.TotMForward[totMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
                    m_summerStats.TotMBackward[totMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
                    m_summerStats.TotMidMForward[totMidMForwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
                    m_summerStats.TotMidMBackward[totMidMBackwardInd - 1].Samples.Add(new Tuple<DateTime, double>(day, pctChg));
                }

            }

            if (uberVxxConfig.TotM_OutlierElimination != OutlierElimination.None)
            {
                //Maybe AdvancedOutlier elimination is not even needed because with the BasicZscore, here are how many samples are eliminated:
                //OutlierBasicZscore_Zscore = 2.7: SPY outliers skipped at 3.21 %.Pos:70,Neg: 57, 2.17 % of samples.
                //OutlierBasicZscore_Zscore = 4.0: SPY outliers skipped at 4.76 %.Pos:17,Neg: 18, 0.6 % of samples.
                //OutlierBasicZscore_Zscore = 5.0: SPY outliers skipped at 5.95 %.Pos:8,Neg: 9, 0.29 % of samples.
                //It is very even.Because there are big panic days, but there are buy upside days too. e.g.: Oct 13, 2008: up + 14.5 %

                double pctOutliers = ((double)(nPositiveOutliers + nNegativeOutliers) / m_spy.Length);
                //Console.WriteLine($"SPY outliers skipped at {outlierBasicZscore_PctThreshold * 100:0.##}%. Pos:{nPositiveOutliers},Neg:{nNegativeOutliers}, {pctOutliers * 100.0:0.##}% of samples.");
                Utils.Logger.Info($"SPY outliers skipped at {outlierBasicZscore_PctThreshold * 100:0.##}%. Pos:{nPositiveOutliers},Neg:{nNegativeOutliers}, {pctOutliers * 100.0:0.##}% of samples.");
                StrongAssert.True(pctOutliers < 0.05, Severity.NoException, "If 5%+ of the samples are eliminated, that means they are not random outliers. This is unexpected.");
            }
            int nUsedSamples = m_spy.Length - nNegativeOutliers - nPositiveOutliers;
            double pctChgTotalAMean = (nUsedSamples <= 0) ? 0.0 : pctChgUsedSamplesTotal / (double)(nUsedSamples);

            CalculateSampleStatsPerRegime(m_winterStats, pctChgTotalAMean);
            CalculateSampleStatsPerRegime(m_summerStats, pctChgTotalAMean);
            CalculateSampleStatsPerRegime(m_allYearStats, pctChgTotalAMean);

            StringBuilder sbStats = new StringBuilder(Environment.NewLine); // looks better in the log file if it starts in a blank new line
            RenderStatsForDisplaying("Winter", m_winterStats, false, sbStats);
            RenderStatsForDisplaying("Summer", m_summerStats, false, sbStats);
            RenderStatsForDisplaying("AllYear", m_allYearStats, false, sbStats);
            Utils.Logger.Info(sbStats.ToString());

            return true;
        }