public static TestSummary ReduceCorrelated(TestSummary testSummary)
{
//Work out the percentage of trades to use to train
DateTime startDate = (DateTime)testSummary.StartDate;
DateTime endDate = (DateTime)testSummary.EndDate;
double trainDays = (endDate - startDate).TotalDays * ((double)testSummary.ReduceCorrelatedParams.TrainTestSplit / 100);
DateTime endTrain = startDate.AddDays(trainDays);
testSummary.EndTrainDate = endTrain;
//Create a copy of the testSummary so we don't modify the original testSummary when we reduce the testsets
TestSummary filteredTestSummary = new TestSummary(testSummary);
//remove any non-performing sets
List <TestSet> nonPerformers = new List <TestSet>();
int totalProfitable = 0;
foreach (TestSet t in testSummary.TestSets)
{
double p = PerformanceResult.CalculateProfitFactor(t.Trades.Where(x => x.CloseTime <= filteredTestSummary.EndTrainDate).ToArray());
if (p > 1)
{
totalProfitable++;
}
if (p < testSummary.ReduceCorrelatedParams.MinMetric)
{
filteredTestSummary.Remove(t);
}
}
//remove all the test sets if at least half are profitable
double pProfitable = (double)totalProfitable / (double)testSummary.TestSets.Length;
if (pProfitable < 0.5)
{
foreach (TestSet t in testSummary.TestSets)
{
filteredTestSummary.Remove(t);
}
}
//Calcualte weekly profit for every week so we can do a correlation based on weekly profits
Dictionary <string, KeyValuePair <int, double>[]> WeeklyProfits = new Dictionary <string, KeyValuePair <int, double>[]>();
foreach (TestSet ts in testSummary.TestSets)
{
//get all the trades in the train period and calculate weekly profit
var result =
from s in ts.Trades.Where(x => x.CloseTime <= endTrain)
group s by new { week = (s.OpenTime.Year - startDate.Year) * 52 + (s.OpenTime.DayOfYear / 7) } into g
select new KeyValuePair <int, double>(g.Key.week, g.Sum(x => x.Profit));
WeeklyProfits.Add(ts.Description, result.ToArray());
}
//Create a grid of r2 values by comparing each testset with each other test set
Dictionary <Pair, double> r2Values = new Dictionary <Pair, double>();
foreach (KeyValuePair <string, KeyValuePair <int, double>[]> wpRow in WeeklyProfits)
{
foreach (KeyValuePair <string, KeyValuePair <int, double>[]> wpColumn in WeeklyProfits)
{
//skip identical resuklt sets
if (wpColumn.Key == wpRow.Key)
{
continue;
}
//calculate the r2 value from these lists
//Line up the weeks to get an x and y for the current pair
Dictionary <int, Point> list = new Dictionary <int, Point>();
foreach (KeyValuePair <int, double> res in wpRow.Value)
{
list.Add(res.Key, new Point(res.Value, 0, wpRow.Key, null));
}
foreach (KeyValuePair <int, double> res in wpColumn.Value)
{
if (!list.ContainsKey(res.Key))
{
list.Add(res.Key, new Point(0, res.Value, null, wpColumn.Key));
}
else
{
list[res.Key].Y = res.Value;
list[res.Key].YLabel = wpColumn.Key;
}
}
double[] x = list.Select(v => v.Value.X).ToArray();
double[] y = list.Select(v => v.Value.Y).ToArray();
//calculate the r2 and store in dictionary with the testset description pair as the Key
r2Values.Add(new Pair(wpRow.Key, wpColumn.Key), Stat.R2(x, y));
}
}
foreach (KeyValuePair <Pair, double> res in r2Values)
{
//if too corelated remove the worst performer
if (res.Value > testSummary.ReduceCorrelatedParams.R2Cutoff)
{
//get the train set of trades only
Trade[] xTrades = filteredTestSummary.GetTradeSet(Convert.ToString(res.Key.P1), TestSummary.TradeSet.Train);
Trade[] yTrades = filteredTestSummary.GetTradeSet(Convert.ToString(res.Key.P2), TestSummary.TradeSet.Train);
//if both exist in our filtered test sets remove worst performer - it may have already been removed from previous pair r2 comparisons
if (xTrades != null && yTrades != null)
{
double xMetric = PerformanceResult.CalculateProfitFactor(xTrades);
double yMetric = PerformanceResult.CalculateProfitFactor(yTrades);
if (xMetric > yMetric)
{
filteredTestSummary.Remove(Convert.ToString(res.Key.P2));
}
else
{
filteredTestSummary.Remove(Convert.ToString(res.Key.P1));
}
}
}
}
return(filteredTestSummary);
}
