public WekaClassifier CreateWekaClassifier()
{
_aco1.Initialize();
_aco1.Work();
//_aco1.PostProcessing();
Dataset reduced = null;
if (_attributeFirst)
{
reduced = _trainingSet.ReduceAttributes(ACO1.BestAnt.Solution.AttributesToRemove());
}
else
{
reduced = _trainingSet.ReduceInstances(ACO1.BestAnt.Solution.InstancesToRemove());
}
_aco2.Initialize(_aco1.BestAnt.Solution);
_aco2.Work();
//_aco2.PostProcessing();
this._bestSolution = ACO2.BestAnt.Solution;
weka.classifiers.Classifier classifier = ((WekaClassificationQualityEvaluator)_aco1.Problem.SolutionQualityEvaluator).CreateClassifier(this._bestSolution);
WekaClassifier wekaClassifier = new WekaClassifier();
wekaClassifier.Classifier = classifier;
wekaClassifier.AttributesToRemove = this._bestSolution.AttributesToRemove();
return(wekaClassifier);
}
public WekaClassifier CreateWekaClassifier()
{
this.Work();
weka.classifiers.Classifier classifier = ((WekaClassificationQualityEvaluator)this._problem.SolutionQualityEvaluator).CreateClassifier(this.BestAnt.Solution);
WekaClassifier wekaClassifier = new WekaClassifier();
wekaClassifier.Classifier = classifier;
wekaClassifier.AttributesToRemove = this.BestAnt.Solution.AttributesToRemove();
return(wekaClassifier);
}
//**************************************************************************************
/// <summary>
/// Calculates average precision as seen by implied simulation.
/// </summary>
private static float CalculateAveragePrecision(List <WekaClassifier> iClassifiers)
{
var weight = new List <float>(iClassifiers.Count);
var precisions = new List <float>();
var classfiers = WekaClassifier.SplitByProfitTime(iClassifiers);
foreach (KeyValuePair <int, List <WekaClassifier> > pair in classfiers)
{
var p = WekaClassifier.GetAveragePrecision(pair.Value);
precisions.Add(p);
weight.Add((float)(Math.Pow(p, 6) / Math.Pow(pair.Value[0].GetProfitsStdDev(), 6)));
}
return(Statistics.WeightedArithmeticMean(precisions.ToArray(), Statistics.Normalize(weight.ToArray())));
}
//**************************************************************************************
/// <summary>
/// Returns calculated parameters.
/// </summary>
/// <param name="iCurrentPrice">Last known price.</param>
/// <param name="iCandlestickIndex">Provides index (current time) for each candlestick list. If this value is set to null, the most recent entry will be used.</param>
/// <param name="iCandlesticks">Maximum amount of daily candlesticks: 270; 12H: 25; 6H: 50; 3H: 100; 2H: 150; 1H: 300; 30m: 600; 15m: 0; 5m: 0; 1m:0</param>
public static float[] CalculateParameters(List <ClassifierParameter> iParameters, CandlestickCollection iCandlesticks, Candlestick.Period iPeriod, float iCurrentPrice, CandlestickIndexCollection iCandlestickIndex)
{
var candlesticks = iCandlesticks[iPeriod];
int candlestickIndex = iCandlestickIndex == null ? candlesticks.Count - 1 : iCandlestickIndex[iPeriod];
if (candlestickIndex >= iCandlesticks[iPeriod].Count)
{
throw new Exception("Candlestick index is higher than total number of candlesticks");
}
// Make sure we have enough periods
for (int i = 0; i < iParameters.Count; i++)
{
if (candlesticks.Count < iParameters[i].Periods + 10)
{
return(null);
}
}
float[] results = new float[iParameters.Count];
var macd = new SortedDictionary <int, MACD>();
var macdCurrentPrice = new SortedDictionary <int, MACD>();
for (int i = 0; i < iParameters.Count; i++)
{
//int candlestickIndex = Candlestick.GetIndex(iParameters[i].Candlesticks, iCandlesticks, iCandlestickIndex);
int periods = iParameters[i].Periods;
switch (iParameters[i].Type)
{
case ParameterType.RSI:
results[i] = TechnicalIndicators.CalculateRSI(candlesticks, periods, candlestickIndex);
break;
case ParameterType.RSIWithCurrentPrice:
results[i] = TechnicalIndicators.CalculateRSI(candlesticks, periods, candlestickIndex, iCurrentPrice);
break;
case ParameterType.RSIInt:
results[i] = TechnicalIndicators.RSIToInt(TechnicalIndicators.CalculateRSI(candlesticks, periods, candlestickIndex));
break;
case ParameterType.RSIIntWithCurrentPrice:
results[i] = TechnicalIndicators.RSIToInt(TechnicalIndicators.CalculateRSI(candlesticks, periods, candlestickIndex, iCurrentPrice));
break;
case ParameterType.LastCriticalRSI:
int startIndex = Math.Max(candlestickIndex - 270 + iParameters[i].Periods, iParameters[i].Periods);
int lastCriticalRSI = 0;
for (int k = startIndex; k >= candlestickIndex; k++)
{
lastCriticalRSI = TechnicalIndicators.CalculateLastCriticalRSI(TechnicalIndicators.RSIToInt(TechnicalIndicators.CalculateRSI(candlesticks, periods, k)), lastCriticalRSI);
}
results[i] = lastCriticalRSI;
break;
case ParameterType.LastCriticalRSIWithCurrentPrice:
startIndex = Math.Max(candlestickIndex - 270 + iParameters[i].Periods, iParameters[i].Periods);
lastCriticalRSI = 0;
for (int k = startIndex; k >= candlestickIndex; k++)
{
lastCriticalRSI = TechnicalIndicators.CalculateLastCriticalRSI(TechnicalIndicators.RSIToInt(TechnicalIndicators.CalculateRSI(candlesticks, periods, k, iCurrentPrice)), lastCriticalRSI);
}
results[i] = lastCriticalRSI;
break;
case ParameterType.MeanToStd:
results[i] = TechnicalIndicators.CalculateMeanToStdDev(candlesticks, periods, candlestickIndex, iCurrentPrice);
break;
case ParameterType.MeanToStdInt:
results[i] = (float)Math.Floor(TechnicalIndicators.CalculateMeanToStdDev(candlesticks, periods, candlestickIndex, iCurrentPrice));
break;
case ParameterType.LinearRegressionSlope:
results[i] = (float)TechnicalIndicators.CalculateLinearRegressionSlope(candlesticks, periods, candlestickIndex);
break;
case ParameterType.LinearRegressionSlopePN:
results[i] = TechnicalIndicators.CalculateLinearRegressionSlope(candlesticks, periods, candlestickIndex) >= 0 ? 1 : -1;
break;
case ParameterType.MarginSlope:
results[i] = TechnicalIndicators.CalculateMarginSlope(candlesticks, periods, candlestickIndex, iCurrentPrice, iParameters[i].Attributes[0]);
break;
case ParameterType.MarginSlopePN:
results[i] = TechnicalIndicators.CalculateMarginSlopePN(candlesticks, periods, candlestickIndex, iCurrentPrice, iParameters[i].Attributes[0]);
break;
case ParameterType.MACDSign:
if (!macd.ContainsKey(periods))
{
macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex));
}
results[i] = macd[periods].Signal;
break;
case ParameterType.MACDSignWithCurrentPrice:
if (!macdCurrentPrice.ContainsKey(periods))
{
macdCurrentPrice.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex, iCurrentPrice));
}
results[i] = macdCurrentPrice[periods].Signal;
break;
case ParameterType.MACDHist:
if (!macd.ContainsKey(periods))
{
macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex));
}
results[i] = macd[periods].Hist;
break;
case ParameterType.MACDHistWithCurrentPrice:
if (!macdCurrentPrice.ContainsKey(periods))
{
macdCurrentPrice.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex, iCurrentPrice));
}
results[i] = macdCurrentPrice[periods].Hist;
break;
case ParameterType.MACDHistChange:
if (!macd.ContainsKey(periods))
{
macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex));
}
MACD macd90 = macd[periods];
MACD previousMACD90 = TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex - 1);
results[i] = previousMACD90.Hist == 0 ? 0 : (macd90.Hist / previousMACD90.Hist - 1);
break;
case ParameterType.MACDHistChangeWithCurrentPrice:
if (!macdCurrentPrice.ContainsKey(periods))
{
macdCurrentPrice.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex, iCurrentPrice));
}
macd90 = macdCurrentPrice[periods];
previousMACD90 = TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex - 1);
results[i] = previousMACD90.Hist == 0 ? 0 : (macd90.Hist / previousMACD90.Hist - 1);
break;
case ParameterType.MACDHistSlope:
float[] hist = new float[(int)iParameters[i].Attributes[0]];
for (int k = hist.Length - 1; k >= 0; k--)
{
hist[hist.Length - 1 - k] = TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex - k).Hist;
}
results[i] = new LinearRegression(hist).Slope;
break;
case ParameterType.MACDHistPN:
if (!macd.ContainsKey(periods))
{
macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex));
}
results[i] = macd[periods].Hist >= 0 ? 1 : -1;
break;
case ParameterType.MACDHistCrossed:
if (!macd.ContainsKey(periods))
{
macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex));
}
macd90 = macd[periods];
previousMACD90 = TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex - 1);
if (macd90.Hist >= 0)
{
results[i] = previousMACD90.Hist >= 0 ? 0 : 1;
}
else
{
results[i] = previousMACD90.Hist < 0 ? 0 : -1;
}
break;
case ParameterType.MACDHistDifference:
if (!macd.ContainsKey(periods))
{
macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex));
}
macd90 = macd[periods];
previousMACD90 = TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex - 1);
results[i] = macd90.Hist - previousMACD90.Hist;
break;
case ParameterType.MACD:
if (!macd.ContainsKey(periods))
{
macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex));
}
results[i] = macd[periods].Macd;
break;
case ParameterType.SlopesEMA:
List <float> slopes = new List <float>();
for (int k = 0; k < iParameters[i].Attributes.Count; k++)
{
int periodLength = (int)iParameters[i].Attributes[k];
slopes.Add(new LinearRegression(TechnicalIndicators.CreatePriceArray(candlesticks, periodLength, candlestickIndex)).Slope);
}
results[i] = Utils.Last(Statistics.EMA(slopes.ToArray(), slopes.Count));
break;
case ParameterType.ABAverage:
results[i] = (TechnicalIndicators.CalculatePriceABaverage(candlesticks, periods, candlestickIndex, iCurrentPrice) ? 1.0f : 0.0f);
break;
case ParameterType.PercentMargin:
results[i] = (TechnicalIndicators.CalculateOnePercentMargin(candlesticks, periods, candlestickIndex, iCurrentPrice, iParameters[i].Attributes[0]) ? 1.0f : 0.0f);
break;
case ParameterType.Classifier:
results[i] = WekaClassifier.Find((int)iParameters[i].Attributes[0]).PredictDFP(iCandlesticks, iCurrentPrice, iCandlestickIndex);
break;
case ParameterType.ClassifierTargetChangeOldest:
var classifier = WekaClassifier.Find((int)iParameters[i].Attributes[0]);
var targetTime = iCandlesticks[WekaClassifier.kTrainingPeriod][iCandlestickIndex[WekaClassifier.kTrainingPeriod] - classifier.ProfitTime].StartTime;
var pastIndex = new CandlestickIndexCollection();
for (int k = (int)WekaClassifier.kTrainingPeriod; k >= 0; k--)
{
pastIndex[k] = Math.Max(0, Math.Min(iCandlesticks[k].Count - 1, Candlestick.FindIndex(iCandlesticks[k], targetTime)));
}
var priceBefore = iCandlesticks[WekaClassifier.kTrainingPeriod][pastIndex[WekaClassifier.kTrainingPeriod]].MedianPrice;
var wfp = classifier.PredictDFP(iCandlesticks, priceBefore, pastIndex);
var targetPriceChange = (float)Math.Exp(wfp);
results[i] = priceBefore * targetPriceChange / iCurrentPrice - 1.0f;
break;
}
}
return(results);
}
//**************************************************************************************
/// <summary>
/// Starts simulation starting from the end of historical candlesticksm and up to prediction duration.
/// </summary>
private static void Start(CandlestickCollection iHistoricalCandlesticks, CandlestickCollection iCandlesticks, List <WekaClassifier> iClassifiers, int iPredictionDuration)
{
var index = new CandlestickIndexCollection();
var candlesticks = iCandlesticks;
var classfiers = WekaClassifier.SplitByProfitTime(iClassifiers);
var initialCount = candlesticks[WekaClassifier.kTrainingPeriod].Count;
var startIndex = Math.Max(initialCount - classfiers.Keys.Max() - 1, 1);
var endIndex = initialCount + iPredictionDuration - 1;
var priceDiffStdDev = Statistics.StandardDeviation(Statistics.CalculateDifferences(iHistoricalCandlesticks[WekaClassifier.kTrainingPeriod].Select(x => x.MedianPrice).ToArray()));
var wfp = new SortedDictionary <int, List <float> >();
var weight = new List <float>(classfiers.Count);
foreach (KeyValuePair <int, List <WekaClassifier> > pair in classfiers)
{
wfp.Add(pair.Key, new List <float>());
var precision = WekaClassifier.GetAveragePrecision(pair.Value);
weight.Add((float)(Math.Pow(precision, 6) / Math.Pow(pair.Value[0].GetProfitsStdDev(), 6.0f)));
}
weight = Statistics.Normalize(weight.ToArray()).ToList();
for (int i = startIndex; i <= endIndex; i++)
{
// Update indexes
for (int k = (int)WekaClassifier.kTrainingPeriod; k >= 0; k--)
{
index[k] = Math.Max(0, Math.Min(candlesticks[k].Count - 1, Candlestick.FindIndex(candlesticks[k], candlesticks[WekaClassifier.kTrainingPeriod][i].StartTime, index[k]) - 1));
}
// Extract current price
var p = candlesticks[WekaClassifier.kTrainingPeriod][i].MedianPrice;
// Calculate WFP
foreach (KeyValuePair <int, List <WekaClassifier> > pair in classfiers)
{
if (i >= initialCount - pair.Key - 1)
{
var predictionsNow = WekaClassifier.PredictFP(pair.Value, p, candlesticks, index);
var wfpNow = WekaClassifier.FPToWFP(pair.Value, predictionsNow);
wfp[pair.Key].Add(wfpNow);
}
}
// Future
if (i + 1 >= initialCount)
{
var lastCandle = candlesticks[WekaClassifier.kTrainingPeriod][candlesticks[WekaClassifier.kTrainingPeriod].Count - 1];
var estimatedPrices = new List <float>(classfiers.Count);
foreach (KeyValuePair <int, List <WekaClassifier> > pair in classfiers)
{
estimatedPrices.Add(EstimatePrice(pair.Key, wfp[pair.Key], candlesticks[WekaClassifier.kTrainingPeriod]));
}
var targetPrice = Statistics.WeightedArithmeticMean(estimatedPrices.ToArray(), weight.ToArray());
targetPrice = Statistics.Clamp(targetPrice, lastCandle.MedianPrice * (1.0f - 3.0f * priceDiffStdDev), lastCandle.MedianPrice * (1.0f + 3.0f * priceDiffStdDev));
var candle = new Candlestick(lastCandle.EndTime, lastCandle.EndTime + 86400000 / (ulong)Candlestick.PeriodToDaily(Candlestick.Period.H1), lastCandle.ClosePrice, targetPrice, Math.Max(lastCandle.ClosePrice, targetPrice), Math.Min(lastCandle.ClosePrice, targetPrice), targetPrice);
candlesticks.Add(candle, WekaClassifier.kTrainingPeriod);
}
}
}
//**************************************************************************************
/// <summary>
/// Starts optimal classifiers combination search.
/// </summary>
public static void Search(CandlestickCollection iHistoricalCandlesticks, List <WekaClassifier> iRootClassifiers)
{
var startTime = new DateTime(2002, 1, 1);
var endTime = new DateTime(2019, 12, 23);
while (true)
{
// Load enabled classifiers
var enabledClassifiers = WekaClassifier.Find(iRootClassifiers, WekaClassifier.LoadEnabledClassifiers());
enabledClassifiers = enabledClassifiers.OrderBy(x => x.GetPrecision()).ToList();
// Performe historical simulation
ImpliedSimulation.SimulateFullHistory(iHistoricalCandlesticks, enabledClassifiers, startTime, endTime, out var bestCorrelation, out var bestStdDev);
var bestPrecision = CalculateAveragePrecision(enabledClassifiers) / 100.0f;
var found = false;
OutputNewBest(bestCorrelation, bestStdDev, bestPrecision);
// Perform historical simulations with one classifier removed
for (int i = 0; i < enabledClassifiers.Count; i++)
{
var currentClassifiers = WekaClassifier.Remove(enabledClassifiers, enabledClassifiers[i].ID);
ImpliedSimulation.SimulateFullHistory(iHistoricalCandlesticks, currentClassifiers, startTime, endTime, out var currentCorrelation, out var currentStdDev, "WithoutClassifier" + enabledClassifiers[i].ID.ToString());
var currentPrecision = CalculateAveragePrecision(currentClassifiers) / 100.0f;
if (IsBetterCombination(bestCorrelation, bestStdDev, bestPrecision, currentCorrelation, currentStdDev, currentPrecision))
{
Console.WriteLine("Weak classifier has been found. Classifier ID: " + enabledClassifiers[i].ID);
OutputNewBest(currentCorrelation, currentStdDev, currentPrecision);
WekaClassifier.DisableClassifier(enabledClassifiers[i].ID);
ClearOutputDirectory();
found = true;
break;
}
}
if (found)
{
continue;
}
// Create optional classifiers list
var optionalClassifiers = new List <WekaClassifier>();
foreach (WekaClassifier cl in iRootClassifiers)
{
if (!enabledClassifiers.Contains(cl))
{
optionalClassifiers.Add(cl);
}
}
optionalClassifiers = optionalClassifiers.OrderBy(x => x.GetPrecision()).Reverse().ToList();
enabledClassifiers = enabledClassifiers.OrderBy(x => x.GetPrecision()).ToList();
// Perform historical simulations by upgrading existing classifiers
for (int i = 0; i < enabledClassifiers.Count; i++)
{
for (int k = 0; k < optionalClassifiers.Count; k++)
{
if (optionalClassifiers[k].ProfitTime == enabledClassifiers[i].ProfitTime)
{
if (optionalClassifiers[k].GetPrecision() <= enabledClassifiers[i].GetPrecision())
{
if (i < 3)
{
Console.WriteLine("Low precision classifier detected. Classifier ID: " + optionalClassifiers[k].ID);
}
continue;
}
var currentClassifiers = WekaClassifier.Remove(enabledClassifiers, enabledClassifiers[i].ID);
currentClassifiers.Add(optionalClassifiers[k]);
ImpliedSimulation.SimulateFullHistory(iHistoricalCandlesticks, currentClassifiers, startTime, endTime, out var currentCorrelation, out var currentStdDev, "ClassifierUpgrade_" + enabledClassifiers[i].ID.ToString() + "_To_" + optionalClassifiers[k].ID.ToString());
var currentPrecision = CalculateAveragePrecision(currentClassifiers) / 100.0f;
if (IsBetterCombination(bestCorrelation, bestStdDev, bestPrecision, currentCorrelation, currentStdDev, currentPrecision))
{
Console.WriteLine("Classifier upgrade has been found. Old classifier ID: " + enabledClassifiers[i].ID + ". New classifier ID:" + optionalClassifiers[k].ID);
OutputNewBest(currentCorrelation, currentStdDev, currentPrecision);
WekaClassifier.DisableClassifier(enabledClassifiers[i].ID);
WekaClassifier.EnableClassifier(optionalClassifiers[k].ID);
ClearOutputDirectory();
found = true;
break;
}
else
{
optionalClassifiers[k].UnloadModel();
}
}
}
if (found)
{
break;
}
}
if (found)
{
continue;
}
// Perform historical simulations with one classifier added
for (int i = 0; i < optionalClassifiers.Count; i++)
{
var currentClassifiers = new List <WekaClassifier>(enabledClassifiers);
currentClassifiers.Add(optionalClassifiers[i]);
ImpliedSimulation.SimulateFullHistory(iHistoricalCandlesticks, currentClassifiers, startTime, endTime, out var currentCorrelation, out var currentStdDev, "WithClassifier" + optionalClassifiers[i].ID.ToString());
var currentPrecision = CalculateAveragePrecision(currentClassifiers) / 100.0f;
if (IsBetterCombination(bestCorrelation, bestStdDev, bestPrecision, currentCorrelation, currentStdDev, currentPrecision))
{
Console.WriteLine("Strong classifier has been found. Classifier ID: " + optionalClassifiers[i].ID);
OutputNewBest(currentCorrelation, currentStdDev, currentPrecision);
WekaClassifier.EnableClassifier(optionalClassifiers[i].ID);
ClearOutputDirectory();
found = true;
break;
}
else
{
optionalClassifiers[i].UnloadModel();
}
}
if (found)
{
continue;
}
else
{
break;
}
}
}
