//************************************************************************************** /// <summary> /// Retrieves data points for each prediction. /// </summary> private SortedList <Prediction, List <int> > GetHistoricalPredictionPoints(CandlestickCollection iCandlestick, List <int> iTrainingPoints) { var results = new SortedList <Prediction, List <int> >(); foreach (var p in (Prediction[])Enum.GetValues(typeof(Prediction))) { results.Add(p, new List <int>()); } WaitTillModelReady(); // Calculate parameters var parameters = CalculateParameters(Parameters, iCandlestick, iTrainingPoints, Period); // Predictions for (int k = 0; k < parameters.Count; k++) { if (parameters[k] is null) { throw new SystemException("Parameter cannot be empty"); } var pred = Predict(parameters[k]); results[pred].Add(iTrainingPoints[k]); } return(results); }
//************************************************************************************** /// <summary> /// Simulates full history broked down into periods and std. dev. and correlation. /// </summary> public static void SimulateFullHistory(CandlestickCollection iHistoricalCandlesticks, List <WekaClassifier> iClassifiers, DateTime iStartTime, DateTime iEndTime, out float oCorrelation, out float oStdDev, string iName = "") { var currentSim = SimulateFullHistory(iHistoricalCandlesticks, iClassifiers, iStartTime, iEndTime, iName); oStdDev = Statistics.StandardDeviation(currentSim.AsEnumerable().Select(r => r.Field <float>("PDifference")).ToArray()); oCorrelation = (float)Math.Pow(Statistics.CalculateCorrelation(currentSim.AsEnumerable().Select(r => r.Field <float>("PRealPriceChange")).ToArray(), currentSim.AsEnumerable().Select(r => r.Field <float>("PSimulationPriceChange")).ToArray()), 2.0f); }
//************************************************************************************** /// <summary> /// Loads and asserts that realtime candlesticks is retrieved. /// </summary> public static CandlestickCollection LoadRealTimeCandlesticks(CandlestickCollection iHistoricalCandlesticks) { CandlestickCollection realtimeCandlesticks = new CandlestickCollection(CandlestickCollection.Type.Realtime); if (realtimeCandlesticks[Candlestick.Period.m30] == null) { throw new Exception("Could not retrieve real time 30 minutes candlesticks."); } CandlesticksConsistencyCheck(realtimeCandlesticks, iHistoricalCandlesticks); Console.WriteLine("Realtime candlesticks retrieved."); return(realtimeCandlesticks); }
//************************************************************************************** /// <summary> /// Creates and fills in weka instances object. /// </summary> protected static weka.core.Instances CreateInstances(CandlestickCollection iCandlesticks, List <int> iTrainingPoints, java.util.ArrayList iAttributes, List <ClassifierParameter> iParameters, Candlestick.Period iPeriod, int iMaxProfitTime) { int dataSize = iTrainingPoints.Count; // Create instance object var instances = new weka.core.Instances("Data", iAttributes, dataSize); // Set class index instances.setClassIndex(instances.numAttributes() - 1); // Fill instances FillInstances(instances, CalculateDecisions(iCandlesticks[kTrainingPeriod], iTrainingPoints, iMaxProfitTime), CalculateParameters(iParameters, iCandlesticks, iTrainingPoints, iPeriod)); return(instances); }
//************************************************************************************** /// <summary> /// Build cllasifier model and save it to a file. /// </summary> public override void Build(CandlestickCollection iCandlestick) { List <int> trainingPoints = null; // Build model if (!File.Exists(ModelFilename)) { OutputMessage("Building model"); trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime); Model = new weka.classifiers.functions.LinearRegression(); Model.setEliminateColinearAttributes(LRInfo.EliminateColinearAttributes); Model.setMinimal(LRInfo.Minimal); Model.buildClassifier(CreateInstances(iCandlestick, trainingPoints, Attributes, Parameters, Period, ProfitTime)); weka.core.SerializationHelper.write(ModelFilename, Model); coefficients = Model.coefficients(); } // Calculate coeficient of determination and profits std dev if (LRInfo.CoefOfDetermination == null || LRInfo.ProfitStdDev == null) { WaitTillModelReady(); OutputMessage("Calculating coeficient of determination and profits std dev"); if (trainingPoints is null) { trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime); } var par = CalculateParameters(Parameters, iCandlestick, trainingPoints, Period); var futureProfits = FullToTraining(new List <float>(CalculateFutureProfits(iCandlestick[kTrainingPeriod], ProfitTime)), trainingPoints).ToArray(); var pred = new List <float>(); for (int j = 0; j < par.Count; j++) { var fp = PredictDFP(par[j]); var targetPriceChange = (float)Math.Exp(fp); pred.Add(targetPriceChange - 1.0f); } LRInfo.CoefOfDetermination = (float)Math.Pow(Statistics.CalculateCorrelation(futureProfits, pred.ToArray()), 2.0f); LRInfo.ProfitStdDev = Statistics.StandardDeviation(futureProfits, pred.ToArray()); WekaLinearRegressionInfo.UpdateDB(LRInfo); } }
//************************************************************************************** /// <summary> /// Checks if retrieved candlesticks is similar to historical data. /// </summary> private static void CandlesticksConsistencyCheck(CandlestickCollection iRealtimeCandlesticks, CandlestickCollection iHistoricalCandlesticks) { List <float> diff = new List <float>(); for (int i = 0; i < 5 * 365; i++) { DateTime referenceDate = new DateTime(2012, 1, 1, 23, 59, 59); referenceDate = referenceDate.AddDays(i); Candlestick directRealtime = iRealtimeCandlesticks[Candlestick.Period.D1][Candlestick.FindIndex(iRealtimeCandlesticks[Candlestick.Period.D1], Utils.DateTimeToUnix(referenceDate))]; Candlestick directHistorical = iHistoricalCandlesticks[Candlestick.Period.H1][Candlestick.FindIndex(iHistoricalCandlesticks[Candlestick.Period.H1], Utils.DateTimeToUnix(referenceDate))]; diff.Add(Math.Abs(directHistorical.ClosePrice - directRealtime.ClosePrice)); } var currentDiff = Statistics.ArithmeticMean(diff.ToArray()); if (currentDiff > 0.000403) // Should be 400!! { Console.WriteLine("Realtime candlesticks incosistent with historical candlesticks"); Console.WriteLine("Continue (c); Quit (q)?"); string answer = Console.ReadLine(); if (answer == "q") { Application.Exit(); } else if (answer != "c") { throw new Exception("Realtime candlesticks incosistent with historical candlesticks: " + currentDiff); } } // Check for same time zone var consolidatedH = Candlestick.Consolidate(iRealtimeCandlesticks[Candlestick.Period.H1], 1); var trueDaily = iRealtimeCandlesticks[Candlestick.Period.D1]; if (consolidatedH[consolidatedH.Count - 1].EndTime != trueDaily[trueDaily.Count - 1].EndTime) { Console.WriteLine("WARNING: Realtime candlesticks timezone incosistent"); } if (consolidatedH[consolidatedH.Count - 1].OpenPrice != trueDaily[trueDaily.Count - 1].OpenPrice && consolidatedH[consolidatedH.Count - 1].ClosePrice != trueDaily[trueDaily.Count - 1].ClosePrice) { Console.WriteLine("WARNING: Realtime candlesticks timezone incosistent"); } }
//************************************************************************************** public void OutputRSQ(CandlestickCollection iCandlesticks, string oFilename) { WaitTillModelReady(); var trainingPoints = LoadTrainingPoints(iCandlesticks, -1, ProfitTime); var par = CalculateParameters(Parameters, iCandlesticks, trainingPoints, Period); var futureProfits = FullToTraining(new List <float>(CalculateFutureProfits(iCandlesticks[kTrainingPeriod], ProfitTime)), trainingPoints).ToArray(); var pred = new List <float>(); for (int j = 0; j < par.Count; j++) { var fp = PredictDFP(par[j]); var targetPriceChange = (float)Math.Exp(fp); pred.Add(targetPriceChange - 1.0f); } var CoefOfDetermination = (float)Math.Pow(Statistics.CalculateCorrelation(futureProfits, pred.ToArray()), 2.0f); var ProfitStdDev = Statistics.StandardDeviation(futureProfits, pred.ToArray()); Console.WriteLine("Classifier " + ID); Console.WriteLine(CoefOfDetermination); Console.WriteLine(ProfitStdDev); using (StreamWriter writer = new StreamWriter(File.OpenWrite(oFilename))) { writer.WriteLine("DateTime;RSQ;"); for (int j = 0; j < trainingPoints.Count; j++) { var time = Utils.UnixToDateTime(iCandlesticks[kTrainingPeriod][trainingPoints[j]].EndTime).ToString(); writer.WriteLine(time + ";" + Math.Pow(futureProfits[j] - pred[j], 2.0) + ";"); } } }
//************************************************************************************** /// <summary> /// Build cllasifier model and save it to a file. /// </summary> public override void Build(CandlestickCollection iCandlestick) { List <int> trainingPoints = null; // Calculate average profit and std dev if (J48Info.ProfitAverage is null || J48Info.ProfitStdDev is null) { trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime); float[] profits = FullToTraining(new List <float>(CalculateFutureProfits(iCandlestick[kTrainingPeriod], ProfitTime)), trainingPoints).ToArray(); J48Info.ProfitStdDev = Statistics.StandardDeviation(profits); J48Info.ProfitAverage = J48Info.ParentID is null ? 0.0f : Statistics.ArithmeticMean(profits); WekaJ48Info.UpdateDB(J48Info); } // Build model if (!File.Exists(ModelFilename)) { OutputMessage("Building model"); if (trainingPoints is null) { trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime); } Model = new weka.classifiers.trees.J48(); Model.buildClassifier(CreateInstances(iCandlestick, trainingPoints, Attributes, Parameters, Period, ProfitTime)); weka.core.SerializationHelper.write(ModelFilename, Model); } // Perfrom crossfold test if (J48Info.Precision is null) { if (Model is null) { LoadModel(); } OutputMessage("Perfroming crossfold"); if (trainingPoints is null) { trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime); } var instances = CreateInstances(iCandlestick, trainingPoints, Attributes, Parameters, Period, ProfitTime); var evaluation = new weka.classifiers.Evaluation(instances); evaluation.crossValidateModel(Model, instances, 10, new java.util.Random(0)); J48Info.Precision = (float)evaluation.pctCorrect(); WekaJ48Info.UpdateDB(J48Info); } // Perfrom singular test if (J48Info.IsSingular == null) { if (Model is null) { LoadModel(); } OutputMessage("Perfroming singular test"); var results = new SortedList <Prediction, List <int> >(); foreach (Prediction p in (Prediction[])Enum.GetValues(typeof(Prediction))) { results.Add(p, new List <int>()); } if (trainingPoints is null) { trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime); } var parameters = CalculateParameters(Parameters, iCandlestick, trainingPoints, Period); for (int k = 0; k < parameters.Count; k++) { var pred = Predict(parameters[k]); results[pred].Add(trainingPoints[k]); } J48Info.IsSingular = results.Count(x => x.Value.Count > 0) <= 1; WekaJ48Info.UpdateDB(J48Info); } // Calculating prediction profits if (J48Info.PredictionProfits.Count(x => x != null) == 0) { if (Model is null) { LoadModel(); } OutputMessage("Calculating prediction profits"); if (trainingPoints is null) { trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime); } var predictionPoints = GetHistoricalPredictionPoints(iCandlestick, trainingPoints); foreach (Prediction p in (Prediction[])Enum.GetValues(typeof(Prediction))) { float[] profits = FullToTraining(new List <float>(CalculateFutureProfits(iCandlestick[kTrainingPeriod], ProfitTime)), predictionPoints[p]).ToArray(); if (profits.Length < 10) { J48Info.PredictionProfits[(int)p] = DecisionToFutureProfit(p, (float)J48Info.ProfitStdDev, (float)J48Info.ProfitAverage); } else { J48Info.PredictionProfits[(int)p] = Statistics.ArithmeticMean(profits); } } WekaJ48Info.UpdateDB(J48Info); } // Create children if (!J48Info.ReproductionComplete.GetValueOrDefault(false)) { lock (this) { if (J48Info.Precision > 50.0f && !J48Info.IsSingular.GetValueOrDefault(false)) { OutputMessage("Creating children"); if (trainingPoints is null) { trainingPoints = LoadTrainingPoints(iCandlestick, ID, ProfitTime); } var predictionPoints = GetHistoricalPredictionPoints(iCandlestick, trainingPoints); foreach (Prediction p in (Prediction[])Enum.GetValues(typeof(Prediction))) { if (predictionPoints[p] != null && predictionPoints[p].Count >= 1000 && J48Info.ChildrenID[(int)p] == null) { var child = CreateNew(ParametersID, Parameters, Period, ProfitTime, predictionPoints[p]); // Set parent child.J48Info.ParentID = ID; WekaJ48Info.UpdateDB(child.J48Info); // Update parent info J48Info.ChildrenID[(int)p] = (int)child.ID; WekaJ48Info.UpdateDB(J48Info); childs[(int)p] = child; } } } J48Info.ReproductionComplete = true; WekaJ48Info.UpdateDB(J48Info); } } }
//************************************************************************************** /// <summary> /// Calculates and returns prediction using provided candlesticks and their last values as current time. /// </summary> /// <param name="iCurrentPrice">Last known price.</param> public Prediction Predict(CandlestickCollection iCandlesticks, float iCurrentPrice, CandlestickIndexCollection iCandlestickIndex = null) { WaitTillModelReady(); return(Predict(ClassifierParameter.CalculateParameters(Parameters, iCandlesticks, Period, iCurrentPrice, iCandlestickIndex))); }
//************************************************************************************** /// <summary> /// Returns calculated parameters. /// </summary> /// <param name="iCurrentPrice">Last known price.</param> /// <param name="iCandlestickIndex">Provides index (current time) for each candlestick list. If this value is set to null, the most recent entry will be used.</param> /// <param name="iCandlesticks">Maximum amount of daily candlesticks: 270; 12H: 25; 6H: 50; 3H: 100; 2H: 150; 1H: 300; 30m: 600; 15m: 0; 5m: 0; 1m:0</param> public static float[] CalculateParameters(List <ClassifierParameter> iParameters, CandlestickCollection iCandlesticks, Candlestick.Period iPeriod, float iCurrentPrice, CandlestickIndexCollection iCandlestickIndex) { var candlesticks = iCandlesticks[iPeriod]; int candlestickIndex = iCandlestickIndex == null ? candlesticks.Count - 1 : iCandlestickIndex[iPeriod]; if (candlestickIndex >= iCandlesticks[iPeriod].Count) { throw new Exception("Candlestick index is higher than total number of candlesticks"); } // Make sure we have enough periods for (int i = 0; i < iParameters.Count; i++) { if (candlesticks.Count < iParameters[i].Periods + 10) { return(null); } } float[] results = new float[iParameters.Count]; var macd = new SortedDictionary <int, MACD>(); var macdCurrentPrice = new SortedDictionary <int, MACD>(); for (int i = 0; i < iParameters.Count; i++) { //int candlestickIndex = Candlestick.GetIndex(iParameters[i].Candlesticks, iCandlesticks, iCandlestickIndex); int periods = iParameters[i].Periods; switch (iParameters[i].Type) { case ParameterType.RSI: results[i] = TechnicalIndicators.CalculateRSI(candlesticks, periods, candlestickIndex); break; case ParameterType.RSIWithCurrentPrice: results[i] = TechnicalIndicators.CalculateRSI(candlesticks, periods, candlestickIndex, iCurrentPrice); break; case ParameterType.RSIInt: results[i] = TechnicalIndicators.RSIToInt(TechnicalIndicators.CalculateRSI(candlesticks, periods, candlestickIndex)); break; case ParameterType.RSIIntWithCurrentPrice: results[i] = TechnicalIndicators.RSIToInt(TechnicalIndicators.CalculateRSI(candlesticks, periods, candlestickIndex, iCurrentPrice)); break; case ParameterType.LastCriticalRSI: int startIndex = Math.Max(candlestickIndex - 270 + iParameters[i].Periods, iParameters[i].Periods); int lastCriticalRSI = 0; for (int k = startIndex; k >= candlestickIndex; k++) { lastCriticalRSI = TechnicalIndicators.CalculateLastCriticalRSI(TechnicalIndicators.RSIToInt(TechnicalIndicators.CalculateRSI(candlesticks, periods, k)), lastCriticalRSI); } results[i] = lastCriticalRSI; break; case ParameterType.LastCriticalRSIWithCurrentPrice: startIndex = Math.Max(candlestickIndex - 270 + iParameters[i].Periods, iParameters[i].Periods); lastCriticalRSI = 0; for (int k = startIndex; k >= candlestickIndex; k++) { lastCriticalRSI = TechnicalIndicators.CalculateLastCriticalRSI(TechnicalIndicators.RSIToInt(TechnicalIndicators.CalculateRSI(candlesticks, periods, k, iCurrentPrice)), lastCriticalRSI); } results[i] = lastCriticalRSI; break; case ParameterType.MeanToStd: results[i] = TechnicalIndicators.CalculateMeanToStdDev(candlesticks, periods, candlestickIndex, iCurrentPrice); break; case ParameterType.MeanToStdInt: results[i] = (float)Math.Floor(TechnicalIndicators.CalculateMeanToStdDev(candlesticks, periods, candlestickIndex, iCurrentPrice)); break; case ParameterType.LinearRegressionSlope: results[i] = (float)TechnicalIndicators.CalculateLinearRegressionSlope(candlesticks, periods, candlestickIndex); break; case ParameterType.LinearRegressionSlopePN: results[i] = TechnicalIndicators.CalculateLinearRegressionSlope(candlesticks, periods, candlestickIndex) >= 0 ? 1 : -1; break; case ParameterType.MarginSlope: results[i] = TechnicalIndicators.CalculateMarginSlope(candlesticks, periods, candlestickIndex, iCurrentPrice, iParameters[i].Attributes[0]); break; case ParameterType.MarginSlopePN: results[i] = TechnicalIndicators.CalculateMarginSlopePN(candlesticks, periods, candlestickIndex, iCurrentPrice, iParameters[i].Attributes[0]); break; case ParameterType.MACDSign: if (!macd.ContainsKey(periods)) { macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex)); } results[i] = macd[periods].Signal; break; case ParameterType.MACDSignWithCurrentPrice: if (!macdCurrentPrice.ContainsKey(periods)) { macdCurrentPrice.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex, iCurrentPrice)); } results[i] = macdCurrentPrice[periods].Signal; break; case ParameterType.MACDHist: if (!macd.ContainsKey(periods)) { macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex)); } results[i] = macd[periods].Hist; break; case ParameterType.MACDHistWithCurrentPrice: if (!macdCurrentPrice.ContainsKey(periods)) { macdCurrentPrice.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex, iCurrentPrice)); } results[i] = macdCurrentPrice[periods].Hist; break; case ParameterType.MACDHistChange: if (!macd.ContainsKey(periods)) { macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex)); } MACD macd90 = macd[periods]; MACD previousMACD90 = TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex - 1); results[i] = previousMACD90.Hist == 0 ? 0 : (macd90.Hist / previousMACD90.Hist - 1); break; case ParameterType.MACDHistChangeWithCurrentPrice: if (!macdCurrentPrice.ContainsKey(periods)) { macdCurrentPrice.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex, iCurrentPrice)); } macd90 = macdCurrentPrice[periods]; previousMACD90 = TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex - 1); results[i] = previousMACD90.Hist == 0 ? 0 : (macd90.Hist / previousMACD90.Hist - 1); break; case ParameterType.MACDHistSlope: float[] hist = new float[(int)iParameters[i].Attributes[0]]; for (int k = hist.Length - 1; k >= 0; k--) { hist[hist.Length - 1 - k] = TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex - k).Hist; } results[i] = new LinearRegression(hist).Slope; break; case ParameterType.MACDHistPN: if (!macd.ContainsKey(periods)) { macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex)); } results[i] = macd[periods].Hist >= 0 ? 1 : -1; break; case ParameterType.MACDHistCrossed: if (!macd.ContainsKey(periods)) { macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex)); } macd90 = macd[periods]; previousMACD90 = TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex - 1); if (macd90.Hist >= 0) { results[i] = previousMACD90.Hist >= 0 ? 0 : 1; } else { results[i] = previousMACD90.Hist < 0 ? 0 : -1; } break; case ParameterType.MACDHistDifference: if (!macd.ContainsKey(periods)) { macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex)); } macd90 = macd[periods]; previousMACD90 = TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex - 1); results[i] = macd90.Hist - previousMACD90.Hist; break; case ParameterType.MACD: if (!macd.ContainsKey(periods)) { macd.Add(periods, TechnicalIndicators.CalculateMACD(candlesticks, periods, candlestickIndex)); } results[i] = macd[periods].Macd; break; case ParameterType.SlopesEMA: List <float> slopes = new List <float>(); for (int k = 0; k < iParameters[i].Attributes.Count; k++) { int periodLength = (int)iParameters[i].Attributes[k]; slopes.Add(new LinearRegression(TechnicalIndicators.CreatePriceArray(candlesticks, periodLength, candlestickIndex)).Slope); } results[i] = Utils.Last(Statistics.EMA(slopes.ToArray(), slopes.Count)); break; case ParameterType.ABAverage: results[i] = (TechnicalIndicators.CalculatePriceABaverage(candlesticks, periods, candlestickIndex, iCurrentPrice) ? 1.0f : 0.0f); break; case ParameterType.PercentMargin: results[i] = (TechnicalIndicators.CalculateOnePercentMargin(candlesticks, periods, candlestickIndex, iCurrentPrice, iParameters[i].Attributes[0]) ? 1.0f : 0.0f); break; case ParameterType.Classifier: results[i] = WekaClassifier.Find((int)iParameters[i].Attributes[0]).PredictDFP(iCandlesticks, iCurrentPrice, iCandlestickIndex); break; case ParameterType.ClassifierTargetChangeOldest: var classifier = WekaClassifier.Find((int)iParameters[i].Attributes[0]); var targetTime = iCandlesticks[WekaClassifier.kTrainingPeriod][iCandlestickIndex[WekaClassifier.kTrainingPeriod] - classifier.ProfitTime].StartTime; var pastIndex = new CandlestickIndexCollection(); for (int k = (int)WekaClassifier.kTrainingPeriod; k >= 0; k--) { pastIndex[k] = Math.Max(0, Math.Min(iCandlesticks[k].Count - 1, Candlestick.FindIndex(iCandlesticks[k], targetTime))); } var priceBefore = iCandlesticks[WekaClassifier.kTrainingPeriod][pastIndex[WekaClassifier.kTrainingPeriod]].MedianPrice; var wfp = classifier.PredictDFP(iCandlesticks, priceBefore, pastIndex); var targetPriceChange = (float)Math.Exp(wfp); results[i] = priceBefore * targetPriceChange / iCurrentPrice - 1.0f; break; } } return(results); }
//************************************************************************************** /// <summary> /// Simulates full history broked down into periods and returns DataTable containing results. /// </summary> private static DataTable SimulateFullHistory(CandlestickCollection iHistoricalCandlesticks, List <WekaClassifier> iClassifiers, DateTime iStartTime, DateTime iEndTime, string iName = "") { if (iHistoricalCandlesticks.type == CandlestickCollection.Type.Realtime) { throw new ArgumentException("Historical simulation can not be performed with realtime candlesticks."); } // Create summary data table. var results = new DataTable(); results.Columns.Add("EndTime", typeof(DateTime)); results.Columns.Add("RealPrice", typeof(float)); results.Columns.Add("SimulationPrice", typeof(float)); results.Columns.Add("PDifference", typeof(float)); results.Columns.Add("PRealPriceChange", typeof(float)); results.Columns.Add("PSimulationPriceChange", typeof(float)); // Construct file name var fileName = "Output/" + iHistoricalCandlesticks.type.ToString() + "Simulation_" + iName + ".csv"; Console.WriteLine("Starting simulation: " + iName); if (File.Exists(fileName)) { // Open file using (StreamReader reader = new StreamReader(fileName)) { // Read header reader.ReadLine(); // Read file contents while (!reader.EndOfStream) { var line = reader.ReadLine(); var contents = line.Split(';'); var row = results.NewRow(); row["EndTime"] = DateTime.Parse(contents[0]); row["RealPrice"] = float.Parse(contents[1]); row["SimulationPrice"] = float.Parse(contents[2]); row["PDifference"] = float.Parse(contents[3]); row["PRealPriceChange"] = float.Parse(contents[4]); row["PSimulationPriceChange"] = float.Parse(contents[5]); results.Rows.Add(row); } } return(results); } // Use only high precision classifiers var classifiers = iClassifiers; int it = 0; var tasks = new List <Task <DataTable> >(); // Iterate through history while (true) { DateTime d = iStartTime; d = d.AddDays(it * 7); if (d > iEndTime) { break; } it++; // Create simulation tasks var t = Task.Run(() => PerformSimulation(iHistoricalCandlesticks, classifiers, d, 7 * 24, Candlestick.Period.D1)); tasks.Add(t); } // Wait for tasks to complete foreach (Task t in tasks) { t.Wait(); } foreach (Task <DataTable> t in tasks) { var simRes = t.Result; if (simRes == null || simRes.Rows.Count < 2) { continue; } for (int i = 1; i < simRes.Rows.Count; i++) { var row = results.NewRow(); row["EndTime"] = simRes.Rows[i]["EndTime"]; row["RealPrice"] = simRes.Rows[i]["RealPrice"]; row["SimulationPrice"] = simRes.Rows[i]["SimulationPrice"]; row["PDifference"] = (float)simRes.Rows[i]["SimulationPrice"] / (float)simRes.Rows[i]["RealPrice"] - 1.0f; row["PRealPriceChange"] = Math.Log((float)simRes.Rows[i]["RealPrice"] / (float)simRes.Rows[i - 1]["RealPrice"]); row["PSimulationPriceChange"] = Math.Log((float)simRes.Rows[i]["SimulationPrice"] / (float)simRes.Rows[i - 1]["SimulationPrice"]); results.Rows.Add(row); } } Utils.SaveDataTableToCSV(results, fileName); return(results); }
//************************************************************************************** /// <summary> /// Starts simulation starting from the end of historical candlesticksm and up to prediction duration. /// </summary> private static void Start(CandlestickCollection iHistoricalCandlesticks, CandlestickCollection iCandlesticks, List <WekaClassifier> iClassifiers, int iPredictionDuration) { var index = new CandlestickIndexCollection(); var candlesticks = iCandlesticks; var classfiers = WekaClassifier.SplitByProfitTime(iClassifiers); var initialCount = candlesticks[WekaClassifier.kTrainingPeriod].Count; var startIndex = Math.Max(initialCount - classfiers.Keys.Max() - 1, 1); var endIndex = initialCount + iPredictionDuration - 1; var priceDiffStdDev = Statistics.StandardDeviation(Statistics.CalculateDifferences(iHistoricalCandlesticks[WekaClassifier.kTrainingPeriod].Select(x => x.MedianPrice).ToArray())); var wfp = new SortedDictionary <int, List <float> >(); var weight = new List <float>(classfiers.Count); foreach (KeyValuePair <int, List <WekaClassifier> > pair in classfiers) { wfp.Add(pair.Key, new List <float>()); var precision = WekaClassifier.GetAveragePrecision(pair.Value); weight.Add((float)(Math.Pow(precision, 6) / Math.Pow(pair.Value[0].GetProfitsStdDev(), 6.0f))); } weight = Statistics.Normalize(weight.ToArray()).ToList(); for (int i = startIndex; i <= endIndex; i++) { // Update indexes for (int k = (int)WekaClassifier.kTrainingPeriod; k >= 0; k--) { index[k] = Math.Max(0, Math.Min(candlesticks[k].Count - 1, Candlestick.FindIndex(candlesticks[k], candlesticks[WekaClassifier.kTrainingPeriod][i].StartTime, index[k]) - 1)); } // Extract current price var p = candlesticks[WekaClassifier.kTrainingPeriod][i].MedianPrice; // Calculate WFP foreach (KeyValuePair <int, List <WekaClassifier> > pair in classfiers) { if (i >= initialCount - pair.Key - 1) { var predictionsNow = WekaClassifier.PredictFP(pair.Value, p, candlesticks, index); var wfpNow = WekaClassifier.FPToWFP(pair.Value, predictionsNow); wfp[pair.Key].Add(wfpNow); } } // Future if (i + 1 >= initialCount) { var lastCandle = candlesticks[WekaClassifier.kTrainingPeriod][candlesticks[WekaClassifier.kTrainingPeriod].Count - 1]; var estimatedPrices = new List <float>(classfiers.Count); foreach (KeyValuePair <int, List <WekaClassifier> > pair in classfiers) { estimatedPrices.Add(EstimatePrice(pair.Key, wfp[pair.Key], candlesticks[WekaClassifier.kTrainingPeriod])); } var targetPrice = Statistics.WeightedArithmeticMean(estimatedPrices.ToArray(), weight.ToArray()); targetPrice = Statistics.Clamp(targetPrice, lastCandle.MedianPrice * (1.0f - 3.0f * priceDiffStdDev), lastCandle.MedianPrice * (1.0f + 3.0f * priceDiffStdDev)); var candle = new Candlestick(lastCandle.EndTime, lastCandle.EndTime + 86400000 / (ulong)Candlestick.PeriodToDaily(Candlestick.Period.H1), lastCandle.ClosePrice, targetPrice, Math.Max(lastCandle.ClosePrice, targetPrice), Math.Min(lastCandle.ClosePrice, targetPrice), targetPrice); candlesticks.Add(candle, WekaClassifier.kTrainingPeriod); } } }
//************************************************************************************** /// <summary> /// Performs simulation and outputs results to a DataTable /// </summary> public static DataTable PerformSimulation(CandlestickCollection iCandlesticks, List <WekaClassifier> iClassifiers, DateTime iStartTime, int iNumberOfHours, Candlestick.Period iOutputPeriod = Candlestick.Period.H1) { // Create new data table. var table = new DataTable(iCandlesticks.type.ToString() + "Simulation_" + iStartTime.Year + "_" + iStartTime.Month + "_" + iStartTime.Day + "_" + iNumberOfHours); table.Columns.Add("EndTime", typeof(DateTime)); if (iCandlesticks.type != CandlestickCollection.Type.Realtime) { table.Columns.Add("RealPrice", typeof(float)); } table.Columns.Add("SimulationPrice", typeof(float)); try { CandlestickCollection.Split(iCandlesticks, iStartTime, Candlestick.Period.H1, out CandlestickCollection results, out CandlestickCollection oPart2); var startIndex = results[iOutputPeriod].Count; Start(iCandlesticks, results, iClassifiers, iNumberOfHours); var endIndex = iCandlesticks.type == CandlestickCollection.Type.Realtime ? results[iOutputPeriod].Count : Math.Min(results[iOutputPeriod].Count, iCandlesticks[iOutputPeriod].Count); for (int i = startIndex; i < endIndex; i++) { var row = table.NewRow(); row["EndTime"] = Utils.UnixToDateTime(results[iOutputPeriod][i].EndTime); if (iCandlesticks.type != CandlestickCollection.Type.Realtime) { row["RealPrice"] = iCandlesticks[iOutputPeriod][i].MedianPrice; } row["SimulationPrice"] = results[iOutputPeriod][i].MedianPrice; table.Rows.Add(row); } if (iCandlesticks.type == CandlestickCollection.Type.Realtime) { var medians = new List <float>(24); var sindex = Candlestick.FindIndex(results[Candlestick.Period.H1], Utils.DateTimeToUnix(iStartTime)); for (int i = sindex; i < Math.Min(sindex + 24, results[Candlestick.Period.H1].Count); i++) { medians.Add(results[Candlestick.Period.H1][i].MedianPrice); } var low = float.MaxValue; var high = float.MinValue; for (int i = sindex; i < results[Candlestick.Period.H1].Count; i++) { var p = results[Candlestick.Period.H1][i].MedianPrice; if (p < low) { low = p; } if (p > high) { high = p; } } Console.WriteLine("Next 24 hours estimated median price: " + medians.Median()); Console.WriteLine("Next 7 days High/Low: " + high.ToString() + "/" + low.ToString()); } } catch (Exception ex) { Console.WriteLine("Exception was thrown while performing simulation: " + table.TableName + " Exp.:" + ex.ToString()); } return(table); }
//************************************************************************************** /// <summary> /// Starts optimal classifiers combination search. /// </summary> public static void Search(CandlestickCollection iHistoricalCandlesticks, List <WekaClassifier> iRootClassifiers) { var startTime = new DateTime(2002, 1, 1); var endTime = new DateTime(2019, 12, 23); while (true) { // Load enabled classifiers var enabledClassifiers = WekaClassifier.Find(iRootClassifiers, WekaClassifier.LoadEnabledClassifiers()); enabledClassifiers = enabledClassifiers.OrderBy(x => x.GetPrecision()).ToList(); // Performe historical simulation ImpliedSimulation.SimulateFullHistory(iHistoricalCandlesticks, enabledClassifiers, startTime, endTime, out var bestCorrelation, out var bestStdDev); var bestPrecision = CalculateAveragePrecision(enabledClassifiers) / 100.0f; var found = false; OutputNewBest(bestCorrelation, bestStdDev, bestPrecision); // Perform historical simulations with one classifier removed for (int i = 0; i < enabledClassifiers.Count; i++) { var currentClassifiers = WekaClassifier.Remove(enabledClassifiers, enabledClassifiers[i].ID); ImpliedSimulation.SimulateFullHistory(iHistoricalCandlesticks, currentClassifiers, startTime, endTime, out var currentCorrelation, out var currentStdDev, "WithoutClassifier" + enabledClassifiers[i].ID.ToString()); var currentPrecision = CalculateAveragePrecision(currentClassifiers) / 100.0f; if (IsBetterCombination(bestCorrelation, bestStdDev, bestPrecision, currentCorrelation, currentStdDev, currentPrecision)) { Console.WriteLine("Weak classifier has been found. Classifier ID: " + enabledClassifiers[i].ID); OutputNewBest(currentCorrelation, currentStdDev, currentPrecision); WekaClassifier.DisableClassifier(enabledClassifiers[i].ID); ClearOutputDirectory(); found = true; break; } } if (found) { continue; } // Create optional classifiers list var optionalClassifiers = new List <WekaClassifier>(); foreach (WekaClassifier cl in iRootClassifiers) { if (!enabledClassifiers.Contains(cl)) { optionalClassifiers.Add(cl); } } optionalClassifiers = optionalClassifiers.OrderBy(x => x.GetPrecision()).Reverse().ToList(); enabledClassifiers = enabledClassifiers.OrderBy(x => x.GetPrecision()).ToList(); // Perform historical simulations by upgrading existing classifiers for (int i = 0; i < enabledClassifiers.Count; i++) { for (int k = 0; k < optionalClassifiers.Count; k++) { if (optionalClassifiers[k].ProfitTime == enabledClassifiers[i].ProfitTime) { if (optionalClassifiers[k].GetPrecision() <= enabledClassifiers[i].GetPrecision()) { if (i < 3) { Console.WriteLine("Low precision classifier detected. Classifier ID: " + optionalClassifiers[k].ID); } continue; } var currentClassifiers = WekaClassifier.Remove(enabledClassifiers, enabledClassifiers[i].ID); currentClassifiers.Add(optionalClassifiers[k]); ImpliedSimulation.SimulateFullHistory(iHistoricalCandlesticks, currentClassifiers, startTime, endTime, out var currentCorrelation, out var currentStdDev, "ClassifierUpgrade_" + enabledClassifiers[i].ID.ToString() + "_To_" + optionalClassifiers[k].ID.ToString()); var currentPrecision = CalculateAveragePrecision(currentClassifiers) / 100.0f; if (IsBetterCombination(bestCorrelation, bestStdDev, bestPrecision, currentCorrelation, currentStdDev, currentPrecision)) { Console.WriteLine("Classifier upgrade has been found. Old classifier ID: " + enabledClassifiers[i].ID + ". New classifier ID:" + optionalClassifiers[k].ID); OutputNewBest(currentCorrelation, currentStdDev, currentPrecision); WekaClassifier.DisableClassifier(enabledClassifiers[i].ID); WekaClassifier.EnableClassifier(optionalClassifiers[k].ID); ClearOutputDirectory(); found = true; break; } else { optionalClassifiers[k].UnloadModel(); } } } if (found) { break; } } if (found) { continue; } // Perform historical simulations with one classifier added for (int i = 0; i < optionalClassifiers.Count; i++) { var currentClassifiers = new List <WekaClassifier>(enabledClassifiers); currentClassifiers.Add(optionalClassifiers[i]); ImpliedSimulation.SimulateFullHistory(iHistoricalCandlesticks, currentClassifiers, startTime, endTime, out var currentCorrelation, out var currentStdDev, "WithClassifier" + optionalClassifiers[i].ID.ToString()); var currentPrecision = CalculateAveragePrecision(currentClassifiers) / 100.0f; if (IsBetterCombination(bestCorrelation, bestStdDev, bestPrecision, currentCorrelation, currentStdDev, currentPrecision)) { Console.WriteLine("Strong classifier has been found. Classifier ID: " + optionalClassifiers[i].ID); OutputNewBest(currentCorrelation, currentStdDev, currentPrecision); WekaClassifier.EnableClassifier(optionalClassifiers[i].ID); ClearOutputDirectory(); found = true; break; } else { optionalClassifiers[i].UnloadModel(); } } if (found) { continue; } else { break; } } }