//public void SetTimeDays(DateTime trainStart, int trainDays, int testDays) //{ // SetTimeMinutesFromTrainStart(trainStart, trainDays * 60 * 24, testDays * 60 * 24); //} public void GenerateEaOrder(CandidateClassifier bestClassifierInfo, weka.core.Instances testInstances) { if (bestClassifierInfo == null) return; //int j = m_bestClassifierInfo.DealType == m_dealType[0] ? 0 : 1; string eaorderFileName = string.Format("{0}\\ea_order.txt", TestParameters.BaseDir); //Instances testInstances = bestClassifierInfo.CurrentTestInstances; //Instances testInstancesNew = null; //Instances testInstances = null; //if (m_saveDataFile) //{ // SetTraining(false); // string testFileName = GetArffFileName(m_dealType[j], m_newFileAppend); // WekaUtils.Instance.WriteLog("Load instance of " + testFileName); // testInstancesNew = WekaUtils.LoadInstances(testFileName); // testInstances = WekaUtils.LoadInstances(GetArffFileName(m_dealType[j])); //} //else //{ // testInstancesNew = m_testInstancesNew[j]; // testInstances = m_testInstances[j]; //} System.Data.DataTable dt = null; if (!testInstances.attribute(0).isDate()) { dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(string.Format("SELECT Date FROM {0} WHERE Date >= '{1}' AND Date < '{2}' {4}", "EURUSD" + "_" + "M1", WekaData.m_testTimeStart.ToString(Parameters.DateTimeFormat), WekaData.m_testTimeEnd.ToString(Parameters.DateTimeFormat), "ORDER BY Time")); if (dt.Rows.Count != testInstances.numInstances()) { WekaUtils.Instance.WriteLog("different count!"); return; } } //else //{ // if (testInstancesNew.numInstances() != testInstances.numInstances()) // { // WekaUtils.Instance.WriteLog("different count!"); // return; // } //} // todo: save info in file //for (int j = 0; j < m_dealType.Length; ++j) //{ // if (m_saveDataFile) // { // string modelFileName = GetModelFileName(m_dealType[j]); // WriteLog("Load model of " + modelFileName); // //ObjectInputStream ois = new ObjectInputStream(new FileInputStream(modelFileName)); // //Classifier cls = (Classifier)ois.readObject(); // //ois.close(); // m_bestClassifier[m_dealType[j]] = (Classifier)weka.core.SerializationHelper.read(modelFileName); // } //} //foreach (string testFileName in System.IO.Directory.GetFiles(m_baseDir, NormalizeFileName(string.Format("{0}_{1}_{2}_*.arff", m_symbol, dt1.ToString(m_dateTimeFormat), dt2.ToString(m_dateTimeFormat))))) //{ // string[] ss2 = System.IO.Path.GetFileNameWithoutExtension(testFileName).Split(new char[] { '_' }, StringSplitOptions.RemoveEmptyEntries); // if (ss2.Length != 5) // continue; // try // { // m_trainTimeStart = ConvertFileNameToDateTime(ss2[1]); // m_trainTimeEnd = ConvertFileNameToDateTime(ss2[2]); // m_testTimeStart = ConvertFileNameToDateTime(ss2[3]); // m_testTimeEnd = ConvertFileNameToDateTime(ss2[4]); // } // catch (Exception) // { // continue; // } double[] tr = bestClassifierInfo.CurrentTestRet; double[] cv = bestClassifierInfo.CurrentClassValue; using (StreamWriter sw = new StreamWriter(eaorderFileName, true)) { for (int i = 0; i < testInstances.numInstances(); i++) { string clsLabels; double v = tr[i]; if (v != double.NaN) { clsLabels = testInstances.classAttribute().value((int)v);// CalcAction(v); } else { clsLabels = "0"; } string clsLabel; if (clsLabels == "2") { clsLabel = bestClassifierInfo.DealType == 'B' ? "1" : "-1"; } else { clsLabel = "0"; } //if (clsLabels.Length >= 2) //{ // if (clsLabels[0] == "1" && clsLabels[1] == "0") // clsLabel = "1"; // else if (clsLabels[0] == "0" && clsLabels[1] == "1") // clsLabel = "-1"; // else if (clsLabels[0] == "1" && clsLabels[1] == "1") // clsLabel = "2"; //} //else if (clsLabels.Length == 1) //{ // if (clsLabels[0] == "1") // { // if (m_dealType[0] == "B") // clsLabel = "1"; // else if (m_dealType[0] == "S") // clsLabel = "-1"; // } //} //else //{ // throw new AssertException("invalid clsLabels.Length"); //} string date; string dateClose = string.Empty; if (testInstances.attribute(0).isDate()) { date = WekaUtils.GetDateValueFromInstances(testInstances, 0, i).ToString(Parameters.DateTimeFormat); dateClose = WekaUtils.GetDateValueFromInstances(testInstances, 1, i).ToString(Parameters.DateTimeFormat); } else { date = ((DateTime)dt.Rows[i]["Date"]).ToString(Parameters.DateTimeFormat); } sw.WriteLine(string.Format("{0}, {1}, {2}, {3}, 0, 0, {4}, {5}", m_actions[clsLabel], date, bestClassifierInfo.Tp, bestClassifierInfo.Sl, dateClose, cv[i] == v ? "Right" : "Wrong")); //ret[ConvertFileNameToDateTime(date)] = clsLabel; } } //// save labeled data //BufferedWriter writer = new BufferedWriter(new FileWriter(string.Format("c:\\eurusd_m1_arff.result"))); //writer.write(labeled.toString()); //writer.newLine(); //writer.flush(); //writer.close(); }
private void TrainandTest() { var c = new CandidateClassifier("Test", m_currentTp, m_currentSl, Parameters.AllDealTypes[0], 0, null); TrainandTest(c, null); //var ev = (new MyEvaluation(TestParameters.CostMatrix)); //ev.evaluateModel(c.Classifier, c.CurrentTrainInstancesNew1); //ev.evaluateModel(c.Classifier, c.CurrentTestInstancesNew1); }
public void TrainandTest(CandidateClassifier classifierInfo, CandidateParameter cp) { //string dealType = classifierInfo.DealType; Classifier cls = null; if (TestParameters.UseTrain) { string modelFileName = GetModelFileName(classifierInfo.Name); if (TestParameters.SaveModel) { cls = WekaUtils.TryLoadClassifier(modelFileName); } Instances trainInstancesNew, trainInstances; trainInstances = m_trainInstances; trainInstancesNew = m_trainInstancesNew; if (cls == null) { if (classifierInfo.Classifier == null) { classifierInfo.Classifier = WekaUtils.CreateClassifier(cp.ClassifierType, m_currentTp, m_currentSl); } cls = classifierInfo.Classifier; } else { if (TestParameters.EnableDetailLog) { System.Console.WriteLine("Model is loaded."); } } if (m_enableTrainSplitTest) { Instances trainTrainInst, trainTestInst; DateTime splitTrainTimeEnd; if (m_trainSplitTestNums != -1) { int trainTrainSize = trainInstancesNew.numInstances() - m_trainSplitTestNums; int trainTestSize = m_trainSplitTestNums; trainTrainInst = new Instances(trainInstancesNew, 0, trainTrainSize); trainTestInst = new Instances(trainInstancesNew, trainTrainSize, trainTestSize); splitTrainTimeEnd = WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainTrainSize); } else if (m_trainSplitPercent != -1) { if (m_trainSplitPercent == 100.0) { int trainTrainSize = trainInstancesNew.numInstances(); trainTrainInst = new Instances(trainInstancesNew, 0, trainTrainSize); trainTestInst = new Instances(trainInstancesNew, 0, trainTrainSize); splitTrainTimeEnd = WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainTrainSize); } else { int trainTrainSize = (int)Math.Round(trainInstancesNew.numInstances() * m_trainSplitPercent / 100); int trainTestSize = trainInstancesNew.numInstances() - trainTrainSize; trainTrainInst = new Instances(trainInstancesNew, 0, trainTrainSize); trainTestInst = new Instances(trainInstancesNew, trainTrainSize, trainTestSize); splitTrainTimeEnd = WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainTrainSize); } } else { trainTrainInst = new Instances(trainInstancesNew, 0); trainTestInst = new Instances(trainInstancesNew, 0); DateTime dt = WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainInstances.numInstances() - 1); splitTrainTimeEnd = m_trainTimeEnd.AddMinutes(-TestParameters.BatchTestMinutes); while (splitTrainTimeEnd > dt) { splitTrainTimeEnd = splitTrainTimeEnd.AddMinutes(-TestParameters.BatchTestMinutes); } for (int i = 0; i < trainInstances.numInstances(); ++i) { dt = WekaUtils.GetDateValueFromInstances(trainInstances, 0, i); if (dt <= splitTrainTimeEnd) { var ins = new DenseInstance(trainInstancesNew.instance(i)); trainTrainInst.add(ins); } else { var ins = new DenseInstance(trainInstancesNew.instance(i)); trainTestInst.add(ins); } } } cls = WekaUtils.TrainInstances(trainTrainInst, TestParameters.SaveModel ? modelFileName : null, cls); //m_classifierQueue[dealType].Enqueue(new ClassifierInfo(cls, splitTrainTimeEnd)); //foreach (var i in m_classifierQueue[dealType]) //{ // var e = WekaUtils.TestInstances(trainTestInst, i.Cls); // i.TotalCost = i.TotalCost * m_classifierQueueFactor + e.totalCost(); // i.TotalNum = (int)(i.TotalNum * m_classifierQueueFactor) + (int)e.numInstances(); //} //WriteEvalSummary(eval1, string.Format("Test Data from {0} to {1}", m_testTimeStart.ToString(Parameters.DateTimeFormat), m_testTimeEnd.ToString(Parameters.DateTimeFormat))); } else { cls = WekaUtils.TrainInstances(trainInstancesNew, TestParameters.SaveModel ? modelFileName : null, cls); //m_classifierQueue[dealType].Enqueue(new ClassifierInfo(cls, m_trainTimeEnd)); //foreach (var i in m_classifierQueue[dealType]) //{ // var e = WekaUtils.TestInstances(trainInstancesNew, i.Cls); // i.TotalCost = i.TotalCost * m_classifierQueueFactor + e.totalCost(); // i.TotalNum = (int)(i.TotalNum * m_classifierQueueFactor) + (int)e.numInstances(); //} } if (TestParameters.EnableDetailLog) { System.Console.WriteLine("Model is trained."); } classifierInfo.Classifier = cls; //if (classifierInfo.CurrentTrainInstances1 != null) //{ // classifierInfo.CurrentTrainInstances1.clear(); //} //if (classifierInfo.CurrentTrainInstancesNew1 != null) //{ // classifierInfo.CurrentTrainInstancesNew1.clear(); //} //classifierInfo.CurrentTrainInstances = new Instances(trainInstances, 0, trainInstances.numInstances()); //classifierInfo.CurrentTrainInstancesNew = new Instances(trainInstancesNew, 0, trainInstancesNew.numInstances()); if (classifierInfo.MoneyManagement == null) { classifierInfo.MoneyManagement = WekaUtils.CreateMoneyManagement(cp.MoneyManagementType, m_currentTp, m_currentSl); } IMoneyManagement mm = WekaUtils.TrainInstances4MM(trainInstancesNew, TestParameters.SaveModel ? modelFileName : null, classifierInfo.MoneyManagement); classifierInfo.MoneyManagement = mm; } else { if (classifierInfo.Classifier == null) { classifierInfo.Classifier = WekaUtils.CreateClassifier(cp.ClassifierType, m_currentTp, m_currentSl); } cls = classifierInfo.Classifier; if (classifierInfo.MoneyManagement == null) { classifierInfo.MoneyManagement = WekaUtils.CreateMoneyManagement(cp.MoneyManagementType, m_currentTp, m_currentSl); } } if (m_enableTest) { Instances testInstancesNew, testInstances; testInstances = m_testInstances; testInstancesNew = m_testInstancesNew; double[] cv = WekaUtils.ClassifyInstances(testInstancesNew, cls); if (TestParameters.EnableExcludeClassifier) { bool hasPositive = false; for (int i = 0; i < cv.Length; ++i) { if (cv[i] == 2) { hasPositive = true; break; } } if (hasPositive) { // Exclude if (classifierInfo.ExcludeClassifier == null) { string modelFileName4Exclude = GetExcludeModelFileName(classifierInfo.Name); classifierInfo.ExcludeClassifier = WekaUtils.TryLoadClassifier(modelFileName4Exclude); } if (classifierInfo.ExcludeClassifier != null) { double[] cv2 = WekaUtils.ClassifyInstances(testInstancesNew, classifierInfo.ExcludeClassifier); // cv2 == 0 -> is class, otherwise = double.NaN; for (int i = 0; i < cv.Length; ++i) { cv[i] = cv[i] == 2 && cv2[i] == 2 ? 2 : 0; } } } } classifierInfo.CurrentTestRet = cv; classifierInfo.CurrentClassValue = new double[testInstances.numInstances()]; for (int i = 0; i < testInstances.numInstances(); ++i) { classifierInfo.CurrentClassValue[i] = testInstances.instance(i).classValue(); } for (int i = 0; i < testInstances.numInstances(); i++) { if (cv[i] == 2) { double openPrice = testInstances.instance(i).value(testInstances.attribute("mainClose")); DateTime openTime = WekaUtils.GetDateValueFromInstances(testInstances, 0, i); if (testInstances.instance(i).classValue() == 2 || testInstances.instance(i).classValue() == 0) { classifierInfo.Deals.AddDeal(openTime, openPrice, classifierInfo.DealType, classifierInfo.MoneyManagement.GetVolume(testInstances.instance(i)), testInstances.instance(i).classValue() == 2 ? -classifierInfo.Tp : classifierInfo.Sl, WekaUtils.GetDateValueFromInstances(testInstances, 1, i)); } else if (testInstances.instance(i).classValue() == 1) { double closePriceTp, closePriceSl; if (classifierInfo.DealType == 'B') { closePriceTp = openPrice + classifierInfo.Tp * DealsInfo.GetPoint(0); closePriceSl = openPrice - classifierInfo.Sl * DealsInfo.GetPoint(0); } else { closePriceTp = openPrice - classifierInfo.Tp * DealsInfo.GetPoint(0); closePriceSl = openPrice + classifierInfo.Sl * DealsInfo.GetPoint(0); } classifierInfo.Deals.AddDeal(openTime, openPrice, classifierInfo.DealType, classifierInfo.MoneyManagement.GetVolume(testInstances.instance(i)), closePriceTp, closePriceSl); } else { throw new AssertException("classValue should be 0,1,2."); } } } } }
protected void InitClassifierInfos(ParameterdCandidateStrategy parent = null) { //int[] sls = m_batchSls != null ? m_batchSls : (m_sl.HasValue ? new int[] { m_sl.Value } : new int[] { 50, 70 }); //int[][] tps = m_batchTps != null ? m_batchTps : (m_tp.HasValue ? new int[][] { new int[] { m_tp.Value } } // : new int[][] { new int[] { 100, 140 }, new int[] { 50, 70 } }); //if (Parameters.AllDealTypes == null || Parameters.AllDealTypes.Length == 0) //{ // Parameters.AllDealTypes = new char[] { 'B', 'S' }; //} if (parent == null) { m_classifierInfos.Clear(); m_classifierInfoIdxs = new CandidateClassifier[Parameters.AllDealTypes.Length, this.CandidateParameter.BatchTps.Length, this.CandidateParameter.BatchSls.Length, Parameters.AllHour]; m_totalScores = new double[m_classifierInfoIdxs.GetLength(0), m_classifierInfoIdxs.GetLength(1), m_classifierInfoIdxs.GetLength(2), m_classifierInfoIdxs.GetLength(3)]; m_totalDeals = new int[m_classifierInfoIdxs.GetLength(0), m_classifierInfoIdxs.GetLength(1), m_classifierInfoIdxs.GetLength(2), m_classifierInfoIdxs.GetLength(3)]; for (int k = 0; k < m_classifierInfoIdxs.GetLength(0); ++k) { for (int i = 0; i < m_classifierInfoIdxs.GetLength(1); ++i) { for (int j = 0; j < m_classifierInfoIdxs.GetLength(2); ++j) { int tp = this.CandidateParameter.BatchTps[i]; int sl = this.CandidateParameter.BatchSls[j]; if (!TestParameters.EnablePerhourTrain) { string name = string.Format("{0}_{1}_{2}", Parameters.AllDealTypes[k], tp, sl); m_classifierInfos[name] = new CandidateClassifier(name, tp, sl, Parameters.AllDealTypes[k], -1, this.CandidateParameter); m_classifierInfoIdxs[k, i, j, 0] = m_classifierInfos[name]; m_totalScores[k, i, j, 0] = 0; m_totalDeals[k, i, j, 0] = 0; } else { for (int h = 0; h < m_classifierInfoIdxs.GetLength(3); ++h) { //if (h != 9) // continue; string name = string.Format("{0}_{1}_{2}_H{3}", Parameters.AllDealTypes[k], tp, sl, h); m_classifierInfos[name] = new CandidateClassifier(name, tp, sl, Parameters.AllDealTypes[k], h, this.CandidateParameter); m_classifierInfoIdxs[k, i, j, h] = m_classifierInfos[name]; m_totalScores[k, i, j, h] = 0; m_totalDeals[k, i, j, h] = 0; } } } } } this.HasParent = false; } else { m_classifierInfoIdxs = parent.m_classifierInfoIdxs; m_totalScores = new double[m_classifierInfoIdxs.GetLength(0), m_classifierInfoIdxs.GetLength(1), m_classifierInfoIdxs.GetLength(2), m_classifierInfoIdxs.GetLength(3)]; m_totalDeals = new int[m_classifierInfoIdxs.GetLength(0), m_classifierInfoIdxs.GetLength(1), m_classifierInfoIdxs.GetLength(2), m_classifierInfoIdxs.GetLength(3)]; m_classifierInfos = parent.m_classifierInfos; this.HasParent = true; } }
public void SaveTestData(CandidateClassifier clsInfo, DateTime nowDate) { if (TestParameters.OnlyNewestTestDataSaved) { string sql = string.Format("DELETE FROM {0}", m_testDataTableName); Feng.Data.DbHelper.Instance.ExecuteNonQuery(sql); } string str = WekaUtils.DoubleArrayToIntString(clsInfo.CurrentTestRet); string scv = WekaUtils.DoubleArrayToIntString(clsInfo.CurrentClassValue); if (m_testDataInsertTable == null) { m_testDataInsertTable = new System.Data.DataTable(); m_testDataInsertTable.Columns.Add("Id", typeof(int)); m_testDataInsertTable.Columns.Add("Time", typeof(long)); //m_testDataInsertTable.Columns.Add("Type", typeof(int)); m_testDataInsertTable.Columns.Add("ClsName", typeof(string)); m_testDataInsertTable.Columns.Add("TestResult", typeof(string)); m_testDataInsertTable.Columns.Add("ClassValue", typeof(string)); m_testDataInsertTable.Columns.Add("DealsInfo", typeof(byte[])); m_testDataInsertTable.Columns.Add("DealsData", typeof(byte[])); } var row = m_testDataInsertTable.NewRow(); row["ClsName"] = clsInfo.Name; //row["Type"] = m_testDataType; row["Time"] = WekaUtils.GetTimeFromDate(nowDate); row["TestResult"] = str; row["ClassValue"] = scv; row["DealsInfo"] = Feng.Windows.Utils.SerializeHelper.Serialize(clsInfo.Deals); bool shouldSave = false; if (nowDate.DayOfWeek != DayOfWeek.Sunday && nowDate.DayOfWeek != DayOfWeek.Saturday && nowDate.Day == 20) { shouldSave = true; } if (nowDate.Day == 21 && nowDate.DayOfWeek == DayOfWeek.Monday) { shouldSave = true; } if (nowDate.Day == 22 && nowDate.DayOfWeek == DayOfWeek.Monday) { shouldSave = true; } if (TestParameters.OnlyNewestTestDataSaved) { shouldSave = true; } if (shouldSave) { byte[] dealsData = Feng.Windows.Utils.SerializeHelper.Serialize(clsInfo.Deals.Deals); System.IO.MemoryStream outStream = new System.IO.MemoryStream(); using (System.IO.Compression.GZipStream zipStream = new System.IO.Compression.GZipStream(outStream, System.IO.Compression.CompressionMode.Compress)) { System.IO.MemoryStream inStream = new System.IO.MemoryStream(dealsData); inStream.CopyTo(zipStream); } row["DealsData"] = outStream.ToArray(); } else { row["DealsData"] = System.DBNull.Value; } m_testDataInsertTable.Rows.Add(row); if (m_testDataInsertTable.Rows.Count > 100) { //WekaUtils.Instance.WriteLog("Save Test Data."); try { Feng.Data.DbHelper.Instance.BulkCopy(m_testDataInsertTable, m_testDataTableName); } catch (Exception ex) { WekaUtils.Instance.WriteLog(ex.Message); } m_testDataInsertTable.Rows.Clear(); } //SqlCommand cmd = new SqlCommand("INSERT INTO TestData ([ClsName],[Time], [Type], TestResult, ClassValue, DealsData) VALUES (@ClsName, @Time, @Type, @TestResult, @ClassValue, @DealsData)"); //cmd.Parameters.AddWithValue("@ClsName", clsInfo.Name); //cmd.Parameters.AddWithValue("@Type", m_testDataType); //cmd.Parameters.AddWithValue("@Time", WekaUtils.GetTimeFromDate(currentDate)); //cmd.Parameters.AddWithValue("@TestResult", str); //cmd.Parameters.AddWithValue("@ClassValue", scv); //cmd.Parameters.AddWithValue("@DealsData", Feng.Utils.SerializeHelper.Serialize(clsInfo.Deals)); //SetTransaction(cmd); }
public static void PrintDealInfoSum(DateTime date) { string sql = ("SELECT Time, ClsName, TestResult, ClassValue, DealsInfo, DealsData FROM TestData WHERE [Time] >= @Time1 AND [Time] < @Time2 ORDER BY Time, ClsName"); SqlCommand cmd = new SqlCommand(string.Format(sql)); cmd.Parameters.AddWithValue("@Time1", WekaUtils.GetTimeFromDate(date)); cmd.Parameters.AddWithValue("@Time2", WekaUtils.GetTimeFromDate(date.AddDays(1))); var dt = DbHelper.Instance.ExecuteDataTable(cmd); StringBuilder sb = new StringBuilder(); foreach (System.Data.DataRow row in dt.Rows) { var clsInfo = new CandidateClassifier(row["ClsName"].ToString(), 0, 0, 'B', -1, null); clsInfo.SetData((string)row["TestResult"], (string)row["ClassValue"], (byte[])row["DealsInfo"], (byte[])row["DealsData"]); string s = clsInfo.Deals.PrintAll(); sb.AppendLine(clsInfo.Name); sb.AppendLine(s); } Console.WriteLine(sb.ToString()); Console.ReadLine(); }
public static void PrintDealInfoDetail(string clsInfoName, DateTime date) { string sql = ("SELECT Time, ClsName, TestResult, ClassValue, DealsInfo, DealsData FROM TestData WHERE [Time] = @Time AND ClsName = @ClsName"); SqlCommand cmd = new SqlCommand(string.Format(sql)); cmd.Parameters.AddWithValue("@Time", WekaUtils.GetTimeFromDate(date)); cmd.Parameters.AddWithValue("@ClsName", clsInfoName); var dt = DbHelper.Instance.ExecuteDataTable(cmd); var row = dt.Rows[0]; var clsInfo = new CandidateClassifier(clsInfoName, 0, 0, 'B', -1, null); clsInfo.SetData((string)row["TestResult"], (string)row["ClassValue"], (byte[])row["DealsInfo"], (byte[])row["DealsData"]); string s = clsInfo.Deals.PrintAll(true); Console.WriteLine(s); Console.ReadLine(); }