public ParameterdCandidateStrategy(CandidateParameter cp = null, ParameterdCandidateStrategy parent = null, IBestCandidateSelector bestCandidateSelector = null) { if (cp == null) { cp = new CandidateParameter("Default"); } this.CandidateParameter = cp; if (cp.ClassifierType == null && !TestParameters.EnableExcludeClassifier) { cp.SymbolCount = 1; cp.PeriodCount = 1; cp.PeriodTimeCount = 0; cp.PrevTimeCount = 0; } InitClassifierInfos(parent); if (TestParameters2.RealTimeMode) { m_enableDetailLogLevel2 = false; } m_bestCandidateSelector = bestCandidateSelector; }
public WekaData(char dealType, int tp, int sl, CandidateParameter cp) { m_currentDealType = dealType; m_currentTp = tp; m_currentSl = sl; m_cp = cp; }
public CandidateParameter Clone() { CandidateParameter that = this.MemberwiseClone() as CandidateParameter; that.BatchTps = new int[this.BatchTps.Length]; this.BatchTps.CopyTo(that.BatchTps, 0); that.BatchSls = new int[this.BatchSls.Length]; this.BatchSls.CopyTo(that.BatchSls, 0); that.AllIndNames = new Dictionary <string, int>(); foreach (var i in this.AllIndNames) { that.AllIndNames.Add(i.Key, i.Value); } that.AllIndNames2 = new Dictionary <string, int>(); foreach (var i in this.AllIndNames2) { that.AllIndNames2.Add(i.Key, i.Value); } that.AllSymbols = new string[this.AllSymbols.Length]; for (int i = 0; i < that.AllSymbols.Length; ++i) { that.AllSymbols[i] = this.AllSymbols[i]; } that.AllPeriods = new string[this.AllPeriods.Length]; for (int i = 0; i < that.AllPeriods.Length; ++i) { that.AllPeriods[i] = this.AllPeriods[i]; } return(that); }
public CandidateClassifier(string name, int tp, int sl, char dealType, int hour, CandidateParameter cp) { this.Name = name; this.Tp = tp; this.Sl = sl; this.DealType = dealType; this.Hour = hour; this.Deals = new DealsInfo(cp.DealInfoLastMinutes); this.WekaData = new WekaData(dealType, tp, sl, cp); }
public static void SaveCCScoresToDb(DateTime nowDate, CandidateParameter cp, ParameterdCandidateStrategy pcs) { string tableName = string.Format("{0}_CCSCORE_{1}", cp.MainSymbol, cp.DealInfoLastMinutes / (7 * 24 * 12 * 5)); long nowTime = WekaUtils.GetTimeFromDate(nowDate); try { Feng.Data.DbHelper.Instance.ExecuteNonQuery(string.Format("SELECT TOP 1 * FROM {0} WHERE TIME = -1", tableName)); } catch (Exception) { string sqlCreate = string.Format(@"CREATE TABLE {0}( [Time] [bigint] NOT NULL, [scores] [varbinary](max) NOT NULL, [nda] [varbinary](max) NOT NULL, [nds] [varbinary](max) NOT NULL, CONSTRAINT [PK_{0}] PRIMARY KEY CLUSTERED ( [Time] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] ) ON [PRIMARY]", tableName); Feng.Data.DbHelper.Instance.ExecuteNonQuery(sqlCreate); } long[, ,] ndas = new long[Parameters.AllDealTypes.Length, cp.BatchTps.Length, cp.BatchSls.Length]; long[, ,] ndss = new long[Parameters.AllDealTypes.Length, cp.BatchTps.Length, cp.BatchSls.Length]; double[, ,] nowScores = new double[Parameters.AllDealTypes.Length, cp.BatchTps.Length, cp.BatchSls.Length]; pcs.IterateClassifierInfos((k, ii, jj, h) => { nowScores[k, ii, jj] = pcs.m_classifierInfoIdxs[k, ii, jj, h].Deals.NowScore; ndas[k, ii, jj] = pcs.m_classifierInfoIdxs[k, ii, jj, h].Deals.DealLastTimeAvg; ndss[k, ii, jj] = pcs.m_classifierInfoIdxs[k, ii, jj, h].Deals.DealLastTimeStd; }); string sql = string.Format("INSERT INTO [{0}] ([Time],[scores],[nda],[nds]) VALUES (@Time, @score, @nda, @nds)", tableName); var cmd = new System.Data.SqlClient.SqlCommand(sql); cmd.Parameters.AddWithValue("@Time", nowTime); cmd.Parameters.AddWithValue("@score", CCScoreData.SerializeScores(nowScores)); cmd.Parameters.AddWithValue("@nda", CCScoreData.SerializeScoreTimes(ndas)); cmd.Parameters.AddWithValue("@nds", CCScoreData.SerializeScoreTimes(ndss)); try { Feng.Data.DbHelper.Instance.ExecuteNonQuery(string.Format("DELETE FROM {0} WHERE TIME = '{1}'", tableName, WekaUtils.GetTimeFromDate(nowDate))); Feng.Data.DbHelper.Instance.ExecuteNonQuery(cmd); } catch (Exception) { } }
private void CreateCds(ref int[, , ,] cds, CandidateParameter cp, int count) { cds = new int[cp.SymbolCount, cp.PeriodCount, count, 61]; for (int s = 0; s < cp.SymbolCount; ++s) { for (int p = 0; p < cp.PeriodCount; ++p) { for (int i = 0; i < cds.GetLength(3); ++i) { for (int j = 0; j < cds.GetLength(2); ++j) { cds[s, p, j, i] = -1; } } } } }
public List<ParameterdCandidateStrategy> AddRealDealCandidates(string symbol, int dealInfoLastWeek, IBestCandidateSelector bcs = null) { List<ParameterdCandidateStrategy> realDealsCandidates = new List<ParameterdCandidateStrategy>(); //TestParameters.InitTpsls(20, 10); //realDealsCandidates.Add(new RealDealCandidate("1M_10", 1 * 4 * 7 * 24 * 12 * 5, this)); //realDealsCandidates.Add(new RealDealCandidate("1.5M_10", (int)(1.5 * 4 * 7 * 24 * 12 * 5), this)); //realDealsCandidates.Add(new RealDealCandidate("2M_10", 2 * 4 * 7 * 24 * 12 * 5, this)); { CandidateParameter cp = new CandidateParameter(symbol); int delta = TestParameters.GetTpSlMinDelta(symbol) * TestParameters2.nTpsl; cp.InitTpsls(TestParameters2.tpStart, delta, TestParameters2.tpCount, TestParameters2.slStart, delta, TestParameters2.slCount); cp.SymbolStart = Array.IndexOf<string>(cp.AllSymbols, symbol); cp.SymbolCount = 1; cp.PeriodStart = Array.IndexOf<string>(cp.AllPeriods, "M1"); cp.PeriodCount = 1; cp.DealInfoLastMinutes = dealInfoLastWeek * 7 * 24 * 12 * 5; cp.Group = 1; InitBatchBatch(cp); if (cp.ClassifierType == null && !TestParameters.EnableExcludeClassifier) { cp.AllIndNames.Clear(); cp.AllIndNames2.Clear(); } else { cp.DeleteUnusedIndicators(); //GenerateBatchEmptyInstance(); } ParameterdCandidateStrategy mainPcs = new ParameterdCandidateStrategy(cp, null, bcs); realDealsCandidates.Add(mainPcs); // for (int i = 2; i <= 3; ++i) // { // var cp2 = cp.Clone(); // cp2.Name += "_" + i.ToString(); // cp2.DealInfoLastMinutes = i * 2 * 4 * 7 * 24 * 12 * 5; // realDealsCandidates.Add(new ParameterdCandidateStrategy(cp2)); // } } { //CandidateParameter cp = new CandidateParameter("GBPUSD"); //cp.InitTpsls(10, 60); //cp.SymbolStart = 1; //cp.DealInfoLastMinutes = 2 * 4 * 7 * 24 * 12 * 5; //cp.Group = 2; //ParameterdCandidateStrategy mainPcs = new ParameterdCandidateStrategy(cp); //mainPcs.m_bestCandidateSelector = new BestCandidateSelector1(0); //realDealsCandidates.Add(mainPcs); // for (int i = 2; i <= 3; ++i) // { // var cp2 = cp.Clone(); // cp2.Name += "_" + i.ToString(); // cp2.DealInfoLastMinutes = i * 2 * 4 * 7 * 24 * 12 * 5; // realDealsCandidates.Add(new ParameterdCandidateStrategy(cp2)); // } } { //CandidateParameter cp = new CandidateParameter("EURGBP"); //cp.InitTpsls(10, 30); //cp.SymbolStart = 6; //cp.DealInfoLastMinutes = 6 * 4 * 7 * 24 * 12 * 5; //cp.Group = 3; //ParameterdCandidateStrategy mainPcs = new ParameterdCandidateStrategy(cp); //mainPcs.m_bestCandidateSelector = new BestCandidateSelector1(0); //realDealsCandidates.Add(mainPcs); // // for (int i = 2; i <= 3; ++i) // // { // // var cp2 = cp.Clone(); // // cp2.Name += "_" + i.ToString(); // // cp2.DealInfoLastMinutes = i * 2 * 4 * 7 * 24 * 12 * 5; // // realDealsCandidates.Add(new ParameterdCandidateStrategy(cp2)); // // } } foreach (var i in realDealsCandidates) { OutputTestInfoCandidate(i.CandidateParameter); WekaUtils.Instance.WriteLog(string.Format("BestCandidateSelector = {0}", i.BestCandidateSelector == null ? "Null" : i.BestCandidateSelector.ToString())); } return realDealsCandidates; }
public static void OutputTestInfoCandidate(CandidateParameter cp) { WekaUtils.Instance.WriteHorizontalLine(); WekaUtils.Instance.WriteLog(string.Format("Candidate Name = {0}", cp.Name)); WekaUtils.Instance.WriteLog(string.Format("MainSymbol = {0}, MainPeriod = {1}", cp.MainSymbol, cp.MainPeriod)); WekaUtils.Instance.WriteLog(string.Format("PeriodCount = {0}, PrevTimeCount = {1}, SymbolCount = {2}", cp.PeriodCount, cp.PrevTimeCount, cp.SymbolCount)); WekaUtils.Instance.WriteLog(string.Format("TakeProfit Length = {0}, StopLoss Length = {1}", cp.BatchTps.Length, cp.BatchSls.Length)); WekaUtils.Instance.WriteLog(string.Format("dealInfoLastMinutes = {0}", cp.DealInfoLastMinutes)); WekaUtils.Instance.WriteLog(string.Format("ClassifierType = {0}", cp.ClassifierType == null ? "Default" : cp.ClassifierType.Name)); WekaUtils.Instance.WriteLog(string.Format("MoneyManagementType = {0}", cp.MoneyManagementType == null ? "Default" : cp.MoneyManagementType.Name)); }
public void InitBatchBatch(CandidateParameter cp) { TestParameters.SaveDataFile = false; TestParameters.EnableDetailLog = false; TestParameters.SaveModel = false; if (cp.ClassifierType == null && !TestParameters.EnableExcludeClassifier) { TestParameters.SaveModel = false; TestParameters.UseFilter = false; if (cp.MoneyManagementType == null) { TestParameters.UseTrain = false; } //GenerateBatchEmptyInstance(); } else { //GenerateBatchEmptyInstance(); } //TestParameters.EnablePerhourTrain = true; OutputTestInfo(); UpdateSettings(); }
public void BuildExcludeModels() { Parameters.TotalCanUseMemory = 600 * 1000 * 1000; TestParameters.SaveDataFile = false; //m_batchBufferMinutes = (new TimeSpan(365 + 5, 0, 0, 0)).TotalMinutes; weka.classifiers.Classifier cls = new weka.classifiers.functions.LibSVM(); (cls as weka.classifiers.AbstractClassifier).setOptions(weka.core.Utils.splitOptions("-S 0 -K 2")); //Classifier cls = new MincostLiblinearClassifier(); //Classifier cls = new weka.classifiers.functions.LibLINEAR(); //(cls as AbstractClassifier).setOptions(weka.core.Utils.splitOptions("-S 0 -P -C 1 -B 1")); //var cls = new SvmLightClassifier(); //cls.setOptions(weka.core.Utils.splitOptions("-c 20 -l 4 -w 1 --p 1 --b 1")); var cp = new CandidateParameter("BuildExcludeModels"); cp.DeleteUnusedIndicators(); ParameterdCandidateStrategy realDealCandidate = new ParameterdCandidateStrategy(cp); SetTrainTime(new DateTime(2009, 1, 1), new DateTime(2009, 4, 30)); //m_generateOneClassHp = 0; realDealCandidate.IterateClassifierInfos2(new Func<CandidateClassifier, bool>((clsInfo) => { string modelFileName = clsInfo.WekaData.GetExcludeModelFileName(clsInfo.Name); //if (!System.IO.File.Exists(modelFileName)) { WekaUtils.Instance.WriteLog(string.Format("{0} is building exclude model.", clsInfo.Name)); //clsInfo.WekaData.m_currentTestHour = clsInfo.Hour; clsInfo.WekaData.GenerateData(true, false); WekaUtils.TrainInstances(clsInfo.WekaData.CurrentTrainInstancesNew, modelFileName, cls); MyEvaluation eval = new MyEvaluation(); eval.evaluateModel(cls, clsInfo.WekaData.CurrentTrainInstancesNew); return true; } //else //{ // m_currentTestHour = h; // GenerateData(true, false); // cls = WekaUtils.TryLoadClassifier(modelFileName); // var eval = WekaUtils.TestInstances(m_trainInstancesNew[k], cls); //} })); }
// tableType = 0: MainTable; 1: OtherDataTable; 2:HpTable public ForexDataRows GetDbData(DateTime dt1, DateTime dt2, string tableName, int tableType, bool isTrain, CandidateParameter cp) { lock (m_lockObject) { //int dealTypeIdx1 = dealType == 'B' ? 0 : 1; DateTime prevDate = dt1.AddMinutes(-2 * cp.PrevTimeCount * WekaUtils.GetMinuteofPeriod(cp.AllPeriods[cp.PeriodCount - 1])); DateTime nextDate = dt2; if (m_useBufferReadData && m_batchBufferMinutes < 0) { int n1 = cp.SymbolCount * cp.PeriodCount * (cp.PeriodTimeCount + 1) * (cp.AllIndNames.Count + cp.AllIndNames2.Count + 5) / WekaUtils.GetMinuteofPeriod(cp.MainPeriod); n1 = Math.Max(n1, 1); int n2 = 1; //n2 = 3 * TestParameters.BatchTps.Length * TestParameters.BatchSls.Length * Parameters.AllDealTypes.Length / WekaUtils.GetMinuteofPeriod(TestParameters.MainPeriod); //n2 /= 6; n2 = 10000 / WekaUtils.GetMinuteofPeriod(cp.MainPeriod); n2 = Math.Max(n2, 1); m_batchBufferMinutes = (Parameters.TotalCanUseBuffer - System.GC.GetTotalMemory(false)) / (n1 + n2) * WekaUtils.GetMinuteofPeriod(cp.MainPeriod); // / 240 int min = (int)(nextDate - prevDate).TotalMinutes + 24 * TestParameters.BatchTestMinutes; int max = (int)(1.1 * TestParameters.BatchTrainMinutes); if (m_batchBufferMinutes < min) { //m_useBufferReadData = false; WekaUtils.Instance.WriteLog(string.Format("Partial buffer. batchGetTime = {0}, min = {1}", m_batchBufferMinutes, min)); m_batchBufferMinutes = min; } else { //m_useBufferReadData = true; WekaUtils.Instance.WriteLog(string.Format("Use buffer. batchGetTime = {0}, max = {1}", m_batchBufferMinutes, max)); m_batchBufferMinutes = Math.Min(m_batchBufferMinutes, max); } } if (m_useBufferReadData) { DateTime bufferDateStart = prevDate; DateTime bufferDateEnd = bufferDateStart.AddMinutes(m_batchBufferMinutes); if (TestParameters2.RealTimeMode) { if (isTrain) { bufferDateStart = dt1.AddDays(-7); bufferDateEnd = dt2.AddDays(7); } else { bufferDateStart = dt1; bufferDateEnd = dt2; } } //System.Console.WriteLine("{0}, {1}, {2}, {3}", bufferDateStart, bufferDateEnd, prevDate, nextDate); if (bufferDateEnd >= nextDate) { string allDataKey = tableName; //if (tableType == 2) //{ // allDataKey += "_" + Parameters.AllDealTypes[dealTypeIdx] + "_" + tp.ToString() + "_" + sl.ToString(); //} if (m_allData == null) { if (m_serializeAllData) { foreach (string s in System.IO.Directory.GetFiles(".", m_allDataCacheFile)) { string[] ss = System.IO.Path.GetFileNameWithoutExtension(s).Split(new char[] { '_' }, StringSplitOptions.RemoveEmptyEntries); DateTime d1 = Convert.ToDateTime(ss[1]); DateTime d2 = Convert.ToDateTime(ss[2]); if (bufferDateStart >= d1 && bufferDateEnd <= d2) { m_allData = Feng.Windows.Utils.SerializeHelper.Deserialize <Dictionary <string, ForexDataTable> >(s); break; } } } if (m_allData == null) { m_allData = new Dictionary <string, ForexDataTable>(); //m_allDateDates = new DateTime[m_dealType.Length][]; } } if (m_allData.ContainsKey(allDataKey)) { Tuple <DateTime, DateTime> ds = m_allDataDates[allDataKey]; bool expired = false; if (!isTrain) { if (ds.Item2 < nextDate || ds.Item1 > prevDate) { //System.Console.WriteLine("{0}, {1}, {2}, {3}", ds.Item1.ToString(), ds.Item2, prevDate, nextDate); expired = true; } } else { if (ds.Item2 <= nextDate.AddMinutes(TestParameters.EnablePerhourTrain ? 2 * Parameters.AllHour * TestParameters.BatchTestMinutes : 2 * TestParameters.BatchTestMinutes) || ds.Item1 > prevDate) { expired = true; } } if (expired) { m_allData.Remove(allDataKey); m_allDataDates.Remove(allDataKey); } } //long usedMemory = System.GC.GetTotalMemory(false); //if (m_cacheData && usedMemory >= Parameters.TotalCanUseMemory) //{ // WekaUtils.Instance.WriteLog(string.Format("Use too many memory. now total memory = {0}", usedMemory)); // m_cacheData = false; //} //else if (!m_cacheData && usedMemory < Parameters.TotalCanUseMemory) //{ // WekaUtils.Instance.WriteLog(string.Format("reset to cache data. now total memory = {0}", usedMemory)); // m_cacheData = true; //} if (m_cacheData && !m_allData.ContainsKey(allDataKey)) { StringBuilder sql = new StringBuilder(); if (tableType == 0) { sql.Append(string.Format("SELECT [Time], [Date], [hour], [dayofweek], [spread], [close] as mainClose")); // , [AskVolume], [BidVolume] } else if (tableType == 1) { sql.Append(string.Format("SELECT [Time], [Date]")); } else if (tableType == 2) { sql.Append(string.Format("SELECT [Time], [hp], [hp_date]")); } if (tableType == 0 || tableType == 1) { foreach (var kvp in cp.AllIndNames2) { sql.Append(string.Format(",[{0}]", kvp.Key)); } foreach (var kvp in cp.AllIndNames) { sql.Append(string.Format(",[{0}]", kvp.Key)); } } sql.Append(string.Format(" FROM {0} WHERE Time >= '{1}' AND Time < '{2}' ", tableName, WekaUtils.GetTimeFromDate(bufferDateStart), WekaUtils.GetTimeFromDate(bufferDateEnd))); //if (tableType == 0 || tableType == 1) //{ // sql.Append(" AND IsActive = 1 "); //} //if (tableType == 2) //{ // sql.Append(string.Format(" AND DealType = '{0}' AND Tp = {1} AND Sl = {2}", Parameters.AllDealTypes[dealTypeIdx], tp, sl)); //} if (!string.IsNullOrEmpty(TestParameters.DbSelectWhere)) { sql.Append(string.Format(" AND {0}", TestParameters.DbSelectWhere)); } sql.Append(string.Format(" {0}", m_selectOrder)); string cmd = sql.ToString(); //m_allData[allDataKey] = DbHelper.Instance.ExecuteDataTable(cmd); //m_allData[allDataKey].PrimaryKey = new System.Data.DataColumn[] { m_allData[allDataKey].Columns["Time"] }; m_allData[allDataKey] = new ForexDataTable(Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd), tableType == 2); m_allDataDates[allDataKey] = new Tuple <DateTime, DateTime>(bufferDateStart, bufferDateEnd); WekaUtils.Instance.WriteLog(string.Format("Buffer Get Data From {0} to {1} of {2}", bufferDateStart, bufferDateEnd, allDataKey)); if (m_serializeAllData) { string allDataCacheFile = m_allDataCacheFile.Replace("*", TestParameters.BatchDateStart.ToString(Parameters.DateTimeFormat) + "_" + TestParameters.BatchDateEnd.ToString(Parameters.DateTimeFormat)); Feng.Windows.Utils.SerializeHelper.Serialize(allDataCacheFile, m_allData); } } if (m_allData.ContainsKey(allDataKey)) { //string filter = GetDbDateFilter(dt1, dt2, tableType); //return m_allData[allDataKey].Select(filter); Tuple <long, long> filter = GetDbDateFilter(dt1, dt2, tableType); return(m_allData[allDataKey].SelectByDate(filter.Item1, filter.Item2)); } } } // no get from cache { StringBuilder sql = new StringBuilder(); if (tableType == 0) { sql.Append(string.Format("SELECT [Time], [Date], [hour], [dayofweek], [spread], [close] as mainClose")); // , [AskVolume], [BidVolume] } else if (tableType == 1) { sql.Append(string.Format("SELECT [Time], [Date]")); } else if (tableType == 2) { sql.Append(string.Format("SELECT [Time], [hp], [hp_date]")); } if (tableType == 0 || tableType == 1) { foreach (var kvp in cp.AllIndNames2) { sql.Append(string.Format(",[{0}]", kvp.Key)); } foreach (var kvp in cp.AllIndNames) { sql.Append(string.Format(",[{0}]", kvp.Key)); } } string filter = GetDbDateFilter4Db(dt1, dt2, tableType); sql.Append(string.Format(" FROM {0} WHERE {1}", tableName, filter)); //if (tableType == 2) //{ // sql.Append(string.Format(" AND DealType = '{0}' AND Tp = {1} AND Sl = {2}", Parameters.AllDealTypes[dealTypeIdx], tp, sl)); //} if (!string.IsNullOrEmpty(TestParameters.DbSelectWhere)) { sql.Append(string.Format(" AND {0}", TestParameters.DbSelectWhere)); } sql.Append(string.Format(" {0}", m_selectOrder)); string cmd = sql.ToString(); WekaUtils.Instance.WriteLog(string.Format("NoBuffer Get Date From {0} to {1} of {2}", prevDate, nextDate, tableName)); System.Data.DataTable dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd); //return dt.Select(); return(new ForexDataRows(new ForexDataTable(dt, tableType == 2), 0, dt.Rows.Count)); } } }
public static void GenerateArffTemplate(bool generateTrainData, bool generateTestData, CandidateParameter cp) { if (!m_trainInstancesTemplates.ContainsKey(cp.Name)) { string arffFileName = string.Format("{0}\\mlea_header_{1}.arff", TestParameters.CommonDir, cp.Name); if (System.IO.File.Exists(arffFileName)) { System.IO.File.Delete(arffFileName); } using (StreamWriter sw = new StreamWriter(arffFileName)) { sw.WriteLine(GetArffHeader(System.DateTime.Today, System.DateTime.Today, cp)); } m_trainInstancesTemplates[cp.Name] = WekaUtils.LoadInstances(arffFileName); m_testInstancesTemplates[cp.Name] = WekaUtils.LoadInstances(arffFileName); } Instances trainInstancesTemplate = m_trainInstancesTemplates[cp.Name]; Instances testInstancesTemplate = m_testInstancesTemplates[cp.Name]; for (int k = 0; k < 2; ++k) { if (k == 0 && !generateTrainData) continue; if (k == 1 && !generateTestData) continue; bool isTrain = (k == 0); DateTime dt1 = isTrain ? m_trainTimeStart : m_testTimeStart; DateTime dt2 = isTrain ? m_trainTimeEnd : m_testTimeEnd; Instances hereInstances; if (isTrain) { hereInstances = trainInstancesTemplate; } else { hereInstances = testInstancesTemplate; } if (!TestParameters.EnablePerhourTrain) { while (hereInstances.numInstances() > 0) { if (WekaUtils.GetDateValueFromInstances(hereInstances, 0, 0) < dt1) { hereInstances.delete(0); } else { break; } } } else { hereInstances.delete(); } hereInstances.setRelationName(string.Format("{0}_{1}", dt1.ToString(Parameters.DateTimeFormat), dt2.ToString(Parameters.DateTimeFormat))); ForexDataRows[, ,] dvs = new ForexDataRows[cp.SymbolCount, cp.PeriodCount, cp.PeriodTimeCount + 1]; int[, ,] dvsIdx = new int[cp.SymbolCount, cp.PeriodCount, cp.PeriodTimeCount + 1]; for (int s = 0; s < cp.SymbolCount; ++s) { for (int i = 0; i < cp.PeriodCount; ++i) { for (int j = -1; j < Math.Max(0, Math.Min(cp.PeriodTimeCount - i, Parameters.PeriodTimeNames[i].Length)); ++j) { string tableName = cp.AllSymbols[s + cp.SymbolStart] + "_" + cp.AllPeriods[i + cp.PeriodStart] + (j < 0 ? string.Empty : "_" + Parameters.PeriodTimeNames[i + cp.PeriodStart][j]); dvs[s, i, j + 1] = DbData.Instance.GetDbData(dt1, dt2, tableName, s == 0 && i == 0 && j == -1 ? 0 : 1, isTrain, cp); dvsIdx[s, i, j + 1] = 0; } } } ForexDataRows mainDv = dvs[0, 0, 0]; if (mainDv.Length == 0) continue; if (dvs[0, cp.PeriodCount - 1, 0].Length <= cp.PrevTimeCount - 1) continue; int startRowIdx = 0; startRowIdx = 0;// FindRowByTime(mainDv, (long)dvs[0, PeriodCount - 1, 0][PrevTimeCount - 1]["Time"], ref startRowIdx); DateTime nowInstanceMaxDate = DateTime.MinValue; if (!TestParameters.EnablePerhourTrain && hereInstances.numInstances() > 0) { nowInstanceMaxDate = WekaUtils.GetDateValueFromInstances(hereInstances, 0, hereInstances.numInstances() - 1); } for (int rowIdx = startRowIdx; rowIdx < mainDv.Length; ++rowIdx) { ForexData mainDrv = mainDv[rowIdx]; //DateTime row_date = (DateTime)mainDrv[1]; // "Date" //long mainTime = (long)mainDrv[0]; // "Time" long mainTime = mainDrv.Time; DateTime row_date = WekaUtils.GetDateFromTime(mainTime); if (row_date <= nowInstanceMaxDate) continue; if (row_date < dt1) continue; if (row_date >= dt2) break; if (TestParameters.EnablePerhourTrain) { if (m_currentTestHour != row_date.Hour) continue; } int hp = 1; double[] instanceValue = new double[hereInstances.numAttributes()]; //instanceValue[0] = hereInstances.attribute(0).parseDate(row_date.ToString(Parameters.DateTimeFormat)); //instanceValue[1] = hereInstances.attribute(1).parseDate(hp_date.ToString(Parameters.DateTimeFormat)); //instance.setValue(1, hereInstances.attribute(1).indexOfValue(mainDrv["hour"].ToString())); //instance.setValue(2, hereInstances.attribute(2).indexOfValue(mainDrv["dayofweek"].ToString())); instanceValue[0] = (row_date - Parameters.MtStartTime).TotalMilliseconds; // if not set to gmt, should -8 * 60 * 60 * 1000; // utc8 instanceValue[1] = (Parameters.MaxDate - Parameters.MtStartTime).TotalMilliseconds; instanceValue[2] = Convert.ToDouble(mainDrv["spread"]); instanceValue[3] = (double)mainDrv["mainClose"]; instanceValue[4] = (int)mainDrv["hour"] / 24.0; instanceValue[5] = (int)mainDrv["dayofweek"] / 5.0; //if (mainDrv["AskVolume"] != System.DBNull.Value && mainDrv["BidVolume"] != System.DBNull.Value) //{ // instanceValue[4] = ((((double)mainDrv["AskVolume"]) - (double)mainDrv["BidVolume"]) / 100000); //} //else { instanceValue[6] = 0; } int start = 7; try { for (int s = 0; s < cp.SymbolCount; ++s) { //double mainClose = (double)mainDrv["close"]; int nowRowIdx_s = FindRowByTime(dvs[s, 0, 0], mainTime, ref dvsIdx[s, 0, 0]); WekaUtils.DebugAssert((long)dvs[s, 0, 0][nowRowIdx_s].Time == mainTime, "(long)dvs[s, 0, 0][nowRowIdx_s].Time == mainTime"); double mainClose = 0; if (cp.AllIndNames2.ContainsKey("close")) { mainClose = (double)dvs[s, 0, 0][nowRowIdx_s]["close"]; } for (int i = 0; i < cp.PeriodCount; ++i) { int periodSeconds = 60 * WekaUtils.GetMinuteofPeriod(cp.AllPeriods[i + cp.PeriodStart]); int nowRowIdx = FindRowByTime(dvs[s, i, 0], mainTime / periodSeconds * periodSeconds, ref dvsIdx[s, i, 0]); for (int p = 0; p < cp.PrevTimeCount; ++p) { if (nowRowIdx - p < 0) { throw new ArgumentException("No prev data!"); } ForexData nowDrv = dvs[s, i, 0][nowRowIdx - p]; foreach (var kvp in cp.AllIndNames2) { double v = Convert.ToDouble(nowDrv[kvp.Key]); double ind = WekaUtils.NormalizeValue(kvp.Key, kvp.Value, v, mainClose, WekaUtils.GetSymbolPoint(cp.AllSymbols[s])); instanceValue[start] = ind; start++; } for (int j = -1; j < Math.Max(0, Math.Min(cp.PeriodTimeCount - i, Parameters.PeriodTimeNames[i].Length)); ++j) { int nowRowIdx2; if (j == -1) nowRowIdx2 = nowRowIdx; else nowRowIdx2 = FindRowByTime(dvs[s, i, j + 1], mainTime / periodSeconds * periodSeconds, ref dvsIdx[s, i, j + 1]); if (nowRowIdx2 - p < 0) { throw new ArgumentException("No prev data!"); } ForexData nowDrv2 = dvs[s, i, j + 1][nowRowIdx2 - p]; foreach (var kvp in cp.AllIndNames) { double v = (double)nowDrv2[kvp.Key]; double ind = WekaUtils.NormalizeValue(kvp.Key, kvp.Value, v, mainClose, WekaUtils.GetSymbolPoint(cp.AllSymbols[s])); instanceValue[start] = ind; start++; } } } } } } catch (ArgumentException) { continue; } if (m_useClassAsAttribute) { instanceValue[hereInstances.numAttributes() - 2] = hp; } instanceValue[hereInstances.numAttributes() - 1] = hp; Instance instance = new weka.core.DenseInstance(1, instanceValue); //if (!hereInstances.checkInstance(instance)) //{ // throw new ArgumentException("Imcompatible instance!"); //} hereInstances.add(instance); } } }
public WekaData(CandidateParameter cp) { m_cp = cp; }
public void TrainandTest(CandidateClassifier classifierInfo, CandidateParameter cp) { //string dealType = classifierInfo.DealType; Classifier cls = null; if (TestParameters.UseTrain) { string modelFileName = GetModelFileName(classifierInfo.Name); if (TestParameters.SaveModel) { cls = WekaUtils.TryLoadClassifier(modelFileName); } Instances trainInstancesNew, trainInstances; trainInstances = m_trainInstances; trainInstancesNew = m_trainInstancesNew; if (cls == null) { if (classifierInfo.Classifier == null) { classifierInfo.Classifier = WekaUtils.CreateClassifier(cp.ClassifierType, m_currentTp, m_currentSl); } cls = classifierInfo.Classifier; } else { if (TestParameters.EnableDetailLog) { System.Console.WriteLine("Model is loaded."); } } if (m_enableTrainSplitTest) { Instances trainTrainInst, trainTestInst; DateTime splitTrainTimeEnd; if (m_trainSplitTestNums != -1) { int trainTrainSize = trainInstancesNew.numInstances() - m_trainSplitTestNums; int trainTestSize = m_trainSplitTestNums; trainTrainInst = new Instances(trainInstancesNew, 0, trainTrainSize); trainTestInst = new Instances(trainInstancesNew, trainTrainSize, trainTestSize); splitTrainTimeEnd = WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainTrainSize); } else if (m_trainSplitPercent != -1) { if (m_trainSplitPercent == 100.0) { int trainTrainSize = trainInstancesNew.numInstances(); trainTrainInst = new Instances(trainInstancesNew, 0, trainTrainSize); trainTestInst = new Instances(trainInstancesNew, 0, trainTrainSize); splitTrainTimeEnd = WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainTrainSize); } else { int trainTrainSize = (int)Math.Round(trainInstancesNew.numInstances() * m_trainSplitPercent / 100); int trainTestSize = trainInstancesNew.numInstances() - trainTrainSize; trainTrainInst = new Instances(trainInstancesNew, 0, trainTrainSize); trainTestInst = new Instances(trainInstancesNew, trainTrainSize, trainTestSize); splitTrainTimeEnd = WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainTrainSize); } } else { trainTrainInst = new Instances(trainInstancesNew, 0); trainTestInst = new Instances(trainInstancesNew, 0); DateTime dt = WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainInstances.numInstances() - 1); splitTrainTimeEnd = m_trainTimeEnd.AddMinutes(-TestParameters.BatchTestMinutes); while (splitTrainTimeEnd > dt) { splitTrainTimeEnd = splitTrainTimeEnd.AddMinutes(-TestParameters.BatchTestMinutes); } for (int i = 0; i < trainInstances.numInstances(); ++i) { dt = WekaUtils.GetDateValueFromInstances(trainInstances, 0, i); if (dt <= splitTrainTimeEnd) { var ins = new DenseInstance(trainInstancesNew.instance(i)); trainTrainInst.add(ins); } else { var ins = new DenseInstance(trainInstancesNew.instance(i)); trainTestInst.add(ins); } } } cls = WekaUtils.TrainInstances(trainTrainInst, TestParameters.SaveModel ? modelFileName : null, cls); //m_classifierQueue[dealType].Enqueue(new ClassifierInfo(cls, splitTrainTimeEnd)); //foreach (var i in m_classifierQueue[dealType]) //{ // var e = WekaUtils.TestInstances(trainTestInst, i.Cls); // i.TotalCost = i.TotalCost * m_classifierQueueFactor + e.totalCost(); // i.TotalNum = (int)(i.TotalNum * m_classifierQueueFactor) + (int)e.numInstances(); //} //WriteEvalSummary(eval1, string.Format("Test Data from {0} to {1}", m_testTimeStart.ToString(Parameters.DateTimeFormat), m_testTimeEnd.ToString(Parameters.DateTimeFormat))); } else { cls = WekaUtils.TrainInstances(trainInstancesNew, TestParameters.SaveModel ? modelFileName : null, cls); //m_classifierQueue[dealType].Enqueue(new ClassifierInfo(cls, m_trainTimeEnd)); //foreach (var i in m_classifierQueue[dealType]) //{ // var e = WekaUtils.TestInstances(trainInstancesNew, i.Cls); // i.TotalCost = i.TotalCost * m_classifierQueueFactor + e.totalCost(); // i.TotalNum = (int)(i.TotalNum * m_classifierQueueFactor) + (int)e.numInstances(); //} } if (TestParameters.EnableDetailLog) { System.Console.WriteLine("Model is trained."); } classifierInfo.Classifier = cls; //if (classifierInfo.CurrentTrainInstances1 != null) //{ // classifierInfo.CurrentTrainInstances1.clear(); //} //if (classifierInfo.CurrentTrainInstancesNew1 != null) //{ // classifierInfo.CurrentTrainInstancesNew1.clear(); //} //classifierInfo.CurrentTrainInstances = new Instances(trainInstances, 0, trainInstances.numInstances()); //classifierInfo.CurrentTrainInstancesNew = new Instances(trainInstancesNew, 0, trainInstancesNew.numInstances()); if (classifierInfo.MoneyManagement == null) { classifierInfo.MoneyManagement = WekaUtils.CreateMoneyManagement(cp.MoneyManagementType, m_currentTp, m_currentSl); } IMoneyManagement mm = WekaUtils.TrainInstances4MM(trainInstancesNew, TestParameters.SaveModel ? modelFileName : null, classifierInfo.MoneyManagement); classifierInfo.MoneyManagement = mm; } else { if (classifierInfo.Classifier == null) { classifierInfo.Classifier = WekaUtils.CreateClassifier(cp.ClassifierType, m_currentTp, m_currentSl); } cls = classifierInfo.Classifier; if (classifierInfo.MoneyManagement == null) { classifierInfo.MoneyManagement = WekaUtils.CreateMoneyManagement(cp.MoneyManagementType, m_currentTp, m_currentSl); } } if (m_enableTest) { Instances testInstancesNew, testInstances; testInstances = m_testInstances; testInstancesNew = m_testInstancesNew; double[] cv = WekaUtils.ClassifyInstances(testInstancesNew, cls); if (TestParameters.EnableExcludeClassifier) { bool hasPositive = false; for (int i = 0; i < cv.Length; ++i) { if (cv[i] == 2) { hasPositive = true; break; } } if (hasPositive) { // Exclude if (classifierInfo.ExcludeClassifier == null) { string modelFileName4Exclude = GetExcludeModelFileName(classifierInfo.Name); classifierInfo.ExcludeClassifier = WekaUtils.TryLoadClassifier(modelFileName4Exclude); } if (classifierInfo.ExcludeClassifier != null) { double[] cv2 = WekaUtils.ClassifyInstances(testInstancesNew, classifierInfo.ExcludeClassifier); // cv2 == 0 -> is class, otherwise = double.NaN; for (int i = 0; i < cv.Length; ++i) { cv[i] = cv[i] == 2 && cv2[i] == 2 ? 2 : 0; } } } } classifierInfo.CurrentTestRet = cv; classifierInfo.CurrentClassValue = new double[testInstances.numInstances()]; for (int i = 0; i < testInstances.numInstances(); ++i) { classifierInfo.CurrentClassValue[i] = testInstances.instance(i).classValue(); } for (int i = 0; i < testInstances.numInstances(); i++) { if (cv[i] == 2) { double openPrice = testInstances.instance(i).value(testInstances.attribute("mainClose")); DateTime openTime = WekaUtils.GetDateValueFromInstances(testInstances, 0, i); if (testInstances.instance(i).classValue() == 2 || testInstances.instance(i).classValue() == 0) { classifierInfo.Deals.AddDeal(openTime, openPrice, classifierInfo.DealType, classifierInfo.MoneyManagement.GetVolume(testInstances.instance(i)), testInstances.instance(i).classValue() == 2 ? -classifierInfo.Tp : classifierInfo.Sl, WekaUtils.GetDateValueFromInstances(testInstances, 1, i)); } else if (testInstances.instance(i).classValue() == 1) { double closePriceTp, closePriceSl; if (classifierInfo.DealType == 'B') { closePriceTp = openPrice + classifierInfo.Tp * DealsInfo.GetPoint(0); closePriceSl = openPrice - classifierInfo.Sl * DealsInfo.GetPoint(0); } else { closePriceTp = openPrice - classifierInfo.Tp * DealsInfo.GetPoint(0); closePriceSl = openPrice + classifierInfo.Sl * DealsInfo.GetPoint(0); } classifierInfo.Deals.AddDeal(openTime, openPrice, classifierInfo.DealType, classifierInfo.MoneyManagement.GetVolume(testInstances.instance(i)), closePriceTp, closePriceSl); } else { throw new AssertException("classValue should be 0,1,2."); } } } } }
public static string GetArffHeader(DateTime dt1, DateTime dt2, CandidateParameter cp) { //string header; //using (StreamReader sr = new StreamReader(string.Format("{0}\\{1}_header_{2}.arff", m_arffHeaderDir, m_symbol, m_symbolPeriod2_cnt))) //{ // header = sr.ReadToEnd(); //} // header = header.Replace("@relation eurusd_m1", string.Format("@relation '{0}_{1}_{2}'", m_symbolPeriod2, dt1, dt2)); StringBuilder header = new StringBuilder(); header.AppendLine(string.Format("@relation '{0}_{1}'", dt1.ToString(Parameters.DateTimeFormat), dt2.ToString(Parameters.DateTimeFormat))); header.AppendLine("@attribute timestamp date \"yyyy-MM-dd'T'HH:mm:ss\""); header.AppendLine("@attribute hpdate date \"yyyy-MM-dd'T'HH:mm:ss\""); //header.AppendLine("@attribute hour {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23}"); //header.AppendLine("@attribute day_of_week {1, 2, 3, 4, 5}"); header.AppendLine("@attribute spread numeric"); header.AppendLine("@attribute mainClose numeric"); header.AppendLine("@attribute hour numeric"); header.AppendLine("@attribute day_of_week numeric"); header.AppendLine("@attribute vol numeric"); for (int s = 0; s < cp.SymbolCount; ++s) { for (int i = 0; i < cp.PeriodCount; ++i) { for (int p = 0; p < cp.PrevTimeCount; ++p) { string postFix1 = cp.AllPeriods[i + cp.PeriodStart] + (p == 0 ? string.Empty : "_P" + p.ToString()); foreach (string name in cp.AllIndNames2.Keys) { header.AppendLine(string.Format("@attribute {0}_{1} {2}", cp.AllSymbols[s + cp.SymbolStart], name + "_" + postFix1, TestParameters.IndicatorUseNumeric ? "numeric" : "{0,1,2,3}")); } for (int j = -1; j < Math.Max(0, Math.Min(cp.PeriodTimeCount - i, Parameters.PeriodTimeNames[i + cp.PeriodStart].Length)); ++j) { string postFix2; if (j >= 0) postFix2 = cp.AllPeriods[i + cp.PeriodStart] + "_" + Parameters.PeriodTimeNames[i + cp.PeriodStart][j] + (p == 0 ? string.Empty : "_P" + p.ToString()); else postFix2 = postFix1; foreach (string name in cp.AllIndNames.Keys) { header.AppendLine(string.Format("@attribute {0}_{1} {2}", cp.AllSymbols[s + cp.SymbolStart], name + "_" + postFix2, TestParameters.IndicatorUseNumeric ? "numeric" : "{0,1,2,3}")); } } } } } if (m_useClassAsAttribute) { header.AppendLine("@attribute propAttr {0,1,2,3}"); } if (!m_generateOneClassHp.HasValue) { // 0: Fail; 1: Unknown 2: Success header.AppendLine("@attribute prop {0,1,2,3}"); } else { header.AppendLine("@attribute prop {0}"); } header.AppendLine(); header.AppendLine("@data"); return header.ToString(); }
private void CreateCds(ref int[, , ,] cds, CandidateParameter cp, int count) { cds = new int[cp.SymbolCount, cp.PeriodCount, count, 61]; for (int s = 0; s < cp.SymbolCount; ++s) for (int p = 0; p < cp.PeriodCount; ++p) for (int i = 0; i < cds.GetLength(3); ++i) for (int j = 0; j < cds.GetLength(2); ++j) cds[s, p, j, i] = -1; }