public ParameterdCandidateStrategy(CandidateParameter cp = null,
ParameterdCandidateStrategy parent = null, IBestCandidateSelector bestCandidateSelector = null)
{
if (cp == null)
{
cp = new CandidateParameter("Default");
}
this.CandidateParameter = cp;
if (cp.ClassifierType == null && !TestParameters.EnableExcludeClassifier)
{
cp.SymbolCount = 1;
cp.PeriodCount = 1;
cp.PeriodTimeCount = 0;
cp.PrevTimeCount = 0;
}
InitClassifierInfos(parent);
if (TestParameters2.RealTimeMode)
{
m_enableDetailLogLevel2 = false;
}
m_bestCandidateSelector = bestCandidateSelector;
}
public WekaData(char dealType, int tp, int sl, CandidateParameter cp)
{
m_currentDealType = dealType;
m_currentTp = tp;
m_currentSl = sl;
m_cp = cp;
}
public ParameterdCandidateStrategy(CandidateParameter cp = null,
ParameterdCandidateStrategy parent = null, IBestCandidateSelector bestCandidateSelector = null)
{
if (cp == null)
{
cp = new CandidateParameter("Default");
}
this.CandidateParameter = cp;
if (cp.ClassifierType == null && !TestParameters.EnableExcludeClassifier)
{
cp.SymbolCount = 1;
cp.PeriodCount = 1;
cp.PeriodTimeCount = 0;
cp.PrevTimeCount = 0;
}
InitClassifierInfos(parent);
if (TestParameters2.RealTimeMode)
{
m_enableDetailLogLevel2 = false;
}
m_bestCandidateSelector = bestCandidateSelector;
}
public CandidateParameter Clone()
{
CandidateParameter that = this.MemberwiseClone() as CandidateParameter;
that.BatchTps = new int[this.BatchTps.Length];
this.BatchTps.CopyTo(that.BatchTps, 0);
that.BatchSls = new int[this.BatchSls.Length];
this.BatchSls.CopyTo(that.BatchSls, 0);
that.AllIndNames = new Dictionary <string, int>();
foreach (var i in this.AllIndNames)
{
that.AllIndNames.Add(i.Key, i.Value);
}
that.AllIndNames2 = new Dictionary <string, int>();
foreach (var i in this.AllIndNames2)
{
that.AllIndNames2.Add(i.Key, i.Value);
}
that.AllSymbols = new string[this.AllSymbols.Length];
for (int i = 0; i < that.AllSymbols.Length; ++i)
{
that.AllSymbols[i] = this.AllSymbols[i];
}
that.AllPeriods = new string[this.AllPeriods.Length];
for (int i = 0; i < that.AllPeriods.Length; ++i)
{
that.AllPeriods[i] = this.AllPeriods[i];
}
return(that);
}
public CandidateClassifier(string name, int tp, int sl, char dealType, int hour, CandidateParameter cp)
{
this.Name = name;
this.Tp = tp;
this.Sl = sl;
this.DealType = dealType;
this.Hour = hour;
this.Deals = new DealsInfo(cp.DealInfoLastMinutes);
this.WekaData = new WekaData(dealType, tp, sl, cp);
}
public static void SaveCCScoresToDb(DateTime nowDate, CandidateParameter cp, ParameterdCandidateStrategy pcs)
{
string tableName = string.Format("{0}_CCSCORE_{1}", cp.MainSymbol, cp.DealInfoLastMinutes / (7 * 24 * 12 * 5));
long nowTime = WekaUtils.GetTimeFromDate(nowDate);
try
{
Feng.Data.DbHelper.Instance.ExecuteNonQuery(string.Format("SELECT TOP 1 * FROM {0} WHERE TIME = -1", tableName));
}
catch (Exception)
{
string sqlCreate = string.Format(@"CREATE TABLE {0}(
[Time] [bigint] NOT NULL,
[scores] [varbinary](max) NOT NULL,
[nda] [varbinary](max) NOT NULL,
[nds] [varbinary](max) NOT NULL,
CONSTRAINT [PK_{0}] PRIMARY KEY CLUSTERED
(
[Time] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]", tableName);
Feng.Data.DbHelper.Instance.ExecuteNonQuery(sqlCreate);
}
long[, ,] ndas = new long[Parameters.AllDealTypes.Length, cp.BatchTps.Length, cp.BatchSls.Length];
long[, ,] ndss = new long[Parameters.AllDealTypes.Length, cp.BatchTps.Length, cp.BatchSls.Length];
double[, ,] nowScores = new double[Parameters.AllDealTypes.Length, cp.BatchTps.Length, cp.BatchSls.Length];
pcs.IterateClassifierInfos((k, ii, jj, h) =>
{
nowScores[k, ii, jj] = pcs.m_classifierInfoIdxs[k, ii, jj, h].Deals.NowScore;
ndas[k, ii, jj] = pcs.m_classifierInfoIdxs[k, ii, jj, h].Deals.DealLastTimeAvg;
ndss[k, ii, jj] = pcs.m_classifierInfoIdxs[k, ii, jj, h].Deals.DealLastTimeStd;
});
string sql = string.Format("INSERT INTO [{0}] ([Time],[scores],[nda],[nds]) VALUES (@Time, @score, @nda, @nds)", tableName);
var cmd = new System.Data.SqlClient.SqlCommand(sql);
cmd.Parameters.AddWithValue("@Time", nowTime);
cmd.Parameters.AddWithValue("@score", CCScoreData.SerializeScores(nowScores));
cmd.Parameters.AddWithValue("@nda", CCScoreData.SerializeScoreTimes(ndas));
cmd.Parameters.AddWithValue("@nds", CCScoreData.SerializeScoreTimes(ndss));
try
{
Feng.Data.DbHelper.Instance.ExecuteNonQuery(string.Format("DELETE FROM {0} WHERE TIME = '{1}'", tableName, WekaUtils.GetTimeFromDate(nowDate)));
Feng.Data.DbHelper.Instance.ExecuteNonQuery(cmd);
}
catch (Exception)
{
}
}
private void CreateCds(ref int[, , ,] cds, CandidateParameter cp, int count)
{
cds = new int[cp.SymbolCount, cp.PeriodCount, count, 61];
for (int s = 0; s < cp.SymbolCount; ++s)
{
for (int p = 0; p < cp.PeriodCount; ++p)
{
for (int i = 0; i < cds.GetLength(3); ++i)
{
for (int j = 0; j < cds.GetLength(2); ++j)
{
cds[s, p, j, i] = -1;
}
}
}
}
}
public List<ParameterdCandidateStrategy> AddRealDealCandidates(string symbol, int dealInfoLastWeek, IBestCandidateSelector bcs = null)
{
List<ParameterdCandidateStrategy> realDealsCandidates = new List<ParameterdCandidateStrategy>();
//TestParameters.InitTpsls(20, 10);
//realDealsCandidates.Add(new RealDealCandidate("1M_10", 1 * 4 * 7 * 24 * 12 * 5, this));
//realDealsCandidates.Add(new RealDealCandidate("1.5M_10", (int)(1.5 * 4 * 7 * 24 * 12 * 5), this));
//realDealsCandidates.Add(new RealDealCandidate("2M_10", 2 * 4 * 7 * 24 * 12 * 5, this));
{
CandidateParameter cp = new CandidateParameter(symbol);
int delta = TestParameters.GetTpSlMinDelta(symbol) * TestParameters2.nTpsl;
cp.InitTpsls(TestParameters2.tpStart, delta, TestParameters2.tpCount, TestParameters2.slStart, delta, TestParameters2.slCount);
cp.SymbolStart = Array.IndexOf<string>(cp.AllSymbols, symbol);
cp.SymbolCount = 1;
cp.PeriodStart = Array.IndexOf<string>(cp.AllPeriods, "M1");
cp.PeriodCount = 1;
cp.DealInfoLastMinutes = dealInfoLastWeek * 7 * 24 * 12 * 5;
cp.Group = 1;
InitBatchBatch(cp);
if (cp.ClassifierType == null && !TestParameters.EnableExcludeClassifier)
{
cp.AllIndNames.Clear();
cp.AllIndNames2.Clear();
}
else
{
cp.DeleteUnusedIndicators();
//GenerateBatchEmptyInstance();
}
ParameterdCandidateStrategy mainPcs = new ParameterdCandidateStrategy(cp, null, bcs);
realDealsCandidates.Add(mainPcs);
// for (int i = 2; i <= 3; ++i)
// {
// var cp2 = cp.Clone();
// cp2.Name += "_" + i.ToString();
// cp2.DealInfoLastMinutes = i * 2 * 4 * 7 * 24 * 12 * 5;
// realDealsCandidates.Add(new ParameterdCandidateStrategy(cp2));
// }
}
{
//CandidateParameter cp = new CandidateParameter("GBPUSD");
//cp.InitTpsls(10, 60);
//cp.SymbolStart = 1;
//cp.DealInfoLastMinutes = 2 * 4 * 7 * 24 * 12 * 5;
//cp.Group = 2;
//ParameterdCandidateStrategy mainPcs = new ParameterdCandidateStrategy(cp);
//mainPcs.m_bestCandidateSelector = new BestCandidateSelector1(0);
//realDealsCandidates.Add(mainPcs);
// for (int i = 2; i <= 3; ++i)
// {
// var cp2 = cp.Clone();
// cp2.Name += "_" + i.ToString();
// cp2.DealInfoLastMinutes = i * 2 * 4 * 7 * 24 * 12 * 5;
// realDealsCandidates.Add(new ParameterdCandidateStrategy(cp2));
// }
}
{
//CandidateParameter cp = new CandidateParameter("EURGBP");
//cp.InitTpsls(10, 30);
//cp.SymbolStart = 6;
//cp.DealInfoLastMinutes = 6 * 4 * 7 * 24 * 12 * 5;
//cp.Group = 3;
//ParameterdCandidateStrategy mainPcs = new ParameterdCandidateStrategy(cp);
//mainPcs.m_bestCandidateSelector = new BestCandidateSelector1(0);
//realDealsCandidates.Add(mainPcs);
// // for (int i = 2; i <= 3; ++i)
// // {
// // var cp2 = cp.Clone();
// // cp2.Name += "_" + i.ToString();
// // cp2.DealInfoLastMinutes = i * 2 * 4 * 7 * 24 * 12 * 5;
// // realDealsCandidates.Add(new ParameterdCandidateStrategy(cp2));
// // }
}
foreach (var i in realDealsCandidates)
{
OutputTestInfoCandidate(i.CandidateParameter);
WekaUtils.Instance.WriteLog(string.Format("BestCandidateSelector = {0}",
i.BestCandidateSelector == null ? "Null" : i.BestCandidateSelector.ToString()));
}
return realDealsCandidates;
}
public static void OutputTestInfoCandidate(CandidateParameter cp)
{
WekaUtils.Instance.WriteHorizontalLine();
WekaUtils.Instance.WriteLog(string.Format("Candidate Name = {0}", cp.Name));
WekaUtils.Instance.WriteLog(string.Format("MainSymbol = {0}, MainPeriod = {1}", cp.MainSymbol, cp.MainPeriod));
WekaUtils.Instance.WriteLog(string.Format("PeriodCount = {0}, PrevTimeCount = {1}, SymbolCount = {2}", cp.PeriodCount, cp.PrevTimeCount, cp.SymbolCount));
WekaUtils.Instance.WriteLog(string.Format("TakeProfit Length = {0}, StopLoss Length = {1}", cp.BatchTps.Length, cp.BatchSls.Length));
WekaUtils.Instance.WriteLog(string.Format("dealInfoLastMinutes = {0}", cp.DealInfoLastMinutes));
WekaUtils.Instance.WriteLog(string.Format("ClassifierType = {0}", cp.ClassifierType == null ? "Default" : cp.ClassifierType.Name));
WekaUtils.Instance.WriteLog(string.Format("MoneyManagementType = {0}", cp.MoneyManagementType == null ? "Default" : cp.MoneyManagementType.Name));
}
public void InitBatchBatch(CandidateParameter cp)
{
TestParameters.SaveDataFile = false;
TestParameters.EnableDetailLog = false;
TestParameters.SaveModel = false;
if (cp.ClassifierType == null && !TestParameters.EnableExcludeClassifier)
{
TestParameters.SaveModel = false;
TestParameters.UseFilter = false;
if (cp.MoneyManagementType == null)
{
TestParameters.UseTrain = false;
}
//GenerateBatchEmptyInstance();
}
else
{
//GenerateBatchEmptyInstance();
}
//TestParameters.EnablePerhourTrain = true;
OutputTestInfo();
UpdateSettings();
}
public void BuildExcludeModels()
{
Parameters.TotalCanUseMemory = 600 * 1000 * 1000;
TestParameters.SaveDataFile = false;
//m_batchBufferMinutes = (new TimeSpan(365 + 5, 0, 0, 0)).TotalMinutes;
weka.classifiers.Classifier cls = new weka.classifiers.functions.LibSVM();
(cls as weka.classifiers.AbstractClassifier).setOptions(weka.core.Utils.splitOptions("-S 0 -K 2"));
//Classifier cls = new MincostLiblinearClassifier();
//Classifier cls = new weka.classifiers.functions.LibLINEAR();
//(cls as AbstractClassifier).setOptions(weka.core.Utils.splitOptions("-S 0 -P -C 1 -B 1"));
//var cls = new SvmLightClassifier();
//cls.setOptions(weka.core.Utils.splitOptions("-c 20 -l 4 -w 1 --p 1 --b 1"));
var cp = new CandidateParameter("BuildExcludeModels");
cp.DeleteUnusedIndicators();
ParameterdCandidateStrategy realDealCandidate = new ParameterdCandidateStrategy(cp);
SetTrainTime(new DateTime(2009, 1, 1), new DateTime(2009, 4, 30));
//m_generateOneClassHp = 0;
realDealCandidate.IterateClassifierInfos2(new Func<CandidateClassifier, bool>((clsInfo) =>
{
string modelFileName = clsInfo.WekaData.GetExcludeModelFileName(clsInfo.Name);
//if (!System.IO.File.Exists(modelFileName))
{
WekaUtils.Instance.WriteLog(string.Format("{0} is building exclude model.", clsInfo.Name));
//clsInfo.WekaData.m_currentTestHour = clsInfo.Hour;
clsInfo.WekaData.GenerateData(true, false);
WekaUtils.TrainInstances(clsInfo.WekaData.CurrentTrainInstancesNew, modelFileName, cls);
MyEvaluation eval = new MyEvaluation();
eval.evaluateModel(cls, clsInfo.WekaData.CurrentTrainInstancesNew);
return true;
}
//else
//{
// m_currentTestHour = h;
// GenerateData(true, false);
// cls = WekaUtils.TryLoadClassifier(modelFileName);
// var eval = WekaUtils.TestInstances(m_trainInstancesNew[k], cls);
//}
}));
}
// tableType = 0: MainTable; 1: OtherDataTable; 2:HpTable
public ForexDataRows GetDbData(DateTime dt1, DateTime dt2, string tableName, int tableType, bool isTrain, CandidateParameter cp)
{
lock (m_lockObject)
{
//int dealTypeIdx1 = dealType == 'B' ? 0 : 1;
DateTime prevDate = dt1.AddMinutes(-2 * cp.PrevTimeCount * WekaUtils.GetMinuteofPeriod(cp.AllPeriods[cp.PeriodCount - 1]));
DateTime nextDate = dt2;
if (m_useBufferReadData && m_batchBufferMinutes < 0)
{
int n1 = cp.SymbolCount * cp.PeriodCount * (cp.PeriodTimeCount + 1)
* (cp.AllIndNames.Count + cp.AllIndNames2.Count + 5) / WekaUtils.GetMinuteofPeriod(cp.MainPeriod);
n1 = Math.Max(n1, 1);
int n2 = 1;
//n2 = 3 * TestParameters.BatchTps.Length * TestParameters.BatchSls.Length * Parameters.AllDealTypes.Length / WekaUtils.GetMinuteofPeriod(TestParameters.MainPeriod);
//n2 /= 6;
n2 = 10000 / WekaUtils.GetMinuteofPeriod(cp.MainPeriod);
n2 = Math.Max(n2, 1);
m_batchBufferMinutes = (Parameters.TotalCanUseBuffer - System.GC.GetTotalMemory(false)) / (n1 + n2) * WekaUtils.GetMinuteofPeriod(cp.MainPeriod); // / 240
int min = (int)(nextDate - prevDate).TotalMinutes + 24 * TestParameters.BatchTestMinutes;
int max = (int)(1.1 * TestParameters.BatchTrainMinutes);
if (m_batchBufferMinutes < min)
{
//m_useBufferReadData = false;
WekaUtils.Instance.WriteLog(string.Format("Partial buffer. batchGetTime = {0}, min = {1}", m_batchBufferMinutes, min));
m_batchBufferMinutes = min;
}
else
{
//m_useBufferReadData = true;
WekaUtils.Instance.WriteLog(string.Format("Use buffer. batchGetTime = {0}, max = {1}", m_batchBufferMinutes, max));
m_batchBufferMinutes = Math.Min(m_batchBufferMinutes, max);
}
}
if (m_useBufferReadData)
{
DateTime bufferDateStart = prevDate;
DateTime bufferDateEnd = bufferDateStart.AddMinutes(m_batchBufferMinutes);
if (TestParameters2.RealTimeMode)
{
if (isTrain)
{
bufferDateStart = dt1.AddDays(-7);
bufferDateEnd = dt2.AddDays(7);
}
else
{
bufferDateStart = dt1;
bufferDateEnd = dt2;
}
}
//System.Console.WriteLine("{0}, {1}, {2}, {3}", bufferDateStart, bufferDateEnd, prevDate, nextDate);
if (bufferDateEnd >= nextDate)
{
string allDataKey = tableName;
//if (tableType == 2)
//{
// allDataKey += "_" + Parameters.AllDealTypes[dealTypeIdx] + "_" + tp.ToString() + "_" + sl.ToString();
//}
if (m_allData == null)
{
if (m_serializeAllData)
{
foreach (string s in System.IO.Directory.GetFiles(".", m_allDataCacheFile))
{
string[] ss = System.IO.Path.GetFileNameWithoutExtension(s).Split(new char[] { '_' }, StringSplitOptions.RemoveEmptyEntries);
DateTime d1 = Convert.ToDateTime(ss[1]);
DateTime d2 = Convert.ToDateTime(ss[2]);
if (bufferDateStart >= d1 && bufferDateEnd <= d2)
{
m_allData = Feng.Windows.Utils.SerializeHelper.Deserialize <Dictionary <string, ForexDataTable> >(s);
break;
}
}
}
if (m_allData == null)
{
m_allData = new Dictionary <string, ForexDataTable>();
//m_allDateDates = new DateTime[m_dealType.Length][];
}
}
if (m_allData.ContainsKey(allDataKey))
{
Tuple <DateTime, DateTime> ds = m_allDataDates[allDataKey];
bool expired = false;
if (!isTrain)
{
if (ds.Item2 < nextDate || ds.Item1 > prevDate)
{
//System.Console.WriteLine("{0}, {1}, {2}, {3}", ds.Item1.ToString(), ds.Item2, prevDate, nextDate);
expired = true;
}
}
else
{
if (ds.Item2 <= nextDate.AddMinutes(TestParameters.EnablePerhourTrain ?
2 * Parameters.AllHour * TestParameters.BatchTestMinutes :
2 * TestParameters.BatchTestMinutes) || ds.Item1 > prevDate)
{
expired = true;
}
}
if (expired)
{
m_allData.Remove(allDataKey);
m_allDataDates.Remove(allDataKey);
}
}
//long usedMemory = System.GC.GetTotalMemory(false);
//if (m_cacheData && usedMemory >= Parameters.TotalCanUseMemory)
//{
// WekaUtils.Instance.WriteLog(string.Format("Use too many memory. now total memory = {0}", usedMemory));
// m_cacheData = false;
//}
//else if (!m_cacheData && usedMemory < Parameters.TotalCanUseMemory)
//{
// WekaUtils.Instance.WriteLog(string.Format("reset to cache data. now total memory = {0}", usedMemory));
// m_cacheData = true;
//}
if (m_cacheData && !m_allData.ContainsKey(allDataKey))
{
StringBuilder sql = new StringBuilder();
if (tableType == 0)
{
sql.Append(string.Format("SELECT [Time], [Date], [hour], [dayofweek], [spread], [close] as mainClose")); // , [AskVolume], [BidVolume]
}
else if (tableType == 1)
{
sql.Append(string.Format("SELECT [Time], [Date]"));
}
else if (tableType == 2)
{
sql.Append(string.Format("SELECT [Time], [hp], [hp_date]"));
}
if (tableType == 0 || tableType == 1)
{
foreach (var kvp in cp.AllIndNames2)
{
sql.Append(string.Format(",[{0}]", kvp.Key));
}
foreach (var kvp in cp.AllIndNames)
{
sql.Append(string.Format(",[{0}]", kvp.Key));
}
}
sql.Append(string.Format(" FROM {0} WHERE Time >= '{1}' AND Time < '{2}' ",
tableName, WekaUtils.GetTimeFromDate(bufferDateStart), WekaUtils.GetTimeFromDate(bufferDateEnd)));
//if (tableType == 0 || tableType == 1)
//{
// sql.Append(" AND IsActive = 1 ");
//}
//if (tableType == 2)
//{
// sql.Append(string.Format(" AND DealType = '{0}' AND Tp = {1} AND Sl = {2}", Parameters.AllDealTypes[dealTypeIdx], tp, sl));
//}
if (!string.IsNullOrEmpty(TestParameters.DbSelectWhere))
{
sql.Append(string.Format(" AND {0}", TestParameters.DbSelectWhere));
}
sql.Append(string.Format(" {0}", m_selectOrder));
string cmd = sql.ToString();
//m_allData[allDataKey] = DbHelper.Instance.ExecuteDataTable(cmd);
//m_allData[allDataKey].PrimaryKey = new System.Data.DataColumn[] { m_allData[allDataKey].Columns["Time"] };
m_allData[allDataKey] = new ForexDataTable(Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd), tableType == 2);
m_allDataDates[allDataKey] = new Tuple <DateTime, DateTime>(bufferDateStart, bufferDateEnd);
WekaUtils.Instance.WriteLog(string.Format("Buffer Get Data From {0} to {1} of {2}", bufferDateStart, bufferDateEnd, allDataKey));
if (m_serializeAllData)
{
string allDataCacheFile = m_allDataCacheFile.Replace("*",
TestParameters.BatchDateStart.ToString(Parameters.DateTimeFormat) + "_"
+ TestParameters.BatchDateEnd.ToString(Parameters.DateTimeFormat));
Feng.Windows.Utils.SerializeHelper.Serialize(allDataCacheFile, m_allData);
}
}
if (m_allData.ContainsKey(allDataKey))
{
//string filter = GetDbDateFilter(dt1, dt2, tableType);
//return m_allData[allDataKey].Select(filter);
Tuple <long, long> filter = GetDbDateFilter(dt1, dt2, tableType);
return(m_allData[allDataKey].SelectByDate(filter.Item1, filter.Item2));
}
}
}
// no get from cache
{
StringBuilder sql = new StringBuilder();
if (tableType == 0)
{
sql.Append(string.Format("SELECT [Time], [Date], [hour], [dayofweek], [spread], [close] as mainClose")); // , [AskVolume], [BidVolume]
}
else if (tableType == 1)
{
sql.Append(string.Format("SELECT [Time], [Date]"));
}
else if (tableType == 2)
{
sql.Append(string.Format("SELECT [Time], [hp], [hp_date]"));
}
if (tableType == 0 || tableType == 1)
{
foreach (var kvp in cp.AllIndNames2)
{
sql.Append(string.Format(",[{0}]", kvp.Key));
}
foreach (var kvp in cp.AllIndNames)
{
sql.Append(string.Format(",[{0}]", kvp.Key));
}
}
string filter = GetDbDateFilter4Db(dt1, dt2, tableType);
sql.Append(string.Format(" FROM {0} WHERE {1}",
tableName, filter));
//if (tableType == 2)
//{
// sql.Append(string.Format(" AND DealType = '{0}' AND Tp = {1} AND Sl = {2}", Parameters.AllDealTypes[dealTypeIdx], tp, sl));
//}
if (!string.IsNullOrEmpty(TestParameters.DbSelectWhere))
{
sql.Append(string.Format(" AND {0}", TestParameters.DbSelectWhere));
}
sql.Append(string.Format(" {0}", m_selectOrder));
string cmd = sql.ToString();
WekaUtils.Instance.WriteLog(string.Format("NoBuffer Get Date From {0} to {1} of {2}", prevDate, nextDate, tableName));
System.Data.DataTable dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd);
//return dt.Select();
return(new ForexDataRows(new ForexDataTable(dt, tableType == 2), 0, dt.Rows.Count));
}
}
}
public static void GenerateArffTemplate(bool generateTrainData, bool generateTestData, CandidateParameter cp)
{
if (!m_trainInstancesTemplates.ContainsKey(cp.Name))
{
string arffFileName = string.Format("{0}\\mlea_header_{1}.arff", TestParameters.CommonDir, cp.Name);
if (System.IO.File.Exists(arffFileName))
{
System.IO.File.Delete(arffFileName);
}
using (StreamWriter sw = new StreamWriter(arffFileName))
{
sw.WriteLine(GetArffHeader(System.DateTime.Today, System.DateTime.Today, cp));
}
m_trainInstancesTemplates[cp.Name] = WekaUtils.LoadInstances(arffFileName);
m_testInstancesTemplates[cp.Name] = WekaUtils.LoadInstances(arffFileName);
}
Instances trainInstancesTemplate = m_trainInstancesTemplates[cp.Name];
Instances testInstancesTemplate = m_testInstancesTemplates[cp.Name];
for (int k = 0; k < 2; ++k)
{
if (k == 0 && !generateTrainData)
continue;
if (k == 1 && !generateTestData)
continue;
bool isTrain = (k == 0);
DateTime dt1 = isTrain ? m_trainTimeStart : m_testTimeStart;
DateTime dt2 = isTrain ? m_trainTimeEnd : m_testTimeEnd;
Instances hereInstances;
if (isTrain)
{
hereInstances = trainInstancesTemplate;
}
else
{
hereInstances = testInstancesTemplate;
}
if (!TestParameters.EnablePerhourTrain)
{
while (hereInstances.numInstances() > 0)
{
if (WekaUtils.GetDateValueFromInstances(hereInstances, 0, 0) < dt1)
{
hereInstances.delete(0);
}
else
{
break;
}
}
}
else
{
hereInstances.delete();
}
hereInstances.setRelationName(string.Format("{0}_{1}", dt1.ToString(Parameters.DateTimeFormat), dt2.ToString(Parameters.DateTimeFormat)));
ForexDataRows[, ,] dvs = new ForexDataRows[cp.SymbolCount, cp.PeriodCount, cp.PeriodTimeCount + 1];
int[, ,] dvsIdx = new int[cp.SymbolCount, cp.PeriodCount, cp.PeriodTimeCount + 1];
for (int s = 0; s < cp.SymbolCount; ++s)
{
for (int i = 0; i < cp.PeriodCount; ++i)
{
for (int j = -1; j < Math.Max(0, Math.Min(cp.PeriodTimeCount - i, Parameters.PeriodTimeNames[i].Length)); ++j)
{
string tableName = cp.AllSymbols[s + cp.SymbolStart] + "_" +
cp.AllPeriods[i + cp.PeriodStart] +
(j < 0 ? string.Empty : "_" + Parameters.PeriodTimeNames[i + cp.PeriodStart][j]);
dvs[s, i, j + 1] = DbData.Instance.GetDbData(dt1, dt2, tableName, s == 0 && i == 0 && j == -1 ? 0 : 1, isTrain, cp);
dvsIdx[s, i, j + 1] = 0;
}
}
}
ForexDataRows mainDv = dvs[0, 0, 0];
if (mainDv.Length == 0)
continue;
if (dvs[0, cp.PeriodCount - 1, 0].Length <= cp.PrevTimeCount - 1)
continue;
int startRowIdx = 0;
startRowIdx = 0;// FindRowByTime(mainDv, (long)dvs[0, PeriodCount - 1, 0][PrevTimeCount - 1]["Time"], ref startRowIdx);
DateTime nowInstanceMaxDate = DateTime.MinValue;
if (!TestParameters.EnablePerhourTrain && hereInstances.numInstances() > 0)
{
nowInstanceMaxDate = WekaUtils.GetDateValueFromInstances(hereInstances, 0, hereInstances.numInstances() - 1);
}
for (int rowIdx = startRowIdx; rowIdx < mainDv.Length; ++rowIdx)
{
ForexData mainDrv = mainDv[rowIdx];
//DateTime row_date = (DateTime)mainDrv[1]; // "Date"
//long mainTime = (long)mainDrv[0]; // "Time"
long mainTime = mainDrv.Time;
DateTime row_date = WekaUtils.GetDateFromTime(mainTime);
if (row_date <= nowInstanceMaxDate)
continue;
if (row_date < dt1)
continue;
if (row_date >= dt2)
break;
if (TestParameters.EnablePerhourTrain)
{
if (m_currentTestHour != row_date.Hour)
continue;
}
int hp = 1;
double[] instanceValue = new double[hereInstances.numAttributes()];
//instanceValue[0] = hereInstances.attribute(0).parseDate(row_date.ToString(Parameters.DateTimeFormat));
//instanceValue[1] = hereInstances.attribute(1).parseDate(hp_date.ToString(Parameters.DateTimeFormat));
//instance.setValue(1, hereInstances.attribute(1).indexOfValue(mainDrv["hour"].ToString()));
//instance.setValue(2, hereInstances.attribute(2).indexOfValue(mainDrv["dayofweek"].ToString()));
instanceValue[0] = (row_date - Parameters.MtStartTime).TotalMilliseconds; // if not set to gmt, should -8 * 60 * 60 * 1000; // utc8
instanceValue[1] = (Parameters.MaxDate - Parameters.MtStartTime).TotalMilliseconds;
instanceValue[2] = Convert.ToDouble(mainDrv["spread"]);
instanceValue[3] = (double)mainDrv["mainClose"];
instanceValue[4] = (int)mainDrv["hour"] / 24.0;
instanceValue[5] = (int)mainDrv["dayofweek"] / 5.0;
//if (mainDrv["AskVolume"] != System.DBNull.Value && mainDrv["BidVolume"] != System.DBNull.Value)
//{
// instanceValue[4] = ((((double)mainDrv["AskVolume"]) - (double)mainDrv["BidVolume"]) / 100000);
//}
//else
{
instanceValue[6] = 0;
}
int start = 7;
try
{
for (int s = 0; s < cp.SymbolCount; ++s)
{
//double mainClose = (double)mainDrv["close"];
int nowRowIdx_s = FindRowByTime(dvs[s, 0, 0], mainTime, ref dvsIdx[s, 0, 0]);
WekaUtils.DebugAssert((long)dvs[s, 0, 0][nowRowIdx_s].Time == mainTime, "(long)dvs[s, 0, 0][nowRowIdx_s].Time == mainTime");
double mainClose = 0;
if (cp.AllIndNames2.ContainsKey("close"))
{
mainClose = (double)dvs[s, 0, 0][nowRowIdx_s]["close"];
}
for (int i = 0; i < cp.PeriodCount; ++i)
{
int periodSeconds = 60 * WekaUtils.GetMinuteofPeriod(cp.AllPeriods[i + cp.PeriodStart]);
int nowRowIdx = FindRowByTime(dvs[s, i, 0], mainTime / periodSeconds * periodSeconds, ref dvsIdx[s, i, 0]);
for (int p = 0; p < cp.PrevTimeCount; ++p)
{
if (nowRowIdx - p < 0)
{
throw new ArgumentException("No prev data!");
}
ForexData nowDrv = dvs[s, i, 0][nowRowIdx - p];
foreach (var kvp in cp.AllIndNames2)
{
double v = Convert.ToDouble(nowDrv[kvp.Key]);
double ind = WekaUtils.NormalizeValue(kvp.Key, kvp.Value, v, mainClose, WekaUtils.GetSymbolPoint(cp.AllSymbols[s]));
instanceValue[start] = ind;
start++;
}
for (int j = -1; j < Math.Max(0, Math.Min(cp.PeriodTimeCount - i, Parameters.PeriodTimeNames[i].Length)); ++j)
{
int nowRowIdx2;
if (j == -1)
nowRowIdx2 = nowRowIdx;
else
nowRowIdx2 = FindRowByTime(dvs[s, i, j + 1], mainTime / periodSeconds * periodSeconds, ref dvsIdx[s, i, j + 1]);
if (nowRowIdx2 - p < 0)
{
throw new ArgumentException("No prev data!");
}
ForexData nowDrv2 = dvs[s, i, j + 1][nowRowIdx2 - p];
foreach (var kvp in cp.AllIndNames)
{
double v = (double)nowDrv2[kvp.Key];
double ind = WekaUtils.NormalizeValue(kvp.Key, kvp.Value, v, mainClose, WekaUtils.GetSymbolPoint(cp.AllSymbols[s]));
instanceValue[start] = ind;
start++;
}
}
}
}
}
}
catch (ArgumentException)
{
continue;
}
if (m_useClassAsAttribute)
{
instanceValue[hereInstances.numAttributes() - 2] = hp;
}
instanceValue[hereInstances.numAttributes() - 1] = hp;
Instance instance = new weka.core.DenseInstance(1, instanceValue);
//if (!hereInstances.checkInstance(instance))
//{
// throw new ArgumentException("Imcompatible instance!");
//}
hereInstances.add(instance);
}
}
}
public WekaData(CandidateParameter cp)
{
m_cp = cp;
}
public void TrainandTest(CandidateClassifier classifierInfo, CandidateParameter cp)
{
//string dealType = classifierInfo.DealType;
Classifier cls = null;
if (TestParameters.UseTrain)
{
string modelFileName = GetModelFileName(classifierInfo.Name);
if (TestParameters.SaveModel)
{
cls = WekaUtils.TryLoadClassifier(modelFileName);
}
Instances trainInstancesNew, trainInstances;
trainInstances = m_trainInstances;
trainInstancesNew = m_trainInstancesNew;
if (cls == null)
{
if (classifierInfo.Classifier == null)
{
classifierInfo.Classifier = WekaUtils.CreateClassifier(cp.ClassifierType, m_currentTp, m_currentSl);
}
cls = classifierInfo.Classifier;
}
else
{
if (TestParameters.EnableDetailLog)
{
System.Console.WriteLine("Model is loaded.");
}
}
if (m_enableTrainSplitTest)
{
Instances trainTrainInst, trainTestInst;
DateTime splitTrainTimeEnd;
if (m_trainSplitTestNums != -1)
{
int trainTrainSize = trainInstancesNew.numInstances() - m_trainSplitTestNums;
int trainTestSize = m_trainSplitTestNums;
trainTrainInst = new Instances(trainInstancesNew, 0, trainTrainSize);
trainTestInst = new Instances(trainInstancesNew, trainTrainSize, trainTestSize);
splitTrainTimeEnd = WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainTrainSize);
}
else if (m_trainSplitPercent != -1)
{
if (m_trainSplitPercent == 100.0)
{
int trainTrainSize = trainInstancesNew.numInstances();
trainTrainInst = new Instances(trainInstancesNew, 0, trainTrainSize);
trainTestInst = new Instances(trainInstancesNew, 0, trainTrainSize);
splitTrainTimeEnd = WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainTrainSize);
}
else
{
int trainTrainSize = (int)Math.Round(trainInstancesNew.numInstances() * m_trainSplitPercent / 100);
int trainTestSize = trainInstancesNew.numInstances() - trainTrainSize;
trainTrainInst = new Instances(trainInstancesNew, 0, trainTrainSize);
trainTestInst = new Instances(trainInstancesNew, trainTrainSize, trainTestSize);
splitTrainTimeEnd = WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainTrainSize);
}
}
else
{
trainTrainInst = new Instances(trainInstancesNew, 0);
trainTestInst = new Instances(trainInstancesNew, 0);
DateTime dt = WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainInstances.numInstances() - 1);
splitTrainTimeEnd = m_trainTimeEnd.AddMinutes(-TestParameters.BatchTestMinutes);
while (splitTrainTimeEnd > dt)
{
splitTrainTimeEnd = splitTrainTimeEnd.AddMinutes(-TestParameters.BatchTestMinutes);
}
for (int i = 0; i < trainInstances.numInstances(); ++i)
{
dt = WekaUtils.GetDateValueFromInstances(trainInstances, 0, i);
if (dt <= splitTrainTimeEnd)
{
var ins = new DenseInstance(trainInstancesNew.instance(i));
trainTrainInst.add(ins);
}
else
{
var ins = new DenseInstance(trainInstancesNew.instance(i));
trainTestInst.add(ins);
}
}
}
cls = WekaUtils.TrainInstances(trainTrainInst, TestParameters.SaveModel ? modelFileName : null, cls);
//m_classifierQueue[dealType].Enqueue(new ClassifierInfo(cls, splitTrainTimeEnd));
//foreach (var i in m_classifierQueue[dealType])
//{
// var e = WekaUtils.TestInstances(trainTestInst, i.Cls);
// i.TotalCost = i.TotalCost * m_classifierQueueFactor + e.totalCost();
// i.TotalNum = (int)(i.TotalNum * m_classifierQueueFactor) + (int)e.numInstances();
//}
//WriteEvalSummary(eval1, string.Format("Test Data from {0} to {1}", m_testTimeStart.ToString(Parameters.DateTimeFormat), m_testTimeEnd.ToString(Parameters.DateTimeFormat)));
}
else
{
cls = WekaUtils.TrainInstances(trainInstancesNew, TestParameters.SaveModel ? modelFileName : null, cls);
//m_classifierQueue[dealType].Enqueue(new ClassifierInfo(cls, m_trainTimeEnd));
//foreach (var i in m_classifierQueue[dealType])
//{
// var e = WekaUtils.TestInstances(trainInstancesNew, i.Cls);
// i.TotalCost = i.TotalCost * m_classifierQueueFactor + e.totalCost();
// i.TotalNum = (int)(i.TotalNum * m_classifierQueueFactor) + (int)e.numInstances();
//}
}
if (TestParameters.EnableDetailLog)
{
System.Console.WriteLine("Model is trained.");
}
classifierInfo.Classifier = cls;
//if (classifierInfo.CurrentTrainInstances1 != null)
//{
// classifierInfo.CurrentTrainInstances1.clear();
//}
//if (classifierInfo.CurrentTrainInstancesNew1 != null)
//{
// classifierInfo.CurrentTrainInstancesNew1.clear();
//}
//classifierInfo.CurrentTrainInstances = new Instances(trainInstances, 0, trainInstances.numInstances());
//classifierInfo.CurrentTrainInstancesNew = new Instances(trainInstancesNew, 0, trainInstancesNew.numInstances());
if (classifierInfo.MoneyManagement == null)
{
classifierInfo.MoneyManagement = WekaUtils.CreateMoneyManagement(cp.MoneyManagementType, m_currentTp, m_currentSl);
}
IMoneyManagement mm = WekaUtils.TrainInstances4MM(trainInstancesNew, TestParameters.SaveModel ? modelFileName : null, classifierInfo.MoneyManagement);
classifierInfo.MoneyManagement = mm;
}
else
{
if (classifierInfo.Classifier == null)
{
classifierInfo.Classifier = WekaUtils.CreateClassifier(cp.ClassifierType, m_currentTp, m_currentSl);
}
cls = classifierInfo.Classifier;
if (classifierInfo.MoneyManagement == null)
{
classifierInfo.MoneyManagement = WekaUtils.CreateMoneyManagement(cp.MoneyManagementType, m_currentTp, m_currentSl);
}
}
if (m_enableTest)
{
Instances testInstancesNew, testInstances;
testInstances = m_testInstances;
testInstancesNew = m_testInstancesNew;
double[] cv = WekaUtils.ClassifyInstances(testInstancesNew, cls);
if (TestParameters.EnableExcludeClassifier)
{
bool hasPositive = false;
for (int i = 0; i < cv.Length; ++i)
{
if (cv[i] == 2)
{
hasPositive = true;
break;
}
}
if (hasPositive)
{
// Exclude
if (classifierInfo.ExcludeClassifier == null)
{
string modelFileName4Exclude = GetExcludeModelFileName(classifierInfo.Name);
classifierInfo.ExcludeClassifier = WekaUtils.TryLoadClassifier(modelFileName4Exclude);
}
if (classifierInfo.ExcludeClassifier != null)
{
double[] cv2 = WekaUtils.ClassifyInstances(testInstancesNew, classifierInfo.ExcludeClassifier);
// cv2 == 0 -> is class, otherwise = double.NaN;
for (int i = 0; i < cv.Length; ++i)
{
cv[i] = cv[i] == 2 && cv2[i] == 2 ? 2 : 0;
}
}
}
}
classifierInfo.CurrentTestRet = cv;
classifierInfo.CurrentClassValue = new double[testInstances.numInstances()];
for (int i = 0; i < testInstances.numInstances(); ++i)
{
classifierInfo.CurrentClassValue[i] = testInstances.instance(i).classValue();
}
for (int i = 0; i < testInstances.numInstances(); i++)
{
if (cv[i] == 2)
{
double openPrice = testInstances.instance(i).value(testInstances.attribute("mainClose"));
DateTime openTime = WekaUtils.GetDateValueFromInstances(testInstances, 0, i);
if (testInstances.instance(i).classValue() == 2 || testInstances.instance(i).classValue() == 0)
{
classifierInfo.Deals.AddDeal(openTime,
openPrice,
classifierInfo.DealType,
classifierInfo.MoneyManagement.GetVolume(testInstances.instance(i)),
testInstances.instance(i).classValue() == 2 ? -classifierInfo.Tp : classifierInfo.Sl,
WekaUtils.GetDateValueFromInstances(testInstances, 1, i));
}
else if (testInstances.instance(i).classValue() == 1)
{
double closePriceTp, closePriceSl;
if (classifierInfo.DealType == 'B')
{
closePriceTp = openPrice + classifierInfo.Tp * DealsInfo.GetPoint(0);
closePriceSl = openPrice - classifierInfo.Sl * DealsInfo.GetPoint(0);
}
else
{
closePriceTp = openPrice - classifierInfo.Tp * DealsInfo.GetPoint(0);
closePriceSl = openPrice + classifierInfo.Sl * DealsInfo.GetPoint(0);
}
classifierInfo.Deals.AddDeal(openTime,
openPrice,
classifierInfo.DealType,
classifierInfo.MoneyManagement.GetVolume(testInstances.instance(i)),
closePriceTp, closePriceSl);
}
else
{
throw new AssertException("classValue should be 0,1,2.");
}
}
}
}
}
public static string GetArffHeader(DateTime dt1, DateTime dt2, CandidateParameter cp)
{
//string header;
//using (StreamReader sr = new StreamReader(string.Format("{0}\\{1}_header_{2}.arff", m_arffHeaderDir, m_symbol, m_symbolPeriod2_cnt)))
//{
// header = sr.ReadToEnd();
//}
// header = header.Replace("@relation eurusd_m1", string.Format("@relation '{0}_{1}_{2}'", m_symbolPeriod2, dt1, dt2));
StringBuilder header = new StringBuilder();
header.AppendLine(string.Format("@relation '{0}_{1}'", dt1.ToString(Parameters.DateTimeFormat), dt2.ToString(Parameters.DateTimeFormat)));
header.AppendLine("@attribute timestamp date \"yyyy-MM-dd'T'HH:mm:ss\"");
header.AppendLine("@attribute hpdate date \"yyyy-MM-dd'T'HH:mm:ss\"");
//header.AppendLine("@attribute hour {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23}");
//header.AppendLine("@attribute day_of_week {1, 2, 3, 4, 5}");
header.AppendLine("@attribute spread numeric");
header.AppendLine("@attribute mainClose numeric");
header.AppendLine("@attribute hour numeric");
header.AppendLine("@attribute day_of_week numeric");
header.AppendLine("@attribute vol numeric");
for (int s = 0; s < cp.SymbolCount; ++s)
{
for (int i = 0; i < cp.PeriodCount; ++i)
{
for (int p = 0; p < cp.PrevTimeCount; ++p)
{
string postFix1 = cp.AllPeriods[i + cp.PeriodStart] + (p == 0 ? string.Empty : "_P" + p.ToString());
foreach (string name in cp.AllIndNames2.Keys)
{
header.AppendLine(string.Format("@attribute {0}_{1} {2}",
cp.AllSymbols[s + cp.SymbolStart], name + "_" + postFix1, TestParameters.IndicatorUseNumeric ? "numeric" : "{0,1,2,3}"));
}
for (int j = -1; j < Math.Max(0, Math.Min(cp.PeriodTimeCount - i, Parameters.PeriodTimeNames[i + cp.PeriodStart].Length)); ++j)
{
string postFix2;
if (j >= 0)
postFix2 = cp.AllPeriods[i + cp.PeriodStart] + "_" + Parameters.PeriodTimeNames[i + cp.PeriodStart][j] + (p == 0 ? string.Empty : "_P" + p.ToString());
else
postFix2 = postFix1;
foreach (string name in cp.AllIndNames.Keys)
{
header.AppendLine(string.Format("@attribute {0}_{1} {2}",
cp.AllSymbols[s + cp.SymbolStart], name + "_" + postFix2, TestParameters.IndicatorUseNumeric ? "numeric" : "{0,1,2,3}"));
}
}
}
}
}
if (m_useClassAsAttribute)
{
header.AppendLine("@attribute propAttr {0,1,2,3}");
}
if (!m_generateOneClassHp.HasValue)
{
// 0: Fail; 1: Unknown 2: Success
header.AppendLine("@attribute prop {0,1,2,3}");
}
else
{
header.AppendLine("@attribute prop {0}");
}
header.AppendLine();
header.AppendLine("@data");
return header.ToString();
}
private void CreateCds(ref int[, , ,] cds, CandidateParameter cp, int count)
{
cds = new int[cp.SymbolCount, cp.PeriodCount, count, 61];
for (int s = 0; s < cp.SymbolCount; ++s)
for (int p = 0; p < cp.PeriodCount; ++p)
for (int i = 0; i < cds.GetLength(3); ++i)
for (int j = 0; j < cds.GetLength(2); ++j)
cds[s, p, j, i] = -1;
}
public CandidateClassifier(string name, int tp, int sl, char dealType, int hour, CandidateParameter cp)
{
this.Name = name;
this.Tp = tp;
this.Sl = sl;
this.DealType = dealType;
this.Hour = hour;
this.Deals = new DealsInfo(cp.DealInfoLastMinutes);
this.WekaData = new WekaData(dealType, tp, sl, cp);
}
