public DateTime?GetNewestTestData()
{
string sql = string.Format("SELECT MAX(Time) FROM {0}", m_testDataTableName);
var r = Feng.Data.DbHelper.Instance.ExecuteScalar(sql);
if (r == System.DBNull.Value)
{
return(null);
}
else
{
return(WekaUtils.GetDateFromTime(Convert.ToInt64(r)));
}
}
public static int GetNonbusinessMinutes(DateTime dateStart, DateTime dateEnd, int stepMinutes = 1)
{
int r = 0;
long t1 = WekaUtils.GetTimeFromDate(dateStart);
long t2 = WekaUtils.GetTimeFromDate(dateEnd);
for (long i = t1; i < t2; i += stepMinutes * 60)
{
DateTime d = WekaUtils.GetDateFromTime(i);
if (d.DayOfWeek == DayOfWeek.Saturday || d.DayOfWeek == DayOfWeek.Sunday)
{
r++;
}
}
return(r);
}
public void UpdateUnclosedDeals(DateTime nowDate)
{
Dictionary <DateTime, System.Data.DataRow> dictHp = new Dictionary <DateTime, System.Data.DataRow>();
foreach (var kvp in m_classifierInfos)
{
foreach (var i in kvp.Value.Deals.Deals)
{
if (i.CloseTime.HasValue)
{
continue;
}
if (!dictHp.ContainsKey(i.OpenTime))
{
var sql = string.Format("SELECT * FROM {0}_HP WHERE TIME = {1}", this.CandidateParameter.MainSymbol, WekaUtils.GetTimeFromDate(i.OpenTime));
var dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql);
if (dt.Rows.Count > 0)
{
dictHp[i.OpenTime] = dt.Rows[0];
}
}
if (!dictHp.ContainsKey(i.OpenTime))
{
continue;
}
var row = dictHp[i.OpenTime];
int tpMinDelta = TestParameters.GetTpSlMinDelta(this.CandidateParameter.MainSymbol);
int slMinDelta = TestParameters.GetTpSlMinDelta(this.CandidateParameter.MainSymbol);
int d = i.DealType == 'B' ? 0 : 1;
sbyte?[, ,] hps = HpData.DeserializeHp((byte[])row["hp"]);
long?[, ,] hpsTime = HpData.DeserializeHpTimes((byte[])row["hp_date"]);
var hp = hps[d, kvp.Value.Tp / tpMinDelta - 1, kvp.Value.Sl / slMinDelta - 1];
if (hp.HasValue &&
hp.Value != -1)
{
i.CloseTime = WekaUtils.GetDateFromTime(hpsTime[d, kvp.Value.Tp / tpMinDelta - 1, kvp.Value.Sl / slMinDelta - 1].Value);
i.Cost = hp.Value == 1 ? -kvp.Value.Tp : kvp.Value.Sl;
}
}
}
}
private bool?Do(DateTime openDate, double openPrice, out DateTime?closeDate, int dealType)
{
closeDate = DateTime.MinValue;
var dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(string.Format("SELECT * FROM {0}_HP WHERE Time = {1}", m_symbol, WekaUtils.GetTimeFromDate(openDate)));
if (dt.Rows.Count == 0)
{
return(null);
}
sbyte?[, ,] hp = HpData.DeserializeHp((byte[])dt.Rows[0]["hp"]);
long?[, ,] hpTime = HpData.DeserializeHpTimes((byte[])dt.Rows[0]["hp_date"]);
int ix = (int)m_tp / TestParameters.GetTpSlMinDelta(m_symbol) - 1;
int iy = (int)m_sl / TestParameters.GetTpSlMinDelta(m_symbol) - 1;
if (hp[dealType, ix, iy].HasValue)
{
closeDate = WekaUtils.GetDateFromTime(hpTime[dealType, ix, iy].Value);
return(hp[dealType, ix, iy].Value == 1);
}
return(null);
}
public void GenerateArff(string arffFileName, string candlePatternFile)
{
int preLength = TestParameters2.PreLength;
int tpStart = TestParameters2.tpStart;
int slStart = TestParameters2.slStart;
int tpCount = TestParameters2.tpCount;
int slCount = TestParameters2.slCount;
var dataDates = TestManager.GetDataDateRange();
int minOutBegIdx = TaLibTest.minOutBegIdx;
var cp = TestParameters2.CandidateParameter;
int[, , ,] cds = null;
ForexDataRows[,] hpdvs = new ForexDataRows[cp.SymbolCount, cp.PeriodCount];
for (int s = 0; s < cp.SymbolCount; ++s)
{
for (int p = 0; p < cp.PeriodCount; ++p)
{
string symbol = cp.AllSymbols[s + cp.SymbolStart];
string period = cp.AllPeriods[p + cp.PeriodStart];
string symbolPeriod = string.Format("{0}_{1}", symbol, period);
hpdvs[s, p] = DbData.Instance.GetDbData(dataDates[0], dataDates[1], symbolPeriod, 0, true, cp);
m_currentPeriod = period;
//int minOutNBElement = m_minOutNBElements[m_currentPeriod];
int minOutNBElement = hpdvs[s, p].Length - minOutBegIdx;
if (cds == null)
{
CreateCds(ref cds, cp, minOutNBElement + 100);
}
using (StreamReader sr = new StreamReader(candlePatternFile))
{
int n = 0;
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
string[] ss = line.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
WekaUtils.DebugAssert(ss.Length == minOutNBElement, "ss.Length == minOutNBElement");
for (int i = 0; i < minOutNBElement; ++i)
{
cds[s, p, i, n] = Convert.ToInt32(ss[i]);
}
n++;
}
}
}
}
if (File.Exists(arffFileName))
{
System.IO.File.Delete(arffFileName);
}
string wekaFileName = string.Format(arffFileName);
using (StreamWriter sw = new StreamWriter(wekaFileName))
{
sw.WriteLine("@relation 'candlePatterns'");
sw.WriteLine("@attribute timestamp date \"yyyy-MM-dd\'T\'HH:mm:ss\"");
sw.WriteLine("@attribute hpdate date \"yyyy-MM-dd\'T\'HH:mm:ss\"");
sw.WriteLine("@attribute spread numeric");
sw.WriteLine("@attribute mainClose numeric");
for (int pre = 0; pre < preLength; ++pre)
{
for (int s = 0; s < cp.SymbolCount; ++s)
{
for (int p = 0; p < cp.PeriodCount; ++p)
{
for (int i = 0; i < 61; ++i)
{
sw.WriteLine(string.Format("@attribute {0}_{1}_{2}_p{3} {4}", cp.AllSymbols[s + cp.SymbolStart],
cp.AllPeriods[p + cp.PeriodStart], i.ToString(), pre.ToString(), "{0,100,200,-100,-200}"));
}
}
}
}
sw.WriteLine("@attribute prop " + " {0,1,2,3}");
sw.WriteLine("@data");
sw.WriteLine();
var hps = HpData.Instance.GetHpSum(cp.MainSymbol, cp.MainPeriod);
var hpdv = hpdvs[0, 0];
for (int i = minOutBegIdx + preLength - 1; i < hpdv.Length; ++i)
{
DateTime nowDate = WekaUtils.GetDateFromTime((long)hpdv[i].Time);
//if (nowDate.Hour % TestParameters2.MainPeriodOfHour != 0)
// continue;
if (!hps.ContainsKey(nowDate) && !(TestParameters2.RealTimeMode && i == hpdv.Length - 1))
{
continue;
}
long hpTime = WekaUtils.GetTimeFromDate(Parameters.MaxDate);
int hp = 0;
if (!(TestParameters2.RealTimeMode && i == hpdv.Length - 1))
{
hpTime = hps[nowDate].Item2;
hp = hps[nowDate].Item1;
}
sw.Write(nowDate.ToString(Parameters.DateTimeFormat));
sw.Write(",");
// hp
sw.Write(WekaUtils.GetDateFromTime(hpTime).ToString(Parameters.DateTimeFormat));
sw.Write(",");
sw.Write((int)hpdv[i]["spread"]);
sw.Write(",");
sw.Write(((double)hpdv[i]["close"]).ToString());
sw.Write(",");
for (int pre = 0; pre < preLength; ++pre)
{
for (int s = 0; s < cp.SymbolCount; ++s)
{
for (int p = 0; p < cp.PeriodCount; ++p)
{
for (int j = 0; j < 61; ++j)
{
int candlePattern = cds[s, p, i - minOutBegIdx - preLength + pre + 1, j];
if (candlePattern == -1)
{
throw new AssertException(string.Format("candle pattern should not be -1.idx={0}", i - minOutBegIdx - preLength + pre + 1));
}
sw.Write(candlePattern);
sw.Write(",");
}
}
}
}
sw.WriteLine(hp.ToString());
}
}
}
public override string ToString()
{
return(WekaUtils.GetDateFromTime(time).ToString());
}
public void RunOnBar(long nowTime)
{
DateTime nowDate = WekaUtils.GetDateFromTime(nowTime);
//TestParameters2.TrainStartTime = new DateTime(2000, 1, 1);
if (nowDate < TestParameters2.TrainStartTime)
{
return;
}
TestParameters2.TrainEndTime = nowDate.AddSeconds(1);
//// only for batchbatch
//var ttts = m_tm.SetTimeMinutesFromTestEnd(nowDate,
// TestParameters.BatchTrainMinutes, TestParameters.BatchTestMinutes);
//WekaUtils.DebugAssert((ttts[3] - nowDate).TotalMinutes < 1, "(ttts[3] - nowDate).TotalMinutes < 1");
//Console.WriteLine(string.Format("TrainTime = {0}, {1}; TestTime = {2}, {3}", ttts[0], ttts[1], ttts[2], ttts[3]));
//if (nowDate >= new DateTime(2004, 3, 29))
//{
//}
//WekaUtils.Instance.WriteLog("Run ExecuteCandidate.");
//if (!TestParameters2.DBDataConsistent)
//{
// _realDealsCandidate.UpdateUnclosedDeals(nowDate);
//}
//_realDealsCandidate.ExecuteCandidate(nowDate);
//WekaUtils.Instance.WriteLog("Run GenerateCCScores.");
//GenerateCCScores(nowDate);
bool inPrepare = false;
if (TestParameters2.InPreparing)
{
inPrepare = true;
}
if ((TestParameters2.TrainEndTime - TestParameters2.TrainStartTime).TotalHours < TestParameters2.MinTrainSize * TestParameters2.MainPeriodOfHour)
{
inPrepare = true;
}
if (!inPrepare)
{
WekaUtils.Instance.WriteLog("Run GenerateHpDataToTxt.");
HpData.Instance.GenerateHpDataToTxt(TestParameters2.CandidateParameter.MainSymbol, TestParameters2.CandidateParameter.MainPeriod);
foreach (string s1 in System.IO.Directory.GetFiles(TestParameters.BaseDir, "*.arff"))
{
System.IO.File.Delete(s1);
}
foreach (string s1 in System.IO.Directory.GetFiles(TestParameters.BaseDir, "IncrementTest_*.txt"))
{
System.IO.File.Delete(s1);
}
//WekaUtils.Instance.WriteLog("Run BuildHpProbDeals.");
m_tm.BuildHpProbDeals();
//WekaUtils.Instance.WriteLog("Run BuildPricePatternDeals.");
m_tm.BuildPricePatternDeals();
//WekaUtils.Instance.WriteLog("Run BuildCandlePatternDeals.");
m_taLibTest.BuildCandlePatternDeals();
////WekaUtils.Instance.WriteLog("Run BuildCCScoreDeals.");
//m_txtTest.BuildCCScoreDeals();
string orderFile = TestParameters.GetBaseFilePath(string.Format("ea_order_{0}.txt", TestParameters2.CandidateParameter.MainSymbol));
System.IO.File.Delete(orderFile);
m_txtTest.MergeAllBuildResult();
if (System.IO.File.Exists(orderFile))
{
string signalResultFile = "C:\\ProgramData\\MetaQuotes\\Terminal\\Common\\Files\\{0}_{1}.txt";
if (!TestParameters2.DBDataConsistent)
{
signalResultFile = string.Format(signalResultFile, "MLEASignal", TestParameters2.CandidateParameter.MainSymbol);
}
else
{
signalResultFile = string.Format(signalResultFile, "MLEASignal_db", TestParameters2.CandidateParameter.MainSymbol);
}
System.IO.File.Copy(orderFile, signalResultFile, true);
}
}
HpData.Instance.Clear();
DbData.Instance.Clear();
m_taLibTest.Clear();
WekaData.ClearTemplates();
CCScoreData.Instance.Clear();
System.GC.Collect();
}
public Tuple <int, long> GetHpSumByM1(string symbol, string period, long nowTime, long getTime)
{
System.Diagnostics.Debug.Assert(getTime != -1, "");
Tuple <int, long> ret = new Tuple <int, long>(0, 0);
if (m_hpDataDb == null)
{
m_hpDataDb = new HpDbData(symbol);
}
long time = getTime - 60;
long hpdate = 0;
double[] sum = new double[2];
sum[0] = sum[1] = 0;
while (true)
{
time += 60;
if (time >= getTime + 60 * WekaUtils.GetMinuteofPeriod(period))
{
break;
}
System.Diagnostics.Debug.Assert(time < nowTime, "");
var hps = m_hpDataDb.GetHpData(WekaUtils.GetDateFromTime(time));
if (hps == null)
{
continue;
}
for (int k = 0; k < 2; ++k)
{
for (int tp = TestParameters2.tpStart; tp < TestParameters2.tpCount; ++tp)
{
for (int sl = TestParameters2.slStart; sl < TestParameters2.slCount; ++sl)
{
var hpItem10 = hps.Item1[k, tp, sl, 0];
var hpItem11 = hps.Item1[k, tp, sl, 1];
var hpItem2 = hps.Item2[k, tp, sl];
WekaUtils.DebugAssert(hpItem10 >= 0, "hp data should >= 0");
WekaUtils.DebugAssert(hpItem11 >= 0, "hp data should >= 0");
WekaUtils.DebugAssert(hpItem2 >= 0, "hp data should >= 0");
// 当用PartialData模式的，还未知道的数据
if (hpItem2 == Parameters.MaxTime)
{
continue;
}
if (hpItem2 > nowTime)
{
continue;
}
int tpp = 1;
int tp1 = tpp * (tp + 1);
int sl1 = tpp * (sl + 1);
int v = 1;
//v = 600 / (sl + 1);
sum[k] -= tp1 * v * hpItem10;
sum[k] += sl1 * v * hpItem11;
hpdate = Math.Max(hpdate, hpItem2);
}
}
}
}
int hpSum = 2;
bool useOneClass = false;
if (useOneClass)
{
if (sum[0] < 0)
{
hpSum = 0;
}
}
else
{
bool enableMultiClass = true;
if (enableMultiClass)
{
if (sum[0] < 0 && sum[1] >= 0)
{
hpSum = 0;
}
else if (sum[1] < 0 && sum[0] >= 0)
{
hpSum = 1;
}
else if (sum[0] >= 0 && sum[1] >= 0)
{
hpSum = 2;
}
else
{
hpSum = 3;
}
}
else
{
double delta = 0 * (TestParameters2.tpCount - TestParameters2.tpStart);
if (sum[0] < sum[1] - delta)
{
hpSum = 0;
}
else if (sum[0] > sum[1] + delta)
{
hpSum = 1;
}
}
}
ret = new Tuple <int, long>(hpSum, hpdate);
return(ret);
}