public static void FilterInstances(weka.core.Instances allInstances)
{
DateTime nextHpDate = DateTime.MinValue;
java.util.LinkedList deleteList = new java.util.LinkedList();
for (int i = 0; i < allInstances.numInstances(); ++i)
{
DateTime nowDate = WekaUtils.GetDateValueFromInstances(allInstances, 0, i);
if (TestParameters2.RealTimeMode && i == allInstances.numInstances() - 1)
{
allInstances.instance(i).setClassValue(0);
allInstances.instance(i).setValue(1, WekaUtils.GetTimeFromDate(Parameters.MaxDate) * 1000);
}
else
{
if (nowDate < nextHpDate)
{
deleteList.Add(allInstances.instance(i));
}
else
{
DateTime hpDate = WekaUtils.GetDateValueFromInstances(allInstances, 1, i);
nextHpDate = hpDate;
}
}
}
allInstances.removeAll(deleteList);
}
public void Now(DateTime nowTime, double?nowPrice)
{
m_nowTime = nowTime;
if (nowPrice.HasValue)
{
m_nowPrice.close = nowPrice.Value;
m_nowPrice.time = WekaUtils.GetTimeFromDate(nowTime);
}
if (!m_isLastForPeriod)
{
if (m_firstLastDate == null)
{
m_firstLastDate = nowTime.AddMinutes(m_lastMinutes);
}
if (m_firstLastDate <= m_nowTime)
{
m_isLastForPeriod = true;
}
}
else
{
RemoveOldDeals();
}
//CloseAll();
CalculateTotal();
CalculateNow();
}
//public void ConvertHpDataPeriod(string symbol, string periodSsrc, string periodDest)
//{
// string hpFileName = TestParameters.GetBaseFilePath(string.Format("{0}_{1}_hpdata.txt", symbol, periodSsrc));
// var hpData = GetHpDateFromTxt(hpFileName, TestParameters2.nTpsl);
// string hpFileName2 = TestParameters.GetBaseFilePath(string.Format("{0}_{1}_hpdata.txt", symbol, periodDest));
// int m = TestParameters.TpSlMaxCount / TestParameters2.nTpsl;
// DateTime date = TestParameters2.TrainStartTime;
// DateTime maxDate = TestParameters2.TrainEndTime;
// using(StreamWriter sw = new StreamWriter(hpFileName2))
// {
// while (true)
// {
// if (!TestParameters2.RealTimeMode)
// {
// Console.WriteLine(date.ToString(Parameters.DateTimeFormat));
// }
// StringBuilder sb = new StringBuilder();
// {
// if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday)
// {
// date = date.AddHours(mainPeriodOfHour);
// continue;
// }
// bool isComplete = true;
// if ((TestParameters2.UsePartialHpData || isComplete))
// {
// sb.Append(date.ToString(Parameters.DateTimeFormat));
// sb.Append(", ");
// int[, , ,] hps = new int[2, m, m, 2];
// long[, ,] hpTimes = new long[2, m, m];
// for (int j = 0; j < hps.GetLength(0); ++j)
// for (int k = 0; k < hps.GetLength(1); ++k)
// for (int l = 0; l < hps.GetLength(2); ++l)
// {
// hps[j, k, l, 0] = hps[j, k, l, 1] = -1;
// hpTimes[j, k, l] = -1;
// }
// List<DateTime> dts = new List<DateTime>();
// foreach (DateTime dt in dts)
// {
// if (!hpData.ContainsKey(dt))
// continue;
// sbyte?[, ,] hp = hpData[dt].Item1;
// long[, ,] hpTime = hpData[dt].Item2;
// WekaUtils.DebugAssert(hp.GetLength(0) == 2, "");
// WekaUtils.DebugAssert(hp.GetLength(1) % m == 0, "");
// WekaUtils.DebugAssert(hp.GetLength(2) % m == 0, "");
// WekaUtils.DebugAssert(hpTime.GetLength(0) == 2, "");
// for (int j = 0; j < hp.GetLength(0); ++j)
// for (int k = 0; k < hp.GetLength(1); ++k)
// for (int l = 0; l < hp.GetLength(2); ++l)
// {
// if (k % n != n - 1 || l % n != n - 1)
// continue;
// if (!hp[j, k, l].HasValue || hp[j, k, l] == -1)
// continue;
// //if (k / n >= TestParameters2.tpCount)
// // continue;
// //if (l / n >= TestParameters2.slCount)
// // continue;
// if (hps[j, k / n, l / n, 0] == -1)
// hps[j, k / n, l / n, 0] = 0;
// if (hps[j, k / n, l / n, 1] == -1)
// hps[j, k / n, l / n, 1] = 0;
// if (hp[j, k, l] == 1)
// hps[j, k / n, l / n, 0]++;
// else if (hp[j, k, l] == 0)
// hps[j, k / n, l / n, 1]++;
// else
// throw new AssertException("hp should be 0 or 1.");
// hpTimes[j, k / n, l / n] = Math.Max(hpTimes[j, k / n, l / n], hpTime[j, k, l].Value);
// }
// }
// for (int j = 0; j < hps.GetLength(0); ++j)
// for (int k = 0; k < hps.GetLength(1); ++k)
// for (int l = 0; l < hps.GetLength(2); ++l)
// {
// if (isComplete)
// {
// if (hps[j, k, l, 0] == -1 || hps[j, k, l, 1] == -1)
// {
// throw new AssertException("hps should not be -1.");
// }
// }
// if (hps[j, k, l, 0] == -1)
// hps[j, k, l, 0] = 0;
// if (hps[j, k, l, 1] == -1)
// hps[j, k, l, 1] = 0;
// sb.Append(hps[j, k, l, 0] + ", " + hps[j, k, l, 1] + ", ");
// }
// long maxHpTime = 0;
// for (int j = 0; j < hpTimes.GetLength(0); ++j)
// for (int k = 0; k < hpTimes.GetLength(1); ++k)
// for (int l = 0; l < hpTimes.GetLength(2); ++l)
// {
// if (isComplete)
// {
// if (hpTimes[j, k, l] == -1)
// {
// throw new AssertException("hpTimes should not be -1.");
// }
// }
// if (hpTimes[j, k, l] == -1)
// hpTimes[j, k, l] = Parameters.MaxTime;
// sb.Append(hpTimes[j, k, l] + ", ");
// maxHpTime = Math.Max(hpTimes[j, k, l], maxHpTime);
// }
// sb.AppendLine();
// sw.Write(sb.ToString());
// }
// DateTime nextDate = date.AddMinutes(WekaUtils.GetMinuteofPeriod(periodDest));
// date = nextDate;
// if (date >= maxDate)
// break;
// }
// }
// }
//}
public static Dictionary <long, int> GetHpDateTimesCountFromInstance(string fileName)
{
Dictionary <long, int> dictTimes = new Dictionary <long, int>();
int dealCount = 0;
var instances = WekaUtils.LoadInstances(TestParameters.GetBaseFilePath(fileName));
for (int i = 0; i < instances.numInstances(); ++i)
{
var date = WekaUtils.GetDateValueFromInstances(instances, 0, i);
var hpdate = WekaUtils.GetDateValueFromInstances(instances, 1, i);
var hptime = WekaUtils.GetTimeFromDate(hpdate);
if (dictTimes.ContainsKey(hptime))
{
dictTimes[hptime]++;
}
else
{
dictTimes[hptime] = 1;
}
dealCount++;
Console.WriteLine(date.ToString(Parameters.DateTimeFormat) + ", " + dictTimes.Count + ", " + dealCount);
}
return(dictTimes);
}
private Tuple <long, long> GetDbDateFilter(DateTime dt1, DateTime dt2, int tableType)
{
if (tableType == 0)
{
return(new Tuple <long, long>(WekaUtils.GetTimeFromDate(dt1.AddDays(-1)), WekaUtils.GetTimeFromDate(dt2)));
}
else
{
return(new Tuple <long, long>(WekaUtils.GetTimeFromDate(dt1.AddDays(-3)), WekaUtils.GetTimeFromDate(dt2)));
}
}
public static void SaveCCScoresToDb(DateTime nowDate, CandidateParameter cp, ParameterdCandidateStrategy pcs)
{
string tableName = string.Format("{0}_CCSCORE_{1}", cp.MainSymbol, cp.DealInfoLastMinutes / (7 * 24 * 12 * 5));
long nowTime = WekaUtils.GetTimeFromDate(nowDate);
try
{
Feng.Data.DbHelper.Instance.ExecuteNonQuery(string.Format("SELECT TOP 1 * FROM {0} WHERE TIME = -1", tableName));
}
catch (Exception)
{
string sqlCreate = string.Format(@"CREATE TABLE {0}(
[Time] [bigint] NOT NULL,
[scores] [varbinary](max) NOT NULL,
[nda] [varbinary](max) NOT NULL,
[nds] [varbinary](max) NOT NULL,
CONSTRAINT [PK_{0}] PRIMARY KEY CLUSTERED
(
[Time] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]", tableName);
Feng.Data.DbHelper.Instance.ExecuteNonQuery(sqlCreate);
}
long[, ,] ndas = new long[Parameters.AllDealTypes.Length, cp.BatchTps.Length, cp.BatchSls.Length];
long[, ,] ndss = new long[Parameters.AllDealTypes.Length, cp.BatchTps.Length, cp.BatchSls.Length];
double[, ,] nowScores = new double[Parameters.AllDealTypes.Length, cp.BatchTps.Length, cp.BatchSls.Length];
pcs.IterateClassifierInfos((k, ii, jj, h) =>
{
nowScores[k, ii, jj] = pcs.m_classifierInfoIdxs[k, ii, jj, h].Deals.NowScore;
ndas[k, ii, jj] = pcs.m_classifierInfoIdxs[k, ii, jj, h].Deals.DealLastTimeAvg;
ndss[k, ii, jj] = pcs.m_classifierInfoIdxs[k, ii, jj, h].Deals.DealLastTimeStd;
});
string sql = string.Format("INSERT INTO [{0}] ([Time],[scores],[nda],[nds]) VALUES (@Time, @score, @nda, @nds)", tableName);
var cmd = new System.Data.SqlClient.SqlCommand(sql);
cmd.Parameters.AddWithValue("@Time", nowTime);
cmd.Parameters.AddWithValue("@score", CCScoreData.SerializeScores(nowScores));
cmd.Parameters.AddWithValue("@nda", CCScoreData.SerializeScoreTimes(ndas));
cmd.Parameters.AddWithValue("@nds", CCScoreData.SerializeScoreTimes(ndss));
try
{
Feng.Data.DbHelper.Instance.ExecuteNonQuery(string.Format("DELETE FROM {0} WHERE TIME = '{1}'", tableName, WekaUtils.GetTimeFromDate(nowDate)));
Feng.Data.DbHelper.Instance.ExecuteNonQuery(cmd);
}
catch (Exception)
{
}
}
private string GetDbDateFilter4Db(DateTime dt1, DateTime dt2, int tableType)
{
string filter;
if (tableType == 0)
{
filter = string.Format("Time >= {0} AND Time < {1}", WekaUtils.GetTimeFromDate(dt1.AddDays(-1)), WekaUtils.GetTimeFromDate(dt2));
}
else
{
filter = string.Format("Time >= {0} AND Time < {1}", WekaUtils.GetTimeFromDate(dt1.AddDays(-3)), WekaUtils.GetTimeFromDate(dt2));
}
return(filter);
}
public void Now(DateTime nowTime, double?nowPrice)
{
m_nowTime = nowTime;
if (nowPrice.HasValue)
{
m_nowPrice.close = nowPrice.Value;
}
m_nowPrice.time = WekaUtils.GetTimeFromDate(nowTime);
RemoveOldDeals();
//CloseAll();
CalculateCurrent();
}
public static int GetNonbusinessMinutes(DateTime dateStart, DateTime dateEnd, int stepMinutes = 1)
{
int r = 0;
long t1 = WekaUtils.GetTimeFromDate(dateStart);
long t2 = WekaUtils.GetTimeFromDate(dateEnd);
for (long i = t1; i < t2; i += stepMinutes * 60)
{
DateTime d = WekaUtils.GetDateFromTime(i);
if (d.DayOfWeek == DayOfWeek.Saturday || d.DayOfWeek == DayOfWeek.Sunday)
{
r++;
}
}
return(r);
}
public Tuple <int[, , , ], long[, , ]> GetHpData(DateTime date)
{
int n = m_buffer.Count - 5 * 15 * 20000;
if (n > 0)
{
List <long> delete = new List <long>();
foreach (var kvp in m_buffer)
{
delete.Add(kvp.Key);
if (delete.Count >= n)
{
break;
}
}
foreach (var i in delete)
{
m_buffer.Remove(i);
}
}
Tuple <int[, , , ], long[, , ]> ret = null;
long time = WekaUtils.GetTimeFromDate(date);
if (!m_buffer.ContainsKey(time))
{
var sql = string.Format("SELECT * FROM {0}_HP WHERE TIME = {1}", m_symbol, time);
var dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql);
if (dt.Rows.Count > 0)
{
ret = HpData.SumHp(dt.Select());
}
m_buffer[time] = ret;
}
return(m_buffer[time]);
}
// tableType = 0: MainTable; 1: OtherDataTable; 2:HpTable
public ForexDataRows GetDbData(DateTime dt1, DateTime dt2, string tableName, int tableType, bool isTrain, CandidateParameter cp)
{
lock (m_lockObject)
{
//int dealTypeIdx1 = dealType == 'B' ? 0 : 1;
DateTime prevDate = dt1.AddMinutes(-2 * cp.PrevTimeCount * WekaUtils.GetMinuteofPeriod(cp.AllPeriods[cp.PeriodCount - 1]));
DateTime nextDate = dt2;
if (m_useBufferReadData && m_batchBufferMinutes < 0)
{
int n1 = cp.SymbolCount * cp.PeriodCount * (cp.PeriodTimeCount + 1)
* (cp.AllIndNames.Count + cp.AllIndNames2.Count + 5) / WekaUtils.GetMinuteofPeriod(cp.MainPeriod);
n1 = Math.Max(n1, 1);
int n2 = 1;
//n2 = 3 * TestParameters.BatchTps.Length * TestParameters.BatchSls.Length * Parameters.AllDealTypes.Length / WekaUtils.GetMinuteofPeriod(TestParameters.MainPeriod);
//n2 /= 6;
n2 = 10000 / WekaUtils.GetMinuteofPeriod(cp.MainPeriod);
n2 = Math.Max(n2, 1);
m_batchBufferMinutes = (Parameters.TotalCanUseBuffer - System.GC.GetTotalMemory(false)) / (n1 + n2) * WekaUtils.GetMinuteofPeriod(cp.MainPeriod); // / 240
int min = (int)(nextDate - prevDate).TotalMinutes + 24 * TestParameters.BatchTestMinutes;
int max = (int)(1.1 * TestParameters.BatchTrainMinutes);
if (m_batchBufferMinutes < min)
{
//m_useBufferReadData = false;
WekaUtils.Instance.WriteLog(string.Format("Partial buffer. batchGetTime = {0}, min = {1}", m_batchBufferMinutes, min));
m_batchBufferMinutes = min;
}
else
{
//m_useBufferReadData = true;
WekaUtils.Instance.WriteLog(string.Format("Use buffer. batchGetTime = {0}, max = {1}", m_batchBufferMinutes, max));
m_batchBufferMinutes = Math.Min(m_batchBufferMinutes, max);
}
}
if (m_useBufferReadData)
{
DateTime bufferDateStart = prevDate;
DateTime bufferDateEnd = bufferDateStart.AddMinutes(m_batchBufferMinutes);
if (TestParameters2.RealTimeMode)
{
if (isTrain)
{
bufferDateStart = dt1.AddDays(-7);
bufferDateEnd = dt2.AddDays(7);
}
else
{
bufferDateStart = dt1;
bufferDateEnd = dt2;
}
}
//System.Console.WriteLine("{0}, {1}, {2}, {3}", bufferDateStart, bufferDateEnd, prevDate, nextDate);
if (bufferDateEnd >= nextDate)
{
string allDataKey = tableName;
//if (tableType == 2)
//{
// allDataKey += "_" + Parameters.AllDealTypes[dealTypeIdx] + "_" + tp.ToString() + "_" + sl.ToString();
//}
if (m_allData == null)
{
if (m_serializeAllData)
{
foreach (string s in System.IO.Directory.GetFiles(".", m_allDataCacheFile))
{
string[] ss = System.IO.Path.GetFileNameWithoutExtension(s).Split(new char[] { '_' }, StringSplitOptions.RemoveEmptyEntries);
DateTime d1 = Convert.ToDateTime(ss[1]);
DateTime d2 = Convert.ToDateTime(ss[2]);
if (bufferDateStart >= d1 && bufferDateEnd <= d2)
{
m_allData = Feng.Windows.Utils.SerializeHelper.Deserialize <Dictionary <string, ForexDataTable> >(s);
break;
}
}
}
if (m_allData == null)
{
m_allData = new Dictionary <string, ForexDataTable>();
//m_allDateDates = new DateTime[m_dealType.Length][];
}
}
if (m_allData.ContainsKey(allDataKey))
{
Tuple <DateTime, DateTime> ds = m_allDataDates[allDataKey];
bool expired = false;
if (!isTrain)
{
if (ds.Item2 < nextDate || ds.Item1 > prevDate)
{
//System.Console.WriteLine("{0}, {1}, {2}, {3}", ds.Item1.ToString(), ds.Item2, prevDate, nextDate);
expired = true;
}
}
else
{
if (ds.Item2 <= nextDate.AddMinutes(TestParameters.EnablePerhourTrain ?
2 * Parameters.AllHour * TestParameters.BatchTestMinutes :
2 * TestParameters.BatchTestMinutes) || ds.Item1 > prevDate)
{
expired = true;
}
}
if (expired)
{
m_allData.Remove(allDataKey);
m_allDataDates.Remove(allDataKey);
}
}
//long usedMemory = System.GC.GetTotalMemory(false);
//if (m_cacheData && usedMemory >= Parameters.TotalCanUseMemory)
//{
// WekaUtils.Instance.WriteLog(string.Format("Use too many memory. now total memory = {0}", usedMemory));
// m_cacheData = false;
//}
//else if (!m_cacheData && usedMemory < Parameters.TotalCanUseMemory)
//{
// WekaUtils.Instance.WriteLog(string.Format("reset to cache data. now total memory = {0}", usedMemory));
// m_cacheData = true;
//}
if (m_cacheData && !m_allData.ContainsKey(allDataKey))
{
StringBuilder sql = new StringBuilder();
if (tableType == 0)
{
sql.Append(string.Format("SELECT [Time], [Date], [hour], [dayofweek], [spread], [close] as mainClose")); // , [AskVolume], [BidVolume]
}
else if (tableType == 1)
{
sql.Append(string.Format("SELECT [Time], [Date]"));
}
else if (tableType == 2)
{
sql.Append(string.Format("SELECT [Time], [hp], [hp_date]"));
}
if (tableType == 0 || tableType == 1)
{
foreach (var kvp in cp.AllIndNames2)
{
sql.Append(string.Format(",[{0}]", kvp.Key));
}
foreach (var kvp in cp.AllIndNames)
{
sql.Append(string.Format(",[{0}]", kvp.Key));
}
}
sql.Append(string.Format(" FROM {0} WHERE Time >= '{1}' AND Time < '{2}' ",
tableName, WekaUtils.GetTimeFromDate(bufferDateStart), WekaUtils.GetTimeFromDate(bufferDateEnd)));
//if (tableType == 0 || tableType == 1)
//{
// sql.Append(" AND IsActive = 1 ");
//}
//if (tableType == 2)
//{
// sql.Append(string.Format(" AND DealType = '{0}' AND Tp = {1} AND Sl = {2}", Parameters.AllDealTypes[dealTypeIdx], tp, sl));
//}
if (!string.IsNullOrEmpty(TestParameters.DbSelectWhere))
{
sql.Append(string.Format(" AND {0}", TestParameters.DbSelectWhere));
}
sql.Append(string.Format(" {0}", m_selectOrder));
string cmd = sql.ToString();
//m_allData[allDataKey] = DbHelper.Instance.ExecuteDataTable(cmd);
//m_allData[allDataKey].PrimaryKey = new System.Data.DataColumn[] { m_allData[allDataKey].Columns["Time"] };
m_allData[allDataKey] = new ForexDataTable(Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd), tableType == 2);
m_allDataDates[allDataKey] = new Tuple <DateTime, DateTime>(bufferDateStart, bufferDateEnd);
WekaUtils.Instance.WriteLog(string.Format("Buffer Get Data From {0} to {1} of {2}", bufferDateStart, bufferDateEnd, allDataKey));
if (m_serializeAllData)
{
string allDataCacheFile = m_allDataCacheFile.Replace("*",
TestParameters.BatchDateStart.ToString(Parameters.DateTimeFormat) + "_"
+ TestParameters.BatchDateEnd.ToString(Parameters.DateTimeFormat));
Feng.Windows.Utils.SerializeHelper.Serialize(allDataCacheFile, m_allData);
}
}
if (m_allData.ContainsKey(allDataKey))
{
//string filter = GetDbDateFilter(dt1, dt2, tableType);
//return m_allData[allDataKey].Select(filter);
Tuple <long, long> filter = GetDbDateFilter(dt1, dt2, tableType);
return(m_allData[allDataKey].SelectByDate(filter.Item1, filter.Item2));
}
}
}
// no get from cache
{
StringBuilder sql = new StringBuilder();
if (tableType == 0)
{
sql.Append(string.Format("SELECT [Time], [Date], [hour], [dayofweek], [spread], [close] as mainClose")); // , [AskVolume], [BidVolume]
}
else if (tableType == 1)
{
sql.Append(string.Format("SELECT [Time], [Date]"));
}
else if (tableType == 2)
{
sql.Append(string.Format("SELECT [Time], [hp], [hp_date]"));
}
if (tableType == 0 || tableType == 1)
{
foreach (var kvp in cp.AllIndNames2)
{
sql.Append(string.Format(",[{0}]", kvp.Key));
}
foreach (var kvp in cp.AllIndNames)
{
sql.Append(string.Format(",[{0}]", kvp.Key));
}
}
string filter = GetDbDateFilter4Db(dt1, dt2, tableType);
sql.Append(string.Format(" FROM {0} WHERE {1}",
tableName, filter));
//if (tableType == 2)
//{
// sql.Append(string.Format(" AND DealType = '{0}' AND Tp = {1} AND Sl = {2}", Parameters.AllDealTypes[dealTypeIdx], tp, sl));
//}
if (!string.IsNullOrEmpty(TestParameters.DbSelectWhere))
{
sql.Append(string.Format(" AND {0}", TestParameters.DbSelectWhere));
}
sql.Append(string.Format(" {0}", m_selectOrder));
string cmd = sql.ToString();
WekaUtils.Instance.WriteLog(string.Format("NoBuffer Get Date From {0} to {1} of {2}", prevDate, nextDate, tableName));
System.Data.DataTable dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd);
//return dt.Select();
return(new ForexDataRows(new ForexDataTable(dt, tableType == 2), 0, dt.Rows.Count));
}
}
}
private bool?Do(DateTime openDate, double openPrice, out DateTime?closeDate, int dealType)
{
closeDate = DateTime.MinValue;
var dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(string.Format("SELECT * FROM {0}_HP WHERE Time = {1}", m_symbol, WekaUtils.GetTimeFromDate(openDate)));
if (dt.Rows.Count == 0)
{
return(null);
}
sbyte?[, ,] hp = HpData.DeserializeHp((byte[])dt.Rows[0]["hp"]);
long?[, ,] hpTime = HpData.DeserializeHpTimes((byte[])dt.Rows[0]["hp_date"]);
int ix = (int)m_tp / TestParameters.GetTpSlMinDelta(m_symbol) - 1;
int iy = (int)m_sl / TestParameters.GetTpSlMinDelta(m_symbol) - 1;
if (hp[dealType, ix, iy].HasValue)
{
closeDate = WekaUtils.GetDateFromTime(hpTime[dealType, ix, iy].Value);
return(hp[dealType, ix, iy].Value == 1);
}
return(null);
}
public Dictionary <DateTime, Tuple <int, long> > GetHpSum(string symbol, string period, long nowTime, long getTime)
{
Dictionary <DateTime, Tuple <int, long> > ret = new Dictionary <DateTime, Tuple <int, long> >();
Dictionary <DateTime, Tuple <int[, , , ], long[, , ]> > hpData;
hpData = GetHpDataFromTxt(symbol, period);
int preHpSum = 2;
foreach (var kvp in hpData)
{
DateTime nowDate = kvp.Key;
var retKey = kvp.Key.AddHours(TestParameters2.HourAhead);
if (getTime != -1)
{
if (WekaUtils.GetTimeFromDate(retKey) < getTime)
{
continue;
}
else if (WekaUtils.GetTimeFromDate(retKey) > getTime)
{
break;
}
}
var hps = kvp.Value;
long hpdate = 0;
double[] sum = new double[2];
sum[0] = sum[1] = 0;
for (int k = 0; k < 2; ++k)
{
for (int tp = TestParameters2.tpStart; tp < TestParameters2.tpCount; ++tp)
{
for (int sl = TestParameters2.slStart; sl < TestParameters2.slCount; ++sl)
{
var hpItem10 = hps.Item1[k, tp, sl, 0];
var hpItem11 = hps.Item1[k, tp, sl, 1];
var hpItem2 = hps.Item2[k, tp, sl];
WekaUtils.DebugAssert(hpItem10 >= 0, "hp data should >= 0");
WekaUtils.DebugAssert(hpItem11 >= 0, "hp data should >= 0");
WekaUtils.DebugAssert(hpItem2 >= 0, "hp data should >= 0");
// 当用PartialData模式的，还未知道的数据
if (hpItem2 == Parameters.MaxTime)
{
continue;
}
if (hpItem2 > nowTime)
{
continue;
}
int tpp = 1;
int tp1 = tpp * (tp + 1);
int sl1 = tpp * (sl + 1);
int v = 1;
//v = 600 / (sl + 1);
sum[k] -= tp1 * v * hpItem10;
sum[k] += sl1 * v * hpItem11;
hpdate = Math.Max(hpdate, hpItem2);
//System.Console.WriteLine(string.Format("{0},{1},{2},{3},{4}", k, tp1, sl1, hps.Item1[k, tp, sl, 0], hps.Item1[k, tp, sl, 1]));
}
}
}
//var dt = Feng.Data.DbHelper.Instance.ExecuteDataTable("SELECT * FROM GBPUSD_HP WHERE TIME >= '" + WekaUtils.GetTimeFromDate(kvp.Key) + "' AND TIME < '" + WekaUtils.GetTimeFromDate(kvp.Key.AddHours(1)) + "'");
//var row = dt.Rows[0];
//sbyte?[, ,] hp = WekaUtils.DeserializeHp((byte[])row["hp"]);
//long?[, ,] hpTime = WekaUtils.DeserializeHpTimes((byte[])row["hp_date"]);
int hpSum = 2;
//if (nowDate >= new DateTime(2010, 4, 12, 0, 30, 0))
//{
//}
bool useOneClass = false;
if (useOneClass)
{
if (sum[0] < 0)
{
hpSum = 0;
}
}
else
{
bool enableMultiClass = true;
if (enableMultiClass)
{
if (sum[0] < 0 && sum[1] >= 0)
{
hpSum = 0;
}
else if (sum[1] < 0 && sum[0] >= 0)
{
hpSum = 1;
}
else if (sum[0] >= 0 && sum[1] >= 0)
{
hpSum = 2;
}
else // <0, <0
{
hpSum = 3;
}
preHpSum = hpSum;
}
else
{
double delta = 0 * (TestParameters2.tpCount - TestParameters2.tpStart);
if (sum[0] < sum[1] - delta)
{
hpSum = 0;
}
else if (sum[0] > sum[1] + delta)
{
hpSum = 1;
}
if (hpSum == 2)
{
hpSum = preHpSum;
}
preHpSum = hpSum;
if (hpSum == -1)
{
continue;
}
}
}
if (hpdate == 0)
{
continue;
}
ret[retKey] = new Tuple <int, long>(hpSum, hpdate);
}
return(ret);
}
public void UpdateUnclosedDeals(DateTime nowDate)
{
Dictionary <DateTime, System.Data.DataRow> dictHp = new Dictionary <DateTime, System.Data.DataRow>();
foreach (var kvp in m_classifierInfos)
{
foreach (var i in kvp.Value.Deals.Deals)
{
if (i.CloseTime.HasValue)
{
continue;
}
if (!dictHp.ContainsKey(i.OpenTime))
{
var sql = string.Format("SELECT * FROM {0}_HP WHERE TIME = {1}", this.CandidateParameter.MainSymbol, WekaUtils.GetTimeFromDate(i.OpenTime));
var dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql);
if (dt.Rows.Count > 0)
{
dictHp[i.OpenTime] = dt.Rows[0];
}
}
if (!dictHp.ContainsKey(i.OpenTime))
{
continue;
}
var row = dictHp[i.OpenTime];
int tpMinDelta = TestParameters.GetTpSlMinDelta(this.CandidateParameter.MainSymbol);
int slMinDelta = TestParameters.GetTpSlMinDelta(this.CandidateParameter.MainSymbol);
int d = i.DealType == 'B' ? 0 : 1;
sbyte?[, ,] hps = HpData.DeserializeHp((byte[])row["hp"]);
long?[, ,] hpsTime = HpData.DeserializeHpTimes((byte[])row["hp_date"]);
var hp = hps[d, kvp.Value.Tp / tpMinDelta - 1, kvp.Value.Sl / slMinDelta - 1];
if (hp.HasValue &&
hp.Value != -1)
{
i.CloseTime = WekaUtils.GetDateFromTime(hpsTime[d, kvp.Value.Tp / tpMinDelta - 1, kvp.Value.Sl / slMinDelta - 1].Value);
i.Cost = hp.Value == 1 ? -kvp.Value.Tp : kvp.Value.Sl;
}
}
}
}
public void GenerateDataToTxt(string symbol, string lastWeek)
{
int n = TestParameters2.nTpsl;
string ccScoreFileName = TestParameters.GetBaseFilePath(string.Format("{0}_ccScores_w{2}_{1}.txt",
TestParameters2.CandidateParameter.MainSymbol, TestParameters2.CandidateParameter.MainPeriod, TestParameters2.lastWeek));
DateTime date = TestParameters2.TrainStartTime;
DateTime maxDate = TestParameters2.TrainEndTime;
if (!TestParameters2.RealTimeMode && System.IO.File.Exists(ccScoreFileName))
{
return;
}
SortedDictionary <DateTime, string> dictAlready = new SortedDictionary <DateTime, string>();
if (System.IO.File.Exists(ccScoreFileName))
{
using (StreamReader sr = new StreamReader(ccScoreFileName))
{
while (!sr.EndOfStream)
{
string s = sr.ReadLine();
int idx = s.IndexOf(',');
DateTime d = Convert.ToDateTime(s.Substring(0, idx));
string s2 = s.Substring(idx + 1);
dictAlready[d] = s2.Trim();
}
}
}
string sql;
System.Data.DataTable allDt = null;
DateTime nextBufferDate = DateTime.MinValue;
while (true)
{
if (!TestParameters2.RealTimeMode)
{
Console.WriteLine(date.ToString(Parameters.DateTimeFormat));
}
if (dictAlready.ContainsKey(date))
{
GenerateDateToTxt(ccScoreFileName, date, dictAlready[date]);
date = date.AddHours(TestParameters2.MainPeriodOfHour);
if (date >= maxDate)
{
break;
}
}
else
{
if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday)
{
date = date.AddHours(TestParameters2.MainPeriodOfHour);
continue;
}
if (nextBufferDate <= date)
{
nextBufferDate = date.AddHours(TestParameters2.MainPeriodOfHour);
string tableName = string.Format("{0}_CCSCORE_{1}", TestParameters2.CandidateParameter.MainSymbol, TestParameters2.lastWeek);
sql = string.Format("SELECT * FROM {0} WHERE TIME >= '{1}' AND TIME < '{2}' AND {3}",
tableName, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextBufferDate),
string.IsNullOrEmpty(TestParameters.DbSelectWhere) ? "1 = 1" : TestParameters.DbSelectWhere);
allDt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql);
}
DateTime nextDate = date.AddHours(TestParameters2.MainPeriodOfHour);
sql = string.Format("TIME >= '{1}' AND TIME < '{2}'",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextDate));
var dt = allDt.Select(sql);
if (dt.Length > 0)
{
double[, ,] scores = new double[2, TestParameters.TpMaxCount / n, TestParameters.SlMaxCount / n];
long[, ,] ndas = new long[2, TestParameters.TpMaxCount / n, TestParameters.SlMaxCount / n];
long[, ,] ndss = new long[2, TestParameters.TpMaxCount / n, TestParameters.SlMaxCount / n];
for (int j = 0; j < scores.GetLength(0); ++j)
{
for (int k = 0; k < scores.GetLength(1); ++k)
{
for (int l = 0; l < scores.GetLength(2); ++l)
{
scores[j, k, l] = -1;
ndas[j, k, l] = -1;
ndss[j, k, l] = -1;
}
}
}
System.Data.DataRow row = dt[0];
{
double[, ,] score = DeserializeScores((byte[])row["scores"]);
long[, ,] nda = DeserializeScoreTimes((byte[])row["nda"]);
long[, ,] nds = DeserializeScoreTimes((byte[])row["nds"]);
WekaUtils.DebugAssert(score.GetLength(0) == 2, "");
WekaUtils.DebugAssert(nda.GetLength(0) == 2, "");
for (int j = 0; j < score.GetLength(0); ++j)
{
for (int k = 0; k < score.GetLength(1); ++k)
{
for (int l = 0; l < score.GetLength(2); ++l)
{
if (k % n != n - 1 || l % n != n - 1)
{
continue;
}
//if (k / n >= TestParameters2.tpCount)
// continue;
//if (l / n >= TestParameters2.slCount)
// continue;
scores[j, k / n, l / n] = score[j, k, l];
ndas[j, k / n, l / n] = nda[j, k, l];
ndss[j, k / n, l / n] = nds[j, k, l];
}
}
}
GenerateDateToTxt(ccScoreFileName, date, scores, ndas, ndss);
}
}
date = nextDate;
if (date >= maxDate)
{
break;
}
}
}
}
public static void IterateHpData(string symbol, string period, Action <DateTime, int, int, int, int, long> action)
{
double mainPeriodOfHour = WekaUtils.GetMinuteofPeriod(period) / 60.0;
int n = TestParameters2.nTpsl;
int m1 = TestParameters.TpMaxCount / n;
int m2 = TestParameters.SlMaxCount / n;
DateTime date = TestParameters2.TrainStartTime;
DateTime maxDate = TestParameters2.TrainEndTime;
string sql;
System.Data.DataTable allDt = null;
DateTime nextBufferDate = DateTime.MinValue;
while (true)
{
if (!TestParameters2.RealTimeMode)
{
Console.WriteLine(date.ToString(Parameters.DateTimeFormat));
}
if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday)
{
date = date.AddHours(mainPeriodOfHour);
continue;
}
if (nextBufferDate <= date)
{
nextBufferDate = date.AddHours(mainPeriodOfHour * 240);
sql = string.Format("SELECT * FROM {0}_HP WHERE TIME >= '{1}' AND TIME < '{2}' AND {3}",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextBufferDate),
string.IsNullOrEmpty(TestParameters.DbSelectWhere) ? "1 = 1" : TestParameters.DbSelectWhere);
allDt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql);
}
DateTime nextDate = date.AddHours(mainPeriodOfHour);
sql = string.Format("TIME >= '{1}' AND TIME < '{2}'",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextDate));
var dt = allDt.Select(sql);
bool isComplete = true;
foreach (System.Data.DataRow row in dt)
{
bool b = (bool)row["IsComplete"];
if (!b)
{
isComplete = false;
break;
}
}
if ((TestParameters2.UsePartialHpData || isComplete) && dt.Length > 0)
{
foreach (System.Data.DataRow row in dt)
{
sbyte?[, ,] hp = HpData.DeserializeHp((byte[])row["hp"]);
long?[, ,] hpTime = HpData.DeserializeHpTimes((byte[])row["hp_date"]);
WekaUtils.DebugAssert(hp.GetLength(0) == 2, "");
WekaUtils.DebugAssert(hp.GetLength(1) % m1 == 0, "");
WekaUtils.DebugAssert(hp.GetLength(2) % m2 == 0, "");
WekaUtils.DebugAssert(hpTime.GetLength(0) == 2, "");
for (int j = 0; j < hp.GetLength(0); ++j)
{
for (int k = 0; k < hp.GetLength(1); ++k)
{
for (int l = 0; l < hp.GetLength(2); ++l)
{
if (k % n != n - 1 || l % n != n - 1)
{
continue;
}
if (!hp[j, k, l].HasValue || hp[j, k, l] == -1)
{
continue;
}
action(date, j, k, l, hp[j, k, l].Value, hpTime[j, k, l].Value);
}
}
}
}
}
date = nextDate;
if (date >= maxDate)
{
break;
}
}
}
public void SaveTestData(CandidateClassifier clsInfo, DateTime nowDate)
{
if (TestParameters.OnlyNewestTestDataSaved)
{
string sql = string.Format("DELETE FROM {0}", m_testDataTableName);
Feng.Data.DbHelper.Instance.ExecuteNonQuery(sql);
}
string str = WekaUtils.DoubleArrayToIntString(clsInfo.CurrentTestRet);
string scv = WekaUtils.DoubleArrayToIntString(clsInfo.CurrentClassValue);
if (m_testDataInsertTable == null)
{
m_testDataInsertTable = new System.Data.DataTable();
m_testDataInsertTable.Columns.Add("Id", typeof(int));
m_testDataInsertTable.Columns.Add("Time", typeof(long));
//m_testDataInsertTable.Columns.Add("Type", typeof(int));
m_testDataInsertTable.Columns.Add("ClsName", typeof(string));
m_testDataInsertTable.Columns.Add("TestResult", typeof(string));
m_testDataInsertTable.Columns.Add("ClassValue", typeof(string));
m_testDataInsertTable.Columns.Add("DealsInfo", typeof(byte[]));
m_testDataInsertTable.Columns.Add("DealsData", typeof(byte[]));
}
var row = m_testDataInsertTable.NewRow();
row["ClsName"] = clsInfo.Name;
//row["Type"] = m_testDataType;
row["Time"] = WekaUtils.GetTimeFromDate(nowDate);
row["TestResult"] = str;
row["ClassValue"] = scv;
row["DealsInfo"] = Feng.Windows.Utils.SerializeHelper.Serialize(clsInfo.Deals);
bool shouldSave = false;
if (nowDate.DayOfWeek != DayOfWeek.Sunday && nowDate.DayOfWeek != DayOfWeek.Saturday && nowDate.Day == 20)
{
shouldSave = true;
}
if (nowDate.Day == 21 && nowDate.DayOfWeek == DayOfWeek.Monday)
{
shouldSave = true;
}
if (nowDate.Day == 22 && nowDate.DayOfWeek == DayOfWeek.Monday)
{
shouldSave = true;
}
if (TestParameters.OnlyNewestTestDataSaved)
{
shouldSave = true;
}
if (shouldSave)
{
byte[] dealsData = Feng.Windows.Utils.SerializeHelper.Serialize(clsInfo.Deals.Deals);
System.IO.MemoryStream outStream = new System.IO.MemoryStream();
using (System.IO.Compression.GZipStream zipStream =
new System.IO.Compression.GZipStream(outStream, System.IO.Compression.CompressionMode.Compress))
{
System.IO.MemoryStream inStream = new System.IO.MemoryStream(dealsData);
inStream.CopyTo(zipStream);
}
row["DealsData"] = outStream.ToArray();
}
else
{
row["DealsData"] = System.DBNull.Value;
}
m_testDataInsertTable.Rows.Add(row);
if (m_testDataInsertTable.Rows.Count > 100)
{
//WekaUtils.Instance.WriteLog("Save Test Data.");
try
{
Feng.Data.DbHelper.Instance.BulkCopy(m_testDataInsertTable, m_testDataTableName);
}
catch (Exception ex)
{
WekaUtils.Instance.WriteLog(ex.Message);
}
m_testDataInsertTable.Rows.Clear();
}
//SqlCommand cmd = new SqlCommand("INSERT INTO TestData ([ClsName],[Time], [Type], TestResult, ClassValue, DealsData) VALUES (@ClsName, @Time, @Type, @TestResult, @ClassValue, @DealsData)");
//cmd.Parameters.AddWithValue("@ClsName", clsInfo.Name);
//cmd.Parameters.AddWithValue("@Type", m_testDataType);
//cmd.Parameters.AddWithValue("@Time", WekaUtils.GetTimeFromDate(currentDate));
//cmd.Parameters.AddWithValue("@TestResult", str);
//cmd.Parameters.AddWithValue("@ClassValue", scv);
//cmd.Parameters.AddWithValue("@DealsData", Feng.Utils.SerializeHelper.Serialize(clsInfo.Deals));
//SetTransaction(cmd);
}
public void LoadTestData(DateTime currentDate, Dictionary <string, CandidateClassifier> clsInfos)
{
//EndTransaction();
if (m_testData == null || currentDate >= m_testDataEndDate)
{
m_testDataEndDate = currentDate.AddDays(7);
if (m_testData == null)
{
m_testData = new Dictionary <long, List <System.Data.DataRow> >();
}
else
{
m_testData.Clear();
System.GC.Collect();
}
WekaUtils.Instance.WriteLog(string.Format("Load Test Data to {0}", m_testDataEndDate));
//SqlCommand cmd = new SqlCommand("SELECT ClsName, TestResult, ClassValue, DealsData FROM TestData WHERE [Time] = @Time AND [Type] = @Type");
//cmd.Parameters.AddWithValue("@Time", GetTimeFromDate(currentDate));
//cmd.Parameters.AddWithValue("@Type", m_testDataType);
//var ret = Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd);
Dictionary <string, string> dict = new Dictionary <string, string>();
foreach (var kvp in clsInfos)
{
dict[kvp.Value.DealType + "_" + kvp.Value.Tp + "_" + kvp.Value.Sl] = null;
}
string[] clsNames = new string[dict.Count];
dict.Keys.CopyTo(clsNames, 0);
string[] paramNames = clsNames.Select(
(s, i) => "ClsName LIKE '" + s + "_%'" //"@tag" + i.ToString()
).ToArray();
string inClause = string.Join(" OR ", paramNames);
string sql = ("SELECT Time, ClsName, TestResult, ClassValue, DealsInfo, DealsData FROM {0} WHERE [Time] >= @Time1 AND [Time] < @Time2 AND ({1})");
SqlCommand cmd = new SqlCommand(string.Format(sql, m_testDataTableName, inClause));
cmd.Parameters.AddWithValue("@Time1", WekaUtils.GetTimeFromDate(currentDate));
cmd.Parameters.AddWithValue("@Time2", WekaUtils.GetTimeFromDate(m_testDataEndDate));
//for (int i = 0; i < paramNames.Length; i++)
//{
// cmd.Parameters.AddWithValue(paramNames[i], clsNames[i]);
//}
var ret = Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd);
foreach (System.Data.DataRow row in ret.Rows)
{
long time = (long)row["Time"];
if (!m_testData.ContainsKey(time))
{
m_testData[time] = new List <System.Data.DataRow>();
}
m_testData[time].Add(row);
}
}
long ctime = WekaUtils.GetTimeFromDate(currentDate);
if (m_testData.ContainsKey(ctime))
{
foreach (System.Data.DataRow row in m_testData[ctime])
{
string name = (string)row["ClsName"];
if (clsInfos.ContainsKey(name))
{
if (row["DealsData"] == System.DBNull.Value)
{
clsInfos[name].SetData((string)row["TestResult"], (string)row["ClassValue"], (byte[])row["DealsInfo"], null);
}
else
{
clsInfos[name].SetData((string)row["TestResult"], (string)row["ClassValue"], (byte[])row["DealsInfo"], (byte[])row["DealsData"]);
}
}
}
}
}
public static Dictionary <long, int> GetHpDateTimesCount(string symbol, string period)
{
double mainPeriodOfHour = WekaUtils.GetMinuteofPeriod(period) / 60.0;
Dictionary <long, int> dictTimes = new Dictionary <long, int>();
int dealCount = 0;
int n = TestParameters2.nTpsl;
DateTime date = TestParameters2.TrainStartTime;
DateTime maxDate = TestParameters2.TrainEndTime;
string sql;
System.Data.DataTable allDt = null;
DateTime nextBufferDate = DateTime.MinValue;
while (true)
{
Console.WriteLine(date.ToString(Parameters.DateTimeFormat) + ", " + dictTimes.Count + ", " + dealCount);
if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday)
{
date = date.AddHours(mainPeriodOfHour);
continue;
}
if (nextBufferDate <= date)
{
nextBufferDate = date.AddHours(mainPeriodOfHour * 100);
sql = string.Format("SELECT * FROM {0}_HP WHERE TIME >= '{1}' AND TIME < '{2}' AND {3}",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextBufferDate),
string.IsNullOrEmpty(TestParameters.DbSelectWhere) ? "1 = 1" : TestParameters.DbSelectWhere);
allDt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql);
}
DateTime nextDate = date.AddHours(mainPeriodOfHour);
sql = string.Format("TIME >= '{1}' AND TIME < '{2}'",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextDate));
var dt = allDt.Select(sql);
bool isComplete = true;
if ((TestParameters2.UsePartialHpData || isComplete) && dt.Length > 0)
{
foreach (System.Data.DataRow row in dt)
{
sbyte?[, ,] hp = HpData.DeserializeHp((byte[])row["hp"]);
long?[, ,] hpTime = HpData.DeserializeHpTimes((byte[])row["hp_date"]);
WekaUtils.DebugAssert(hp.GetLength(0) == 2, "");
WekaUtils.DebugAssert(hpTime.GetLength(0) == 2, "");
for (int j = 0; j < hp.GetLength(0); ++j)
{
for (int k = 0; k < hp.GetLength(1); ++k)
{
for (int l = 0; l < hp.GetLength(2); ++l)
{
if (k % n != n - 1 || l % n != n - 1)
{
continue;
}
if (!hp[j, k, l].HasValue || hp[j, k, l] == -1)
{
continue;
}
//if (k / n >= TestParameters2.tpCount)
// continue;
//if (l / n >= TestParameters2.slCount)
// continue;
if (dictTimes.ContainsKey(hpTime[j, k, l].Value))
{
dictTimes[hpTime[j, k, l].Value]++;
}
else
{
dictTimes[hpTime[j, k, l].Value] = 1;
}
dealCount++;
}
}
}
}
}
date = nextDate;
if (date >= maxDate)
{
break;
}
}
return(dictTimes);
}
public void GenerateHpDataToTxt(string symbol, string period)
{
double mainPeriodOfHour = WekaUtils.GetMinuteofPeriod(period) / 60.0;
int n = TestParameters2.nTpsl;
int m1 = TestParameters.TpMaxCount / n;
int m2 = TestParameters.SlMaxCount / n;
var dataDates = TestManager.GetDataDateRange();
DateTime date = dataDates[0];
DateTime maxDate = dataDates[1];
date = new DateTime(2000, 1, 1);
string hpFileName = TestParameters.GetBaseFilePath(string.Format("{0}_{1}_hpdata.txt", symbol, period));
if (!TestParameters2.RealTimeMode && System.IO.File.Exists(hpFileName))
{
return;
}
SortedDictionary <DateTime, string> dictAlready = new SortedDictionary <DateTime, string>();
string hpFileNameAlready = hpFileName + ".full";
if (!TestParameters2.DBDataConsistent)
{
if (System.IO.File.Exists(hpFileNameAlready))
{
using (StreamReader sr = new StreamReader(hpFileNameAlready))
{
while (!sr.EndOfStream)
{
string s = sr.ReadLine();
int idx = s.IndexOf(',');
DateTime d = Convert.ToDateTime(s.Substring(0, idx));
string s2 = s.Substring(idx + 1);
dictAlready[d] = s2.Trim();
}
}
}
}
string sql;
System.Data.DataTable allDt = null;
DateTime nextBufferDate = DateTime.MinValue;
using (StreamWriter sw = new StreamWriter(hpFileName))
using (StreamWriter sw2 = new StreamWriter(hpFileNameAlready))
{
while (true)
{
if (!TestParameters2.RealTimeMode)
{
Console.WriteLine(date.ToString(Parameters.DateTimeFormat));
}
StringBuilder sb = new StringBuilder();
if (dictAlready.ContainsKey(date))
{
sb.Append(date.ToString(Parameters.DateTimeFormat));
sb.Append(", ");
sb.AppendLine(dictAlready[date]);
sw.Write(sb.ToString());
sw2.Write(sb.ToString());
date = date.AddHours(mainPeriodOfHour);
if (date >= maxDate)
{
break;
}
}
else
{
if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday)
{
date = date.AddHours(mainPeriodOfHour);
continue;
}
if (nextBufferDate <= date)
{
nextBufferDate = date.AddHours(mainPeriodOfHour);
sql = string.Format("SELECT * FROM {0}_HP WHERE TIME >= '{1}' AND TIME < '{2}' AND {3}",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextBufferDate),
string.IsNullOrEmpty(TestParameters.DbSelectWhere) ? "1 = 1" : TestParameters.DbSelectWhere);
//if (!TestParameters2.UsePartialHpData && TestParameters2.RealTimeMode)
//{
// sql += " AND IsComplete = 1";
//}
allDt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql);
}
DateTime nextDate = date.AddHours(mainPeriodOfHour);
sql = string.Format("TIME >= '{1}' AND TIME < '{2}'",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextDate));
var dt = allDt.Select(sql);
bool isComplete = true;
foreach (System.Data.DataRow row in dt)
{
bool b = (bool)row["IsComplete"];
if (!b)
{
isComplete = false;
break;
}
}
if ((TestParameters2.UsePartialHpData || isComplete) && dt.Length > 0)
{
sb.Append(date.ToString(Parameters.DateTimeFormat));
sb.Append(", ");
var hpAll = SumHp(dt);
var hps = hpAll.Item1;
var hpTimes = hpAll.Item2;
for (int j = 0; j < hps.GetLength(0); ++j)
{
for (int k = 0; k < hps.GetLength(1); ++k)
{
for (int l = 0; l < hps.GetLength(2); ++l)
{
if (isComplete)
{
if (hps[j, k, l, 0] == -1 || hps[j, k, l, 1] == -1)
{
throw new AssertException("hps should not be -1.");
}
}
if (hps[j, k, l, 0] == -1)
{
hps[j, k, l, 0] = 0;
}
if (hps[j, k, l, 1] == -1)
{
hps[j, k, l, 1] = 0;
}
sb.Append(hps[j, k, l, 0] + ", " + hps[j, k, l, 1] + ", ");
}
}
}
long maxHpTime = 0;
for (int j = 0; j < hpTimes.GetLength(0); ++j)
{
for (int k = 0; k < hpTimes.GetLength(1); ++k)
{
for (int l = 0; l < hpTimes.GetLength(2); ++l)
{
if (isComplete)
{
if (hpTimes[j, k, l] == -1)
{
throw new AssertException("hpTimes should not be -1.");
}
}
if (hpTimes[j, k, l] == -1)
{
hpTimes[j, k, l] = Parameters.MaxTime;
}
sb.Append(hpTimes[j, k, l] + ", ");
maxHpTime = Math.Max(hpTimes[j, k, l], maxHpTime);
}
}
}
sb.AppendLine();
sw.Write(sb.ToString());
if (isComplete && !TestParameters2.DBDataConsistent)
{
sw2.Write(sb.ToString());
}
}
date = nextDate;
if (date >= maxDate)
{
break;
}
}
}
}
if (TestParameters2.DBDataConsistent)
{
File.Delete(hpFileNameAlready);
}
}
public void GenerateArff(string arffFileName, string candlePatternFile)
{
int preLength = TestParameters2.PreLength;
int tpStart = TestParameters2.tpStart;
int slStart = TestParameters2.slStart;
int tpCount = TestParameters2.tpCount;
int slCount = TestParameters2.slCount;
var dataDates = TestManager.GetDataDateRange();
int minOutBegIdx = TaLibTest.minOutBegIdx;
var cp = TestParameters2.CandidateParameter;
int[, , ,] cds = null;
ForexDataRows[,] hpdvs = new ForexDataRows[cp.SymbolCount, cp.PeriodCount];
for (int s = 0; s < cp.SymbolCount; ++s)
{
for (int p = 0; p < cp.PeriodCount; ++p)
{
string symbol = cp.AllSymbols[s + cp.SymbolStart];
string period = cp.AllPeriods[p + cp.PeriodStart];
string symbolPeriod = string.Format("{0}_{1}", symbol, period);
hpdvs[s, p] = DbData.Instance.GetDbData(dataDates[0], dataDates[1], symbolPeriod, 0, true, cp);
m_currentPeriod = period;
//int minOutNBElement = m_minOutNBElements[m_currentPeriod];
int minOutNBElement = hpdvs[s, p].Length - minOutBegIdx;
if (cds == null)
{
CreateCds(ref cds, cp, minOutNBElement + 100);
}
using (StreamReader sr = new StreamReader(candlePatternFile))
{
int n = 0;
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
string[] ss = line.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
WekaUtils.DebugAssert(ss.Length == minOutNBElement, "ss.Length == minOutNBElement");
for (int i = 0; i < minOutNBElement; ++i)
{
cds[s, p, i, n] = Convert.ToInt32(ss[i]);
}
n++;
}
}
}
}
if (File.Exists(arffFileName))
{
System.IO.File.Delete(arffFileName);
}
string wekaFileName = string.Format(arffFileName);
using (StreamWriter sw = new StreamWriter(wekaFileName))
{
sw.WriteLine("@relation 'candlePatterns'");
sw.WriteLine("@attribute timestamp date \"yyyy-MM-dd\'T\'HH:mm:ss\"");
sw.WriteLine("@attribute hpdate date \"yyyy-MM-dd\'T\'HH:mm:ss\"");
sw.WriteLine("@attribute spread numeric");
sw.WriteLine("@attribute mainClose numeric");
for (int pre = 0; pre < preLength; ++pre)
{
for (int s = 0; s < cp.SymbolCount; ++s)
{
for (int p = 0; p < cp.PeriodCount; ++p)
{
for (int i = 0; i < 61; ++i)
{
sw.WriteLine(string.Format("@attribute {0}_{1}_{2}_p{3} {4}", cp.AllSymbols[s + cp.SymbolStart],
cp.AllPeriods[p + cp.PeriodStart], i.ToString(), pre.ToString(), "{0,100,200,-100,-200}"));
}
}
}
}
sw.WriteLine("@attribute prop " + " {0,1,2,3}");
sw.WriteLine("@data");
sw.WriteLine();
var hps = HpData.Instance.GetHpSum(cp.MainSymbol, cp.MainPeriod);
var hpdv = hpdvs[0, 0];
for (int i = minOutBegIdx + preLength - 1; i < hpdv.Length; ++i)
{
DateTime nowDate = WekaUtils.GetDateFromTime((long)hpdv[i].Time);
//if (nowDate.Hour % TestParameters2.MainPeriodOfHour != 0)
// continue;
if (!hps.ContainsKey(nowDate) && !(TestParameters2.RealTimeMode && i == hpdv.Length - 1))
{
continue;
}
long hpTime = WekaUtils.GetTimeFromDate(Parameters.MaxDate);
int hp = 0;
if (!(TestParameters2.RealTimeMode && i == hpdv.Length - 1))
{
hpTime = hps[nowDate].Item2;
hp = hps[nowDate].Item1;
}
sw.Write(nowDate.ToString(Parameters.DateTimeFormat));
sw.Write(",");
// hp
sw.Write(WekaUtils.GetDateFromTime(hpTime).ToString(Parameters.DateTimeFormat));
sw.Write(",");
sw.Write((int)hpdv[i]["spread"]);
sw.Write(",");
sw.Write(((double)hpdv[i]["close"]).ToString());
sw.Write(",");
for (int pre = 0; pre < preLength; ++pre)
{
for (int s = 0; s < cp.SymbolCount; ++s)
{
for (int p = 0; p < cp.PeriodCount; ++p)
{
for (int j = 0; j < 61; ++j)
{
int candlePattern = cds[s, p, i - minOutBegIdx - preLength + pre + 1, j];
if (candlePattern == -1)
{
throw new AssertException(string.Format("candle pattern should not be -1.idx={0}", i - minOutBegIdx - preLength + pre + 1));
}
sw.Write(candlePattern);
sw.Write(",");
}
}
}
}
sw.WriteLine(hp.ToString());
}
}
}
public static MqlRates[] ReadRates(string symbolPeriod)
{
string[] ss1 = symbolPeriod.Split('_');
string symbol = ss1[0];
string period = ss1[1] == "M1" ? "min_1" : ss1[1];
string rateFileName = string.Format("E:\\Forex\\Dukas\\{0}_BID_candles_{1}.csv", symbol, period);
List <MqlRates> rates = new List <MqlRates>();
using (System.IO.StreamReader sr = new StreamReader(rateFileName))
{
while (true)
{
if (sr.EndOfStream)
{
break;
}
// Date Time, Open, Close, Low, High, Volume
string s = sr.ReadLine();
string[] ss = s.Split(new char[] { ',' });
long v = (long)Convert.ToDouble(ss[5]);
DateTime date;
//date = Convert.ToDateTime(ss[0]);
bool r = DateTime.TryParseExact(ss[0], new string[] { "dd.MM.yyyy HH:mm:ss.000", "yyyy.MM.dd HH:mm:ss" }, null, System.Globalization.DateTimeStyles.None, out date);
if (!r)
{
throw new InvalidCastException(string.Format("{0} is invalid as datetime", ss[0]));
}
if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday)
{
continue;
}
////if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday)
//{
// if (v == 0)
// {
// if (ss[1] == ss[2] && ss[2] == ss[3] && ss[3] == ss[4])
// {
// DateTime d1 = date.AddHours(5);
// DateTime d2 = date.AddHours(-5);
// if ((d1.DayOfWeek == DayOfWeek.Saturday || d1.DayOfWeek == DayOfWeek.Sunday)
// && (d2.DayOfWeek == DayOfWeek.Saturday || d2.DayOfWeek == DayOfWeek.Sunday))
// continue;
// }
// else
// {
// System.Console.WriteLine(s);
// }
// }
// else
// {
// }
//}
rates.Add(new MqlRates
{
time = WekaUtils.GetTimeFromDate(date),
open = Convert.ToDouble(ss[1]),
high = Convert.ToDouble(ss[4]),
low = Convert.ToDouble(ss[3]),
close = Convert.ToDouble(ss[2]),
tick_volume = 0,
spread = 50,
real_volume = (long)Convert.ToDouble(ss[5]),
});
}
}
return(rates.ToArray());
}