public static void InitParameters(string symbol = "EURUSD", string period = "D1", int ntpsl = 1)
{
//TestParameters.TpSlMaxCount = 30;
//nTpsl = 2;
nTpsl = ntpsl;
if (tpCount == -1)
{
tpCount = TestParameters.TpMaxCount / nTpsl;
}
if (slCount == -1)
{
slCount = TestParameters.SlMaxCount / nTpsl;
}
//tpStart = slStart = 0;
//tpCount = slCount = 10;
var cp = CandidateParameter;
cp.AllSymbols = new string[] { symbol };
cp.SymbolStart = 0;
cp.SymbolCount = cp.AllSymbols.Length;
cp.AllPeriods = new string[] { period };
cp.PeriodStart = 0;
cp.PeriodCount = cp.AllPeriods.Length;
cp.PrevTimeCount = PreLength;
//int delta = TestParameters.GetTpSlMinDelta(symbol) * TestParameters2.nTpsl;
//cp.InitTpsls(TestParameters2.tpStart, delta, TestParameters2.tpCount, TestParameters2.slStart, delta, TestParameters2.slCount);
MainPeriodOfHour = WekaUtils.GetMinuteofPeriod(cp.MainPeriod) / 60.0; // 24=D
cp.InitIndicators();
}
// tableType = 0: MainTable; 1: OtherDataTable; 2:HpTable
public ForexDataRows GetDbData(DateTime dt1, DateTime dt2, string tableName, int tableType, bool isTrain, CandidateParameter cp)
{
lock (m_lockObject)
{
//int dealTypeIdx1 = dealType == 'B' ? 0 : 1;
DateTime prevDate = dt1.AddMinutes(-2 * cp.PrevTimeCount * WekaUtils.GetMinuteofPeriod(cp.AllPeriods[cp.PeriodCount - 1]));
DateTime nextDate = dt2;
if (m_useBufferReadData && m_batchBufferMinutes < 0)
{
int n1 = cp.SymbolCount * cp.PeriodCount * (cp.PeriodTimeCount + 1)
* (cp.AllIndNames.Count + cp.AllIndNames2.Count + 5) / WekaUtils.GetMinuteofPeriod(cp.MainPeriod);
n1 = Math.Max(n1, 1);
int n2 = 1;
//n2 = 3 * TestParameters.BatchTps.Length * TestParameters.BatchSls.Length * Parameters.AllDealTypes.Length / WekaUtils.GetMinuteofPeriod(TestParameters.MainPeriod);
//n2 /= 6;
n2 = 10000 / WekaUtils.GetMinuteofPeriod(cp.MainPeriod);
n2 = Math.Max(n2, 1);
m_batchBufferMinutes = (Parameters.TotalCanUseBuffer - System.GC.GetTotalMemory(false)) / (n1 + n2) * WekaUtils.GetMinuteofPeriod(cp.MainPeriod); // / 240
int min = (int)(nextDate - prevDate).TotalMinutes + 24 * TestParameters.BatchTestMinutes;
int max = (int)(1.1 * TestParameters.BatchTrainMinutes);
if (m_batchBufferMinutes < min)
{
//m_useBufferReadData = false;
WekaUtils.Instance.WriteLog(string.Format("Partial buffer. batchGetTime = {0}, min = {1}", m_batchBufferMinutes, min));
m_batchBufferMinutes = min;
}
else
{
//m_useBufferReadData = true;
WekaUtils.Instance.WriteLog(string.Format("Use buffer. batchGetTime = {0}, max = {1}", m_batchBufferMinutes, max));
m_batchBufferMinutes = Math.Min(m_batchBufferMinutes, max);
}
}
if (m_useBufferReadData)
{
DateTime bufferDateStart = prevDate;
DateTime bufferDateEnd = bufferDateStart.AddMinutes(m_batchBufferMinutes);
if (TestParameters2.RealTimeMode)
{
if (isTrain)
{
bufferDateStart = dt1.AddDays(-7);
bufferDateEnd = dt2.AddDays(7);
}
else
{
bufferDateStart = dt1;
bufferDateEnd = dt2;
}
}
//System.Console.WriteLine("{0}, {1}, {2}, {3}", bufferDateStart, bufferDateEnd, prevDate, nextDate);
if (bufferDateEnd >= nextDate)
{
string allDataKey = tableName;
//if (tableType == 2)
//{
// allDataKey += "_" + Parameters.AllDealTypes[dealTypeIdx] + "_" + tp.ToString() + "_" + sl.ToString();
//}
if (m_allData == null)
{
if (m_serializeAllData)
{
foreach (string s in System.IO.Directory.GetFiles(".", m_allDataCacheFile))
{
string[] ss = System.IO.Path.GetFileNameWithoutExtension(s).Split(new char[] { '_' }, StringSplitOptions.RemoveEmptyEntries);
DateTime d1 = Convert.ToDateTime(ss[1]);
DateTime d2 = Convert.ToDateTime(ss[2]);
if (bufferDateStart >= d1 && bufferDateEnd <= d2)
{
m_allData = Feng.Windows.Utils.SerializeHelper.Deserialize <Dictionary <string, ForexDataTable> >(s);
break;
}
}
}
if (m_allData == null)
{
m_allData = new Dictionary <string, ForexDataTable>();
//m_allDateDates = new DateTime[m_dealType.Length][];
}
}
if (m_allData.ContainsKey(allDataKey))
{
Tuple <DateTime, DateTime> ds = m_allDataDates[allDataKey];
bool expired = false;
if (!isTrain)
{
if (ds.Item2 < nextDate || ds.Item1 > prevDate)
{
//System.Console.WriteLine("{0}, {1}, {2}, {3}", ds.Item1.ToString(), ds.Item2, prevDate, nextDate);
expired = true;
}
}
else
{
if (ds.Item2 <= nextDate.AddMinutes(TestParameters.EnablePerhourTrain ?
2 * Parameters.AllHour * TestParameters.BatchTestMinutes :
2 * TestParameters.BatchTestMinutes) || ds.Item1 > prevDate)
{
expired = true;
}
}
if (expired)
{
m_allData.Remove(allDataKey);
m_allDataDates.Remove(allDataKey);
}
}
//long usedMemory = System.GC.GetTotalMemory(false);
//if (m_cacheData && usedMemory >= Parameters.TotalCanUseMemory)
//{
// WekaUtils.Instance.WriteLog(string.Format("Use too many memory. now total memory = {0}", usedMemory));
// m_cacheData = false;
//}
//else if (!m_cacheData && usedMemory < Parameters.TotalCanUseMemory)
//{
// WekaUtils.Instance.WriteLog(string.Format("reset to cache data. now total memory = {0}", usedMemory));
// m_cacheData = true;
//}
if (m_cacheData && !m_allData.ContainsKey(allDataKey))
{
StringBuilder sql = new StringBuilder();
if (tableType == 0)
{
sql.Append(string.Format("SELECT [Time], [Date], [hour], [dayofweek], [spread], [close] as mainClose")); // , [AskVolume], [BidVolume]
}
else if (tableType == 1)
{
sql.Append(string.Format("SELECT [Time], [Date]"));
}
else if (tableType == 2)
{
sql.Append(string.Format("SELECT [Time], [hp], [hp_date]"));
}
if (tableType == 0 || tableType == 1)
{
foreach (var kvp in cp.AllIndNames2)
{
sql.Append(string.Format(",[{0}]", kvp.Key));
}
foreach (var kvp in cp.AllIndNames)
{
sql.Append(string.Format(",[{0}]", kvp.Key));
}
}
sql.Append(string.Format(" FROM {0} WHERE Time >= '{1}' AND Time < '{2}' ",
tableName, WekaUtils.GetTimeFromDate(bufferDateStart), WekaUtils.GetTimeFromDate(bufferDateEnd)));
//if (tableType == 0 || tableType == 1)
//{
// sql.Append(" AND IsActive = 1 ");
//}
//if (tableType == 2)
//{
// sql.Append(string.Format(" AND DealType = '{0}' AND Tp = {1} AND Sl = {2}", Parameters.AllDealTypes[dealTypeIdx], tp, sl));
//}
if (!string.IsNullOrEmpty(TestParameters.DbSelectWhere))
{
sql.Append(string.Format(" AND {0}", TestParameters.DbSelectWhere));
}
sql.Append(string.Format(" {0}", m_selectOrder));
string cmd = sql.ToString();
//m_allData[allDataKey] = DbHelper.Instance.ExecuteDataTable(cmd);
//m_allData[allDataKey].PrimaryKey = new System.Data.DataColumn[] { m_allData[allDataKey].Columns["Time"] };
m_allData[allDataKey] = new ForexDataTable(Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd), tableType == 2);
m_allDataDates[allDataKey] = new Tuple <DateTime, DateTime>(bufferDateStart, bufferDateEnd);
WekaUtils.Instance.WriteLog(string.Format("Buffer Get Data From {0} to {1} of {2}", bufferDateStart, bufferDateEnd, allDataKey));
if (m_serializeAllData)
{
string allDataCacheFile = m_allDataCacheFile.Replace("*",
TestParameters.BatchDateStart.ToString(Parameters.DateTimeFormat) + "_"
+ TestParameters.BatchDateEnd.ToString(Parameters.DateTimeFormat));
Feng.Windows.Utils.SerializeHelper.Serialize(allDataCacheFile, m_allData);
}
}
if (m_allData.ContainsKey(allDataKey))
{
//string filter = GetDbDateFilter(dt1, dt2, tableType);
//return m_allData[allDataKey].Select(filter);
Tuple <long, long> filter = GetDbDateFilter(dt1, dt2, tableType);
return(m_allData[allDataKey].SelectByDate(filter.Item1, filter.Item2));
}
}
}
// no get from cache
{
StringBuilder sql = new StringBuilder();
if (tableType == 0)
{
sql.Append(string.Format("SELECT [Time], [Date], [hour], [dayofweek], [spread], [close] as mainClose")); // , [AskVolume], [BidVolume]
}
else if (tableType == 1)
{
sql.Append(string.Format("SELECT [Time], [Date]"));
}
else if (tableType == 2)
{
sql.Append(string.Format("SELECT [Time], [hp], [hp_date]"));
}
if (tableType == 0 || tableType == 1)
{
foreach (var kvp in cp.AllIndNames2)
{
sql.Append(string.Format(",[{0}]", kvp.Key));
}
foreach (var kvp in cp.AllIndNames)
{
sql.Append(string.Format(",[{0}]", kvp.Key));
}
}
string filter = GetDbDateFilter4Db(dt1, dt2, tableType);
sql.Append(string.Format(" FROM {0} WHERE {1}",
tableName, filter));
//if (tableType == 2)
//{
// sql.Append(string.Format(" AND DealType = '{0}' AND Tp = {1} AND Sl = {2}", Parameters.AllDealTypes[dealTypeIdx], tp, sl));
//}
if (!string.IsNullOrEmpty(TestParameters.DbSelectWhere))
{
sql.Append(string.Format(" AND {0}", TestParameters.DbSelectWhere));
}
sql.Append(string.Format(" {0}", m_selectOrder));
string cmd = sql.ToString();
WekaUtils.Instance.WriteLog(string.Format("NoBuffer Get Date From {0} to {1} of {2}", prevDate, nextDate, tableName));
System.Data.DataTable dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd);
//return dt.Select();
return(new ForexDataRows(new ForexDataTable(dt, tableType == 2), 0, dt.Rows.Count));
}
}
}
public Tuple <int, long> GetHpSumByM1(string symbol, string period, long nowTime, long getTime)
{
System.Diagnostics.Debug.Assert(getTime != -1, "");
Tuple <int, long> ret = new Tuple <int, long>(0, 0);
if (m_hpDataDb == null)
{
m_hpDataDb = new HpDbData(symbol);
}
long time = getTime - 60;
long hpdate = 0;
double[] sum = new double[2];
sum[0] = sum[1] = 0;
while (true)
{
time += 60;
if (time >= getTime + 60 * WekaUtils.GetMinuteofPeriod(period))
{
break;
}
System.Diagnostics.Debug.Assert(time < nowTime, "");
var hps = m_hpDataDb.GetHpData(WekaUtils.GetDateFromTime(time));
if (hps == null)
{
continue;
}
for (int k = 0; k < 2; ++k)
{
for (int tp = TestParameters2.tpStart; tp < TestParameters2.tpCount; ++tp)
{
for (int sl = TestParameters2.slStart; sl < TestParameters2.slCount; ++sl)
{
var hpItem10 = hps.Item1[k, tp, sl, 0];
var hpItem11 = hps.Item1[k, tp, sl, 1];
var hpItem2 = hps.Item2[k, tp, sl];
WekaUtils.DebugAssert(hpItem10 >= 0, "hp data should >= 0");
WekaUtils.DebugAssert(hpItem11 >= 0, "hp data should >= 0");
WekaUtils.DebugAssert(hpItem2 >= 0, "hp data should >= 0");
// 当用PartialData模式的，还未知道的数据
if (hpItem2 == Parameters.MaxTime)
{
continue;
}
if (hpItem2 > nowTime)
{
continue;
}
int tpp = 1;
int tp1 = tpp * (tp + 1);
int sl1 = tpp * (sl + 1);
int v = 1;
//v = 600 / (sl + 1);
sum[k] -= tp1 * v * hpItem10;
sum[k] += sl1 * v * hpItem11;
hpdate = Math.Max(hpdate, hpItem2);
}
}
}
}
int hpSum = 2;
bool useOneClass = false;
if (useOneClass)
{
if (sum[0] < 0)
{
hpSum = 0;
}
}
else
{
bool enableMultiClass = true;
if (enableMultiClass)
{
if (sum[0] < 0 && sum[1] >= 0)
{
hpSum = 0;
}
else if (sum[1] < 0 && sum[0] >= 0)
{
hpSum = 1;
}
else if (sum[0] >= 0 && sum[1] >= 0)
{
hpSum = 2;
}
else
{
hpSum = 3;
}
}
else
{
double delta = 0 * (TestParameters2.tpCount - TestParameters2.tpStart);
if (sum[0] < sum[1] - delta)
{
hpSum = 0;
}
else if (sum[0] > sum[1] + delta)
{
hpSum = 1;
}
}
}
ret = new Tuple <int, long>(hpSum, hpdate);
return(ret);
}
public void GenerateHpDataToTxt(string symbol, string period)
{
double mainPeriodOfHour = WekaUtils.GetMinuteofPeriod(period) / 60.0;
int n = TestParameters2.nTpsl;
int m1 = TestParameters.TpMaxCount / n;
int m2 = TestParameters.SlMaxCount / n;
var dataDates = TestManager.GetDataDateRange();
DateTime date = dataDates[0];
DateTime maxDate = dataDates[1];
date = new DateTime(2000, 1, 1);
string hpFileName = TestParameters.GetBaseFilePath(string.Format("{0}_{1}_hpdata.txt", symbol, period));
if (!TestParameters2.RealTimeMode && System.IO.File.Exists(hpFileName))
{
return;
}
SortedDictionary <DateTime, string> dictAlready = new SortedDictionary <DateTime, string>();
string hpFileNameAlready = hpFileName + ".full";
if (!TestParameters2.DBDataConsistent)
{
if (System.IO.File.Exists(hpFileNameAlready))
{
using (StreamReader sr = new StreamReader(hpFileNameAlready))
{
while (!sr.EndOfStream)
{
string s = sr.ReadLine();
int idx = s.IndexOf(',');
DateTime d = Convert.ToDateTime(s.Substring(0, idx));
string s2 = s.Substring(idx + 1);
dictAlready[d] = s2.Trim();
}
}
}
}
string sql;
System.Data.DataTable allDt = null;
DateTime nextBufferDate = DateTime.MinValue;
using (StreamWriter sw = new StreamWriter(hpFileName))
using (StreamWriter sw2 = new StreamWriter(hpFileNameAlready))
{
while (true)
{
if (!TestParameters2.RealTimeMode)
{
Console.WriteLine(date.ToString(Parameters.DateTimeFormat));
}
StringBuilder sb = new StringBuilder();
if (dictAlready.ContainsKey(date))
{
sb.Append(date.ToString(Parameters.DateTimeFormat));
sb.Append(", ");
sb.AppendLine(dictAlready[date]);
sw.Write(sb.ToString());
sw2.Write(sb.ToString());
date = date.AddHours(mainPeriodOfHour);
if (date >= maxDate)
{
break;
}
}
else
{
if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday)
{
date = date.AddHours(mainPeriodOfHour);
continue;
}
if (nextBufferDate <= date)
{
nextBufferDate = date.AddHours(mainPeriodOfHour);
sql = string.Format("SELECT * FROM {0}_HP WHERE TIME >= '{1}' AND TIME < '{2}' AND {3}",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextBufferDate),
string.IsNullOrEmpty(TestParameters.DbSelectWhere) ? "1 = 1" : TestParameters.DbSelectWhere);
//if (!TestParameters2.UsePartialHpData && TestParameters2.RealTimeMode)
//{
// sql += " AND IsComplete = 1";
//}
allDt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql);
}
DateTime nextDate = date.AddHours(mainPeriodOfHour);
sql = string.Format("TIME >= '{1}' AND TIME < '{2}'",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextDate));
var dt = allDt.Select(sql);
bool isComplete = true;
foreach (System.Data.DataRow row in dt)
{
bool b = (bool)row["IsComplete"];
if (!b)
{
isComplete = false;
break;
}
}
if ((TestParameters2.UsePartialHpData || isComplete) && dt.Length > 0)
{
sb.Append(date.ToString(Parameters.DateTimeFormat));
sb.Append(", ");
var hpAll = SumHp(dt);
var hps = hpAll.Item1;
var hpTimes = hpAll.Item2;
for (int j = 0; j < hps.GetLength(0); ++j)
{
for (int k = 0; k < hps.GetLength(1); ++k)
{
for (int l = 0; l < hps.GetLength(2); ++l)
{
if (isComplete)
{
if (hps[j, k, l, 0] == -1 || hps[j, k, l, 1] == -1)
{
throw new AssertException("hps should not be -1.");
}
}
if (hps[j, k, l, 0] == -1)
{
hps[j, k, l, 0] = 0;
}
if (hps[j, k, l, 1] == -1)
{
hps[j, k, l, 1] = 0;
}
sb.Append(hps[j, k, l, 0] + ", " + hps[j, k, l, 1] + ", ");
}
}
}
long maxHpTime = 0;
for (int j = 0; j < hpTimes.GetLength(0); ++j)
{
for (int k = 0; k < hpTimes.GetLength(1); ++k)
{
for (int l = 0; l < hpTimes.GetLength(2); ++l)
{
if (isComplete)
{
if (hpTimes[j, k, l] == -1)
{
throw new AssertException("hpTimes should not be -1.");
}
}
if (hpTimes[j, k, l] == -1)
{
hpTimes[j, k, l] = Parameters.MaxTime;
}
sb.Append(hpTimes[j, k, l] + ", ");
maxHpTime = Math.Max(hpTimes[j, k, l], maxHpTime);
}
}
}
sb.AppendLine();
sw.Write(sb.ToString());
if (isComplete && !TestParameters2.DBDataConsistent)
{
sw2.Write(sb.ToString());
}
}
date = nextDate;
if (date >= maxDate)
{
break;
}
}
}
}
if (TestParameters2.DBDataConsistent)
{
File.Delete(hpFileNameAlready);
}
}
public static Dictionary <long, int> GetHpDateTimesCount(string symbol, string period)
{
double mainPeriodOfHour = WekaUtils.GetMinuteofPeriod(period) / 60.0;
Dictionary <long, int> dictTimes = new Dictionary <long, int>();
int dealCount = 0;
int n = TestParameters2.nTpsl;
DateTime date = TestParameters2.TrainStartTime;
DateTime maxDate = TestParameters2.TrainEndTime;
string sql;
System.Data.DataTable allDt = null;
DateTime nextBufferDate = DateTime.MinValue;
while (true)
{
Console.WriteLine(date.ToString(Parameters.DateTimeFormat) + ", " + dictTimes.Count + ", " + dealCount);
if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday)
{
date = date.AddHours(mainPeriodOfHour);
continue;
}
if (nextBufferDate <= date)
{
nextBufferDate = date.AddHours(mainPeriodOfHour * 100);
sql = string.Format("SELECT * FROM {0}_HP WHERE TIME >= '{1}' AND TIME < '{2}' AND {3}",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextBufferDate),
string.IsNullOrEmpty(TestParameters.DbSelectWhere) ? "1 = 1" : TestParameters.DbSelectWhere);
allDt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql);
}
DateTime nextDate = date.AddHours(mainPeriodOfHour);
sql = string.Format("TIME >= '{1}' AND TIME < '{2}'",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextDate));
var dt = allDt.Select(sql);
bool isComplete = true;
if ((TestParameters2.UsePartialHpData || isComplete) && dt.Length > 0)
{
foreach (System.Data.DataRow row in dt)
{
sbyte?[, ,] hp = HpData.DeserializeHp((byte[])row["hp"]);
long?[, ,] hpTime = HpData.DeserializeHpTimes((byte[])row["hp_date"]);
WekaUtils.DebugAssert(hp.GetLength(0) == 2, "");
WekaUtils.DebugAssert(hpTime.GetLength(0) == 2, "");
for (int j = 0; j < hp.GetLength(0); ++j)
{
for (int k = 0; k < hp.GetLength(1); ++k)
{
for (int l = 0; l < hp.GetLength(2); ++l)
{
if (k % n != n - 1 || l % n != n - 1)
{
continue;
}
if (!hp[j, k, l].HasValue || hp[j, k, l] == -1)
{
continue;
}
//if (k / n >= TestParameters2.tpCount)
// continue;
//if (l / n >= TestParameters2.slCount)
// continue;
if (dictTimes.ContainsKey(hpTime[j, k, l].Value))
{
dictTimes[hpTime[j, k, l].Value]++;
}
else
{
dictTimes[hpTime[j, k, l].Value] = 1;
}
dealCount++;
}
}
}
}
}
date = nextDate;
if (date >= maxDate)
{
break;
}
}
return(dictTimes);
}
public static void IterateHpData(string symbol, string period, Action <DateTime, int, int, int, int, long> action)
{
double mainPeriodOfHour = WekaUtils.GetMinuteofPeriod(period) / 60.0;
int n = TestParameters2.nTpsl;
int m1 = TestParameters.TpMaxCount / n;
int m2 = TestParameters.SlMaxCount / n;
DateTime date = TestParameters2.TrainStartTime;
DateTime maxDate = TestParameters2.TrainEndTime;
string sql;
System.Data.DataTable allDt = null;
DateTime nextBufferDate = DateTime.MinValue;
while (true)
{
if (!TestParameters2.RealTimeMode)
{
Console.WriteLine(date.ToString(Parameters.DateTimeFormat));
}
if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday)
{
date = date.AddHours(mainPeriodOfHour);
continue;
}
if (nextBufferDate <= date)
{
nextBufferDate = date.AddHours(mainPeriodOfHour * 240);
sql = string.Format("SELECT * FROM {0}_HP WHERE TIME >= '{1}' AND TIME < '{2}' AND {3}",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextBufferDate),
string.IsNullOrEmpty(TestParameters.DbSelectWhere) ? "1 = 1" : TestParameters.DbSelectWhere);
allDt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql);
}
DateTime nextDate = date.AddHours(mainPeriodOfHour);
sql = string.Format("TIME >= '{1}' AND TIME < '{2}'",
symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextDate));
var dt = allDt.Select(sql);
bool isComplete = true;
foreach (System.Data.DataRow row in dt)
{
bool b = (bool)row["IsComplete"];
if (!b)
{
isComplete = false;
break;
}
}
if ((TestParameters2.UsePartialHpData || isComplete) && dt.Length > 0)
{
foreach (System.Data.DataRow row in dt)
{
sbyte?[, ,] hp = HpData.DeserializeHp((byte[])row["hp"]);
long?[, ,] hpTime = HpData.DeserializeHpTimes((byte[])row["hp_date"]);
WekaUtils.DebugAssert(hp.GetLength(0) == 2, "");
WekaUtils.DebugAssert(hp.GetLength(1) % m1 == 0, "");
WekaUtils.DebugAssert(hp.GetLength(2) % m2 == 0, "");
WekaUtils.DebugAssert(hpTime.GetLength(0) == 2, "");
for (int j = 0; j < hp.GetLength(0); ++j)
{
for (int k = 0; k < hp.GetLength(1); ++k)
{
for (int l = 0; l < hp.GetLength(2); ++l)
{
if (k % n != n - 1 || l % n != n - 1)
{
continue;
}
if (!hp[j, k, l].HasValue || hp[j, k, l] == -1)
{
continue;
}
action(date, j, k, l, hp[j, k, l].Value, hpTime[j, k, l].Value);
}
}
}
}
}
date = nextDate;
if (date >= maxDate)
{
break;
}
}
}
public Dictionary <DateTime, Tuple <int[, , , ], long[, , ]> > GetHpDataFromTxt(string symbol, string period)
{
string hpFile = TestParameters.GetBaseFilePath(string.Format("{0}_{1}_hpdata.txt", symbol, period));
int n = TestParameters2.nTpsl;
int m1 = TestParameters.TpMaxCount / n;
int m2 = TestParameters.SlMaxCount / n;
lock (this)
{
string bufferKey = symbol + "." + period + "." + TestParameters2.nTpsl;
if (m_hpBuffer.ContainsKey(bufferKey))
{
return(m_hpBuffer[bufferKey]);
}
Dictionary <DateTime, Tuple <int[, , , ], long[, , ]> > ret;
string serializeFile = hpFile + ".serialize";
if (false && File.Exists(serializeFile))
{
ret = Feng.Windows.Utils.SerializeHelper.Deserialize <Dictionary <DateTime, Tuple <int[, , , ], long[, , ]> > >(serializeFile);
}
else
{
DateTime lastWriteDate = DateTime.MinValue;
ret = new Dictionary <DateTime, Tuple <int[, , , ], long[, , ]> >();
using (StreamReader sr = new StreamReader(hpFile))
{
while (true)
{
if (sr.EndOfStream)
{
break;
}
string s = sr.ReadLine().Trim();
string[] ss = s.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
int ssLength = 1 + 2 * m1 * m2 * 2 + 2 * m1 * m2;
if (ss.Length != ssLength && sr.EndOfStream)
{
break;
}
WekaUtils.DebugAssert(ss.Length == ssLength, string.Format("HpData LineLength should be {0}, but is {1}", ssLength, ss.Length));
DateTime nowDate = Convert.ToDateTime(ss[0]);
if (nowDate.Day == 1 && (nowDate - lastWriteDate).Days > 1)
{
System.Console.WriteLine("Read HpData at {0}", nowDate.ToString());
lastWriteDate = nowDate;
}
if (nowDate < TestParameters2.TrainStartTime.AddMinutes(-(TestParameters2.MinTrainPeriod + 500) * WekaUtils.GetMinuteofPeriod(TestParameters2.CandidateParameter.MainPeriod)))
{
continue;
}
if (nowDate > TestParameters2.TrainEndTime)
{
break;
}
int nn = 1;
int[, , ,] hps = new int[2, m1, m2, 2];
for (int i = 0; i < hps.GetLength(0); ++i)
{
for (int j = 0; j < hps.GetLength(1); ++j)
{
for (int k = 0; k < hps.GetLength(2); ++k)
{
for (int l = 0; l < 2; ++l)
{
hps[i, j, k, l] = Convert.ToInt32(ss[nn]);
nn++;
}
}
}
}
long[, ,] hpTimes = new long[2, m1, m2];
for (int i = 0; i < hpTimes.GetLength(0); ++i)
{
for (int j = 0; j < hpTimes.GetLength(1); ++j)
{
for (int k = 0; k < hpTimes.GetLength(2); ++k)
{
hpTimes[i, j, k] = Convert.ToInt64(ss[nn]);
nn++;
}
}
}
ret[nowDate] = new Tuple <int[, , , ], long[, , ]>(hps, hpTimes);
}
}
//Feng.Utils.SerializeHelper.Serialize(serializeFile, ret);
}
m_hpBuffer[bufferKey] = ret;
return(ret);
}
}