public static void FilterInstances(weka.core.Instances allInstances) { DateTime nextHpDate = DateTime.MinValue; java.util.LinkedList deleteList = new java.util.LinkedList(); for (int i = 0; i < allInstances.numInstances(); ++i) { DateTime nowDate = WekaUtils.GetDateValueFromInstances(allInstances, 0, i); if (TestParameters2.RealTimeMode && i == allInstances.numInstances() - 1) { allInstances.instance(i).setClassValue(0); allInstances.instance(i).setValue(1, WekaUtils.GetTimeFromDate(Parameters.MaxDate) * 1000); } else { if (nowDate < nextHpDate) { deleteList.Add(allInstances.instance(i)); } else { DateTime hpDate = WekaUtils.GetDateValueFromInstances(allInstances, 1, i); nextHpDate = hpDate; } } } allInstances.removeAll(deleteList); }
public void Now(DateTime nowTime, double?nowPrice) { m_nowTime = nowTime; if (nowPrice.HasValue) { m_nowPrice.close = nowPrice.Value; m_nowPrice.time = WekaUtils.GetTimeFromDate(nowTime); } if (!m_isLastForPeriod) { if (m_firstLastDate == null) { m_firstLastDate = nowTime.AddMinutes(m_lastMinutes); } if (m_firstLastDate <= m_nowTime) { m_isLastForPeriod = true; } } else { RemoveOldDeals(); } //CloseAll(); CalculateTotal(); CalculateNow(); }
//public void ConvertHpDataPeriod(string symbol, string periodSsrc, string periodDest) //{ // string hpFileName = TestParameters.GetBaseFilePath(string.Format("{0}_{1}_hpdata.txt", symbol, periodSsrc)); // var hpData = GetHpDateFromTxt(hpFileName, TestParameters2.nTpsl); // string hpFileName2 = TestParameters.GetBaseFilePath(string.Format("{0}_{1}_hpdata.txt", symbol, periodDest)); // int m = TestParameters.TpSlMaxCount / TestParameters2.nTpsl; // DateTime date = TestParameters2.TrainStartTime; // DateTime maxDate = TestParameters2.TrainEndTime; // using(StreamWriter sw = new StreamWriter(hpFileName2)) // { // while (true) // { // if (!TestParameters2.RealTimeMode) // { // Console.WriteLine(date.ToString(Parameters.DateTimeFormat)); // } // StringBuilder sb = new StringBuilder(); // { // if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday) // { // date = date.AddHours(mainPeriodOfHour); // continue; // } // bool isComplete = true; // if ((TestParameters2.UsePartialHpData || isComplete)) // { // sb.Append(date.ToString(Parameters.DateTimeFormat)); // sb.Append(", "); // int[, , ,] hps = new int[2, m, m, 2]; // long[, ,] hpTimes = new long[2, m, m]; // for (int j = 0; j < hps.GetLength(0); ++j) // for (int k = 0; k < hps.GetLength(1); ++k) // for (int l = 0; l < hps.GetLength(2); ++l) // { // hps[j, k, l, 0] = hps[j, k, l, 1] = -1; // hpTimes[j, k, l] = -1; // } // List<DateTime> dts = new List<DateTime>(); // foreach (DateTime dt in dts) // { // if (!hpData.ContainsKey(dt)) // continue; // sbyte?[, ,] hp = hpData[dt].Item1; // long[, ,] hpTime = hpData[dt].Item2; // WekaUtils.DebugAssert(hp.GetLength(0) == 2, ""); // WekaUtils.DebugAssert(hp.GetLength(1) % m == 0, ""); // WekaUtils.DebugAssert(hp.GetLength(2) % m == 0, ""); // WekaUtils.DebugAssert(hpTime.GetLength(0) == 2, ""); // for (int j = 0; j < hp.GetLength(0); ++j) // for (int k = 0; k < hp.GetLength(1); ++k) // for (int l = 0; l < hp.GetLength(2); ++l) // { // if (k % n != n - 1 || l % n != n - 1) // continue; // if (!hp[j, k, l].HasValue || hp[j, k, l] == -1) // continue; // //if (k / n >= TestParameters2.tpCount) // // continue; // //if (l / n >= TestParameters2.slCount) // // continue; // if (hps[j, k / n, l / n, 0] == -1) // hps[j, k / n, l / n, 0] = 0; // if (hps[j, k / n, l / n, 1] == -1) // hps[j, k / n, l / n, 1] = 0; // if (hp[j, k, l] == 1) // hps[j, k / n, l / n, 0]++; // else if (hp[j, k, l] == 0) // hps[j, k / n, l / n, 1]++; // else // throw new AssertException("hp should be 0 or 1."); // hpTimes[j, k / n, l / n] = Math.Max(hpTimes[j, k / n, l / n], hpTime[j, k, l].Value); // } // } // for (int j = 0; j < hps.GetLength(0); ++j) // for (int k = 0; k < hps.GetLength(1); ++k) // for (int l = 0; l < hps.GetLength(2); ++l) // { // if (isComplete) // { // if (hps[j, k, l, 0] == -1 || hps[j, k, l, 1] == -1) // { // throw new AssertException("hps should not be -1."); // } // } // if (hps[j, k, l, 0] == -1) // hps[j, k, l, 0] = 0; // if (hps[j, k, l, 1] == -1) // hps[j, k, l, 1] = 0; // sb.Append(hps[j, k, l, 0] + ", " + hps[j, k, l, 1] + ", "); // } // long maxHpTime = 0; // for (int j = 0; j < hpTimes.GetLength(0); ++j) // for (int k = 0; k < hpTimes.GetLength(1); ++k) // for (int l = 0; l < hpTimes.GetLength(2); ++l) // { // if (isComplete) // { // if (hpTimes[j, k, l] == -1) // { // throw new AssertException("hpTimes should not be -1."); // } // } // if (hpTimes[j, k, l] == -1) // hpTimes[j, k, l] = Parameters.MaxTime; // sb.Append(hpTimes[j, k, l] + ", "); // maxHpTime = Math.Max(hpTimes[j, k, l], maxHpTime); // } // sb.AppendLine(); // sw.Write(sb.ToString()); // } // DateTime nextDate = date.AddMinutes(WekaUtils.GetMinuteofPeriod(periodDest)); // date = nextDate; // if (date >= maxDate) // break; // } // } // } //} public static Dictionary <long, int> GetHpDateTimesCountFromInstance(string fileName) { Dictionary <long, int> dictTimes = new Dictionary <long, int>(); int dealCount = 0; var instances = WekaUtils.LoadInstances(TestParameters.GetBaseFilePath(fileName)); for (int i = 0; i < instances.numInstances(); ++i) { var date = WekaUtils.GetDateValueFromInstances(instances, 0, i); var hpdate = WekaUtils.GetDateValueFromInstances(instances, 1, i); var hptime = WekaUtils.GetTimeFromDate(hpdate); if (dictTimes.ContainsKey(hptime)) { dictTimes[hptime]++; } else { dictTimes[hptime] = 1; } dealCount++; Console.WriteLine(date.ToString(Parameters.DateTimeFormat) + ", " + dictTimes.Count + ", " + dealCount); } return(dictTimes); }
private Tuple <long, long> GetDbDateFilter(DateTime dt1, DateTime dt2, int tableType) { if (tableType == 0) { return(new Tuple <long, long>(WekaUtils.GetTimeFromDate(dt1.AddDays(-1)), WekaUtils.GetTimeFromDate(dt2))); } else { return(new Tuple <long, long>(WekaUtils.GetTimeFromDate(dt1.AddDays(-3)), WekaUtils.GetTimeFromDate(dt2))); } }
public static void SaveCCScoresToDb(DateTime nowDate, CandidateParameter cp, ParameterdCandidateStrategy pcs) { string tableName = string.Format("{0}_CCSCORE_{1}", cp.MainSymbol, cp.DealInfoLastMinutes / (7 * 24 * 12 * 5)); long nowTime = WekaUtils.GetTimeFromDate(nowDate); try { Feng.Data.DbHelper.Instance.ExecuteNonQuery(string.Format("SELECT TOP 1 * FROM {0} WHERE TIME = -1", tableName)); } catch (Exception) { string sqlCreate = string.Format(@"CREATE TABLE {0}( [Time] [bigint] NOT NULL, [scores] [varbinary](max) NOT NULL, [nda] [varbinary](max) NOT NULL, [nds] [varbinary](max) NOT NULL, CONSTRAINT [PK_{0}] PRIMARY KEY CLUSTERED ( [Time] ASC )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] ) ON [PRIMARY]", tableName); Feng.Data.DbHelper.Instance.ExecuteNonQuery(sqlCreate); } long[, ,] ndas = new long[Parameters.AllDealTypes.Length, cp.BatchTps.Length, cp.BatchSls.Length]; long[, ,] ndss = new long[Parameters.AllDealTypes.Length, cp.BatchTps.Length, cp.BatchSls.Length]; double[, ,] nowScores = new double[Parameters.AllDealTypes.Length, cp.BatchTps.Length, cp.BatchSls.Length]; pcs.IterateClassifierInfos((k, ii, jj, h) => { nowScores[k, ii, jj] = pcs.m_classifierInfoIdxs[k, ii, jj, h].Deals.NowScore; ndas[k, ii, jj] = pcs.m_classifierInfoIdxs[k, ii, jj, h].Deals.DealLastTimeAvg; ndss[k, ii, jj] = pcs.m_classifierInfoIdxs[k, ii, jj, h].Deals.DealLastTimeStd; }); string sql = string.Format("INSERT INTO [{0}] ([Time],[scores],[nda],[nds]) VALUES (@Time, @score, @nda, @nds)", tableName); var cmd = new System.Data.SqlClient.SqlCommand(sql); cmd.Parameters.AddWithValue("@Time", nowTime); cmd.Parameters.AddWithValue("@score", CCScoreData.SerializeScores(nowScores)); cmd.Parameters.AddWithValue("@nda", CCScoreData.SerializeScoreTimes(ndas)); cmd.Parameters.AddWithValue("@nds", CCScoreData.SerializeScoreTimes(ndss)); try { Feng.Data.DbHelper.Instance.ExecuteNonQuery(string.Format("DELETE FROM {0} WHERE TIME = '{1}'", tableName, WekaUtils.GetTimeFromDate(nowDate))); Feng.Data.DbHelper.Instance.ExecuteNonQuery(cmd); } catch (Exception) { } }
private string GetDbDateFilter4Db(DateTime dt1, DateTime dt2, int tableType) { string filter; if (tableType == 0) { filter = string.Format("Time >= {0} AND Time < {1}", WekaUtils.GetTimeFromDate(dt1.AddDays(-1)), WekaUtils.GetTimeFromDate(dt2)); } else { filter = string.Format("Time >= {0} AND Time < {1}", WekaUtils.GetTimeFromDate(dt1.AddDays(-3)), WekaUtils.GetTimeFromDate(dt2)); } return(filter); }
public void Now(DateTime nowTime, double?nowPrice) { m_nowTime = nowTime; if (nowPrice.HasValue) { m_nowPrice.close = nowPrice.Value; } m_nowPrice.time = WekaUtils.GetTimeFromDate(nowTime); RemoveOldDeals(); //CloseAll(); CalculateCurrent(); }
public static int GetNonbusinessMinutes(DateTime dateStart, DateTime dateEnd, int stepMinutes = 1) { int r = 0; long t1 = WekaUtils.GetTimeFromDate(dateStart); long t2 = WekaUtils.GetTimeFromDate(dateEnd); for (long i = t1; i < t2; i += stepMinutes * 60) { DateTime d = WekaUtils.GetDateFromTime(i); if (d.DayOfWeek == DayOfWeek.Saturday || d.DayOfWeek == DayOfWeek.Sunday) { r++; } } return(r); }
public Tuple <int[, , , ], long[, , ]> GetHpData(DateTime date) { int n = m_buffer.Count - 5 * 15 * 20000; if (n > 0) { List <long> delete = new List <long>(); foreach (var kvp in m_buffer) { delete.Add(kvp.Key); if (delete.Count >= n) { break; } } foreach (var i in delete) { m_buffer.Remove(i); } } Tuple <int[, , , ], long[, , ]> ret = null; long time = WekaUtils.GetTimeFromDate(date); if (!m_buffer.ContainsKey(time)) { var sql = string.Format("SELECT * FROM {0}_HP WHERE TIME = {1}", m_symbol, time); var dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql); if (dt.Rows.Count > 0) { ret = HpData.SumHp(dt.Select()); } m_buffer[time] = ret; } return(m_buffer[time]); }
// tableType = 0: MainTable; 1: OtherDataTable; 2:HpTable public ForexDataRows GetDbData(DateTime dt1, DateTime dt2, string tableName, int tableType, bool isTrain, CandidateParameter cp) { lock (m_lockObject) { //int dealTypeIdx1 = dealType == 'B' ? 0 : 1; DateTime prevDate = dt1.AddMinutes(-2 * cp.PrevTimeCount * WekaUtils.GetMinuteofPeriod(cp.AllPeriods[cp.PeriodCount - 1])); DateTime nextDate = dt2; if (m_useBufferReadData && m_batchBufferMinutes < 0) { int n1 = cp.SymbolCount * cp.PeriodCount * (cp.PeriodTimeCount + 1) * (cp.AllIndNames.Count + cp.AllIndNames2.Count + 5) / WekaUtils.GetMinuteofPeriod(cp.MainPeriod); n1 = Math.Max(n1, 1); int n2 = 1; //n2 = 3 * TestParameters.BatchTps.Length * TestParameters.BatchSls.Length * Parameters.AllDealTypes.Length / WekaUtils.GetMinuteofPeriod(TestParameters.MainPeriod); //n2 /= 6; n2 = 10000 / WekaUtils.GetMinuteofPeriod(cp.MainPeriod); n2 = Math.Max(n2, 1); m_batchBufferMinutes = (Parameters.TotalCanUseBuffer - System.GC.GetTotalMemory(false)) / (n1 + n2) * WekaUtils.GetMinuteofPeriod(cp.MainPeriod); // / 240 int min = (int)(nextDate - prevDate).TotalMinutes + 24 * TestParameters.BatchTestMinutes; int max = (int)(1.1 * TestParameters.BatchTrainMinutes); if (m_batchBufferMinutes < min) { //m_useBufferReadData = false; WekaUtils.Instance.WriteLog(string.Format("Partial buffer. batchGetTime = {0}, min = {1}", m_batchBufferMinutes, min)); m_batchBufferMinutes = min; } else { //m_useBufferReadData = true; WekaUtils.Instance.WriteLog(string.Format("Use buffer. batchGetTime = {0}, max = {1}", m_batchBufferMinutes, max)); m_batchBufferMinutes = Math.Min(m_batchBufferMinutes, max); } } if (m_useBufferReadData) { DateTime bufferDateStart = prevDate; DateTime bufferDateEnd = bufferDateStart.AddMinutes(m_batchBufferMinutes); if (TestParameters2.RealTimeMode) { if (isTrain) { bufferDateStart = dt1.AddDays(-7); bufferDateEnd = dt2.AddDays(7); } else { bufferDateStart = dt1; bufferDateEnd = dt2; } } //System.Console.WriteLine("{0}, {1}, {2}, {3}", bufferDateStart, bufferDateEnd, prevDate, nextDate); if (bufferDateEnd >= nextDate) { string allDataKey = tableName; //if (tableType == 2) //{ // allDataKey += "_" + Parameters.AllDealTypes[dealTypeIdx] + "_" + tp.ToString() + "_" + sl.ToString(); //} if (m_allData == null) { if (m_serializeAllData) { foreach (string s in System.IO.Directory.GetFiles(".", m_allDataCacheFile)) { string[] ss = System.IO.Path.GetFileNameWithoutExtension(s).Split(new char[] { '_' }, StringSplitOptions.RemoveEmptyEntries); DateTime d1 = Convert.ToDateTime(ss[1]); DateTime d2 = Convert.ToDateTime(ss[2]); if (bufferDateStart >= d1 && bufferDateEnd <= d2) { m_allData = Feng.Windows.Utils.SerializeHelper.Deserialize <Dictionary <string, ForexDataTable> >(s); break; } } } if (m_allData == null) { m_allData = new Dictionary <string, ForexDataTable>(); //m_allDateDates = new DateTime[m_dealType.Length][]; } } if (m_allData.ContainsKey(allDataKey)) { Tuple <DateTime, DateTime> ds = m_allDataDates[allDataKey]; bool expired = false; if (!isTrain) { if (ds.Item2 < nextDate || ds.Item1 > prevDate) { //System.Console.WriteLine("{0}, {1}, {2}, {3}", ds.Item1.ToString(), ds.Item2, prevDate, nextDate); expired = true; } } else { if (ds.Item2 <= nextDate.AddMinutes(TestParameters.EnablePerhourTrain ? 2 * Parameters.AllHour * TestParameters.BatchTestMinutes : 2 * TestParameters.BatchTestMinutes) || ds.Item1 > prevDate) { expired = true; } } if (expired) { m_allData.Remove(allDataKey); m_allDataDates.Remove(allDataKey); } } //long usedMemory = System.GC.GetTotalMemory(false); //if (m_cacheData && usedMemory >= Parameters.TotalCanUseMemory) //{ // WekaUtils.Instance.WriteLog(string.Format("Use too many memory. now total memory = {0}", usedMemory)); // m_cacheData = false; //} //else if (!m_cacheData && usedMemory < Parameters.TotalCanUseMemory) //{ // WekaUtils.Instance.WriteLog(string.Format("reset to cache data. now total memory = {0}", usedMemory)); // m_cacheData = true; //} if (m_cacheData && !m_allData.ContainsKey(allDataKey)) { StringBuilder sql = new StringBuilder(); if (tableType == 0) { sql.Append(string.Format("SELECT [Time], [Date], [hour], [dayofweek], [spread], [close] as mainClose")); // , [AskVolume], [BidVolume] } else if (tableType == 1) { sql.Append(string.Format("SELECT [Time], [Date]")); } else if (tableType == 2) { sql.Append(string.Format("SELECT [Time], [hp], [hp_date]")); } if (tableType == 0 || tableType == 1) { foreach (var kvp in cp.AllIndNames2) { sql.Append(string.Format(",[{0}]", kvp.Key)); } foreach (var kvp in cp.AllIndNames) { sql.Append(string.Format(",[{0}]", kvp.Key)); } } sql.Append(string.Format(" FROM {0} WHERE Time >= '{1}' AND Time < '{2}' ", tableName, WekaUtils.GetTimeFromDate(bufferDateStart), WekaUtils.GetTimeFromDate(bufferDateEnd))); //if (tableType == 0 || tableType == 1) //{ // sql.Append(" AND IsActive = 1 "); //} //if (tableType == 2) //{ // sql.Append(string.Format(" AND DealType = '{0}' AND Tp = {1} AND Sl = {2}", Parameters.AllDealTypes[dealTypeIdx], tp, sl)); //} if (!string.IsNullOrEmpty(TestParameters.DbSelectWhere)) { sql.Append(string.Format(" AND {0}", TestParameters.DbSelectWhere)); } sql.Append(string.Format(" {0}", m_selectOrder)); string cmd = sql.ToString(); //m_allData[allDataKey] = DbHelper.Instance.ExecuteDataTable(cmd); //m_allData[allDataKey].PrimaryKey = new System.Data.DataColumn[] { m_allData[allDataKey].Columns["Time"] }; m_allData[allDataKey] = new ForexDataTable(Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd), tableType == 2); m_allDataDates[allDataKey] = new Tuple <DateTime, DateTime>(bufferDateStart, bufferDateEnd); WekaUtils.Instance.WriteLog(string.Format("Buffer Get Data From {0} to {1} of {2}", bufferDateStart, bufferDateEnd, allDataKey)); if (m_serializeAllData) { string allDataCacheFile = m_allDataCacheFile.Replace("*", TestParameters.BatchDateStart.ToString(Parameters.DateTimeFormat) + "_" + TestParameters.BatchDateEnd.ToString(Parameters.DateTimeFormat)); Feng.Windows.Utils.SerializeHelper.Serialize(allDataCacheFile, m_allData); } } if (m_allData.ContainsKey(allDataKey)) { //string filter = GetDbDateFilter(dt1, dt2, tableType); //return m_allData[allDataKey].Select(filter); Tuple <long, long> filter = GetDbDateFilter(dt1, dt2, tableType); return(m_allData[allDataKey].SelectByDate(filter.Item1, filter.Item2)); } } } // no get from cache { StringBuilder sql = new StringBuilder(); if (tableType == 0) { sql.Append(string.Format("SELECT [Time], [Date], [hour], [dayofweek], [spread], [close] as mainClose")); // , [AskVolume], [BidVolume] } else if (tableType == 1) { sql.Append(string.Format("SELECT [Time], [Date]")); } else if (tableType == 2) { sql.Append(string.Format("SELECT [Time], [hp], [hp_date]")); } if (tableType == 0 || tableType == 1) { foreach (var kvp in cp.AllIndNames2) { sql.Append(string.Format(",[{0}]", kvp.Key)); } foreach (var kvp in cp.AllIndNames) { sql.Append(string.Format(",[{0}]", kvp.Key)); } } string filter = GetDbDateFilter4Db(dt1, dt2, tableType); sql.Append(string.Format(" FROM {0} WHERE {1}", tableName, filter)); //if (tableType == 2) //{ // sql.Append(string.Format(" AND DealType = '{0}' AND Tp = {1} AND Sl = {2}", Parameters.AllDealTypes[dealTypeIdx], tp, sl)); //} if (!string.IsNullOrEmpty(TestParameters.DbSelectWhere)) { sql.Append(string.Format(" AND {0}", TestParameters.DbSelectWhere)); } sql.Append(string.Format(" {0}", m_selectOrder)); string cmd = sql.ToString(); WekaUtils.Instance.WriteLog(string.Format("NoBuffer Get Date From {0} to {1} of {2}", prevDate, nextDate, tableName)); System.Data.DataTable dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd); //return dt.Select(); return(new ForexDataRows(new ForexDataTable(dt, tableType == 2), 0, dt.Rows.Count)); } } }
private bool?Do(DateTime openDate, double openPrice, out DateTime?closeDate, int dealType) { closeDate = DateTime.MinValue; var dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(string.Format("SELECT * FROM {0}_HP WHERE Time = {1}", m_symbol, WekaUtils.GetTimeFromDate(openDate))); if (dt.Rows.Count == 0) { return(null); } sbyte?[, ,] hp = HpData.DeserializeHp((byte[])dt.Rows[0]["hp"]); long?[, ,] hpTime = HpData.DeserializeHpTimes((byte[])dt.Rows[0]["hp_date"]); int ix = (int)m_tp / TestParameters.GetTpSlMinDelta(m_symbol) - 1; int iy = (int)m_sl / TestParameters.GetTpSlMinDelta(m_symbol) - 1; if (hp[dealType, ix, iy].HasValue) { closeDate = WekaUtils.GetDateFromTime(hpTime[dealType, ix, iy].Value); return(hp[dealType, ix, iy].Value == 1); } return(null); }
public Dictionary <DateTime, Tuple <int, long> > GetHpSum(string symbol, string period, long nowTime, long getTime) { Dictionary <DateTime, Tuple <int, long> > ret = new Dictionary <DateTime, Tuple <int, long> >(); Dictionary <DateTime, Tuple <int[, , , ], long[, , ]> > hpData; hpData = GetHpDataFromTxt(symbol, period); int preHpSum = 2; foreach (var kvp in hpData) { DateTime nowDate = kvp.Key; var retKey = kvp.Key.AddHours(TestParameters2.HourAhead); if (getTime != -1) { if (WekaUtils.GetTimeFromDate(retKey) < getTime) { continue; } else if (WekaUtils.GetTimeFromDate(retKey) > getTime) { break; } } var hps = kvp.Value; long hpdate = 0; double[] sum = new double[2]; sum[0] = sum[1] = 0; for (int k = 0; k < 2; ++k) { for (int tp = TestParameters2.tpStart; tp < TestParameters2.tpCount; ++tp) { for (int sl = TestParameters2.slStart; sl < TestParameters2.slCount; ++sl) { var hpItem10 = hps.Item1[k, tp, sl, 0]; var hpItem11 = hps.Item1[k, tp, sl, 1]; var hpItem2 = hps.Item2[k, tp, sl]; WekaUtils.DebugAssert(hpItem10 >= 0, "hp data should >= 0"); WekaUtils.DebugAssert(hpItem11 >= 0, "hp data should >= 0"); WekaUtils.DebugAssert(hpItem2 >= 0, "hp data should >= 0"); // 当用PartialData模式的,还未知道的数据 if (hpItem2 == Parameters.MaxTime) { continue; } if (hpItem2 > nowTime) { continue; } int tpp = 1; int tp1 = tpp * (tp + 1); int sl1 = tpp * (sl + 1); int v = 1; //v = 600 / (sl + 1); sum[k] -= tp1 * v * hpItem10; sum[k] += sl1 * v * hpItem11; hpdate = Math.Max(hpdate, hpItem2); //System.Console.WriteLine(string.Format("{0},{1},{2},{3},{4}", k, tp1, sl1, hps.Item1[k, tp, sl, 0], hps.Item1[k, tp, sl, 1])); } } } //var dt = Feng.Data.DbHelper.Instance.ExecuteDataTable("SELECT * FROM GBPUSD_HP WHERE TIME >= '" + WekaUtils.GetTimeFromDate(kvp.Key) + "' AND TIME < '" + WekaUtils.GetTimeFromDate(kvp.Key.AddHours(1)) + "'"); //var row = dt.Rows[0]; //sbyte?[, ,] hp = WekaUtils.DeserializeHp((byte[])row["hp"]); //long?[, ,] hpTime = WekaUtils.DeserializeHpTimes((byte[])row["hp_date"]); int hpSum = 2; //if (nowDate >= new DateTime(2010, 4, 12, 0, 30, 0)) //{ //} bool useOneClass = false; if (useOneClass) { if (sum[0] < 0) { hpSum = 0; } } else { bool enableMultiClass = true; if (enableMultiClass) { if (sum[0] < 0 && sum[1] >= 0) { hpSum = 0; } else if (sum[1] < 0 && sum[0] >= 0) { hpSum = 1; } else if (sum[0] >= 0 && sum[1] >= 0) { hpSum = 2; } else // <0, <0 { hpSum = 3; } preHpSum = hpSum; } else { double delta = 0 * (TestParameters2.tpCount - TestParameters2.tpStart); if (sum[0] < sum[1] - delta) { hpSum = 0; } else if (sum[0] > sum[1] + delta) { hpSum = 1; } if (hpSum == 2) { hpSum = preHpSum; } preHpSum = hpSum; if (hpSum == -1) { continue; } } } if (hpdate == 0) { continue; } ret[retKey] = new Tuple <int, long>(hpSum, hpdate); } return(ret); }
public void UpdateUnclosedDeals(DateTime nowDate) { Dictionary <DateTime, System.Data.DataRow> dictHp = new Dictionary <DateTime, System.Data.DataRow>(); foreach (var kvp in m_classifierInfos) { foreach (var i in kvp.Value.Deals.Deals) { if (i.CloseTime.HasValue) { continue; } if (!dictHp.ContainsKey(i.OpenTime)) { var sql = string.Format("SELECT * FROM {0}_HP WHERE TIME = {1}", this.CandidateParameter.MainSymbol, WekaUtils.GetTimeFromDate(i.OpenTime)); var dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql); if (dt.Rows.Count > 0) { dictHp[i.OpenTime] = dt.Rows[0]; } } if (!dictHp.ContainsKey(i.OpenTime)) { continue; } var row = dictHp[i.OpenTime]; int tpMinDelta = TestParameters.GetTpSlMinDelta(this.CandidateParameter.MainSymbol); int slMinDelta = TestParameters.GetTpSlMinDelta(this.CandidateParameter.MainSymbol); int d = i.DealType == 'B' ? 0 : 1; sbyte?[, ,] hps = HpData.DeserializeHp((byte[])row["hp"]); long?[, ,] hpsTime = HpData.DeserializeHpTimes((byte[])row["hp_date"]); var hp = hps[d, kvp.Value.Tp / tpMinDelta - 1, kvp.Value.Sl / slMinDelta - 1]; if (hp.HasValue && hp.Value != -1) { i.CloseTime = WekaUtils.GetDateFromTime(hpsTime[d, kvp.Value.Tp / tpMinDelta - 1, kvp.Value.Sl / slMinDelta - 1].Value); i.Cost = hp.Value == 1 ? -kvp.Value.Tp : kvp.Value.Sl; } } } }
public void GenerateDataToTxt(string symbol, string lastWeek) { int n = TestParameters2.nTpsl; string ccScoreFileName = TestParameters.GetBaseFilePath(string.Format("{0}_ccScores_w{2}_{1}.txt", TestParameters2.CandidateParameter.MainSymbol, TestParameters2.CandidateParameter.MainPeriod, TestParameters2.lastWeek)); DateTime date = TestParameters2.TrainStartTime; DateTime maxDate = TestParameters2.TrainEndTime; if (!TestParameters2.RealTimeMode && System.IO.File.Exists(ccScoreFileName)) { return; } SortedDictionary <DateTime, string> dictAlready = new SortedDictionary <DateTime, string>(); if (System.IO.File.Exists(ccScoreFileName)) { using (StreamReader sr = new StreamReader(ccScoreFileName)) { while (!sr.EndOfStream) { string s = sr.ReadLine(); int idx = s.IndexOf(','); DateTime d = Convert.ToDateTime(s.Substring(0, idx)); string s2 = s.Substring(idx + 1); dictAlready[d] = s2.Trim(); } } } string sql; System.Data.DataTable allDt = null; DateTime nextBufferDate = DateTime.MinValue; while (true) { if (!TestParameters2.RealTimeMode) { Console.WriteLine(date.ToString(Parameters.DateTimeFormat)); } if (dictAlready.ContainsKey(date)) { GenerateDateToTxt(ccScoreFileName, date, dictAlready[date]); date = date.AddHours(TestParameters2.MainPeriodOfHour); if (date >= maxDate) { break; } } else { if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday) { date = date.AddHours(TestParameters2.MainPeriodOfHour); continue; } if (nextBufferDate <= date) { nextBufferDate = date.AddHours(TestParameters2.MainPeriodOfHour); string tableName = string.Format("{0}_CCSCORE_{1}", TestParameters2.CandidateParameter.MainSymbol, TestParameters2.lastWeek); sql = string.Format("SELECT * FROM {0} WHERE TIME >= '{1}' AND TIME < '{2}' AND {3}", tableName, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextBufferDate), string.IsNullOrEmpty(TestParameters.DbSelectWhere) ? "1 = 1" : TestParameters.DbSelectWhere); allDt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql); } DateTime nextDate = date.AddHours(TestParameters2.MainPeriodOfHour); sql = string.Format("TIME >= '{1}' AND TIME < '{2}'", symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextDate)); var dt = allDt.Select(sql); if (dt.Length > 0) { double[, ,] scores = new double[2, TestParameters.TpMaxCount / n, TestParameters.SlMaxCount / n]; long[, ,] ndas = new long[2, TestParameters.TpMaxCount / n, TestParameters.SlMaxCount / n]; long[, ,] ndss = new long[2, TestParameters.TpMaxCount / n, TestParameters.SlMaxCount / n]; for (int j = 0; j < scores.GetLength(0); ++j) { for (int k = 0; k < scores.GetLength(1); ++k) { for (int l = 0; l < scores.GetLength(2); ++l) { scores[j, k, l] = -1; ndas[j, k, l] = -1; ndss[j, k, l] = -1; } } } System.Data.DataRow row = dt[0]; { double[, ,] score = DeserializeScores((byte[])row["scores"]); long[, ,] nda = DeserializeScoreTimes((byte[])row["nda"]); long[, ,] nds = DeserializeScoreTimes((byte[])row["nds"]); WekaUtils.DebugAssert(score.GetLength(0) == 2, ""); WekaUtils.DebugAssert(nda.GetLength(0) == 2, ""); for (int j = 0; j < score.GetLength(0); ++j) { for (int k = 0; k < score.GetLength(1); ++k) { for (int l = 0; l < score.GetLength(2); ++l) { if (k % n != n - 1 || l % n != n - 1) { continue; } //if (k / n >= TestParameters2.tpCount) // continue; //if (l / n >= TestParameters2.slCount) // continue; scores[j, k / n, l / n] = score[j, k, l]; ndas[j, k / n, l / n] = nda[j, k, l]; ndss[j, k / n, l / n] = nds[j, k, l]; } } } GenerateDateToTxt(ccScoreFileName, date, scores, ndas, ndss); } } date = nextDate; if (date >= maxDate) { break; } } } }
public static void IterateHpData(string symbol, string period, Action <DateTime, int, int, int, int, long> action) { double mainPeriodOfHour = WekaUtils.GetMinuteofPeriod(period) / 60.0; int n = TestParameters2.nTpsl; int m1 = TestParameters.TpMaxCount / n; int m2 = TestParameters.SlMaxCount / n; DateTime date = TestParameters2.TrainStartTime; DateTime maxDate = TestParameters2.TrainEndTime; string sql; System.Data.DataTable allDt = null; DateTime nextBufferDate = DateTime.MinValue; while (true) { if (!TestParameters2.RealTimeMode) { Console.WriteLine(date.ToString(Parameters.DateTimeFormat)); } if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday) { date = date.AddHours(mainPeriodOfHour); continue; } if (nextBufferDate <= date) { nextBufferDate = date.AddHours(mainPeriodOfHour * 240); sql = string.Format("SELECT * FROM {0}_HP WHERE TIME >= '{1}' AND TIME < '{2}' AND {3}", symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextBufferDate), string.IsNullOrEmpty(TestParameters.DbSelectWhere) ? "1 = 1" : TestParameters.DbSelectWhere); allDt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql); } DateTime nextDate = date.AddHours(mainPeriodOfHour); sql = string.Format("TIME >= '{1}' AND TIME < '{2}'", symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextDate)); var dt = allDt.Select(sql); bool isComplete = true; foreach (System.Data.DataRow row in dt) { bool b = (bool)row["IsComplete"]; if (!b) { isComplete = false; break; } } if ((TestParameters2.UsePartialHpData || isComplete) && dt.Length > 0) { foreach (System.Data.DataRow row in dt) { sbyte?[, ,] hp = HpData.DeserializeHp((byte[])row["hp"]); long?[, ,] hpTime = HpData.DeserializeHpTimes((byte[])row["hp_date"]); WekaUtils.DebugAssert(hp.GetLength(0) == 2, ""); WekaUtils.DebugAssert(hp.GetLength(1) % m1 == 0, ""); WekaUtils.DebugAssert(hp.GetLength(2) % m2 == 0, ""); WekaUtils.DebugAssert(hpTime.GetLength(0) == 2, ""); for (int j = 0; j < hp.GetLength(0); ++j) { for (int k = 0; k < hp.GetLength(1); ++k) { for (int l = 0; l < hp.GetLength(2); ++l) { if (k % n != n - 1 || l % n != n - 1) { continue; } if (!hp[j, k, l].HasValue || hp[j, k, l] == -1) { continue; } action(date, j, k, l, hp[j, k, l].Value, hpTime[j, k, l].Value); } } } } } date = nextDate; if (date >= maxDate) { break; } } }
public void SaveTestData(CandidateClassifier clsInfo, DateTime nowDate) { if (TestParameters.OnlyNewestTestDataSaved) { string sql = string.Format("DELETE FROM {0}", m_testDataTableName); Feng.Data.DbHelper.Instance.ExecuteNonQuery(sql); } string str = WekaUtils.DoubleArrayToIntString(clsInfo.CurrentTestRet); string scv = WekaUtils.DoubleArrayToIntString(clsInfo.CurrentClassValue); if (m_testDataInsertTable == null) { m_testDataInsertTable = new System.Data.DataTable(); m_testDataInsertTable.Columns.Add("Id", typeof(int)); m_testDataInsertTable.Columns.Add("Time", typeof(long)); //m_testDataInsertTable.Columns.Add("Type", typeof(int)); m_testDataInsertTable.Columns.Add("ClsName", typeof(string)); m_testDataInsertTable.Columns.Add("TestResult", typeof(string)); m_testDataInsertTable.Columns.Add("ClassValue", typeof(string)); m_testDataInsertTable.Columns.Add("DealsInfo", typeof(byte[])); m_testDataInsertTable.Columns.Add("DealsData", typeof(byte[])); } var row = m_testDataInsertTable.NewRow(); row["ClsName"] = clsInfo.Name; //row["Type"] = m_testDataType; row["Time"] = WekaUtils.GetTimeFromDate(nowDate); row["TestResult"] = str; row["ClassValue"] = scv; row["DealsInfo"] = Feng.Windows.Utils.SerializeHelper.Serialize(clsInfo.Deals); bool shouldSave = false; if (nowDate.DayOfWeek != DayOfWeek.Sunday && nowDate.DayOfWeek != DayOfWeek.Saturday && nowDate.Day == 20) { shouldSave = true; } if (nowDate.Day == 21 && nowDate.DayOfWeek == DayOfWeek.Monday) { shouldSave = true; } if (nowDate.Day == 22 && nowDate.DayOfWeek == DayOfWeek.Monday) { shouldSave = true; } if (TestParameters.OnlyNewestTestDataSaved) { shouldSave = true; } if (shouldSave) { byte[] dealsData = Feng.Windows.Utils.SerializeHelper.Serialize(clsInfo.Deals.Deals); System.IO.MemoryStream outStream = new System.IO.MemoryStream(); using (System.IO.Compression.GZipStream zipStream = new System.IO.Compression.GZipStream(outStream, System.IO.Compression.CompressionMode.Compress)) { System.IO.MemoryStream inStream = new System.IO.MemoryStream(dealsData); inStream.CopyTo(zipStream); } row["DealsData"] = outStream.ToArray(); } else { row["DealsData"] = System.DBNull.Value; } m_testDataInsertTable.Rows.Add(row); if (m_testDataInsertTable.Rows.Count > 100) { //WekaUtils.Instance.WriteLog("Save Test Data."); try { Feng.Data.DbHelper.Instance.BulkCopy(m_testDataInsertTable, m_testDataTableName); } catch (Exception ex) { WekaUtils.Instance.WriteLog(ex.Message); } m_testDataInsertTable.Rows.Clear(); } //SqlCommand cmd = new SqlCommand("INSERT INTO TestData ([ClsName],[Time], [Type], TestResult, ClassValue, DealsData) VALUES (@ClsName, @Time, @Type, @TestResult, @ClassValue, @DealsData)"); //cmd.Parameters.AddWithValue("@ClsName", clsInfo.Name); //cmd.Parameters.AddWithValue("@Type", m_testDataType); //cmd.Parameters.AddWithValue("@Time", WekaUtils.GetTimeFromDate(currentDate)); //cmd.Parameters.AddWithValue("@TestResult", str); //cmd.Parameters.AddWithValue("@ClassValue", scv); //cmd.Parameters.AddWithValue("@DealsData", Feng.Utils.SerializeHelper.Serialize(clsInfo.Deals)); //SetTransaction(cmd); }
public void LoadTestData(DateTime currentDate, Dictionary <string, CandidateClassifier> clsInfos) { //EndTransaction(); if (m_testData == null || currentDate >= m_testDataEndDate) { m_testDataEndDate = currentDate.AddDays(7); if (m_testData == null) { m_testData = new Dictionary <long, List <System.Data.DataRow> >(); } else { m_testData.Clear(); System.GC.Collect(); } WekaUtils.Instance.WriteLog(string.Format("Load Test Data to {0}", m_testDataEndDate)); //SqlCommand cmd = new SqlCommand("SELECT ClsName, TestResult, ClassValue, DealsData FROM TestData WHERE [Time] = @Time AND [Type] = @Type"); //cmd.Parameters.AddWithValue("@Time", GetTimeFromDate(currentDate)); //cmd.Parameters.AddWithValue("@Type", m_testDataType); //var ret = Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd); Dictionary <string, string> dict = new Dictionary <string, string>(); foreach (var kvp in clsInfos) { dict[kvp.Value.DealType + "_" + kvp.Value.Tp + "_" + kvp.Value.Sl] = null; } string[] clsNames = new string[dict.Count]; dict.Keys.CopyTo(clsNames, 0); string[] paramNames = clsNames.Select( (s, i) => "ClsName LIKE '" + s + "_%'" //"@tag" + i.ToString() ).ToArray(); string inClause = string.Join(" OR ", paramNames); string sql = ("SELECT Time, ClsName, TestResult, ClassValue, DealsInfo, DealsData FROM {0} WHERE [Time] >= @Time1 AND [Time] < @Time2 AND ({1})"); SqlCommand cmd = new SqlCommand(string.Format(sql, m_testDataTableName, inClause)); cmd.Parameters.AddWithValue("@Time1", WekaUtils.GetTimeFromDate(currentDate)); cmd.Parameters.AddWithValue("@Time2", WekaUtils.GetTimeFromDate(m_testDataEndDate)); //for (int i = 0; i < paramNames.Length; i++) //{ // cmd.Parameters.AddWithValue(paramNames[i], clsNames[i]); //} var ret = Feng.Data.DbHelper.Instance.ExecuteDataTable(cmd); foreach (System.Data.DataRow row in ret.Rows) { long time = (long)row["Time"]; if (!m_testData.ContainsKey(time)) { m_testData[time] = new List <System.Data.DataRow>(); } m_testData[time].Add(row); } } long ctime = WekaUtils.GetTimeFromDate(currentDate); if (m_testData.ContainsKey(ctime)) { foreach (System.Data.DataRow row in m_testData[ctime]) { string name = (string)row["ClsName"]; if (clsInfos.ContainsKey(name)) { if (row["DealsData"] == System.DBNull.Value) { clsInfos[name].SetData((string)row["TestResult"], (string)row["ClassValue"], (byte[])row["DealsInfo"], null); } else { clsInfos[name].SetData((string)row["TestResult"], (string)row["ClassValue"], (byte[])row["DealsInfo"], (byte[])row["DealsData"]); } } } } }
public static Dictionary <long, int> GetHpDateTimesCount(string symbol, string period) { double mainPeriodOfHour = WekaUtils.GetMinuteofPeriod(period) / 60.0; Dictionary <long, int> dictTimes = new Dictionary <long, int>(); int dealCount = 0; int n = TestParameters2.nTpsl; DateTime date = TestParameters2.TrainStartTime; DateTime maxDate = TestParameters2.TrainEndTime; string sql; System.Data.DataTable allDt = null; DateTime nextBufferDate = DateTime.MinValue; while (true) { Console.WriteLine(date.ToString(Parameters.DateTimeFormat) + ", " + dictTimes.Count + ", " + dealCount); if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday) { date = date.AddHours(mainPeriodOfHour); continue; } if (nextBufferDate <= date) { nextBufferDate = date.AddHours(mainPeriodOfHour * 100); sql = string.Format("SELECT * FROM {0}_HP WHERE TIME >= '{1}' AND TIME < '{2}' AND {3}", symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextBufferDate), string.IsNullOrEmpty(TestParameters.DbSelectWhere) ? "1 = 1" : TestParameters.DbSelectWhere); allDt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql); } DateTime nextDate = date.AddHours(mainPeriodOfHour); sql = string.Format("TIME >= '{1}' AND TIME < '{2}'", symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextDate)); var dt = allDt.Select(sql); bool isComplete = true; if ((TestParameters2.UsePartialHpData || isComplete) && dt.Length > 0) { foreach (System.Data.DataRow row in dt) { sbyte?[, ,] hp = HpData.DeserializeHp((byte[])row["hp"]); long?[, ,] hpTime = HpData.DeserializeHpTimes((byte[])row["hp_date"]); WekaUtils.DebugAssert(hp.GetLength(0) == 2, ""); WekaUtils.DebugAssert(hpTime.GetLength(0) == 2, ""); for (int j = 0; j < hp.GetLength(0); ++j) { for (int k = 0; k < hp.GetLength(1); ++k) { for (int l = 0; l < hp.GetLength(2); ++l) { if (k % n != n - 1 || l % n != n - 1) { continue; } if (!hp[j, k, l].HasValue || hp[j, k, l] == -1) { continue; } //if (k / n >= TestParameters2.tpCount) // continue; //if (l / n >= TestParameters2.slCount) // continue; if (dictTimes.ContainsKey(hpTime[j, k, l].Value)) { dictTimes[hpTime[j, k, l].Value]++; } else { dictTimes[hpTime[j, k, l].Value] = 1; } dealCount++; } } } } } date = nextDate; if (date >= maxDate) { break; } } return(dictTimes); }
public void GenerateHpDataToTxt(string symbol, string period) { double mainPeriodOfHour = WekaUtils.GetMinuteofPeriod(period) / 60.0; int n = TestParameters2.nTpsl; int m1 = TestParameters.TpMaxCount / n; int m2 = TestParameters.SlMaxCount / n; var dataDates = TestManager.GetDataDateRange(); DateTime date = dataDates[0]; DateTime maxDate = dataDates[1]; date = new DateTime(2000, 1, 1); string hpFileName = TestParameters.GetBaseFilePath(string.Format("{0}_{1}_hpdata.txt", symbol, period)); if (!TestParameters2.RealTimeMode && System.IO.File.Exists(hpFileName)) { return; } SortedDictionary <DateTime, string> dictAlready = new SortedDictionary <DateTime, string>(); string hpFileNameAlready = hpFileName + ".full"; if (!TestParameters2.DBDataConsistent) { if (System.IO.File.Exists(hpFileNameAlready)) { using (StreamReader sr = new StreamReader(hpFileNameAlready)) { while (!sr.EndOfStream) { string s = sr.ReadLine(); int idx = s.IndexOf(','); DateTime d = Convert.ToDateTime(s.Substring(0, idx)); string s2 = s.Substring(idx + 1); dictAlready[d] = s2.Trim(); } } } } string sql; System.Data.DataTable allDt = null; DateTime nextBufferDate = DateTime.MinValue; using (StreamWriter sw = new StreamWriter(hpFileName)) using (StreamWriter sw2 = new StreamWriter(hpFileNameAlready)) { while (true) { if (!TestParameters2.RealTimeMode) { Console.WriteLine(date.ToString(Parameters.DateTimeFormat)); } StringBuilder sb = new StringBuilder(); if (dictAlready.ContainsKey(date)) { sb.Append(date.ToString(Parameters.DateTimeFormat)); sb.Append(", "); sb.AppendLine(dictAlready[date]); sw.Write(sb.ToString()); sw2.Write(sb.ToString()); date = date.AddHours(mainPeriodOfHour); if (date >= maxDate) { break; } } else { if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday) { date = date.AddHours(mainPeriodOfHour); continue; } if (nextBufferDate <= date) { nextBufferDate = date.AddHours(mainPeriodOfHour); sql = string.Format("SELECT * FROM {0}_HP WHERE TIME >= '{1}' AND TIME < '{2}' AND {3}", symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextBufferDate), string.IsNullOrEmpty(TestParameters.DbSelectWhere) ? "1 = 1" : TestParameters.DbSelectWhere); //if (!TestParameters2.UsePartialHpData && TestParameters2.RealTimeMode) //{ // sql += " AND IsComplete = 1"; //} allDt = Feng.Data.DbHelper.Instance.ExecuteDataTable(sql); } DateTime nextDate = date.AddHours(mainPeriodOfHour); sql = string.Format("TIME >= '{1}' AND TIME < '{2}'", symbol, WekaUtils.GetTimeFromDate(date), WekaUtils.GetTimeFromDate(nextDate)); var dt = allDt.Select(sql); bool isComplete = true; foreach (System.Data.DataRow row in dt) { bool b = (bool)row["IsComplete"]; if (!b) { isComplete = false; break; } } if ((TestParameters2.UsePartialHpData || isComplete) && dt.Length > 0) { sb.Append(date.ToString(Parameters.DateTimeFormat)); sb.Append(", "); var hpAll = SumHp(dt); var hps = hpAll.Item1; var hpTimes = hpAll.Item2; for (int j = 0; j < hps.GetLength(0); ++j) { for (int k = 0; k < hps.GetLength(1); ++k) { for (int l = 0; l < hps.GetLength(2); ++l) { if (isComplete) { if (hps[j, k, l, 0] == -1 || hps[j, k, l, 1] == -1) { throw new AssertException("hps should not be -1."); } } if (hps[j, k, l, 0] == -1) { hps[j, k, l, 0] = 0; } if (hps[j, k, l, 1] == -1) { hps[j, k, l, 1] = 0; } sb.Append(hps[j, k, l, 0] + ", " + hps[j, k, l, 1] + ", "); } } } long maxHpTime = 0; for (int j = 0; j < hpTimes.GetLength(0); ++j) { for (int k = 0; k < hpTimes.GetLength(1); ++k) { for (int l = 0; l < hpTimes.GetLength(2); ++l) { if (isComplete) { if (hpTimes[j, k, l] == -1) { throw new AssertException("hpTimes should not be -1."); } } if (hpTimes[j, k, l] == -1) { hpTimes[j, k, l] = Parameters.MaxTime; } sb.Append(hpTimes[j, k, l] + ", "); maxHpTime = Math.Max(hpTimes[j, k, l], maxHpTime); } } } sb.AppendLine(); sw.Write(sb.ToString()); if (isComplete && !TestParameters2.DBDataConsistent) { sw2.Write(sb.ToString()); } } date = nextDate; if (date >= maxDate) { break; } } } } if (TestParameters2.DBDataConsistent) { File.Delete(hpFileNameAlready); } }
public void GenerateArff(string arffFileName, string candlePatternFile) { int preLength = TestParameters2.PreLength; int tpStart = TestParameters2.tpStart; int slStart = TestParameters2.slStart; int tpCount = TestParameters2.tpCount; int slCount = TestParameters2.slCount; var dataDates = TestManager.GetDataDateRange(); int minOutBegIdx = TaLibTest.minOutBegIdx; var cp = TestParameters2.CandidateParameter; int[, , ,] cds = null; ForexDataRows[,] hpdvs = new ForexDataRows[cp.SymbolCount, cp.PeriodCount]; for (int s = 0; s < cp.SymbolCount; ++s) { for (int p = 0; p < cp.PeriodCount; ++p) { string symbol = cp.AllSymbols[s + cp.SymbolStart]; string period = cp.AllPeriods[p + cp.PeriodStart]; string symbolPeriod = string.Format("{0}_{1}", symbol, period); hpdvs[s, p] = DbData.Instance.GetDbData(dataDates[0], dataDates[1], symbolPeriod, 0, true, cp); m_currentPeriod = period; //int minOutNBElement = m_minOutNBElements[m_currentPeriod]; int minOutNBElement = hpdvs[s, p].Length - minOutBegIdx; if (cds == null) { CreateCds(ref cds, cp, minOutNBElement + 100); } using (StreamReader sr = new StreamReader(candlePatternFile)) { int n = 0; while (!sr.EndOfStream) { string line = sr.ReadLine(); string[] ss = line.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); WekaUtils.DebugAssert(ss.Length == minOutNBElement, "ss.Length == minOutNBElement"); for (int i = 0; i < minOutNBElement; ++i) { cds[s, p, i, n] = Convert.ToInt32(ss[i]); } n++; } } } } if (File.Exists(arffFileName)) { System.IO.File.Delete(arffFileName); } string wekaFileName = string.Format(arffFileName); using (StreamWriter sw = new StreamWriter(wekaFileName)) { sw.WriteLine("@relation 'candlePatterns'"); sw.WriteLine("@attribute timestamp date \"yyyy-MM-dd\'T\'HH:mm:ss\""); sw.WriteLine("@attribute hpdate date \"yyyy-MM-dd\'T\'HH:mm:ss\""); sw.WriteLine("@attribute spread numeric"); sw.WriteLine("@attribute mainClose numeric"); for (int pre = 0; pre < preLength; ++pre) { for (int s = 0; s < cp.SymbolCount; ++s) { for (int p = 0; p < cp.PeriodCount; ++p) { for (int i = 0; i < 61; ++i) { sw.WriteLine(string.Format("@attribute {0}_{1}_{2}_p{3} {4}", cp.AllSymbols[s + cp.SymbolStart], cp.AllPeriods[p + cp.PeriodStart], i.ToString(), pre.ToString(), "{0,100,200,-100,-200}")); } } } } sw.WriteLine("@attribute prop " + " {0,1,2,3}"); sw.WriteLine("@data"); sw.WriteLine(); var hps = HpData.Instance.GetHpSum(cp.MainSymbol, cp.MainPeriod); var hpdv = hpdvs[0, 0]; for (int i = minOutBegIdx + preLength - 1; i < hpdv.Length; ++i) { DateTime nowDate = WekaUtils.GetDateFromTime((long)hpdv[i].Time); //if (nowDate.Hour % TestParameters2.MainPeriodOfHour != 0) // continue; if (!hps.ContainsKey(nowDate) && !(TestParameters2.RealTimeMode && i == hpdv.Length - 1)) { continue; } long hpTime = WekaUtils.GetTimeFromDate(Parameters.MaxDate); int hp = 0; if (!(TestParameters2.RealTimeMode && i == hpdv.Length - 1)) { hpTime = hps[nowDate].Item2; hp = hps[nowDate].Item1; } sw.Write(nowDate.ToString(Parameters.DateTimeFormat)); sw.Write(","); // hp sw.Write(WekaUtils.GetDateFromTime(hpTime).ToString(Parameters.DateTimeFormat)); sw.Write(","); sw.Write((int)hpdv[i]["spread"]); sw.Write(","); sw.Write(((double)hpdv[i]["close"]).ToString()); sw.Write(","); for (int pre = 0; pre < preLength; ++pre) { for (int s = 0; s < cp.SymbolCount; ++s) { for (int p = 0; p < cp.PeriodCount; ++p) { for (int j = 0; j < 61; ++j) { int candlePattern = cds[s, p, i - minOutBegIdx - preLength + pre + 1, j]; if (candlePattern == -1) { throw new AssertException(string.Format("candle pattern should not be -1.idx={0}", i - minOutBegIdx - preLength + pre + 1)); } sw.Write(candlePattern); sw.Write(","); } } } } sw.WriteLine(hp.ToString()); } } }
public static MqlRates[] ReadRates(string symbolPeriod) { string[] ss1 = symbolPeriod.Split('_'); string symbol = ss1[0]; string period = ss1[1] == "M1" ? "min_1" : ss1[1]; string rateFileName = string.Format("E:\\Forex\\Dukas\\{0}_BID_candles_{1}.csv", symbol, period); List <MqlRates> rates = new List <MqlRates>(); using (System.IO.StreamReader sr = new StreamReader(rateFileName)) { while (true) { if (sr.EndOfStream) { break; } // Date Time, Open, Close, Low, High, Volume string s = sr.ReadLine(); string[] ss = s.Split(new char[] { ',' }); long v = (long)Convert.ToDouble(ss[5]); DateTime date; //date = Convert.ToDateTime(ss[0]); bool r = DateTime.TryParseExact(ss[0], new string[] { "dd.MM.yyyy HH:mm:ss.000", "yyyy.MM.dd HH:mm:ss" }, null, System.Globalization.DateTimeStyles.None, out date); if (!r) { throw new InvalidCastException(string.Format("{0} is invalid as datetime", ss[0])); } if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday) { continue; } ////if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday) //{ // if (v == 0) // { // if (ss[1] == ss[2] && ss[2] == ss[3] && ss[3] == ss[4]) // { // DateTime d1 = date.AddHours(5); // DateTime d2 = date.AddHours(-5); // if ((d1.DayOfWeek == DayOfWeek.Saturday || d1.DayOfWeek == DayOfWeek.Sunday) // && (d2.DayOfWeek == DayOfWeek.Saturday || d2.DayOfWeek == DayOfWeek.Sunday)) // continue; // } // else // { // System.Console.WriteLine(s); // } // } // else // { // } //} rates.Add(new MqlRates { time = WekaUtils.GetTimeFromDate(date), open = Convert.ToDouble(ss[1]), high = Convert.ToDouble(ss[4]), low = Convert.ToDouble(ss[3]), close = Convert.ToDouble(ss[2]), tick_volume = 0, spread = 50, real_volume = (long)Convert.ToDouble(ss[5]), }); } } return(rates.ToArray()); }