public static void FilterInstances(weka.core.Instances allInstances) { DateTime nextHpDate = DateTime.MinValue; java.util.LinkedList deleteList = new java.util.LinkedList(); for (int i = 0; i < allInstances.numInstances(); ++i) { DateTime nowDate = WekaUtils.GetDateValueFromInstances(allInstances, 0, i); if (TestParameters2.RealTimeMode && i == allInstances.numInstances() - 1) { allInstances.instance(i).setClassValue(0); allInstances.instance(i).setValue(1, WekaUtils.GetTimeFromDate(Parameters.MaxDate) * 1000); } else { if (nowDate < nextHpDate) { deleteList.Add(allInstances.instance(i)); } else { DateTime hpDate = WekaUtils.GetDateValueFromInstances(allInstances, 1, i); nextHpDate = hpDate; } } } allInstances.removeAll(deleteList); }
//public void ConvertHpDataPeriod(string symbol, string periodSsrc, string periodDest) //{ // string hpFileName = TestParameters.GetBaseFilePath(string.Format("{0}_{1}_hpdata.txt", symbol, periodSsrc)); // var hpData = GetHpDateFromTxt(hpFileName, TestParameters2.nTpsl); // string hpFileName2 = TestParameters.GetBaseFilePath(string.Format("{0}_{1}_hpdata.txt", symbol, periodDest)); // int m = TestParameters.TpSlMaxCount / TestParameters2.nTpsl; // DateTime date = TestParameters2.TrainStartTime; // DateTime maxDate = TestParameters2.TrainEndTime; // using(StreamWriter sw = new StreamWriter(hpFileName2)) // { // while (true) // { // if (!TestParameters2.RealTimeMode) // { // Console.WriteLine(date.ToString(Parameters.DateTimeFormat)); // } // StringBuilder sb = new StringBuilder(); // { // if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday) // { // date = date.AddHours(mainPeriodOfHour); // continue; // } // bool isComplete = true; // if ((TestParameters2.UsePartialHpData || isComplete)) // { // sb.Append(date.ToString(Parameters.DateTimeFormat)); // sb.Append(", "); // int[, , ,] hps = new int[2, m, m, 2]; // long[, ,] hpTimes = new long[2, m, m]; // for (int j = 0; j < hps.GetLength(0); ++j) // for (int k = 0; k < hps.GetLength(1); ++k) // for (int l = 0; l < hps.GetLength(2); ++l) // { // hps[j, k, l, 0] = hps[j, k, l, 1] = -1; // hpTimes[j, k, l] = -1; // } // List<DateTime> dts = new List<DateTime>(); // foreach (DateTime dt in dts) // { // if (!hpData.ContainsKey(dt)) // continue; // sbyte?[, ,] hp = hpData[dt].Item1; // long[, ,] hpTime = hpData[dt].Item2; // WekaUtils.DebugAssert(hp.GetLength(0) == 2, ""); // WekaUtils.DebugAssert(hp.GetLength(1) % m == 0, ""); // WekaUtils.DebugAssert(hp.GetLength(2) % m == 0, ""); // WekaUtils.DebugAssert(hpTime.GetLength(0) == 2, ""); // for (int j = 0; j < hp.GetLength(0); ++j) // for (int k = 0; k < hp.GetLength(1); ++k) // for (int l = 0; l < hp.GetLength(2); ++l) // { // if (k % n != n - 1 || l % n != n - 1) // continue; // if (!hp[j, k, l].HasValue || hp[j, k, l] == -1) // continue; // //if (k / n >= TestParameters2.tpCount) // // continue; // //if (l / n >= TestParameters2.slCount) // // continue; // if (hps[j, k / n, l / n, 0] == -1) // hps[j, k / n, l / n, 0] = 0; // if (hps[j, k / n, l / n, 1] == -1) // hps[j, k / n, l / n, 1] = 0; // if (hp[j, k, l] == 1) // hps[j, k / n, l / n, 0]++; // else if (hp[j, k, l] == 0) // hps[j, k / n, l / n, 1]++; // else // throw new AssertException("hp should be 0 or 1."); // hpTimes[j, k / n, l / n] = Math.Max(hpTimes[j, k / n, l / n], hpTime[j, k, l].Value); // } // } // for (int j = 0; j < hps.GetLength(0); ++j) // for (int k = 0; k < hps.GetLength(1); ++k) // for (int l = 0; l < hps.GetLength(2); ++l) // { // if (isComplete) // { // if (hps[j, k, l, 0] == -1 || hps[j, k, l, 1] == -1) // { // throw new AssertException("hps should not be -1."); // } // } // if (hps[j, k, l, 0] == -1) // hps[j, k, l, 0] = 0; // if (hps[j, k, l, 1] == -1) // hps[j, k, l, 1] = 0; // sb.Append(hps[j, k, l, 0] + ", " + hps[j, k, l, 1] + ", "); // } // long maxHpTime = 0; // for (int j = 0; j < hpTimes.GetLength(0); ++j) // for (int k = 0; k < hpTimes.GetLength(1); ++k) // for (int l = 0; l < hpTimes.GetLength(2); ++l) // { // if (isComplete) // { // if (hpTimes[j, k, l] == -1) // { // throw new AssertException("hpTimes should not be -1."); // } // } // if (hpTimes[j, k, l] == -1) // hpTimes[j, k, l] = Parameters.MaxTime; // sb.Append(hpTimes[j, k, l] + ", "); // maxHpTime = Math.Max(hpTimes[j, k, l], maxHpTime); // } // sb.AppendLine(); // sw.Write(sb.ToString()); // } // DateTime nextDate = date.AddMinutes(WekaUtils.GetMinuteofPeriod(periodDest)); // date = nextDate; // if (date >= maxDate) // break; // } // } // } //} public static Dictionary <long, int> GetHpDateTimesCountFromInstance(string fileName) { Dictionary <long, int> dictTimes = new Dictionary <long, int>(); int dealCount = 0; var instances = WekaUtils.LoadInstances(TestParameters.GetBaseFilePath(fileName)); for (int i = 0; i < instances.numInstances(); ++i) { var date = WekaUtils.GetDateValueFromInstances(instances, 0, i); var hpdate = WekaUtils.GetDateValueFromInstances(instances, 1, i); var hptime = WekaUtils.GetTimeFromDate(hpdate); if (dictTimes.ContainsKey(hptime)) { dictTimes[hptime]++; } else { dictTimes[hptime] = 1; } dealCount++; Console.WriteLine(date.ToString(Parameters.DateTimeFormat) + ", " + dictTimes.Count + ", " + dealCount); } return(dictTimes); }
public void ExecuteBest(DateTime nowDate, List <CandidateClassifier> minScoreInfos) { OutputRealDealSummary(nowDate); float totalCost = 0; int totalDeal = 0; if (minScoreInfos != null) { List <DealInfo> candidateDeals = new List <DealInfo>(); foreach (var minScoreInfo in minScoreInfos) { minScoreInfo.WekaData.GenerateData(false, true); weka.core.Instances minTestInstances = minScoreInfo.WekaData.CurrentTestInstances; weka.core.Instances minTestInstancesNew = minScoreInfo.WekaData.CurrentTestInstancesNew; if (minTestInstances.numInstances() > 0) { MyEvaluation eval = new MyEvaluation(minScoreInfo.CostMatrix); eval.evaluateModel(minScoreInfo.CurrentTestRet, minScoreInfo.CurrentClassValue); float vol; // vol = (float)minScoreInfo.MoneyManagement.GetVolume(null); vol = 0.1F; //vol = (float)Math.Round(minScore / -20000.0, 1); //WekaUtils.DebugAssert(vol > 0); int tp = (int)eval.numTruePositives(1); int fp = (int)eval.numFalsePositives(1); double minScore = minScoreInfo.Deals.NowScore; //WekaUtils.Instance.WriteLog(string.Format("Best Classifier: N={0},TC={1},TP={2},FP={3},TD={4},TV={5},TTP={6},TFP={7},", // minScoreInfo.Name, minScoreInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString), // minScoreInfo.Deals.NowTp, minScoreInfo.Deals.NowFp, minScoreInfo.Deals.NowDeal, // minScoreInfo.Deals.TotalVolume.ToString("N2"), tp, fp), // true, ConsoleColor.DarkGreen); // Exclude //if (TestParameters.EnableExcludeClassifier) //{ //if (minScoreInfo.ExcludeClassifier == null) //{ // string modelFileName4Exclude = GetExcludeModelFileName(minScoreInfo.Name); // minScoreInfo.ExcludeClassifier = WekaUtils.TryLoadClassifier(modelFileName4Exclude); //} //} for (int i = 0; i < minScoreInfo.CurrentTestRet.Length; i++) { if (minScoreInfo.CurrentTestRet[i] == 2) { if (minScoreInfo.ExcludeClassifier != null) { double cv2 = minScoreInfo.ExcludeClassifier.classifyInstance(minTestInstancesNew.instance(i)); if (cv2 != 2) { continue; } } candidateDeals.Add(new DealInfo(WekaUtils.GetDateValueFromInstances(minTestInstances, 0, i), (float)WekaUtils.GetValueFromInstance(minTestInstances, "mainClose", i), minScoreInfo.DealType, (float)vol, (float)(minScoreInfo.CurrentClassValue[i] == 2 ? -minScoreInfo.Tp : minScoreInfo.Sl), WekaUtils.GetDateValueFromInstances(minTestInstances, 1, i))); } } float nowCost = (float)eval.totalCost(); int nowDeal = tp + fp; totalCost += nowCost * vol; totalDeal += nowDeal; //float diff = Math.Abs(totalCost - realDealsInfo.TotalCost); //WekaUtils.DebugAssert(diff < 5); //WekaUtils.DebugAssert(Math.Abs(totalDeal - (realDealsInfo.NowDeal + realDealsInfo.CurrentDeal)) == 0); //if (diff > 0.5) // totalCost = realDealsInfo.TotalCost; IterateClassifierInfos((k, i, j, h) => { if (m_classifierInfoIdxs[k, i, j, h] == minScoreInfo) { m_totalScores[k, i, j, h] += nowCost; m_totalDeals[k, i, j, h] += nowDeal; return(false); } else { return(true); } }); } } int selectCount = 50; for (int i = 0; i < selectCount; ++i) { if (candidateDeals.Count == 0) { break; } int selectedDealIdx = (int)Math.Round(m_randomGenerator.NextDouble() * candidateDeals.Count); if (selectedDealIdx == candidateDeals.Count) { selectedDealIdx = candidateDeals.Count - 1; } m_realDealsInfo.AddDeal(candidateDeals[selectedDealIdx]); candidateDeals.RemoveAt(selectedDealIdx); } } //if (m_enableDetailLogLevel2) //{ // IterateClassifierInfos((k, i, j, h) => // { // if (h == m_currentTestHour && m_totalScores[k, i, j, h] != 0) // { // WekaUtils.Instance.WriteLog(string.Format("Predict score for {0}: TC={1}, TD={2}", m_classifierInfoIdxs[k, i, j, h].Name, m_totalScores[k, i, j, h].ToString(Parameters.DoubleFormatString), m_totalDeals[k, i, j, h])); // } // }); //} if (!string.IsNullOrEmpty(currentSummary)) { WekaUtils.Instance.WriteLog(currentSummary, true, ConsoleColor.Red); System.Console.Title = nowDate.ToString(Parameters.DateTimeFormat) + ":" + currentSummary; } //if (m_enableDetailLogLevel2) //{ // if (TestParameters.EnablePerhourTrain) // { // double[] totalCostPerHour = new double[m_classifierInfoIdxs.GetLength(3)]; // int[] totalDealPerHour = new int[m_classifierInfoIdxs.GetLength(3)]; // IterateClassifierInfos((k, i, j, h) => // { // totalCostPerHour[h] += m_totalScores[k, i, j, h]; // totalDealPerHour[h] += m_totalDeals[k, i, j, h]; // }); // for (int i = 0; i < totalCostPerHour.Length; ++i) // { // if (totalCostPerHour[i] == 0) // continue; // WekaUtils.Instance.WriteLog(string.Format("Predict score Per hour of {0}: TC={1}, TD={2}", i, totalCostPerHour[i].ToString(Parameters.DoubleFormatString), totalDealPerHour[i])); // } // } //if ((m_testTimeStart.Month == 4 || m_testTimeStart.Month == 3) && m_testTimeStart.Day == 20) //{ // foreach (var cls in m_classifierInfos) // { // WekaUtils.Instance.WriteLog(cls.Value.Deals.PrintAll()); // } //} //if (m_currentTestHour == 0) //{ // var cls = m_classifierInfoIdxs[0, 0, 0, 0]; // WekaUtils.Instance.WriteLog((cls.Classifier as RandomClassifier).GetCountInfo()); //} //} //if (m_saveDataFile) //{ // var files = System.IO.Directory.GetFiles(m_baseDir, "*.arff"); // if (files.Length > 100) // { // foreach (string fileName in files) // { // try // { // System.IO.File.Delete(fileName); // } // catch (Exception) // { // } // } // } //} }
public void ExecuteCandidate(DateTime nowDate) { m_currentTestHour = nowDate.Hour; foreach (var kvp in m_classifierInfos) { kvp.Value.Initialized = false; } if (TestParameters.EnnableLoadTestData) { if (!TestParameters.OnlyNewestTestDataSaved) { CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos); } else { DateTime?maxTestDate = CCScoreData.Instance.GetNewestTestData(); if (maxTestDate.HasValue && maxTestDate.Value > nowDate) { return; } if (maxTestDate.HasValue && maxTestDate.Value == nowDate) { CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos); } } } WekaData.GenerateArffTemplate(true, true, this.CandidateParameter); string candidateClsInfoSummary = "CC:N={0},TrN={1},TeN={4},NC={7},NTP={8},NFP={9},NDA={14},NDS={15},TD={10},TV={11},Cls={12},MM={13}"; //Instances testInstancesWithoutClassValue = null; / wrong, should use closeTime System.Threading.Tasks.Task[] tasks; System.Threading.Tasks.TaskFactory taskFactory = null; if (TestParameters.EnableMultiThread) { int cpuCount = Environment.ProcessorCount - 2; cpuCount = Math.Max(cpuCount, 1); LimitedConcurrencyLevelTaskScheduler lcts = new LimitedConcurrencyLevelTaskScheduler(1); taskFactory = new System.Threading.Tasks.TaskFactory(lcts); } if (!TestParameters.EnablePerhourTrain) { tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count]; } else { tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count / Parameters.AllHour]; } int taskIdx = 0; foreach (var kvp in m_classifierInfos) { if (TestParameters.EnablePerhourTrain) { if (kvp.Value.Hour != m_currentTestHour) { continue; } } var clsInfo = kvp.Value; Action action = () => { if (clsInfo.Initialized || nowDate > System.DateTime.Now) { if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name, -1, 0, 0, clsInfo.CurrentClassValue.Length, 0, 0, clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString), clsInfo.Deals.NowTp, clsInfo.Deals.NowFp, clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"), clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(), clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(), clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd)); } } else { weka.core.Instances trainInstances; weka.core.Instances testInstances; bool noData = false; if (TestParameters.UseTrain) { clsInfo.WekaData.GenerateData(true, true); trainInstances = clsInfo.WekaData.CurrentTrainInstances; testInstances = clsInfo.WekaData.CurrentTestInstances; if (trainInstances.numInstances() == 0) { if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format("{0} - No Train Data", clsInfo.Name)); } noData = true; } } else { clsInfo.WekaData.GenerateData(false, true); trainInstances = WekaData.GetTrainInstancesTemplate(this.CandidateParameter.Name); testInstances = clsInfo.WekaData.CurrentTestInstances; } if (testInstances.numInstances() == 0) { if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format("{0} - No Test Data", clsInfo.Name)); } noData = true; } if (!noData) { clsInfo.WekaData.TrainandTest(clsInfo, this.CandidateParameter); clsInfo.Deals.Now(nowDate, WekaUtils.GetValueFromInstance(testInstances, "mainClose", 0)); if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name, trainInstances.numInstances(), WekaUtils.GetDateValueFromInstances(trainInstances, 0, 0), WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainInstances.numInstances() - 1), testInstances.numInstances(), WekaUtils.GetDateValueFromInstances(testInstances, 0, 0), WekaUtils.GetDateValueFromInstances(testInstances, 0, testInstances.numInstances() - 1), clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString), clsInfo.Deals.NowTp, clsInfo.Deals.NowFp, clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"), clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(), clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(), clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd)); } } else { clsInfo.CurrentClassValue = Parameters.DoubleArrayEmpty; clsInfo.CurrentTestRet = Parameters.DoubleArrayEmpty; clsInfo.Deals.Now(nowDate, null); if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name, trainInstances.numInstances(), 0, 0, testInstances.numInstances(), 0, 0, clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString), clsInfo.Deals.NowTp, clsInfo.Deals.NowFp, clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"), clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(), clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(), clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd)); } } if (TestParameters.EnnableLoadTestData) { CCScoreData.Instance.SaveTestData(clsInfo, nowDate); } if (TestParameters.EnableDetailLog) { // Check TestInstance Result SortedDictionary <int, int> closeNums = new SortedDictionary <int, int>(); for (int i = 0; i < testInstances.numInstances(); ++i) { if (clsInfo.CurrentTestRet[i] != 2) { continue; } DateTime openDate = WekaUtils.GetDateValueFromInstances(testInstances, 0, i); DateTime closeDate = WekaUtils.GetDateValueFromInstances(testInstances, 1, i); WekaUtils.DebugAssert(openDate >= nowDate, "openDate >= nowDate"); int n = (int)(closeDate - nowDate).TotalHours / (TestParameters.BatchTestMinutes / 60) + 1; if (!closeNums.ContainsKey(n)) { closeNums[n] = 1; } else { closeNums[n]++; } } foreach (var cn in closeNums) { WekaUtils.Instance.WriteLog(string.Format(" Next {0} Period-{1} has {2} deals", cn.Key, nowDate.AddHours(TestParameters.BatchTestMinutes / 60 * cn.Key), cn.Value)); } } } clsInfo.WekaData.Clear(); }; if (TestParameters.EnableMultiThread) { tasks[taskIdx] = taskFactory.StartNew(action); taskIdx++; } else { action(); } } if (TestParameters.EnableMultiThread) { try { System.Threading.Tasks.Task.WaitAll(tasks); } catch (AggregateException ex) { foreach (var i in ex.InnerExceptions) { WekaUtils.Instance.WriteLog(i.Message); WekaUtils.Instance.WriteLog(i.StackTrace); } } } if (TestParameters.SaveCCScoresToDb) { CCScoreData.SaveCCScoresToDb(nowDate, this.CandidateParameter, this); } }