public void ExecuteCandidate(DateTime nowDate) { m_currentTestHour = nowDate.Hour; foreach (var kvp in m_classifierInfos) { kvp.Value.Initialized = false; } if (TestParameters.EnnableLoadTestData) { if (!TestParameters.OnlyNewestTestDataSaved) { CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos); } else { DateTime? maxTestDate = CCScoreData.Instance.GetNewestTestData(); if (maxTestDate.HasValue && maxTestDate.Value > nowDate) return; if (maxTestDate.HasValue && maxTestDate.Value == nowDate) CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos); } } WekaData.GenerateArffTemplate(true, true, this.CandidateParameter); string candidateClsInfoSummary = "CC:N={0},TrN={1},TeN={4},NC={7},NTP={8},NFP={9},NDA={14},NDS={15},TD={10},TV={11},Cls={12},MM={13}"; //Instances testInstancesWithoutClassValue = null; / wrong, should use closeTime System.Threading.Tasks.Task[] tasks; System.Threading.Tasks.TaskFactory taskFactory = null; if (TestParameters.EnableMultiThread) { int cpuCount = Environment.ProcessorCount - 2; cpuCount = Math.Max(cpuCount, 1); LimitedConcurrencyLevelTaskScheduler lcts = new LimitedConcurrencyLevelTaskScheduler(1); taskFactory = new System.Threading.Tasks.TaskFactory(lcts); } if (!TestParameters.EnablePerhourTrain) { tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count]; } else { tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count / Parameters.AllHour]; } int taskIdx = 0; foreach (var kvp in m_classifierInfos) { if (TestParameters.EnablePerhourTrain) { if (kvp.Value.Hour != m_currentTestHour) continue; } var clsInfo = kvp.Value; Action action = () => { if (clsInfo.Initialized || nowDate > System.DateTime.Now) { if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name, -1, 0, 0, clsInfo.CurrentClassValue.Length, 0, 0, clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString), clsInfo.Deals.NowTp, clsInfo.Deals.NowFp, clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"), clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(), clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(), clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd)); } } else { weka.core.Instances trainInstances; weka.core.Instances testInstances; bool noData = false; if (TestParameters.UseTrain) { clsInfo.WekaData.GenerateData(true, true); trainInstances = clsInfo.WekaData.CurrentTrainInstances; testInstances = clsInfo.WekaData.CurrentTestInstances; if (trainInstances.numInstances() == 0) { if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format("{0} - No Train Data", clsInfo.Name)); } noData = true; } } else { clsInfo.WekaData.GenerateData(false, true); trainInstances = WekaData.GetTrainInstancesTemplate(this.CandidateParameter.Name); testInstances = clsInfo.WekaData.CurrentTestInstances; } if (testInstances.numInstances() == 0) { if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format("{0} - No Test Data", clsInfo.Name)); } noData = true; } if (!noData) { clsInfo.WekaData.TrainandTest(clsInfo, this.CandidateParameter); clsInfo.Deals.Now(nowDate, WekaUtils.GetValueFromInstance(testInstances, "mainClose", 0)); if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name, trainInstances.numInstances(), WekaUtils.GetDateValueFromInstances(trainInstances, 0, 0), WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainInstances.numInstances() - 1), testInstances.numInstances(), WekaUtils.GetDateValueFromInstances(testInstances, 0, 0), WekaUtils.GetDateValueFromInstances(testInstances, 0, testInstances.numInstances() - 1), clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString), clsInfo.Deals.NowTp, clsInfo.Deals.NowFp, clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"), clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(), clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(), clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd)); } } else { clsInfo.CurrentClassValue = Parameters.DoubleArrayEmpty; clsInfo.CurrentTestRet = Parameters.DoubleArrayEmpty; clsInfo.Deals.Now(nowDate, null); if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name, trainInstances.numInstances(), 0, 0, testInstances.numInstances(), 0, 0, clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString), clsInfo.Deals.NowTp, clsInfo.Deals.NowFp, clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"), clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(), clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(), clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd)); } } if (TestParameters.EnnableLoadTestData) { CCScoreData.Instance.SaveTestData(clsInfo, nowDate); } if (TestParameters.EnableDetailLog) { // Check TestInstance Result SortedDictionary<int, int> closeNums = new SortedDictionary<int, int>(); for (int i = 0; i < testInstances.numInstances(); ++i) { if (clsInfo.CurrentTestRet[i] != 2) continue; DateTime openDate = WekaUtils.GetDateValueFromInstances(testInstances, 0, i); DateTime closeDate = WekaUtils.GetDateValueFromInstances(testInstances, 1, i); WekaUtils.DebugAssert(openDate >= nowDate, "openDate >= nowDate"); int n = (int)(closeDate - nowDate).TotalHours / (TestParameters.BatchTestMinutes / 60) + 1; if (!closeNums.ContainsKey(n)) { closeNums[n] = 1; } else { closeNums[n]++; } } foreach (var cn in closeNums) { WekaUtils.Instance.WriteLog(string.Format(" Next {0} Period-{1} has {2} deals", cn.Key, nowDate.AddHours(TestParameters.BatchTestMinutes / 60 * cn.Key), cn.Value)); } } } clsInfo.WekaData.Clear(); }; if (TestParameters.EnableMultiThread) { tasks[taskIdx] = taskFactory.StartNew(action); taskIdx++; } else { action(); } } if (TestParameters.EnableMultiThread) { try { System.Threading.Tasks.Task.WaitAll(tasks); } catch (AggregateException ex) { foreach (var i in ex.InnerExceptions) { WekaUtils.Instance.WriteLog(i.Message); WekaUtils.Instance.WriteLog(i.StackTrace); } } } if (TestParameters.SaveCCScoresToDb) { CCScoreData.SaveCCScoresToDb(nowDate, this.CandidateParameter, this); } }
public void ExecuteCandidate(DateTime nowDate) { m_currentTestHour = nowDate.Hour; foreach (var kvp in m_classifierInfos) { kvp.Value.Initialized = false; } if (TestParameters.EnnableLoadTestData) { if (!TestParameters.OnlyNewestTestDataSaved) { CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos); } else { DateTime?maxTestDate = CCScoreData.Instance.GetNewestTestData(); if (maxTestDate.HasValue && maxTestDate.Value > nowDate) { return; } if (maxTestDate.HasValue && maxTestDate.Value == nowDate) { CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos); } } } WekaData.GenerateArffTemplate(true, true, this.CandidateParameter); string candidateClsInfoSummary = "CC:N={0},TrN={1},TeN={4},NC={7},NTP={8},NFP={9},NDA={14},NDS={15},TD={10},TV={11},Cls={12},MM={13}"; //Instances testInstancesWithoutClassValue = null; / wrong, should use closeTime System.Threading.Tasks.Task[] tasks; System.Threading.Tasks.TaskFactory taskFactory = null; if (TestParameters.EnableMultiThread) { int cpuCount = Environment.ProcessorCount - 2; cpuCount = Math.Max(cpuCount, 1); LimitedConcurrencyLevelTaskScheduler lcts = new LimitedConcurrencyLevelTaskScheduler(1); taskFactory = new System.Threading.Tasks.TaskFactory(lcts); } if (!TestParameters.EnablePerhourTrain) { tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count]; } else { tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count / Parameters.AllHour]; } int taskIdx = 0; foreach (var kvp in m_classifierInfos) { if (TestParameters.EnablePerhourTrain) { if (kvp.Value.Hour != m_currentTestHour) { continue; } } var clsInfo = kvp.Value; Action action = () => { if (clsInfo.Initialized || nowDate > System.DateTime.Now) { if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name, -1, 0, 0, clsInfo.CurrentClassValue.Length, 0, 0, clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString), clsInfo.Deals.NowTp, clsInfo.Deals.NowFp, clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"), clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(), clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(), clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd)); } } else { weka.core.Instances trainInstances; weka.core.Instances testInstances; bool noData = false; if (TestParameters.UseTrain) { clsInfo.WekaData.GenerateData(true, true); trainInstances = clsInfo.WekaData.CurrentTrainInstances; testInstances = clsInfo.WekaData.CurrentTestInstances; if (trainInstances.numInstances() == 0) { if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format("{0} - No Train Data", clsInfo.Name)); } noData = true; } } else { clsInfo.WekaData.GenerateData(false, true); trainInstances = WekaData.GetTrainInstancesTemplate(this.CandidateParameter.Name); testInstances = clsInfo.WekaData.CurrentTestInstances; } if (testInstances.numInstances() == 0) { if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format("{0} - No Test Data", clsInfo.Name)); } noData = true; } if (!noData) { clsInfo.WekaData.TrainandTest(clsInfo, this.CandidateParameter); clsInfo.Deals.Now(nowDate, WekaUtils.GetValueFromInstance(testInstances, "mainClose", 0)); if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name, trainInstances.numInstances(), WekaUtils.GetDateValueFromInstances(trainInstances, 0, 0), WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainInstances.numInstances() - 1), testInstances.numInstances(), WekaUtils.GetDateValueFromInstances(testInstances, 0, 0), WekaUtils.GetDateValueFromInstances(testInstances, 0, testInstances.numInstances() - 1), clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString), clsInfo.Deals.NowTp, clsInfo.Deals.NowFp, clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"), clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(), clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(), clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd)); } } else { clsInfo.CurrentClassValue = Parameters.DoubleArrayEmpty; clsInfo.CurrentTestRet = Parameters.DoubleArrayEmpty; clsInfo.Deals.Now(nowDate, null); if (m_enableDetailLogLevel2) { WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name, trainInstances.numInstances(), 0, 0, testInstances.numInstances(), 0, 0, clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString), clsInfo.Deals.NowTp, clsInfo.Deals.NowFp, clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"), clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(), clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(), clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd)); } } if (TestParameters.EnnableLoadTestData) { CCScoreData.Instance.SaveTestData(clsInfo, nowDate); } if (TestParameters.EnableDetailLog) { // Check TestInstance Result SortedDictionary <int, int> closeNums = new SortedDictionary <int, int>(); for (int i = 0; i < testInstances.numInstances(); ++i) { if (clsInfo.CurrentTestRet[i] != 2) { continue; } DateTime openDate = WekaUtils.GetDateValueFromInstances(testInstances, 0, i); DateTime closeDate = WekaUtils.GetDateValueFromInstances(testInstances, 1, i); WekaUtils.DebugAssert(openDate >= nowDate, "openDate >= nowDate"); int n = (int)(closeDate - nowDate).TotalHours / (TestParameters.BatchTestMinutes / 60) + 1; if (!closeNums.ContainsKey(n)) { closeNums[n] = 1; } else { closeNums[n]++; } } foreach (var cn in closeNums) { WekaUtils.Instance.WriteLog(string.Format(" Next {0} Period-{1} has {2} deals", cn.Key, nowDate.AddHours(TestParameters.BatchTestMinutes / 60 * cn.Key), cn.Value)); } } } clsInfo.WekaData.Clear(); }; if (TestParameters.EnableMultiThread) { tasks[taskIdx] = taskFactory.StartNew(action); taskIdx++; } else { action(); } } if (TestParameters.EnableMultiThread) { try { System.Threading.Tasks.Task.WaitAll(tasks); } catch (AggregateException ex) { foreach (var i in ex.InnerExceptions) { WekaUtils.Instance.WriteLog(i.Message); WekaUtils.Instance.WriteLog(i.StackTrace); } } } if (TestParameters.SaveCCScoresToDb) { CCScoreData.SaveCCScoresToDb(nowDate, this.CandidateParameter, this); } }