Пример #1
0
        public void ExecuteCandidate(DateTime nowDate)
        {
            m_currentTestHour = nowDate.Hour;

            foreach (var kvp in m_classifierInfos)
            {
                kvp.Value.Initialized = false;
            }
            if (TestParameters.EnnableLoadTestData)
            {
                if (!TestParameters.OnlyNewestTestDataSaved)
                {
                    CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos);
                }
                else
                {
                    DateTime? maxTestDate = CCScoreData.Instance.GetNewestTestData();
                    if (maxTestDate.HasValue && maxTestDate.Value > nowDate)
                        return;
                    if (maxTestDate.HasValue && maxTestDate.Value == nowDate)
                        CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos);
                }
            }

            WekaData.GenerateArffTemplate(true, true, this.CandidateParameter);

            string candidateClsInfoSummary = "CC:N={0},TrN={1},TeN={4},NC={7},NTP={8},NFP={9},NDA={14},NDS={15},TD={10},TV={11},Cls={12},MM={13}";
            //Instances testInstancesWithoutClassValue = null;  / wrong, should use closeTime

            System.Threading.Tasks.Task[] tasks;
            System.Threading.Tasks.TaskFactory taskFactory = null;
            if (TestParameters.EnableMultiThread)
            {
                int cpuCount = Environment.ProcessorCount - 2;
                cpuCount = Math.Max(cpuCount, 1);
                LimitedConcurrencyLevelTaskScheduler lcts = new LimitedConcurrencyLevelTaskScheduler(1);
                taskFactory = new System.Threading.Tasks.TaskFactory(lcts);
            }
            if (!TestParameters.EnablePerhourTrain)
            {
                tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count];
            }
            else
            {
                tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count / Parameters.AllHour];
            }
            int taskIdx = 0;
            foreach (var kvp in m_classifierInfos)
            {
                if (TestParameters.EnablePerhourTrain)
                {
                    if (kvp.Value.Hour != m_currentTestHour)
                        continue;
                }
                var clsInfo = kvp.Value;

                Action action = () =>
                    {
                        if (clsInfo.Initialized || nowDate > System.DateTime.Now)
                        {
                            if (m_enableDetailLogLevel2)
                            {
                                WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
                                    -1, 0, 0,
                                    clsInfo.CurrentClassValue.Length, 0, 0,
                                    clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
                                    clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
                                    clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
                                    clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
                                    clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
                                    clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
                            }
                        }
                        else
                        {
                            weka.core.Instances trainInstances;
                            weka.core.Instances testInstances;
                            bool noData = false;
                            if (TestParameters.UseTrain)
                            {
                                clsInfo.WekaData.GenerateData(true, true);
                                trainInstances = clsInfo.WekaData.CurrentTrainInstances;
                                testInstances = clsInfo.WekaData.CurrentTestInstances;

                                if (trainInstances.numInstances() == 0)
                                {
                                    if (m_enableDetailLogLevel2)
                                    {
                                        WekaUtils.Instance.WriteLog(string.Format("{0} - No Train Data", clsInfo.Name));
                                    }
                                    noData = true;
                                }
                            }
                            else
                            {
                                clsInfo.WekaData.GenerateData(false, true);
                                trainInstances = WekaData.GetTrainInstancesTemplate(this.CandidateParameter.Name);
                                testInstances = clsInfo.WekaData.CurrentTestInstances;
                            }

                            if (testInstances.numInstances() == 0)
                            {
                                if (m_enableDetailLogLevel2)
                                {
                                    WekaUtils.Instance.WriteLog(string.Format("{0} - No Test Data", clsInfo.Name));
                                }
                                noData = true;
                            }

                            if (!noData)
                            {
                                clsInfo.WekaData.TrainandTest(clsInfo, this.CandidateParameter);

                                clsInfo.Deals.Now(nowDate, WekaUtils.GetValueFromInstance(testInstances, "mainClose", 0));
                                if (m_enableDetailLogLevel2)
                                {
                                    WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
                                        trainInstances.numInstances(), WekaUtils.GetDateValueFromInstances(trainInstances, 0, 0), WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainInstances.numInstances() - 1),
                                        testInstances.numInstances(), WekaUtils.GetDateValueFromInstances(testInstances, 0, 0), WekaUtils.GetDateValueFromInstances(testInstances, 0, testInstances.numInstances() - 1),
                                        clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
                                         clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
                                         clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
                                         clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
                                         clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
                                         clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
                                }
                            }
                            else
                            {
                                clsInfo.CurrentClassValue = Parameters.DoubleArrayEmpty;
                                clsInfo.CurrentTestRet = Parameters.DoubleArrayEmpty;

                                clsInfo.Deals.Now(nowDate, null);
                                if (m_enableDetailLogLevel2)
                                {
                                    WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
                                        trainInstances.numInstances(), 0, 0,
                                        testInstances.numInstances(), 0, 0,
                                        clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
                                         clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
                                         clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
                                         clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
                                         clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
                                         clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
                                }
                            }

                            if (TestParameters.EnnableLoadTestData)
                            {
                                CCScoreData.Instance.SaveTestData(clsInfo, nowDate);
                            }

                            if (TestParameters.EnableDetailLog)
                            {
                                // Check TestInstance Result
                                SortedDictionary<int, int> closeNums = new SortedDictionary<int, int>();
                                for (int i = 0; i < testInstances.numInstances(); ++i)
                                {
                                    if (clsInfo.CurrentTestRet[i] != 2)
                                        continue;

                                    DateTime openDate = WekaUtils.GetDateValueFromInstances(testInstances, 0, i);
                                    DateTime closeDate = WekaUtils.GetDateValueFromInstances(testInstances, 1, i);
                                    WekaUtils.DebugAssert(openDate >= nowDate, "openDate >= nowDate");

                                    int n = (int)(closeDate - nowDate).TotalHours / (TestParameters.BatchTestMinutes / 60) + 1;
                                    if (!closeNums.ContainsKey(n))
                                    {
                                        closeNums[n] = 1;
                                    }
                                    else
                                    {
                                        closeNums[n]++;
                                    }
                                }
                                foreach (var cn in closeNums)
                                {
                                    WekaUtils.Instance.WriteLog(string.Format("  Next {0} Period-{1} has {2} deals",
                                        cn.Key, nowDate.AddHours(TestParameters.BatchTestMinutes / 60 * cn.Key), cn.Value));
                                }
                            }
                        }

                        clsInfo.WekaData.Clear();
                    };

                if (TestParameters.EnableMultiThread)
                {
                    tasks[taskIdx] = taskFactory.StartNew(action);

                    taskIdx++;
                }
                else
                {
                    action();
                }
            }

            if (TestParameters.EnableMultiThread)
            {
                try
                {
                    System.Threading.Tasks.Task.WaitAll(tasks);
                }
                catch (AggregateException ex)
                {
                    foreach (var i in ex.InnerExceptions)
                    {
                        WekaUtils.Instance.WriteLog(i.Message);
                        WekaUtils.Instance.WriteLog(i.StackTrace);
                    }
                }
            }

            if (TestParameters.SaveCCScoresToDb)
            {
                CCScoreData.SaveCCScoresToDb(nowDate, this.CandidateParameter, this);
            }
        }
Пример #2
0
        public void ExecuteCandidate(DateTime nowDate)
        {
            m_currentTestHour = nowDate.Hour;

            foreach (var kvp in m_classifierInfos)
            {
                kvp.Value.Initialized = false;
            }
            if (TestParameters.EnnableLoadTestData)
            {
                if (!TestParameters.OnlyNewestTestDataSaved)
                {
                    CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos);
                }
                else
                {
                    DateTime?maxTestDate = CCScoreData.Instance.GetNewestTestData();
                    if (maxTestDate.HasValue && maxTestDate.Value > nowDate)
                    {
                        return;
                    }
                    if (maxTestDate.HasValue && maxTestDate.Value == nowDate)
                    {
                        CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos);
                    }
                }
            }

            WekaData.GenerateArffTemplate(true, true, this.CandidateParameter);

            string candidateClsInfoSummary = "CC:N={0},TrN={1},TeN={4},NC={7},NTP={8},NFP={9},NDA={14},NDS={15},TD={10},TV={11},Cls={12},MM={13}";

            //Instances testInstancesWithoutClassValue = null;  / wrong, should use closeTime

            System.Threading.Tasks.Task[]      tasks;
            System.Threading.Tasks.TaskFactory taskFactory = null;
            if (TestParameters.EnableMultiThread)
            {
                int cpuCount = Environment.ProcessorCount - 2;
                cpuCount = Math.Max(cpuCount, 1);
                LimitedConcurrencyLevelTaskScheduler lcts = new LimitedConcurrencyLevelTaskScheduler(1);
                taskFactory = new System.Threading.Tasks.TaskFactory(lcts);
            }
            if (!TestParameters.EnablePerhourTrain)
            {
                tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count];
            }
            else
            {
                tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count / Parameters.AllHour];
            }
            int taskIdx = 0;

            foreach (var kvp in m_classifierInfos)
            {
                if (TestParameters.EnablePerhourTrain)
                {
                    if (kvp.Value.Hour != m_currentTestHour)
                    {
                        continue;
                    }
                }
                var clsInfo = kvp.Value;

                Action action = () =>
                {
                    if (clsInfo.Initialized || nowDate > System.DateTime.Now)
                    {
                        if (m_enableDetailLogLevel2)
                        {
                            WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
                                                                      -1, 0, 0,
                                                                      clsInfo.CurrentClassValue.Length, 0, 0,
                                                                      clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
                                                                      clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
                                                                      clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
                                                                      clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
                                                                      clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
                                                                      clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
                        }
                    }
                    else
                    {
                        weka.core.Instances trainInstances;
                        weka.core.Instances testInstances;
                        bool noData = false;
                        if (TestParameters.UseTrain)
                        {
                            clsInfo.WekaData.GenerateData(true, true);
                            trainInstances = clsInfo.WekaData.CurrentTrainInstances;
                            testInstances  = clsInfo.WekaData.CurrentTestInstances;

                            if (trainInstances.numInstances() == 0)
                            {
                                if (m_enableDetailLogLevel2)
                                {
                                    WekaUtils.Instance.WriteLog(string.Format("{0} - No Train Data", clsInfo.Name));
                                }
                                noData = true;
                            }
                        }
                        else
                        {
                            clsInfo.WekaData.GenerateData(false, true);
                            trainInstances = WekaData.GetTrainInstancesTemplate(this.CandidateParameter.Name);
                            testInstances  = clsInfo.WekaData.CurrentTestInstances;
                        }

                        if (testInstances.numInstances() == 0)
                        {
                            if (m_enableDetailLogLevel2)
                            {
                                WekaUtils.Instance.WriteLog(string.Format("{0} - No Test Data", clsInfo.Name));
                            }
                            noData = true;
                        }

                        if (!noData)
                        {
                            clsInfo.WekaData.TrainandTest(clsInfo, this.CandidateParameter);

                            clsInfo.Deals.Now(nowDate, WekaUtils.GetValueFromInstance(testInstances, "mainClose", 0));
                            if (m_enableDetailLogLevel2)
                            {
                                WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
                                                                          trainInstances.numInstances(), WekaUtils.GetDateValueFromInstances(trainInstances, 0, 0), WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainInstances.numInstances() - 1),
                                                                          testInstances.numInstances(), WekaUtils.GetDateValueFromInstances(testInstances, 0, 0), WekaUtils.GetDateValueFromInstances(testInstances, 0, testInstances.numInstances() - 1),
                                                                          clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
                                                                          clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
                                                                          clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
                                                                          clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
                                                                          clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
                                                                          clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
                            }
                        }
                        else
                        {
                            clsInfo.CurrentClassValue = Parameters.DoubleArrayEmpty;
                            clsInfo.CurrentTestRet    = Parameters.DoubleArrayEmpty;

                            clsInfo.Deals.Now(nowDate, null);
                            if (m_enableDetailLogLevel2)
                            {
                                WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
                                                                          trainInstances.numInstances(), 0, 0,
                                                                          testInstances.numInstances(), 0, 0,
                                                                          clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
                                                                          clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
                                                                          clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
                                                                          clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
                                                                          clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
                                                                          clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
                            }
                        }

                        if (TestParameters.EnnableLoadTestData)
                        {
                            CCScoreData.Instance.SaveTestData(clsInfo, nowDate);
                        }

                        if (TestParameters.EnableDetailLog)
                        {
                            // Check TestInstance Result
                            SortedDictionary <int, int> closeNums = new SortedDictionary <int, int>();
                            for (int i = 0; i < testInstances.numInstances(); ++i)
                            {
                                if (clsInfo.CurrentTestRet[i] != 2)
                                {
                                    continue;
                                }

                                DateTime openDate  = WekaUtils.GetDateValueFromInstances(testInstances, 0, i);
                                DateTime closeDate = WekaUtils.GetDateValueFromInstances(testInstances, 1, i);
                                WekaUtils.DebugAssert(openDate >= nowDate, "openDate >= nowDate");

                                int n = (int)(closeDate - nowDate).TotalHours / (TestParameters.BatchTestMinutes / 60) + 1;
                                if (!closeNums.ContainsKey(n))
                                {
                                    closeNums[n] = 1;
                                }
                                else
                                {
                                    closeNums[n]++;
                                }
                            }
                            foreach (var cn in closeNums)
                            {
                                WekaUtils.Instance.WriteLog(string.Format("  Next {0} Period-{1} has {2} deals",
                                                                          cn.Key, nowDate.AddHours(TestParameters.BatchTestMinutes / 60 * cn.Key), cn.Value));
                            }
                        }
                    }

                    clsInfo.WekaData.Clear();
                };

                if (TestParameters.EnableMultiThread)
                {
                    tasks[taskIdx] = taskFactory.StartNew(action);

                    taskIdx++;
                }
                else
                {
                    action();
                }
            }

            if (TestParameters.EnableMultiThread)
            {
                try
                {
                    System.Threading.Tasks.Task.WaitAll(tasks);
                }
                catch (AggregateException ex)
                {
                    foreach (var i in ex.InnerExceptions)
                    {
                        WekaUtils.Instance.WriteLog(i.Message);
                        WekaUtils.Instance.WriteLog(i.StackTrace);
                    }
                }
            }

            if (TestParameters.SaveCCScoresToDb)
            {
                CCScoreData.SaveCCScoresToDb(nowDate, this.CandidateParameter, this);
            }
        }