public static void FilterInstances(weka.core.Instances allInstances)
{
DateTime nextHpDate = DateTime.MinValue;
java.util.LinkedList deleteList = new java.util.LinkedList();
for (int i = 0; i < allInstances.numInstances(); ++i)
{
DateTime nowDate = WekaUtils.GetDateValueFromInstances(allInstances, 0, i);
if (TestParameters2.RealTimeMode && i == allInstances.numInstances() - 1)
{
allInstances.instance(i).setClassValue(0);
allInstances.instance(i).setValue(1, WekaUtils.GetTimeFromDate(Parameters.MaxDate) * 1000);
}
else
{
if (nowDate < nextHpDate)
{
deleteList.Add(allInstances.instance(i));
}
else
{
DateTime hpDate = WekaUtils.GetDateValueFromInstances(allInstances, 1, i);
nextHpDate = hpDate;
}
}
}
allInstances.removeAll(deleteList);
}
//public void ConvertHpDataPeriod(string symbol, string periodSsrc, string periodDest)
//{
// string hpFileName = TestParameters.GetBaseFilePath(string.Format("{0}_{1}_hpdata.txt", symbol, periodSsrc));
// var hpData = GetHpDateFromTxt(hpFileName, TestParameters2.nTpsl);
// string hpFileName2 = TestParameters.GetBaseFilePath(string.Format("{0}_{1}_hpdata.txt", symbol, periodDest));
// int m = TestParameters.TpSlMaxCount / TestParameters2.nTpsl;
// DateTime date = TestParameters2.TrainStartTime;
// DateTime maxDate = TestParameters2.TrainEndTime;
// using(StreamWriter sw = new StreamWriter(hpFileName2))
// {
// while (true)
// {
// if (!TestParameters2.RealTimeMode)
// {
// Console.WriteLine(date.ToString(Parameters.DateTimeFormat));
// }
// StringBuilder sb = new StringBuilder();
// {
// if (date.DayOfWeek == DayOfWeek.Saturday || date.DayOfWeek == DayOfWeek.Sunday)
// {
// date = date.AddHours(mainPeriodOfHour);
// continue;
// }
// bool isComplete = true;
// if ((TestParameters2.UsePartialHpData || isComplete))
// {
// sb.Append(date.ToString(Parameters.DateTimeFormat));
// sb.Append(", ");
// int[, , ,] hps = new int[2, m, m, 2];
// long[, ,] hpTimes = new long[2, m, m];
// for (int j = 0; j < hps.GetLength(0); ++j)
// for (int k = 0; k < hps.GetLength(1); ++k)
// for (int l = 0; l < hps.GetLength(2); ++l)
// {
// hps[j, k, l, 0] = hps[j, k, l, 1] = -1;
// hpTimes[j, k, l] = -1;
// }
// List<DateTime> dts = new List<DateTime>();
// foreach (DateTime dt in dts)
// {
// if (!hpData.ContainsKey(dt))
// continue;
// sbyte?[, ,] hp = hpData[dt].Item1;
// long[, ,] hpTime = hpData[dt].Item2;
// WekaUtils.DebugAssert(hp.GetLength(0) == 2, "");
// WekaUtils.DebugAssert(hp.GetLength(1) % m == 0, "");
// WekaUtils.DebugAssert(hp.GetLength(2) % m == 0, "");
// WekaUtils.DebugAssert(hpTime.GetLength(0) == 2, "");
// for (int j = 0; j < hp.GetLength(0); ++j)
// for (int k = 0; k < hp.GetLength(1); ++k)
// for (int l = 0; l < hp.GetLength(2); ++l)
// {
// if (k % n != n - 1 || l % n != n - 1)
// continue;
// if (!hp[j, k, l].HasValue || hp[j, k, l] == -1)
// continue;
// //if (k / n >= TestParameters2.tpCount)
// // continue;
// //if (l / n >= TestParameters2.slCount)
// // continue;
// if (hps[j, k / n, l / n, 0] == -1)
// hps[j, k / n, l / n, 0] = 0;
// if (hps[j, k / n, l / n, 1] == -1)
// hps[j, k / n, l / n, 1] = 0;
// if (hp[j, k, l] == 1)
// hps[j, k / n, l / n, 0]++;
// else if (hp[j, k, l] == 0)
// hps[j, k / n, l / n, 1]++;
// else
// throw new AssertException("hp should be 0 or 1.");
// hpTimes[j, k / n, l / n] = Math.Max(hpTimes[j, k / n, l / n], hpTime[j, k, l].Value);
// }
// }
// for (int j = 0; j < hps.GetLength(0); ++j)
// for (int k = 0; k < hps.GetLength(1); ++k)
// for (int l = 0; l < hps.GetLength(2); ++l)
// {
// if (isComplete)
// {
// if (hps[j, k, l, 0] == -1 || hps[j, k, l, 1] == -1)
// {
// throw new AssertException("hps should not be -1.");
// }
// }
// if (hps[j, k, l, 0] == -1)
// hps[j, k, l, 0] = 0;
// if (hps[j, k, l, 1] == -1)
// hps[j, k, l, 1] = 0;
// sb.Append(hps[j, k, l, 0] + ", " + hps[j, k, l, 1] + ", ");
// }
// long maxHpTime = 0;
// for (int j = 0; j < hpTimes.GetLength(0); ++j)
// for (int k = 0; k < hpTimes.GetLength(1); ++k)
// for (int l = 0; l < hpTimes.GetLength(2); ++l)
// {
// if (isComplete)
// {
// if (hpTimes[j, k, l] == -1)
// {
// throw new AssertException("hpTimes should not be -1.");
// }
// }
// if (hpTimes[j, k, l] == -1)
// hpTimes[j, k, l] = Parameters.MaxTime;
// sb.Append(hpTimes[j, k, l] + ", ");
// maxHpTime = Math.Max(hpTimes[j, k, l], maxHpTime);
// }
// sb.AppendLine();
// sw.Write(sb.ToString());
// }
// DateTime nextDate = date.AddMinutes(WekaUtils.GetMinuteofPeriod(periodDest));
// date = nextDate;
// if (date >= maxDate)
// break;
// }
// }
// }
//}
public static Dictionary <long, int> GetHpDateTimesCountFromInstance(string fileName)
{
Dictionary <long, int> dictTimes = new Dictionary <long, int>();
int dealCount = 0;
var instances = WekaUtils.LoadInstances(TestParameters.GetBaseFilePath(fileName));
for (int i = 0; i < instances.numInstances(); ++i)
{
var date = WekaUtils.GetDateValueFromInstances(instances, 0, i);
var hpdate = WekaUtils.GetDateValueFromInstances(instances, 1, i);
var hptime = WekaUtils.GetTimeFromDate(hpdate);
if (dictTimes.ContainsKey(hptime))
{
dictTimes[hptime]++;
}
else
{
dictTimes[hptime] = 1;
}
dealCount++;
Console.WriteLine(date.ToString(Parameters.DateTimeFormat) + ", " + dictTimes.Count + ", " + dealCount);
}
return(dictTimes);
}
public void ExecuteBest(DateTime nowDate, List <CandidateClassifier> minScoreInfos)
{
OutputRealDealSummary(nowDate);
float totalCost = 0;
int totalDeal = 0;
if (minScoreInfos != null)
{
List <DealInfo> candidateDeals = new List <DealInfo>();
foreach (var minScoreInfo in minScoreInfos)
{
minScoreInfo.WekaData.GenerateData(false, true);
weka.core.Instances minTestInstances = minScoreInfo.WekaData.CurrentTestInstances;
weka.core.Instances minTestInstancesNew = minScoreInfo.WekaData.CurrentTestInstancesNew;
if (minTestInstances.numInstances() > 0)
{
MyEvaluation eval = new MyEvaluation(minScoreInfo.CostMatrix);
eval.evaluateModel(minScoreInfo.CurrentTestRet, minScoreInfo.CurrentClassValue);
float vol;
// vol = (float)minScoreInfo.MoneyManagement.GetVolume(null);
vol = 0.1F;
//vol = (float)Math.Round(minScore / -20000.0, 1);
//WekaUtils.DebugAssert(vol > 0);
int tp = (int)eval.numTruePositives(1);
int fp = (int)eval.numFalsePositives(1);
double minScore = minScoreInfo.Deals.NowScore;
//WekaUtils.Instance.WriteLog(string.Format("Best Classifier: N={0},TC={1},TP={2},FP={3},TD={4},TV={5},TTP={6},TFP={7},",
// minScoreInfo.Name, minScoreInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
// minScoreInfo.Deals.NowTp, minScoreInfo.Deals.NowFp, minScoreInfo.Deals.NowDeal,
// minScoreInfo.Deals.TotalVolume.ToString("N2"), tp, fp),
// true, ConsoleColor.DarkGreen);
// Exclude
//if (TestParameters.EnableExcludeClassifier)
//{
//if (minScoreInfo.ExcludeClassifier == null)
//{
// string modelFileName4Exclude = GetExcludeModelFileName(minScoreInfo.Name);
// minScoreInfo.ExcludeClassifier = WekaUtils.TryLoadClassifier(modelFileName4Exclude);
//}
//}
for (int i = 0; i < minScoreInfo.CurrentTestRet.Length; i++)
{
if (minScoreInfo.CurrentTestRet[i] == 2)
{
if (minScoreInfo.ExcludeClassifier != null)
{
double cv2 = minScoreInfo.ExcludeClassifier.classifyInstance(minTestInstancesNew.instance(i));
if (cv2 != 2)
{
continue;
}
}
candidateDeals.Add(new DealInfo(WekaUtils.GetDateValueFromInstances(minTestInstances, 0, i),
(float)WekaUtils.GetValueFromInstance(minTestInstances, "mainClose", i),
minScoreInfo.DealType,
(float)vol,
(float)(minScoreInfo.CurrentClassValue[i] == 2 ? -minScoreInfo.Tp : minScoreInfo.Sl),
WekaUtils.GetDateValueFromInstances(minTestInstances, 1, i)));
}
}
float nowCost = (float)eval.totalCost();
int nowDeal = tp + fp;
totalCost += nowCost * vol;
totalDeal += nowDeal;
//float diff = Math.Abs(totalCost - realDealsInfo.TotalCost);
//WekaUtils.DebugAssert(diff < 5);
//WekaUtils.DebugAssert(Math.Abs(totalDeal - (realDealsInfo.NowDeal + realDealsInfo.CurrentDeal)) == 0);
//if (diff > 0.5)
// totalCost = realDealsInfo.TotalCost;
IterateClassifierInfos((k, i, j, h) =>
{
if (m_classifierInfoIdxs[k, i, j, h] == minScoreInfo)
{
m_totalScores[k, i, j, h] += nowCost;
m_totalDeals[k, i, j, h] += nowDeal;
return(false);
}
else
{
return(true);
}
});
}
}
int selectCount = 50;
for (int i = 0; i < selectCount; ++i)
{
if (candidateDeals.Count == 0)
{
break;
}
int selectedDealIdx = (int)Math.Round(m_randomGenerator.NextDouble() * candidateDeals.Count);
if (selectedDealIdx == candidateDeals.Count)
{
selectedDealIdx = candidateDeals.Count - 1;
}
m_realDealsInfo.AddDeal(candidateDeals[selectedDealIdx]);
candidateDeals.RemoveAt(selectedDealIdx);
}
}
//if (m_enableDetailLogLevel2)
//{
// IterateClassifierInfos((k, i, j, h) =>
// {
// if (h == m_currentTestHour && m_totalScores[k, i, j, h] != 0)
// {
// WekaUtils.Instance.WriteLog(string.Format("Predict score for {0}: TC={1}, TD={2}", m_classifierInfoIdxs[k, i, j, h].Name, m_totalScores[k, i, j, h].ToString(Parameters.DoubleFormatString), m_totalDeals[k, i, j, h]));
// }
// });
//}
if (!string.IsNullOrEmpty(currentSummary))
{
WekaUtils.Instance.WriteLog(currentSummary, true, ConsoleColor.Red);
System.Console.Title = nowDate.ToString(Parameters.DateTimeFormat) + ":" + currentSummary;
}
//if (m_enableDetailLogLevel2)
//{
// if (TestParameters.EnablePerhourTrain)
// {
// double[] totalCostPerHour = new double[m_classifierInfoIdxs.GetLength(3)];
// int[] totalDealPerHour = new int[m_classifierInfoIdxs.GetLength(3)];
// IterateClassifierInfos((k, i, j, h) =>
// {
// totalCostPerHour[h] += m_totalScores[k, i, j, h];
// totalDealPerHour[h] += m_totalDeals[k, i, j, h];
// });
// for (int i = 0; i < totalCostPerHour.Length; ++i)
// {
// if (totalCostPerHour[i] == 0)
// continue;
// WekaUtils.Instance.WriteLog(string.Format("Predict score Per hour of {0}: TC={1}, TD={2}", i, totalCostPerHour[i].ToString(Parameters.DoubleFormatString), totalDealPerHour[i]));
// }
// }
//if ((m_testTimeStart.Month == 4 || m_testTimeStart.Month == 3) && m_testTimeStart.Day == 20)
//{
// foreach (var cls in m_classifierInfos)
// {
// WekaUtils.Instance.WriteLog(cls.Value.Deals.PrintAll());
// }
//}
//if (m_currentTestHour == 0)
//{
// var cls = m_classifierInfoIdxs[0, 0, 0, 0];
// WekaUtils.Instance.WriteLog((cls.Classifier as RandomClassifier).GetCountInfo());
//}
//}
//if (m_saveDataFile)
//{
// var files = System.IO.Directory.GetFiles(m_baseDir, "*.arff");
// if (files.Length > 100)
// {
// foreach (string fileName in files)
// {
// try
// {
// System.IO.File.Delete(fileName);
// }
// catch (Exception)
// {
// }
// }
// }
//}
}
public void ExecuteCandidate(DateTime nowDate)
{
m_currentTestHour = nowDate.Hour;
foreach (var kvp in m_classifierInfos)
{
kvp.Value.Initialized = false;
}
if (TestParameters.EnnableLoadTestData)
{
if (!TestParameters.OnlyNewestTestDataSaved)
{
CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos);
}
else
{
DateTime?maxTestDate = CCScoreData.Instance.GetNewestTestData();
if (maxTestDate.HasValue && maxTestDate.Value > nowDate)
{
return;
}
if (maxTestDate.HasValue && maxTestDate.Value == nowDate)
{
CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos);
}
}
}
WekaData.GenerateArffTemplate(true, true, this.CandidateParameter);
string candidateClsInfoSummary = "CC:N={0},TrN={1},TeN={4},NC={7},NTP={8},NFP={9},NDA={14},NDS={15},TD={10},TV={11},Cls={12},MM={13}";
//Instances testInstancesWithoutClassValue = null; / wrong, should use closeTime
System.Threading.Tasks.Task[] tasks;
System.Threading.Tasks.TaskFactory taskFactory = null;
if (TestParameters.EnableMultiThread)
{
int cpuCount = Environment.ProcessorCount - 2;
cpuCount = Math.Max(cpuCount, 1);
LimitedConcurrencyLevelTaskScheduler lcts = new LimitedConcurrencyLevelTaskScheduler(1);
taskFactory = new System.Threading.Tasks.TaskFactory(lcts);
}
if (!TestParameters.EnablePerhourTrain)
{
tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count];
}
else
{
tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count / Parameters.AllHour];
}
int taskIdx = 0;
foreach (var kvp in m_classifierInfos)
{
if (TestParameters.EnablePerhourTrain)
{
if (kvp.Value.Hour != m_currentTestHour)
{
continue;
}
}
var clsInfo = kvp.Value;
Action action = () =>
{
if (clsInfo.Initialized || nowDate > System.DateTime.Now)
{
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
-1, 0, 0,
clsInfo.CurrentClassValue.Length, 0, 0,
clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
}
}
else
{
weka.core.Instances trainInstances;
weka.core.Instances testInstances;
bool noData = false;
if (TestParameters.UseTrain)
{
clsInfo.WekaData.GenerateData(true, true);
trainInstances = clsInfo.WekaData.CurrentTrainInstances;
testInstances = clsInfo.WekaData.CurrentTestInstances;
if (trainInstances.numInstances() == 0)
{
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format("{0} - No Train Data", clsInfo.Name));
}
noData = true;
}
}
else
{
clsInfo.WekaData.GenerateData(false, true);
trainInstances = WekaData.GetTrainInstancesTemplate(this.CandidateParameter.Name);
testInstances = clsInfo.WekaData.CurrentTestInstances;
}
if (testInstances.numInstances() == 0)
{
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format("{0} - No Test Data", clsInfo.Name));
}
noData = true;
}
if (!noData)
{
clsInfo.WekaData.TrainandTest(clsInfo, this.CandidateParameter);
clsInfo.Deals.Now(nowDate, WekaUtils.GetValueFromInstance(testInstances, "mainClose", 0));
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
trainInstances.numInstances(), WekaUtils.GetDateValueFromInstances(trainInstances, 0, 0), WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainInstances.numInstances() - 1),
testInstances.numInstances(), WekaUtils.GetDateValueFromInstances(testInstances, 0, 0), WekaUtils.GetDateValueFromInstances(testInstances, 0, testInstances.numInstances() - 1),
clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
}
}
else
{
clsInfo.CurrentClassValue = Parameters.DoubleArrayEmpty;
clsInfo.CurrentTestRet = Parameters.DoubleArrayEmpty;
clsInfo.Deals.Now(nowDate, null);
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
trainInstances.numInstances(), 0, 0,
testInstances.numInstances(), 0, 0,
clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
}
}
if (TestParameters.EnnableLoadTestData)
{
CCScoreData.Instance.SaveTestData(clsInfo, nowDate);
}
if (TestParameters.EnableDetailLog)
{
// Check TestInstance Result
SortedDictionary <int, int> closeNums = new SortedDictionary <int, int>();
for (int i = 0; i < testInstances.numInstances(); ++i)
{
if (clsInfo.CurrentTestRet[i] != 2)
{
continue;
}
DateTime openDate = WekaUtils.GetDateValueFromInstances(testInstances, 0, i);
DateTime closeDate = WekaUtils.GetDateValueFromInstances(testInstances, 1, i);
WekaUtils.DebugAssert(openDate >= nowDate, "openDate >= nowDate");
int n = (int)(closeDate - nowDate).TotalHours / (TestParameters.BatchTestMinutes / 60) + 1;
if (!closeNums.ContainsKey(n))
{
closeNums[n] = 1;
}
else
{
closeNums[n]++;
}
}
foreach (var cn in closeNums)
{
WekaUtils.Instance.WriteLog(string.Format(" Next {0} Period-{1} has {2} deals",
cn.Key, nowDate.AddHours(TestParameters.BatchTestMinutes / 60 * cn.Key), cn.Value));
}
}
}
clsInfo.WekaData.Clear();
};
if (TestParameters.EnableMultiThread)
{
tasks[taskIdx] = taskFactory.StartNew(action);
taskIdx++;
}
else
{
action();
}
}
if (TestParameters.EnableMultiThread)
{
try
{
System.Threading.Tasks.Task.WaitAll(tasks);
}
catch (AggregateException ex)
{
foreach (var i in ex.InnerExceptions)
{
WekaUtils.Instance.WriteLog(i.Message);
WekaUtils.Instance.WriteLog(i.StackTrace);
}
}
}
if (TestParameters.SaveCCScoresToDb)
{
CCScoreData.SaveCCScoresToDb(nowDate, this.CandidateParameter, this);
}
}