public static void FilterInstances(weka.core.Instances allInstances)
{
DateTime nextHpDate = DateTime.MinValue;
java.util.LinkedList deleteList = new java.util.LinkedList();
for (int i = 0; i < allInstances.numInstances(); ++i)
{
DateTime nowDate = WekaUtils.GetDateValueFromInstances(allInstances, 0, i);
if (TestParameters2.RealTimeMode && i == allInstances.numInstances() - 1)
{
allInstances.instance(i).setClassValue(0);
allInstances.instance(i).setValue(1, WekaUtils.GetTimeFromDate(Parameters.MaxDate) * 1000);
}
else
{
if (nowDate < nextHpDate)
{
deleteList.Add(allInstances.instance(i));
}
else
{
DateTime hpDate = WekaUtils.GetDateValueFromInstances(allInstances, 1, i);
nextHpDate = hpDate;
}
}
}
allInstances.removeAll(deleteList);
}
//private weka.core.Instances m_instances;
public void evaluateModel(double[] v, weka.core.Instances instances)
{
double[] c = new double[instances.numInstances()];
for (int i = 0; i < c.Length; ++i)
{
c[i] = instances.instance(i).classValue();
}
evaluateModel(v, c);
}
private static void writeFile(weka.core.Instances features, string filename)
{
if(FILE_FORMAT == "csv")
{
using (TextWriter writer = new StreamWriter(filename))
{
if(features.numInstances() == 0)
return;
for(int i = 0; i < features.numAttributes(); i++)
{
writer.Write(features.attribute(i));
if(i < features.numAttributes() - 1)
writer.Write(DELIMITER);
}
writer.WriteLine();
for(int i = 0; i < features.numInstances(); i++)
{
for(int j = 0; j < features.numAttributes(); j++)
{
writer.Write(features.instance(i).value(j));
if(j < features.numAttributes() - 1)
writer.Write(DELIMITER);
}
writer.WriteLine();
}
}
} else if (FILE_FORMAT == "arff")
{
using (TextWriter writer = new StreamWriter(filename))
{
writer.WriteLine(features.toString());
}
}
}
public static void GenerateHpDateSpan(weka.core.Instances allInstances)
{
using (StreamWriter sw = new StreamWriter("d:\\a.txt"))
{
for (int i = 0; i < allInstances.numInstances(); ++i)
{
DateTime nowDate = WekaUtils.GetDateValueFromInstances(allInstances, 0, i);
DateTime nowHpDate = WekaUtils.GetDateValueFromInstances(allInstances, 1, i);
sw.WriteLine((nowHpDate - nowDate).TotalMinutes);
}
}
}
private static object[,] IncrementTestTrain(bool newjHpTime, Func<weka.classifiers.Classifier> clsCreator, string resultFile, int step,
weka.core.Instances trainInstances, weka.core.Instances trainInstancesWithDate,
weka.core.Instances allInstances, weka.core.Instances allInstancesWithDate,
long maxjHpTime, int maxjHpTimeHp,
Tuple<int, int> tuple)
{
if (!newjHpTime)
return null;
int timeAfter = 0;
object[,] toRet = new object[step, 9];
for (int j = 0; j < step; ++j)
toRet[j, 0] = 2;
var cls = clsCreator();
bool b = true;
b = WekaUtils.TrainInstances(cls, trainInstances);
if (!b)
{
WekaUtils.Instance.WriteLog(string.Format("TrainInstance error!"));
return null;
}
long totolCost = 0;
//var eval = WekaUtils.TestInstances(trainInstances, cls, null);
//totolCost = (long)eval.totalCost();
// get hp last time
long[] maxLastTimes = new long[trainInstances.numClasses()];
long[] minLastTimes = new long[trainInstances.numClasses()];
for (int j = 0; j < trainInstances.numClasses(); ++j)
{
maxLastTimes[j] = 0;
minLastTimes[j] = long.MaxValue;
}
for (int j = 0; j < trainInstances.numInstances(); ++j)
{
DateTime openTime = WekaUtils.GetDateValueFromInstances(trainInstancesWithDate, 0, j);
DateTime closeTime = WekaUtils.GetDateValueFromInstances(trainInstancesWithDate, 1, j);
int v = (int)trainInstances.instance(j).classValue();
long lastTime = (long)(closeTime - openTime).TotalSeconds;
if (lastTime > maxLastTimes[v])
maxLastTimes[v] = lastTime;
if (lastTime < minLastTimes[v])
minLastTimes[v] = lastTime;
}
var ai = tuple.Item1;
DateTime nowDate = WekaUtils.GetDateValueFromInstances(allInstancesWithDate, 0, ai);
//var i2 = tuple.Item2;
for (int j = 0; j < step; ++j)
{
int idx = ai + timeAfter + j;
DateTime nextDate = WekaUtils.GetDateValueFromInstances(allInstancesWithDate, 0, idx);
double d = 2;
d = cls.classifyInstance(allInstances.instance(idx));
double[] v = cls.distributionForInstance(allInstances.instance(idx));
//if (d == 2)
//{
// WekaUtils.Instance.WriteLog(string.Format("{0} classifyInstance result = 2", nowDate.ToString(Parameters.DateTimeFormat)));
// continue;
//}
//if ((d == 1 && maxjHpTimeHp == 1)
// || (d == 0 && maxjHpTimeHp == 0))
// d = 2;
//if ((d == 1 && maxjHpTimeHp == 0)
// || (d == 0 && maxjHpTimeHp == 1))
// d = 2;
toRet[j, 0] = (int)d;
toRet[j, 1] = (int)allInstances.instance(idx).classValue();
toRet[j, 2] = WekaUtils.GetTimeFromDate(nextDate);
toRet[j, 3] = WekaUtils.GetTimeFromDate(nowDate);
toRet[j, 4] = trainInstances.numInstances();
toRet[j, 5] = totolCost;
toRet[j, 6] = maxjHpTime;
toRet[j, 7] = v[(int)d];
if (d != 2 && (maxLastTimes[(int)d] == 0
|| maxLastTimes[(int)d] == long.MaxValue))
{
maxLastTimes[(int)d] = maxLastTimes[3];
}
toRet[j, 8] = maxLastTimes[(int)d];
//using (StreamWriter sw = new StreamWriter("d:\\p.txt", true))
//{
// sw.WriteLine(string.Format("{0}", (int)allInstancesNoDate.instance(idx).classValue()));
//}
}
return toRet;
}
//public void SetTimeDays(DateTime trainStart, int trainDays, int testDays)
//{
// SetTimeMinutesFromTrainStart(trainStart, trainDays * 60 * 24, testDays * 60 * 24);
//}
public void GenerateEaOrder(CandidateClassifier bestClassifierInfo, weka.core.Instances testInstances)
{
if (bestClassifierInfo == null)
return;
//int j = m_bestClassifierInfo.DealType == m_dealType[0] ? 0 : 1;
string eaorderFileName = string.Format("{0}\\ea_order.txt", TestParameters.BaseDir);
//Instances testInstances = bestClassifierInfo.CurrentTestInstances;
//Instances testInstancesNew = null;
//Instances testInstances = null;
//if (m_saveDataFile)
//{
// SetTraining(false);
// string testFileName = GetArffFileName(m_dealType[j], m_newFileAppend);
// WekaUtils.Instance.WriteLog("Load instance of " + testFileName);
// testInstancesNew = WekaUtils.LoadInstances(testFileName);
// testInstances = WekaUtils.LoadInstances(GetArffFileName(m_dealType[j]));
//}
//else
//{
// testInstancesNew = m_testInstancesNew[j];
// testInstances = m_testInstances[j];
//}
System.Data.DataTable dt = null;
if (!testInstances.attribute(0).isDate())
{
dt = Feng.Data.DbHelper.Instance.ExecuteDataTable(string.Format("SELECT Date FROM {0} WHERE Date >= '{1}' AND Date < '{2}' {4}",
"EURUSD" + "_" + "M1",
WekaData.m_testTimeStart.ToString(Parameters.DateTimeFormat), WekaData.m_testTimeEnd.ToString(Parameters.DateTimeFormat),
"ORDER BY Time"));
if (dt.Rows.Count != testInstances.numInstances())
{
WekaUtils.Instance.WriteLog("different count!");
return;
}
}
//else
//{
// if (testInstancesNew.numInstances() != testInstances.numInstances())
// {
// WekaUtils.Instance.WriteLog("different count!");
// return;
// }
//}
// todo: save info in file
//for (int j = 0; j < m_dealType.Length; ++j)
//{
// if (m_saveDataFile)
// {
// string modelFileName = GetModelFileName(m_dealType[j]);
// WriteLog("Load model of " + modelFileName);
// //ObjectInputStream ois = new ObjectInputStream(new FileInputStream(modelFileName));
// //Classifier cls = (Classifier)ois.readObject();
// //ois.close();
// m_bestClassifier[m_dealType[j]] = (Classifier)weka.core.SerializationHelper.read(modelFileName);
// }
//}
//foreach (string testFileName in System.IO.Directory.GetFiles(m_baseDir, NormalizeFileName(string.Format("{0}_{1}_{2}_*.arff", m_symbol, dt1.ToString(m_dateTimeFormat), dt2.ToString(m_dateTimeFormat)))))
//{
// string[] ss2 = System.IO.Path.GetFileNameWithoutExtension(testFileName).Split(new char[] { '_' }, StringSplitOptions.RemoveEmptyEntries);
// if (ss2.Length != 5)
// continue;
// try
// {
// m_trainTimeStart = ConvertFileNameToDateTime(ss2[1]);
// m_trainTimeEnd = ConvertFileNameToDateTime(ss2[2]);
// m_testTimeStart = ConvertFileNameToDateTime(ss2[3]);
// m_testTimeEnd = ConvertFileNameToDateTime(ss2[4]);
// }
// catch (Exception)
// {
// continue;
// }
double[] tr = bestClassifierInfo.CurrentTestRet;
double[] cv = bestClassifierInfo.CurrentClassValue;
using (StreamWriter sw = new StreamWriter(eaorderFileName, true))
{
for (int i = 0; i < testInstances.numInstances(); i++)
{
string clsLabels;
double v = tr[i];
if (v != double.NaN)
{
clsLabels = testInstances.classAttribute().value((int)v);// CalcAction(v);
}
else
{
clsLabels = "0";
}
string clsLabel;
if (clsLabels == "2")
{
clsLabel = bestClassifierInfo.DealType == 'B' ? "1" : "-1";
}
else
{
clsLabel = "0";
}
//if (clsLabels.Length >= 2)
//{
// if (clsLabels[0] == "1" && clsLabels[1] == "0")
// clsLabel = "1";
// else if (clsLabels[0] == "0" && clsLabels[1] == "1")
// clsLabel = "-1";
// else if (clsLabels[0] == "1" && clsLabels[1] == "1")
// clsLabel = "2";
//}
//else if (clsLabels.Length == 1)
//{
// if (clsLabels[0] == "1")
// {
// if (m_dealType[0] == "B")
// clsLabel = "1";
// else if (m_dealType[0] == "S")
// clsLabel = "-1";
// }
//}
//else
//{
// throw new AssertException("invalid clsLabels.Length");
//}
string date;
string dateClose = string.Empty;
if (testInstances.attribute(0).isDate())
{
date = WekaUtils.GetDateValueFromInstances(testInstances, 0, i).ToString(Parameters.DateTimeFormat);
dateClose = WekaUtils.GetDateValueFromInstances(testInstances, 1, i).ToString(Parameters.DateTimeFormat);
}
else
{
date = ((DateTime)dt.Rows[i]["Date"]).ToString(Parameters.DateTimeFormat);
}
sw.WriteLine(string.Format("{0}, {1}, {2}, {3}, 0, 0, {4}, {5}",
m_actions[clsLabel], date, bestClassifierInfo.Tp, bestClassifierInfo.Sl, dateClose, cv[i] == v ? "Right" : "Wrong"));
//ret[ConvertFileNameToDateTime(date)] = clsLabel;
}
}
//// save labeled data
//BufferedWriter writer = new BufferedWriter(new FileWriter(string.Format("c:\\eurusd_m1_arff.result")));
//writer.write(labeled.toString());
//writer.newLine();
//writer.flush();
//writer.close();
}
public static string IncrementTest(weka.core.Instances allInstancesWithDate,
Func<weka.classifiers.Classifier> clsCreator, string removeAttributes, string resultFile, int step)
{
//if (!(TestParameters2.UsePartialHpDataM1 || TestParameters2.UsePartialHpData))
//{
// HpData.Instance.Clear();
//}
int trainMinutes = TestParameters2.MinTrainPeriod * WekaUtils.GetMinuteofPeriod(TestParameters2.CandidateParameter.MainPeriod);
string ret = string.Empty;
string sampleFile = null; // resultFile.Replace("Increment", "sample");
bool useInstanceWeight = false;
bool enablePerHour = false;
bool enableDiffClass = false;
int sameClassCount = -1;// TestParameters2.MaxTrainSize / 3; // allInstances.numClasses();
bool enableDiffHpTime = false;
bool enableRemoveLittle = false;
bool enableRemoveLargeThanMid = true;
bool enableFilter = false;
if (!TestParameters2.RealTimeMode && File.Exists(resultFile))
File.Delete(resultFile);
weka.core.Instances allInstances;
var filter = new weka.filters.MultiFilter();
//filter.setOptions(weka.core.Utils.splitOptions("-F \"weka.filters.unsupervised.attribute.Remove -R 1,4\" -F \"weka.filters.unsupervised.attribute.Discretize -B 10 -M -1.0 -R first-last\""));
filter.setOptions(weka.core.Utils.splitOptions(string.Format("-F \"weka.filters.unsupervised.attribute.Remove -R {0} \"", removeAttributes)));
filter.setInputFormat(allInstancesWithDate);
allInstances = weka.filters.Filter.useFilter(allInstancesWithDate, filter);
long[] jHpTimes = new long[allInstancesWithDate.numInstances()];
DateTime[] jDates = new DateTime[allInstancesWithDate.numInstances()];
DateTime[] jHpDates = new DateTime[allInstancesWithDate.numInstances()];
int[] jHps = new int[allInstancesWithDate.numInstances()];
for (int j = 0; j < jDates.Length; ++j)
{
jDates[j] = WekaUtils.GetDateValueFromInstances(allInstancesWithDate, 0, j);
jHpTimes[j] = WekaUtils.GetTimeValueFromInstances(allInstancesWithDate, 1, j);
jHpDates[j] = WekaUtils.GetDateFromTime(jHpTimes[j]);
jHps[j] = (int)allInstances.instance(j).classValue();
}
#region "action"
Func<Tuple<int, int>, Tuple<weka.core.Instances, weka.core.Instances, long, int, int>> action = (tuple) =>
{
var ai = tuple.Item1;
DateTime nowDate = WekaUtils.GetDateValueFromInstances(allInstancesWithDate, 0, ai);
if (nowDate < TestParameters2.TrainStartTime || nowDate > TestParameters2.TrainEndTime)
return null;
DateTime nowHpDate = WekaUtils.GetDateValueFromInstances(allInstancesWithDate, 1, ai);
double nowClass = allInstancesWithDate.instance(ai).classValue();
double preJClass = -1;
List<weka.core.Instance> listTrainInstances = new List<weka.core.Instance>(ai / 2);
List<weka.core.Instance> listTrainInstancesWithDate = new List<weka.core.Instance>(ai / 2);
int[] counts = new int[allInstancesWithDate.numClasses()];
long maxjHpTime = -1;
int maxjHpTimeHp = 2;
int maxjHpTimeCount = 0;
bool enoughTrainMinutes = false;
DateTime firstDate = nowDate.AddMinutes(-trainMinutes);
int classIdxWithDate = allInstancesWithDate.classIndex();
int classIdx = allInstances.classIndex();
for (int j = ai - 1; j >= 0; --j)
{
long jHpTime = jHpTimes[j]; // WekaUtils.GetTimeValueFromInstances(allInstancesWithDate, 1, j);
DateTime jHpDate = jHpDates[j];// WekaUtils.GetDateFromTime(jHpTime);
DateTime jDate = jDates[j];// WekaUtils.GetDateValueFromInstances(allInstancesWithDate, 0, j);
int jHp = jHps[j]; // (int)allInstancesWithDate.instance(j).value(classIdxWithDate);
if (enablePerHour)
{
if (nowDate.Hour != jDate.Hour)
continue;
}
weka.core.Instance instInsert = null;
weka.core.Instance instInsertWithDate = null;
if (jHpDate <= nowDate)
{
if (jHpTime > maxjHpTime)
{
maxjHpTime = jHpTime;
maxjHpTimeHp = jHp;
maxjHpTimeCount = 1;
}
else if (jHpTime == maxjHpTime)
{
maxjHpTimeCount++;
}
instInsert = new weka.core.DenseInstance(allInstances.instance(j));
//instInsert.setDataset(trainInstances);
instInsertWithDate = new weka.core.DenseInstance(allInstancesWithDate.instance(j));
//instInsertWithDate.setDataset(trainInstancesWithDate);
}
else
{
if (TestParameters2.UsePartialHpDataM1 || TestParameters2.UsePartialHpData)
{
Tuple<int, long> hp = null;
if (TestParameters2.UsePartialHpDataM1)
{
hp = HpData.Instance.GetHpSumByM1(TestParameters2.CandidateParameter.MainSymbol, TestParameters2.CandidateParameter.MainPeriod,
WekaUtils.GetTimeFromDate(nowDate), WekaUtils.GetTimeFromDate(jDate));
if (hp.Item2 == 0)
hp = null;
}
else if (TestParameters2.UsePartialHpData)
{
var hps = HpData.Instance.GetHpSum(TestParameters2.CandidateParameter.MainSymbol, TestParameters2.CandidateParameter.MainPeriod,
WekaUtils.GetTimeFromDate(nowDate), WekaUtils.GetTimeFromDate(jDate));
if (hps.ContainsKey(jDate))
{
hp = hps[jDate];
}
}
if (hp != null)
{
if (WekaUtils.GetDateFromTime(hp.Item2) > nowDate)
{
throw new AssertException("hpdate should less than now");
}
jHp = hp.Item1;
jHpTime = hp.Item2;
if (jHpTime > maxjHpTime)
{
maxjHpTime = jHpTime;
maxjHpTimeHp = jHp;
maxjHpTimeCount = 0;
}
else if (jHpTime == maxjHpTime)
{
maxjHpTimeCount++;
}
instInsert = new weka.core.DenseInstance(allInstances.instance(j));
//instInsert.setDataset(trainInstances);
//instInsert.setClassValue(jHp);
instInsert.setValue(classIdx, jHp);
instInsertWithDate = new weka.core.DenseInstance(allInstancesWithDate.instance(j));
//instInsertWithDate.setDataset(trainInstancesWithDate);
//instInsertWithDate.setClassValue(jHp);
instInsertWithDate.setValue(classIdxWithDate, jHp);
instInsertWithDate.setValue(1, jHpTime * 1000);
}
}
}
if (instInsert == null)
continue;
double jClass = jHp;
if (enableDiffClass && jClass == preJClass)
continue;
if (sameClassCount > 0)
{
if (counts[(int)jClass] >= sameClassCount)
continue;
counts[(int)jClass]++;
}
if (enableFilter && j > 0 && Filter(jDate, allInstancesWithDate.instance(j), allInstancesWithDate.instance(j - 1)))
continue;
if (useInstanceWeight)
instInsert.setWeight((nowDate - jDate).TotalMinutes);
listTrainInstances.Add(instInsert);
listTrainInstancesWithDate.Add(instInsertWithDate);
preJClass = jClass;
if (jDate <= firstDate)
{
enoughTrainMinutes = true;
break;
}
}
//weka.core.Instances trainInstances2 = new weka.core.Instances(allInstancesNoDate, 0);
//for (int x = trainInstances.numInstances() - 1; x >= 0; --x)
//{
// weka.core.Instance inst = new weka.core.DenseInstance(trainInstances.instance(x));
// trainInstances2.add(inst);
//}
//WekaUtils.SaveInstances(trainInstances2, "d:\\a.arff");
//if (trainInstances.numInstances() >= trainLength)
// break;
if (!enoughTrainMinutes)
{
Console.WriteLine(string.Format("{0}, not enough trainMinutes",
nowDate.ToString(Parameters.DateTimeFormat)));
return null;
}
if (listTrainInstances.Count < TestParameters2.MinTrainSize)
{
Console.WriteLine(string.Format("{0}, numInstances {1} < minTrainSize {2}",
nowDate.ToString(Parameters.DateTimeFormat), listTrainInstances.Count, TestParameters2.MinTrainSize));
return null;
}
//else if (listTrainInstances.Count == 1)
//{
// lock (WekaUtils.Instance)
// {
// WekaUtils.Instance.WriteLog("trainInstances.numInstances() == 1, nowDate = " + nowDate.ToString());
// if (!System.IO.File.Exists("d:\\a.arff"))
// {
// WekaUtils.SaveInstances(trainInstances, "d:\\a.arff");
// }
// }
//}
weka.core.Instances trainInstances = new weka.core.Instances(allInstances, listTrainInstances.Count);
weka.core.Instances trainInstancesWithDate = new weka.core.Instances(allInstancesWithDate, listTrainInstancesWithDate.Count);
WekaUtils.AddInstanceQuickly(trainInstances, listTrainInstances);
WekaUtils.AddInstanceQuickly(trainInstancesWithDate, listTrainInstancesWithDate);
if (enableRemoveLittle)
{
double preClass = 2;
for (int ii = 0; ii < trainInstances.numInstances(); ++ii)
{
var iiClass = trainInstances.instance(ii).classValue();
if (iiClass == 2)
continue;
int jj = ii + 1;
while (jj < trainInstances.numInstances())
{
if (trainInstances.instance(jj).classValue() == iiClass)
jj++;
else
break;
}
int count = jj - ii;
if (count < 5)
{
for (jj = 0; jj < count; ++jj)
{
trainInstances.instance(ii + jj).setClassValue(preClass);
}
}
else
{
preClass = iiClass;
ii += count;
}
}
}
if (enableDiffHpTime)
{
Dictionary<long, int> jDictHpTimes = new Dictionary<long, int>();
int n = trainInstances.numInstances();
List<weka.core.Instance> list1 = new List<weka.core.Instance>(n);
List<weka.core.Instance> list2 = new List<weka.core.Instance>(n);
//java.util.LinkedList deleteList = new java.util.LinkedList();
for (int j = 0; j < n; ++j)
{
long jHpTime = WekaUtils.GetTimeValueFromInstances(trainInstancesWithDate, 1, j);
if (jDictHpTimes.ContainsKey(jHpTime))
{
continue;
}
else
{
jDictHpTimes[jHpTime] = list1.Count;
list1.Add(trainInstances.instance(j));
list2.Add(trainInstancesWithDate.instance(j));
}
}
weka.core.Instances newTrainInstances = new weka.core.Instances(trainInstances, list1.Count);
weka.core.Instances newTrainInstancesWithDate = new weka.core.Instances(trainInstancesWithDate, list2.Count);
WekaUtils.AddInstanceQuickly(newTrainInstances, list1);
WekaUtils.AddInstanceQuickly(newTrainInstancesWithDate, list2);
trainInstances = newTrainInstances;
trainInstancesWithDate = newTrainInstancesWithDate;
}
if (enableRemoveLargeThanMid)
{
int n = trainInstances.numInstances();
long[] lastTimes = new long[n];
for (int j = 0; j < n; ++j)
{
long openTime = WekaUtils.GetTimeValueFromInstances(trainInstancesWithDate, 0, j);
long closeTime = WekaUtils.GetTimeValueFromInstances(trainInstancesWithDate, 1, j);
lastTimes[j] = (long)(closeTime - openTime);
}
Array.Sort(lastTimes);
long midLastTime = lastTimes[lastTimes.Count() / 2];
List<weka.core.Instance> list1 = new List<weka.core.Instance>(n);
List<weka.core.Instance> list2 = new List<weka.core.Instance>(n);
//java.util.LinkedList deleteList = new java.util.LinkedList();
for (int j = 0; j < n; ++j)
{
if (lastTimes[j] > midLastTime)
{
//deleteList.add(trainInstances.instance(j));
}
else
{
list1.Add(trainInstances.instance(j));
list2.Add(trainInstancesWithDate.instance(j));
}
}
weka.core.Instances newTrainInstances = new weka.core.Instances(trainInstances, list1.Count);
weka.core.Instances newTrainInstancesWithDate = new weka.core.Instances(trainInstancesWithDate, list2.Count);
WekaUtils.AddInstanceQuickly(newTrainInstances, list1);
WekaUtils.AddInstanceQuickly(newTrainInstancesWithDate, list2);
trainInstances = newTrainInstances;
trainInstancesWithDate = newTrainInstancesWithDate;
//trainInstances.removeAll(deleteList);
}
if (!string.IsNullOrEmpty(sampleFile))
{
lock (sampleFile)
{
if (!System.IO.File.Exists(sampleFile))
{
WekaUtils.SaveInstances(trainInstancesWithDate, sampleFile);
}
}
}
//using (StreamWriter sw = new StreamWriter("d:\\p.txt", true))
//{
// sw.Write("{0},{1},", nowDate.ToString(Parameters.DateTimeFormat), nowHpDate.ToString(Parameters.DateTimeFormat));
//}
return new Tuple<weka.core.Instances, weka.core.Instances, long, int, int>(trainInstances, trainInstancesWithDate, maxjHpTime, maxjHpTimeHp, maxjHpTimeCount);
};
#endregion
//allInstancesNoDate = allInstances;
if (!TestParameters2.RealTimeMode)
{
int tpb = 0, fpb = 0, tps = 0, fps = 0;
int db = 0, ds = 0, dn = 0;
int parallelStep = 1;
if (TestParameters.EnableMultiThread)
{
parallelStep = 100;
}
int startIdx = TestParameters2.MinTrainPeriod * 2 / 3;
for (int i0 = startIdx; i0 < allInstancesWithDate.numInstances() - step; i0 += step * parallelStep)
{
List<Tuple<int, int>> toTest = new List<Tuple<int, int>>();
for (int i = i0; i < Math.Min(i0 + step * parallelStep, allInstancesWithDate.numInstances() - step); i += step)
{
toTest.Add(new Tuple<int, int>(i, toTest.Count));
}
var toRet0 = new Tuple<weka.core.Instances, weka.core.Instances, long, int, int>[toTest.Count];
if (TestParameters.EnableMultiThread)
{
Parallel.ForEach(toTest, (tuple) =>
{
int i = tuple.Item1 - i0;
toRet0[i] = action(tuple);
});
}
else
{
for (int i = 0; i < toTest.Count; ++i)
{
var r = action(toTest[i]);
toRet0[i] = r;
}
}
object[, ,] toRet = new object[toTest.Count, step, 9];
bool[] toRet1 = new bool[toTest.Count];
for (int i = 0; i < toTest.Count; ++i)
{
for (int j = 0; j < step; ++j)
toRet[i, j, 0] = 2;
if (toRet0[i] != null)
{
var maxjHpTime = toRet0[i].Item3;
var maxjHpTimeHp = toRet0[i].Item4;
var maxjHpTimeCount = toRet0[i].Item5;
toRet1[i] = IncrementTestTrainCheck(resultFile, maxjHpTime, maxjHpTimeHp, maxjHpTimeCount, toTest[i]);
}
}
if (TestParameters.EnableMultiThread)
{
Parallel.ForEach(toTest, (tuple) =>
{
int i = tuple.Item1 - i0;
if (toRet0[i] != null)
{
var trainInstances = toRet0[i].Item1;
var trainInstancesWithDate = toRet0[i].Item2;
var maxjHpTime = toRet0[i].Item3;
var maxjHpTimeHp = toRet0[i].Item4;
var r = IncrementTestTrain(toRet1[i], clsCreator, resultFile, step, trainInstances, trainInstancesWithDate, allInstances, allInstancesWithDate,
maxjHpTime, maxjHpTimeHp, toTest[i]);
if (r != null)
{
for (int j = 0; j < toRet.GetLength(1); ++j)
for (int k = 0; k < toRet.GetLength(2); ++k)
toRet[i, j, k] = r[j, k];
}
}
});
}
else
{
for (int i = 0; i < toTest.Count; ++i)
{
if (toRet0[i] != null)
{
var trainInstances = toRet0[i].Item1;
var trainInstancesWithDate = toRet0[i].Item2;
var maxjHpTime = toRet0[i].Item3;
var maxjHpTimeHp = toRet0[i].Item4;
var r = IncrementTestTrain(toRet1[i], clsCreator, resultFile, step, trainInstances, trainInstancesWithDate, allInstances, allInstancesWithDate,
maxjHpTime, maxjHpTimeHp, toTest[i]);
if (r != null)
{
for (int j = 0; j < toRet.GetLength(1); ++j)
for (int k = 0; k < toRet.GetLength(2); ++k)
toRet[i, j, k] = r[j, k];
}
}
}
}
for (int i = 0; i < toTest.Count; ++i)
{
//action(toTest[i]);
for (int j = 0; j < step; ++j)
{
int d = (int)toRet[i, j, 0];
if (d != 0 && d != 1 && d != 2)
throw new AssertException("d should be -1, 0, 1 or 2. but it's" + d.ToString());
if (toRet[i, j, 6] == null)
{
//Console.WriteLine("toRet[i, j, 6] == null");
continue;
}
long maxJHpTime = (long)toRet[i, j, 6];
//if (lastMaxJHpTime >= (long)toRet[i, j, 6])
// continue;
//lastMaxJHpTime = (long)toRet[i, j, 6];
if (d == 0)
db++;
else if (d == 1)
ds++;
else
dn++;
int v = (int)toRet[i, j, 1];
if (d != 2)
{
if (v == 3 || d == v)
{
if (d == 0)
{
tpb++;
}
else if (d == 1)
{
tps++;
}
}
else
{
if (d == 0)
{
fpb++;
}
else if (d == 1)
{
fps++;
}
}
}
DateTime nowDate = WekaUtils.GetDateFromTime((long)toRet[i, j, 3]);
DateTime nextDate = WekaUtils.GetDateFromTime((long)toRet[i, j, 2]);
//nextDate = WekaUtils.GetDateFromTime(maxJHpTime);
//if (j == 0)
{
if (tpb + fpb + tps + fps != 0)
{
ret = (string.Format("{0}, tn={1},tc={2}, db={3},ds={4},dn={5}, tpb={6},fpb={7},tps={8},fps={9},p={10}",
nextDate.ToString(Parameters.DateTimeFormat), toRet[i, j, 4], toRet[i, j, 5],
db, ds, dn,
tpb, fpb, tps, fps, ((double)(tpb+tps) / (tpb + fpb + tps + fps)).ToString("F2")));
}
else
{
ret = (string.Format("{0}, tn={1},tc={2}, db={3},ds={4},dn={5}, tpb={6},fpb={7},tps={8},fps={9},p={10}",
nextDate.ToString(Parameters.DateTimeFormat), toRet[i, j, 4], toRet[i, j, 5],
db, ds, dn,
tpb, fpb, tps, fps, 0));
}
WekaUtils.Instance.WriteLog(ret);
}
//if (d != 2)
{
using (StreamWriter sw = new StreamWriter(resultFile, true))
{
sw.WriteLine(string.Format("{1}, {2}, {6}, {3}, {4}, {5}", nowDate,
nextDate.ToString(Parameters.DateTimeFormat),
d, v, (double)toRet[i, j, 7],
WekaUtils.GetDateFromTime(maxJHpTime).ToString(Parameters.DateTimeFormat),
toRet[i, j, 8]));
}
}
}
}
}
}
else
{
for (int i0 = allInstancesWithDate.numInstances() - 1; i0 < allInstancesWithDate.numInstances() && i0 >= 0; i0++)
{
List<Tuple<int, int>> toTest = new List<Tuple<int, int>>();
toTest.Add(new Tuple<int, int>(i0, toTest.Count));
var toRet0 = new Tuple<weka.core.Instances, weka.core.Instances, long, int, int>[toTest.Count];
for (int i = 0; i < toTest.Count; ++i)
{
var r = action(toTest[i]);
toRet0[i] = r;
}
object[, ,] toRet = new object[toTest.Count, step, 9];
for (int i = 0; i < toTest.Count; ++i)
{
for (int j = 0; j < toRet.GetLength(1); ++j)
toRet[i, j, 0] = 2;
if (toRet0[i] != null)
{
var trainInstances = toRet0[i].Item1;
var trainInstancesWithDate = toRet0[i].Item2;
var maxjHpTime = toRet0[i].Item3;
var maxjHpTimeHp = toRet0[i].Item4;
var maxjHpTimeCount = toRet0[i].Item5;
bool b = IncrementTestTrainCheck(resultFile, maxjHpTime, maxjHpTimeHp, maxjHpTimeCount, toTest[i]);
var r = IncrementTestTrain(b, clsCreator, resultFile, step, trainInstances, trainInstancesWithDate, allInstances, allInstancesWithDate,
maxjHpTime, maxjHpTimeHp, toTest[i]);
if (r != null)
{
for (int j = 0; j < toRet.GetLength(1); ++j)
for (int k = 0; k < toRet.GetLength(2); ++k)
toRet[i, j, k] = r[j, k];
}
}
}
for (int j = 0; j < step; ++j)
{
int d = (int)toRet[0, j, 0];
if (d != 0 && d != 1 && d != 2)
throw new AssertException("d should be -1, 0, 1 or 2.");
if (toRet[0, j, 6] == null)
continue;
long maxJHpTime = (long)toRet[0, j, 6];
//if (lastMaxJHpTime >= (long)toRet[i, j, 6])
// continue;
//lastMaxJHpTime = (long)toRet[i, j, 6];
int v = (int)toRet[0, j, 1];
DateTime nowDate = WekaUtils.GetDateFromTime((long)toRet[0, j, 3]);
DateTime nextDate = WekaUtils.GetDateFromTime((long)toRet[0, j, 2]);
//nextDate = WekaUtils.GetDateFromTime(maxJHpTime);
//if (d != 2)
{
using (StreamWriter sw = new StreamWriter(resultFile, true))
{
sw.WriteLine(string.Format("{1}, {2}, {6}, {3}, {4}, {5}", nowDate,
nextDate.ToString(Parameters.DateTimeFormat),
d, v, (double)toRet[0, j, 7],
WekaUtils.GetDateFromTime(maxJHpTime).ToString(Parameters.DateTimeFormat),
toRet[0, j, 8]));
}
}
}
}
}
return ret;
}
public double GetProb(weka.core.Instances instances)
{
x = 0; y = 0; z = 0;
for (int i = 0; i < instances.numInstances(); ++i)
{
double v = instances.instance(i).classValue();
if (v == 0)
x++;
else if (v == 1)
y++;
else if (v == 2)
z++;
else if (v == 3)
w++;
else
throw new ArgumentException("invalid v");
}
int n = x + y + z + w;
double tp = (TestParameters2.tpStart + 1 + TestParameters2.tpCount) / 2.0;
double sl = (TestParameters2.slStart + 1 + TestParameters2.slCount) / 2.0;
double r1 = (x * tp - y * sl + w / 2 * tp - w / 2 * sl) * x
+ (y * tp - x * sl + w / 2 * tp - w / 2 * sl) * y
- (x * sl + y * sl + w * sl) * z
+ tp * w * n;
r1 = r1 / n / n;
double r;
if (x > y)
r = x * tp - y * sl - z * sl + w * tp;
else if (x < y)
r = -x * sl + y * tp - z * sl + w * tp;
else
r = 0;
r = r / n;
//DateTime date = WekaUtils.GetDateTimeValueFromInstances(instances, 0, 0);
//DateTime hpdate = WekaUtils.GetDateTimeValueFromInstances(instances, 1, 0);
//using (StreamWriter sw = new StreamWriter("d:\\p.txt", true))
//{
// sw.Write(string.Format("{2}, {3}, {4}, {5}, ",
// date.ToString(Parameters.DateTimeFormat), hpdate.ToString(Parameters.DateTimeFormat),
// x, y, z, w, r));
//}
return r1;
}