///
/// <summary> * Signify that this batch of input to the filter is finished. If
/// * the filter requires all instances prior to filtering, output()
/// * may now be called to retrieve the filtered instances. Any
/// * subsequent instances filtered should be filtered based on setting
/// * obtained from the first batch (unless the inputFormat has been
/// * re-assigned or new options have been set). This default
/// * implementation assumes all instance processing occurs during
/// * inputFormat() and input().
/// * </summary>
/// * <returns> true if there are instances pending output </returns>
/// * <exception cref="NullPointerException"> if no input structure has been defined, </exception>
/// * <exception cref="Exception"> if there was a problem finishing the batch. </exception>
///
public override bool batchFinished()
{
if (getInputFormat() == null)
{
throw new java.lang.IllegalStateException("No input instance format defined");
}
// throw if all attributes are not numeric
Instances instances = getInputFormat();
if (instances.numInstances() < getNumCoeffs())
{
throw new java.lang.IllegalStateException("Number of coeffs cannot be greater " + "than the total number of instances");
}
m_attsFT = new AttFTHolder [instances.numAttributes()];
int nearestPower2;
for (int attr = 0; attr < instances.numAttributes(); ++attr)
{
m_attsFT[attr] = new AttFTHolder();
double[] array = instances.attributeToDoubleArray(attr);
// get the nearest power of 2 for the FT
for (nearestPower2 = 1; array.Length > nearestPower2; nearestPower2 <<= 1)
{
;
}
// initialize the complex numbers
m_attsFT[attr].re = new double[nearestPower2];
m_attsFT[attr].im = new double[nearestPower2];
int j = 0;
for (int i=0; i<nearestPower2; ++i, ++j)
{
m_attsFT[attr].re[i] = (j < array.Length) ? array[i] : 0;
m_attsFT[attr].im[i] = 0;
}
// inplace FT
if (m_useFFT)
{
computeFFT(m_attsFT[attr].re, m_attsFT[attr].im);
}
else
{
computeDFT(m_attsFT[attr].re, m_attsFT[attr].im);
}
}
// set instances of the new dataset
for (int i=0; i<getNumCoeffs(); ++i)
{
double[] vals = new double[instances.numAttributes() * 2];
for (int j=0; j<instances.numAttributes(); ++j)
{
vals[2*j] = m_attsFT[j].re[i];
vals[2*j+1] = m_attsFT[j].im[i];
}
Instance inst = new weka.core.DenseInstance(instances.instance(i).weight(), vals);
// Modified
//inst.setDataset(instances.instance(i).dataset());
push(inst);
}
flushInput();
m_NewBatch = true;
return (numPendingOutput() != 0);
}
public static void GenerateArffTemplate(bool generateTrainData, bool generateTestData, CandidateParameter cp)
{
if (!m_trainInstancesTemplates.ContainsKey(cp.Name))
{
string arffFileName = string.Format("{0}\\mlea_header_{1}.arff", TestParameters.CommonDir, cp.Name);
if (System.IO.File.Exists(arffFileName))
{
System.IO.File.Delete(arffFileName);
}
using (StreamWriter sw = new StreamWriter(arffFileName))
{
sw.WriteLine(GetArffHeader(System.DateTime.Today, System.DateTime.Today, cp));
}
m_trainInstancesTemplates[cp.Name] = WekaUtils.LoadInstances(arffFileName);
m_testInstancesTemplates[cp.Name] = WekaUtils.LoadInstances(arffFileName);
}
Instances trainInstancesTemplate = m_trainInstancesTemplates[cp.Name];
Instances testInstancesTemplate = m_testInstancesTemplates[cp.Name];
for (int k = 0; k < 2; ++k)
{
if (k == 0 && !generateTrainData)
continue;
if (k == 1 && !generateTestData)
continue;
bool isTrain = (k == 0);
DateTime dt1 = isTrain ? m_trainTimeStart : m_testTimeStart;
DateTime dt2 = isTrain ? m_trainTimeEnd : m_testTimeEnd;
Instances hereInstances;
if (isTrain)
{
hereInstances = trainInstancesTemplate;
}
else
{
hereInstances = testInstancesTemplate;
}
if (!TestParameters.EnablePerhourTrain)
{
while (hereInstances.numInstances() > 0)
{
if (WekaUtils.GetDateValueFromInstances(hereInstances, 0, 0) < dt1)
{
hereInstances.delete(0);
}
else
{
break;
}
}
}
else
{
hereInstances.delete();
}
hereInstances.setRelationName(string.Format("{0}_{1}", dt1.ToString(Parameters.DateTimeFormat), dt2.ToString(Parameters.DateTimeFormat)));
ForexDataRows[, ,] dvs = new ForexDataRows[cp.SymbolCount, cp.PeriodCount, cp.PeriodTimeCount + 1];
int[, ,] dvsIdx = new int[cp.SymbolCount, cp.PeriodCount, cp.PeriodTimeCount + 1];
for (int s = 0; s < cp.SymbolCount; ++s)
{
for (int i = 0; i < cp.PeriodCount; ++i)
{
for (int j = -1; j < Math.Max(0, Math.Min(cp.PeriodTimeCount - i, Parameters.PeriodTimeNames[i].Length)); ++j)
{
string tableName = cp.AllSymbols[s + cp.SymbolStart] + "_" +
cp.AllPeriods[i + cp.PeriodStart] +
(j < 0 ? string.Empty : "_" + Parameters.PeriodTimeNames[i + cp.PeriodStart][j]);
dvs[s, i, j + 1] = DbData.Instance.GetDbData(dt1, dt2, tableName, s == 0 && i == 0 && j == -1 ? 0 : 1, isTrain, cp);
dvsIdx[s, i, j + 1] = 0;
}
}
}
ForexDataRows mainDv = dvs[0, 0, 0];
if (mainDv.Length == 0)
continue;
if (dvs[0, cp.PeriodCount - 1, 0].Length <= cp.PrevTimeCount - 1)
continue;
int startRowIdx = 0;
startRowIdx = 0;// FindRowByTime(mainDv, (long)dvs[0, PeriodCount - 1, 0][PrevTimeCount - 1]["Time"], ref startRowIdx);
DateTime nowInstanceMaxDate = DateTime.MinValue;
if (!TestParameters.EnablePerhourTrain && hereInstances.numInstances() > 0)
{
nowInstanceMaxDate = WekaUtils.GetDateValueFromInstances(hereInstances, 0, hereInstances.numInstances() - 1);
}
for (int rowIdx = startRowIdx; rowIdx < mainDv.Length; ++rowIdx)
{
ForexData mainDrv = mainDv[rowIdx];
//DateTime row_date = (DateTime)mainDrv[1]; // "Date"
//long mainTime = (long)mainDrv[0]; // "Time"
long mainTime = mainDrv.Time;
DateTime row_date = WekaUtils.GetDateFromTime(mainTime);
if (row_date <= nowInstanceMaxDate)
continue;
if (row_date < dt1)
continue;
if (row_date >= dt2)
break;
if (TestParameters.EnablePerhourTrain)
{
if (m_currentTestHour != row_date.Hour)
continue;
}
int hp = 1;
double[] instanceValue = new double[hereInstances.numAttributes()];
//instanceValue[0] = hereInstances.attribute(0).parseDate(row_date.ToString(Parameters.DateTimeFormat));
//instanceValue[1] = hereInstances.attribute(1).parseDate(hp_date.ToString(Parameters.DateTimeFormat));
//instance.setValue(1, hereInstances.attribute(1).indexOfValue(mainDrv["hour"].ToString()));
//instance.setValue(2, hereInstances.attribute(2).indexOfValue(mainDrv["dayofweek"].ToString()));
instanceValue[0] = (row_date - Parameters.MtStartTime).TotalMilliseconds; // if not set to gmt, should -8 * 60 * 60 * 1000; // utc8
instanceValue[1] = (Parameters.MaxDate - Parameters.MtStartTime).TotalMilliseconds;
instanceValue[2] = Convert.ToDouble(mainDrv["spread"]);
instanceValue[3] = (double)mainDrv["mainClose"];
instanceValue[4] = (int)mainDrv["hour"] / 24.0;
instanceValue[5] = (int)mainDrv["dayofweek"] / 5.0;
//if (mainDrv["AskVolume"] != System.DBNull.Value && mainDrv["BidVolume"] != System.DBNull.Value)
//{
// instanceValue[4] = ((((double)mainDrv["AskVolume"]) - (double)mainDrv["BidVolume"]) / 100000);
//}
//else
{
instanceValue[6] = 0;
}
int start = 7;
try
{
for (int s = 0; s < cp.SymbolCount; ++s)
{
//double mainClose = (double)mainDrv["close"];
int nowRowIdx_s = FindRowByTime(dvs[s, 0, 0], mainTime, ref dvsIdx[s, 0, 0]);
WekaUtils.DebugAssert((long)dvs[s, 0, 0][nowRowIdx_s].Time == mainTime, "(long)dvs[s, 0, 0][nowRowIdx_s].Time == mainTime");
double mainClose = 0;
if (cp.AllIndNames2.ContainsKey("close"))
{
mainClose = (double)dvs[s, 0, 0][nowRowIdx_s]["close"];
}
for (int i = 0; i < cp.PeriodCount; ++i)
{
int periodSeconds = 60 * WekaUtils.GetMinuteofPeriod(cp.AllPeriods[i + cp.PeriodStart]);
int nowRowIdx = FindRowByTime(dvs[s, i, 0], mainTime / periodSeconds * periodSeconds, ref dvsIdx[s, i, 0]);
for (int p = 0; p < cp.PrevTimeCount; ++p)
{
if (nowRowIdx - p < 0)
{
throw new ArgumentException("No prev data!");
}
ForexData nowDrv = dvs[s, i, 0][nowRowIdx - p];
foreach (var kvp in cp.AllIndNames2)
{
double v = Convert.ToDouble(nowDrv[kvp.Key]);
double ind = WekaUtils.NormalizeValue(kvp.Key, kvp.Value, v, mainClose, WekaUtils.GetSymbolPoint(cp.AllSymbols[s]));
instanceValue[start] = ind;
start++;
}
for (int j = -1; j < Math.Max(0, Math.Min(cp.PeriodTimeCount - i, Parameters.PeriodTimeNames[i].Length)); ++j)
{
int nowRowIdx2;
if (j == -1)
nowRowIdx2 = nowRowIdx;
else
nowRowIdx2 = FindRowByTime(dvs[s, i, j + 1], mainTime / periodSeconds * periodSeconds, ref dvsIdx[s, i, j + 1]);
if (nowRowIdx2 - p < 0)
{
throw new ArgumentException("No prev data!");
}
ForexData nowDrv2 = dvs[s, i, j + 1][nowRowIdx2 - p];
foreach (var kvp in cp.AllIndNames)
{
double v = (double)nowDrv2[kvp.Key];
double ind = WekaUtils.NormalizeValue(kvp.Key, kvp.Value, v, mainClose, WekaUtils.GetSymbolPoint(cp.AllSymbols[s]));
instanceValue[start] = ind;
start++;
}
}
}
}
}
}
catch (ArgumentException)
{
continue;
}
if (m_useClassAsAttribute)
{
instanceValue[hereInstances.numAttributes() - 2] = hp;
}
instanceValue[hereInstances.numAttributes() - 1] = hp;
Instance instance = new weka.core.DenseInstance(1, instanceValue);
//if (!hereInstances.checkInstance(instance))
//{
// throw new ArgumentException("Imcompatible instance!");
//}
hereInstances.add(instance);
}
}
}
///
/// <summary> * Signify that this batch of input to the filter is finished. If
/// * the filter requires all instances prior to filtering, output()
/// * may now be called to retrieve the filtered instances. Any
/// * subsequent instances filtered should be filtered based on setting
/// * obtained from the first batch (unless the inputFormat has been
/// * re-assigned or new options have been set). This default
/// * implementation assumes all instance processing occurs during
/// * inputFormat() and input().
/// * </summary>
/// * <returns> true if there are instances pending output </returns>
/// * <exception cref="NullPointerException"> if no input structure has been defined, </exception>
/// * <exception cref="Exception"> if there was a problem finishing the batch. </exception>
///
public override bool batchFinished()
{
if (getInputFormat() == null)
{
throw new java.lang.IllegalStateException("No input instance format defined");
}
// throw if all attributes are not numeric
Instances instances = getInputFormat();
if (instances.numInstances() < getNumCoeffs())
{
throw new java.lang.IllegalStateException("Number of coeffs cannot be greater " + "than the total number of instances");
}
m_attsFT = new AttFTHolder [instances.numAttributes()];
int nearestPower2;
for (int attr = 0; attr < instances.numAttributes(); ++attr)
{
m_attsFT[attr] = new AttFTHolder();
double[] array = instances.attributeToDoubleArray(attr);
// get the nearest power of 2 for the FT
for (nearestPower2 = 1; array.Length > nearestPower2; nearestPower2 <<= 1)
{
;
}
// initialize the complex numbers
m_attsFT[attr].re = new double[nearestPower2];
m_attsFT[attr].im = new double[nearestPower2];
int j = 0;
for (int i = 0; i < nearestPower2; ++i, ++j)
{
m_attsFT[attr].re[i] = (j < array.Length) ? array[i] : 0;
m_attsFT[attr].im[i] = 0;
}
// inplace FT
if (m_useFFT)
{
computeFFT(m_attsFT[attr].re, m_attsFT[attr].im);
}
else
{
computeDFT(m_attsFT[attr].re, m_attsFT[attr].im);
}
}
// set instances of the new dataset
for (int i = 0; i < getNumCoeffs(); ++i)
{
double[] vals = new double[instances.numAttributes() * 2];
for (int j = 0; j < instances.numAttributes(); ++j)
{
vals[2 * j] = m_attsFT[j].re[i];
vals[2 * j + 1] = m_attsFT[j].im[i];
}
Instance inst = new weka.core.DenseInstance(instances.instance(i).weight(), vals);
// Modified
//inst.setDataset(instances.instance(i).dataset());
push(inst);
}
flushInput();
m_NewBatch = true;
return(numPendingOutput() != 0);
}