Ejemplo n.º 1
0
        ///   
        ///    <summary> * Signify that this batch of input to the filter is finished. If
        ///    * the filter requires all instances prior to filtering, output()
        ///    * may now be called to retrieve the filtered instances. Any
        ///    * subsequent instances filtered should be filtered based on setting
        ///    * obtained from the first batch (unless the inputFormat has been
        ///    * re-assigned or new options have been set). This default
        ///    * implementation assumes all instance processing occurs during
        ///    * inputFormat() and input().
        ///    * </summary>
        ///    * <returns> true if there are instances pending output </returns>
        ///    * <exception cref="NullPointerException"> if no input structure has been defined, </exception>
        ///    * <exception cref="Exception"> if there was a problem finishing the batch. </exception>
        ///    
        public override bool batchFinished()
        {
            if (getInputFormat() == null)
            {
                throw new java.lang.IllegalStateException("No input instance format defined");
            }

            // throw if all attributes are not numeric
            Instances instances = getInputFormat();
            if (instances.numInstances() < getNumCoeffs())
            {
                throw new java.lang.IllegalStateException("Number of coeffs cannot be greater " + "than the total number of instances");
            }

            m_attsFT = new AttFTHolder [instances.numAttributes()];

            int nearestPower2;
            for (int attr = 0; attr < instances.numAttributes(); ++attr)
            {
                m_attsFT[attr] = new AttFTHolder();

                double[] array = instances.attributeToDoubleArray(attr);
            // get the nearest power of 2 for the FT
                for (nearestPower2 = 1; array.Length > nearestPower2; nearestPower2 <<= 1)
                {
                    ;
                }

            // initialize the complex numbers
                m_attsFT[attr].re = new double[nearestPower2];
                m_attsFT[attr].im = new double[nearestPower2];
                int j = 0;
                for (int i=0; i<nearestPower2; ++i, ++j)
                {
                    m_attsFT[attr].re[i] = (j < array.Length) ? array[i] : 0;
                    m_attsFT[attr].im[i] = 0;
                }

            // inplace FT
                if (m_useFFT)
                {
                    computeFFT(m_attsFT[attr].re, m_attsFT[attr].im);
                }
                else
                {
                    computeDFT(m_attsFT[attr].re, m_attsFT[attr].im);
                }
            }

            // set instances of the new dataset
            for (int i=0; i<getNumCoeffs(); ++i)
            {
                double[] vals = new double[instances.numAttributes() * 2];
                for (int j=0; j<instances.numAttributes(); ++j)
                {
                    vals[2*j] = m_attsFT[j].re[i];
                    vals[2*j+1] = m_attsFT[j].im[i];
                }

                Instance inst = new weka.core.DenseInstance(instances.instance(i).weight(), vals);

                // Modified
                //inst.setDataset(instances.instance(i).dataset());
                push(inst);
            }

            flushInput();
            m_NewBatch = true;
            return (numPendingOutput() != 0);
        }
Ejemplo n.º 2
0
        public static void GenerateArffTemplate(bool generateTrainData, bool generateTestData,  CandidateParameter cp)
        {
            if (!m_trainInstancesTemplates.ContainsKey(cp.Name))
            {
                string arffFileName = string.Format("{0}\\mlea_header_{1}.arff", TestParameters.CommonDir, cp.Name);
                if (System.IO.File.Exists(arffFileName))
                {
                    System.IO.File.Delete(arffFileName);
                }
                using (StreamWriter sw = new StreamWriter(arffFileName))
                {
                    sw.WriteLine(GetArffHeader(System.DateTime.Today, System.DateTime.Today, cp));
                }

                m_trainInstancesTemplates[cp.Name] = WekaUtils.LoadInstances(arffFileName);
                m_testInstancesTemplates[cp.Name] = WekaUtils.LoadInstances(arffFileName);
            }
            Instances trainInstancesTemplate = m_trainInstancesTemplates[cp.Name];
            Instances testInstancesTemplate = m_testInstancesTemplates[cp.Name];

            for (int k = 0; k < 2; ++k)
            {
                if (k == 0 && !generateTrainData)
                    continue;
                if (k == 1 && !generateTestData)
                    continue;

                bool isTrain = (k == 0);

                DateTime dt1 = isTrain ? m_trainTimeStart : m_testTimeStart;
                DateTime dt2 = isTrain ? m_trainTimeEnd : m_testTimeEnd;

                Instances hereInstances;
                if (isTrain)
                {
                    hereInstances = trainInstancesTemplate;
                }
                else
                {
                    hereInstances = testInstancesTemplate;
                }

                if (!TestParameters.EnablePerhourTrain)
                {
                    while (hereInstances.numInstances() > 0)
                    {
                        if (WekaUtils.GetDateValueFromInstances(hereInstances, 0, 0) < dt1)
                        {
                            hereInstances.delete(0);
                        }
                        else
                        {
                            break;
                        }
                    }
                }
                else
                {
                    hereInstances.delete();
                }

                hereInstances.setRelationName(string.Format("{0}_{1}", dt1.ToString(Parameters.DateTimeFormat), dt2.ToString(Parameters.DateTimeFormat)));

                ForexDataRows[, ,] dvs = new ForexDataRows[cp.SymbolCount, cp.PeriodCount, cp.PeriodTimeCount + 1];
                int[, ,] dvsIdx = new int[cp.SymbolCount, cp.PeriodCount, cp.PeriodTimeCount + 1];
                for (int s = 0; s < cp.SymbolCount; ++s)
                {
                    for (int i = 0; i < cp.PeriodCount; ++i)
                    {
                        for (int j = -1; j < Math.Max(0, Math.Min(cp.PeriodTimeCount - i, Parameters.PeriodTimeNames[i].Length)); ++j)
                        {
                            string tableName = cp.AllSymbols[s + cp.SymbolStart] + "_" +
                                cp.AllPeriods[i + cp.PeriodStart] +
                                (j < 0 ? string.Empty : "_" + Parameters.PeriodTimeNames[i + cp.PeriodStart][j]);
                            dvs[s, i, j + 1] = DbData.Instance.GetDbData(dt1, dt2, tableName, s == 0 && i == 0 && j == -1 ? 0 : 1, isTrain, cp);

                            dvsIdx[s, i, j + 1] = 0;
                        }
                    }
                }

                ForexDataRows mainDv = dvs[0, 0, 0];
                if (mainDv.Length == 0)
                    continue;
                if (dvs[0, cp.PeriodCount - 1, 0].Length <= cp.PrevTimeCount - 1)
                    continue;

                int startRowIdx = 0;
                startRowIdx = 0;// FindRowByTime(mainDv, (long)dvs[0, PeriodCount - 1, 0][PrevTimeCount - 1]["Time"], ref startRowIdx);

                DateTime nowInstanceMaxDate = DateTime.MinValue;
                if (!TestParameters.EnablePerhourTrain && hereInstances.numInstances() > 0)
                {
                    nowInstanceMaxDate = WekaUtils.GetDateValueFromInstances(hereInstances, 0, hereInstances.numInstances() - 1);
                }

                for (int rowIdx = startRowIdx; rowIdx < mainDv.Length; ++rowIdx)
                {
                    ForexData mainDrv = mainDv[rowIdx];
                    //DateTime row_date = (DateTime)mainDrv[1];   // "Date"
                    //long mainTime = (long)mainDrv[0];    // "Time"
                    long mainTime = mainDrv.Time;
                    DateTime row_date = WekaUtils.GetDateFromTime(mainTime);

                    if (row_date <= nowInstanceMaxDate)
                        continue;

                    if (row_date < dt1)
                        continue;
                    if (row_date >= dt2)
                        break;
                    if (TestParameters.EnablePerhourTrain)
                    {
                        if (m_currentTestHour != row_date.Hour)
                            continue;
                    }

                    int hp = 1;

                    double[] instanceValue = new double[hereInstances.numAttributes()];

                    //instanceValue[0] = hereInstances.attribute(0).parseDate(row_date.ToString(Parameters.DateTimeFormat));
                    //instanceValue[1] = hereInstances.attribute(1).parseDate(hp_date.ToString(Parameters.DateTimeFormat));
                    //instance.setValue(1, hereInstances.attribute(1).indexOfValue(mainDrv["hour"].ToString()));
                    //instance.setValue(2, hereInstances.attribute(2).indexOfValue(mainDrv["dayofweek"].ToString()));
                    instanceValue[0] = (row_date - Parameters.MtStartTime).TotalMilliseconds; // if not set to gmt, should -8 * 60 * 60 * 1000;  // utc8
                    instanceValue[1] = (Parameters.MaxDate - Parameters.MtStartTime).TotalMilliseconds;
                    instanceValue[2] = Convert.ToDouble(mainDrv["spread"]);
                    instanceValue[3] = (double)mainDrv["mainClose"];

                    instanceValue[4] = (int)mainDrv["hour"] / 24.0;
                    instanceValue[5] = (int)mainDrv["dayofweek"] / 5.0;

                    //if (mainDrv["AskVolume"] != System.DBNull.Value && mainDrv["BidVolume"] != System.DBNull.Value)
                    //{
                    //    instanceValue[4] = ((((double)mainDrv["AskVolume"]) - (double)mainDrv["BidVolume"]) / 100000);
                    //}
                    //else
                    {
                        instanceValue[6] = 0;
                    }

                    int start = 7;
                    try
                    {
                        for (int s = 0; s < cp.SymbolCount; ++s)
                        {
                            //double mainClose = (double)mainDrv["close"];
                            int nowRowIdx_s = FindRowByTime(dvs[s, 0, 0], mainTime, ref dvsIdx[s, 0, 0]);
                            WekaUtils.DebugAssert((long)dvs[s, 0, 0][nowRowIdx_s].Time == mainTime, "(long)dvs[s, 0, 0][nowRowIdx_s].Time == mainTime");
                            double mainClose = 0;
                            if (cp.AllIndNames2.ContainsKey("close"))
                            {
                                mainClose = (double)dvs[s, 0, 0][nowRowIdx_s]["close"];
                            }

                            for (int i = 0; i < cp.PeriodCount; ++i)
                            {
                                int periodSeconds = 60 * WekaUtils.GetMinuteofPeriod(cp.AllPeriods[i + cp.PeriodStart]);
                                int nowRowIdx = FindRowByTime(dvs[s, i, 0], mainTime / periodSeconds * periodSeconds, ref dvsIdx[s, i, 0]);

                                for (int p = 0; p < cp.PrevTimeCount; ++p)
                                {
                                    if (nowRowIdx - p < 0)
                                    {
                                        throw new ArgumentException("No prev data!");
                                    }
                                    ForexData nowDrv = dvs[s, i, 0][nowRowIdx - p];

                                    foreach (var kvp in cp.AllIndNames2)
                                    {
                                        double v = Convert.ToDouble(nowDrv[kvp.Key]);
                                        double ind = WekaUtils.NormalizeValue(kvp.Key, kvp.Value, v, mainClose, WekaUtils.GetSymbolPoint(cp.AllSymbols[s]));

                                        instanceValue[start] = ind;
                                        start++;
                                    }

                                    for (int j = -1; j < Math.Max(0, Math.Min(cp.PeriodTimeCount - i, Parameters.PeriodTimeNames[i].Length)); ++j)
                                    {
                                        int nowRowIdx2;
                                        if (j == -1)
                                            nowRowIdx2 = nowRowIdx;
                                        else
                                            nowRowIdx2 = FindRowByTime(dvs[s, i, j + 1], mainTime / periodSeconds * periodSeconds, ref dvsIdx[s, i, j + 1]);
                                        if (nowRowIdx2 - p < 0)
                                        {
                                            throw new ArgumentException("No prev data!");
                                        }
                                        ForexData nowDrv2 = dvs[s, i, j + 1][nowRowIdx2 - p];

                                        foreach (var kvp in cp.AllIndNames)
                                        {
                                            double v = (double)nowDrv2[kvp.Key];
                                            double ind = WekaUtils.NormalizeValue(kvp.Key, kvp.Value, v, mainClose, WekaUtils.GetSymbolPoint(cp.AllSymbols[s]));

                                            instanceValue[start] = ind;
                                            start++;
                                        }
                                    }
                                }
                            }
                        }
                    }
                    catch (ArgumentException)
                    {
                        continue;
                    }

                    if (m_useClassAsAttribute)
                    {
                        instanceValue[hereInstances.numAttributes() - 2] = hp;
                    }
                    instanceValue[hereInstances.numAttributes() - 1] = hp;

                    Instance instance = new weka.core.DenseInstance(1, instanceValue);
                    //if (!hereInstances.checkInstance(instance))
                    //{
                    //    throw new ArgumentException("Imcompatible instance!");
                    //}
                    hereInstances.add(instance);
                }
            }
        }
Ejemplo n.º 3
0
///
///    <summary> * Signify that this batch of input to the filter is finished. If
///    * the filter requires all instances prior to filtering, output()
///    * may now be called to retrieve the filtered instances. Any
///    * subsequent instances filtered should be filtered based on setting
///    * obtained from the first batch (unless the inputFormat has been
///    * re-assigned or new options have been set). This default
///    * implementation assumes all instance processing occurs during
///    * inputFormat() and input().
///    * </summary>
///    * <returns> true if there are instances pending output </returns>
///    * <exception cref="NullPointerException"> if no input structure has been defined, </exception>
///    * <exception cref="Exception"> if there was a problem finishing the batch. </exception>
///
        public override bool batchFinished()
        {
            if (getInputFormat() == null)
            {
                throw new java.lang.IllegalStateException("No input instance format defined");
            }

            // throw if all attributes are not numeric
            Instances instances = getInputFormat();

            if (instances.numInstances() < getNumCoeffs())
            {
                throw new java.lang.IllegalStateException("Number of coeffs cannot be greater " + "than the total number of instances");
            }

            m_attsFT = new AttFTHolder [instances.numAttributes()];

            int nearestPower2;

            for (int attr = 0; attr < instances.numAttributes(); ++attr)
            {
                m_attsFT[attr] = new AttFTHolder();

                double[] array = instances.attributeToDoubleArray(attr);
                // get the nearest power of 2 for the FT
                for (nearestPower2 = 1; array.Length > nearestPower2; nearestPower2 <<= 1)
                {
                    ;
                }

                // initialize the complex numbers
                m_attsFT[attr].re = new double[nearestPower2];
                m_attsFT[attr].im = new double[nearestPower2];
                int j = 0;
                for (int i = 0; i < nearestPower2; ++i, ++j)
                {
                    m_attsFT[attr].re[i] = (j < array.Length) ? array[i] : 0;
                    m_attsFT[attr].im[i] = 0;
                }

                // inplace FT
                if (m_useFFT)
                {
                    computeFFT(m_attsFT[attr].re, m_attsFT[attr].im);
                }
                else
                {
                    computeDFT(m_attsFT[attr].re, m_attsFT[attr].im);
                }
            }

            // set instances of the new dataset
            for (int i = 0; i < getNumCoeffs(); ++i)
            {
                double[] vals = new double[instances.numAttributes() * 2];
                for (int j = 0; j < instances.numAttributes(); ++j)
                {
                    vals[2 * j]     = m_attsFT[j].re[i];
                    vals[2 * j + 1] = m_attsFT[j].im[i];
                }

                Instance inst = new weka.core.DenseInstance(instances.instance(i).weight(), vals);

                // Modified
                //inst.setDataset(instances.instance(i).dataset());
                push(inst);
            }

            flushInput();
            m_NewBatch = true;
            return(numPendingOutput() != 0);
        }