This class is not memory based, so very long files can be used, without running out of memory. This dataset uses a Encog binary training file as a buffer. When used with a slower access dataset, such as CSV, XML or SQL, where parsing must occur, this dataset can be used to load from the slower dataset and train at much higher speeds. If you are going to create a binary file, by using the add methods, you must call beginLoad to cause Encog to open an output file. Once the data has been loaded, call endLoad. You can also use the BinaryDataLoader class, with a CODEC, to load many other popular external formats. The binary files produced by this class are in the Encog binary training format, and can be used with any Encog platform. Encog binary files are stored using "little endian" numbers.
Inheritance: INeuralDataSet, IIndexable
BufferedNeuralDataSet Class Documentation
Example #1
0
 /// <summary>
 /// Remove an additional dataset that was created.
 /// </summary>
 /// <param name="child">The additional dataset to remove.</param>
 public void RemoveAdditional(BufferedNeuralDataSet child)
 {
     lock (this)
     {
         this.additional.Remove(child);
     }
 }
Example #2
0
        /// <summary>
        /// Open an additional training set.
        /// </summary>
        /// <returns>An additional training set.</returns>
        public IEngineIndexableSet OpenAdditional()
        {
            BufferedNeuralDataSet result = new BufferedNeuralDataSet(this.file);

            result.owner = this;
            this.additional.Add(result);
            return(result);
        }
Example #3
0
        /// <summary>
        /// Convert a CSV file to a binary training file.
        /// </summary>
        /// <param name="csvFile">The CSV file.</param>
        /// <param name="binFile">The binary file.</param>
        /// <param name="inputCount">The number of input values.</param>
        /// <param name="outputCount">The number of output values.</param>
        /// <param name="headers">True, if there are headers on the3 CSV.</param>
        public static void ConvertCSV2Binary(String csvFile,
                 String binFile, int inputCount, int outputCount,
                 bool headers)
        {

            File.Delete(binFile);
            CSVNeuralDataSet csv = new CSVNeuralDataSet(csvFile.ToString(),
                   inputCount, outputCount, false);
            BufferedNeuralDataSet buffer = new BufferedNeuralDataSet(binFile);
            buffer.BeginLoad(50, 6);
            foreach (INeuralDataPair pair in csv)
            {
                buffer.Add(pair);
            }
            buffer.EndLoad();
        }
Example #4
0
        /// <summary>
        /// Close the dataset.
        /// </summary>
        public void Close()
        {
            Object[] obj = this.additional.ToArray();

            for (int i = 0; i < obj.Length; i++)
            {
                BufferedNeuralDataSet set = (BufferedNeuralDataSet)obj[i];
                set.Close();
            }

            this.additional.Clear();

            if (this.owner != null)
            {
                this.owner.RemoveAdditional(this);
            }

            this.egb.Close();
            this.egb = null;
        }
Example #5
0
        /// <summary>
        /// Evaluate disk.
        /// </summary>
        private void EvalBinary()
        {
            String file = "temp.egb";

            BasicNeuralDataSet training = RandomTrainingFactory.Generate(
                    1000, 10000, 10, 10, -1, 1);

            // create the binary file

            File.Delete(file);

            BufferedNeuralDataSet training2 = new BufferedNeuralDataSet(file);
            training2.Load(training);

            long stop = (10 * Evaluate.MILIS);
            int record = 0;

            INeuralDataPair pair = BasicNeuralDataPair.CreatePair(10, 10);

            Stopwatch watch = new Stopwatch();
            watch.Start();

            int iterations = 0;
            while (watch.ElapsedMilliseconds < stop)
            {
                iterations++;
                training2.GetRecord(record++, pair);
                if (record >= training2.Count)
                    record = 0;
            }

            training2.Close();

            iterations /= 100000;

            this.report.Report(EncogBenchmark.STEPS, EncogBenchmark.STEP4,
                    "Disk(binary) dataset, result: "
                            + Format.FormatInteger(iterations));

            File.Delete(file);
            this.binaryScore = iterations;
        }
Example #6
0
 /// <summary>
 /// Remove an additional dataset that was created. 
 /// </summary>
 /// <param name="child">The additional dataset to remove.</param>
 public void RemoveAdditional(BufferedNeuralDataSet child)
 {
     lock (this)
     {
         this.additional.Remove(child);
     }
 }
Example #7
0
        /// <summary>
        /// Open an additional training set.
        /// </summary>
        /// <returns>An additional training set.</returns>
        public IEngineIndexableSet OpenAdditional()
        {

            BufferedNeuralDataSet result = new BufferedNeuralDataSet(this.file);
            result.owner = this;
            this.additional.Add(result);
            return result;
        }
Example #8
0
 /// <summary>
 /// Construct the buffered enumerator. This is where the file is actually
 /// opened.
 /// </summary>
 /// <param name="owner">The object that created this enumeration.</param>
 public BufferedNeuralDataSetEnumerator(BufferedNeuralDataSet owner)
 {
     this.data    = owner;
     this.current = 0;
 }