public void TestBufferData() { File.Delete(Filename); var set = new BufferedMLDataSet(Filename); set.BeginLoad(2, 1); for (int i = 0; i < XOR.XORInput.Length; i++) { var input = new BasicMLData(XOR.XORInput[i]); var ideal = new BasicMLData(XOR.XORIdeal[i]); set.Add(input, ideal); } set.EndLoad(); XOR.TestXORDataSet(set); }
/// <summary> /// Convert a CSV file to a binary training file. /// </summary> /// <param name="csvFile">The CSV file.</param> /// <param name="format">The format.</param> /// <param name="binFile">The binary file.</param> /// <param name="inputCount">The number of input values.</param> /// <param name="outputCount">The number of output values.</param> /// <param name="headers">True, if there are headers on the3 CSV.</param> /// <param name="expectSignificance">Should a significance column be expected.</param> public static void ConvertCSV2Binary(String csvFile, CSVFormat format, String binFile, int inputCount, int outputCount, bool headers, bool expectSignificance) { new FileInfo(binFile).Delete(); var csv = new CSVMLDataSet(csvFile, inputCount, outputCount, false, format, expectSignificance); var buffer = new BufferedMLDataSet(binFile); buffer.BeginLoad(inputCount, outputCount); foreach (IMLDataPair pair in csv) { buffer.Add(pair); } buffer.EndLoad(); }
public void Train(bool useGui) { // load, or create the neural network BasicNetwork network; if (!_config.TrainedNetworkFile.Exists) { throw new EncogError(@"Can't find neural network file, please generate data"); } network = (BasicNetwork) EncogDirectoryPersistence.LoadObject(_config.TrainedNetworkFile); // convert training data Console.WriteLine(@"Converting training file to binary"); EncogUtility.ConvertCSV2Binary( _config.NormalizedDataFile.ToString(), CSVFormat.English, _config.BinaryFile.ToString(), network.InputCount, network.OutputCount, false, false); var trainingSet = new BufferedMLDataSet( _config.BinaryFile.ToString()); if (useGui) { EncogUtility.TrainDialog(network, trainingSet); } else { EncogUtility.TrainConsole(network, trainingSet, _config.TrainingMinutes); } Console.WriteLine(@"Training complete, saving network..."); EncogDirectoryPersistence.SaveObject(_config.TrainedNetworkFile, network); }
/// <summary> /// Save the training set to an EGB file. /// </summary> /// <param name="egbFile">The EGB file to save to.</param> /// <param name="data">The training data to save.</param> public static void SaveEGB(FileInfo egbFile, IMLDataSet data) { var binary = new BufferedMLDataSet(egbFile.ToString()); binary.Load(data); data.Close(); }
/// <summary> /// Load an EGB file to memory. /// </summary> /// <param name="filename">The file to load.</param> /// <returns>A memory data set.</returns> public static IMLDataSet LoadEGB2Memory(FileInfo filename) { var buffer = new BufferedMLDataSet(filename.ToString()); return buffer.LoadToMemory(); }
/// <summary> /// Evaluate disk. /// </summary> private void EvalBinary() { FileInfo file = FileUtil.CombinePath( new FileInfo(Path.GetTempPath()), "temp.egb" ); BasicMLDataSet training = RandomTrainingFactory.Generate( 1000, 10000, 10, 10, -1, 1); // create the binary file if (file.Exists) { file.Delete(); } var training2 = new BufferedMLDataSet(file.ToString()); training2.Load(training); const long stop = (10*Evaluate.Milis); int record = 0; IMLDataPair pair; var watch = new Stopwatch(); watch.Start(); int iterations = 0; while(true) { iterations++; pair = training[record++]; if(record >= training.Count) record = 0; if((iterations & 0xff) == 0 && watch.ElapsedMilliseconds >= stop) break; } training2.Close(); iterations /= 100000; _report.Report(Steps, Step3, "Disk(binary) dataset, result: " + Format.FormatInteger(iterations)); if (file.Exists) { file.Delete(); } _binaryScore = iterations; }
/// <summary> /// Remove an additional dataset that was created. /// </summary> /// <param name="child">The additional dataset to remove.</param> public void RemoveAdditional(BufferedMLDataSet child) { lock (this) { _additional.Remove(child); } }
/// <summary> /// Open an additional training set. /// </summary> /// <returns>An additional training set.</returns> public IMLDataSet OpenAdditional() { var result = new BufferedMLDataSet(_file) {_owner = this}; _additional.Add(result); return result; }
/// <summary> /// Called to generate the training file. /// </summary> public void Generate() { string[] list = Directory.GetFiles(_path); _trainingFile.Delete(); var output = new BufferedMLDataSet(_trainingFile.ToString()); output.BeginLoad(Config.InputWindow, 1); foreach (string file in list) { var fn = new FileInfo(file); if (fn.Name.StartsWith("collected") && fn.Name.EndsWith(".csv")) { ProcessFile(file, output); } } output.EndLoad(); output.Close(); }
public IMLDataSet OpenAdditional() { BufferedMLDataSet item = new BufferedMLDataSet(this.xb44380e048627945) { _x071bde1041617fce = this }; this.xaa0d3e5126463e13.Add(item); return item; }
/// <summary> /// Load an EGB file to memory. /// </summary> /// <param name="filename">The file to load.</param> /// <returns>A memory data set.</returns> public static IMLDataSet LoadEGB2Memory(FileInfo filename) { var buffer = new BufferedMLDataSet(filename.ToString()); var result = buffer.LoadToMemory(); buffer.Close(); return result; }
/// <summary> /// Construct the buffered enumerator. This is where the file is actually /// opened. /// </summary> /// <param name="owner">The object that created this enumeration.</param> public BufferedNeuralDataSetEnumerator(BufferedMLDataSet owner) { _data = owner; _current = 0; }
public static void ConvertCSV2Binary(FileInfo csvFile, CSVFormat format, FileInfo binFile, int[] input, int[] ideal, bool headers) { ReadCSV dcsv; BufferedMLDataSet set; BasicMLData data; BasicMLData data2; int num; int num2; binFile.Delete(); goto Label_00FB; Label_0021: if (dcsv.Next() || ((((uint) num) - ((uint) num2)) > uint.MaxValue)) { data = new BasicMLData(input.Length); if ((((uint) headers) | uint.MaxValue) != 0) { data2 = new BasicMLData(ideal.Length); if (4 != 0) { if (((uint) num) <= uint.MaxValue) { goto Label_0073; } goto Label_00FB; } } goto Label_00C0; } set.EndLoad(); if (0 == 0) { return; } Label_0073: num = 0; while (num < input.Length) { data[num] = dcsv.GetDouble(input[num]); num++; } for (num2 = 0; num2 < ideal.Length; num2++) { data2[num2] = dcsv.GetDouble(ideal[num2]); } set.Add(data, data2); goto Label_0021; Label_00C0: set = new BufferedMLDataSet(binFile.ToString()); set.BeginLoad(input.Length, ideal.Length); goto Label_0021; Label_00FB: dcsv = new ReadCSV(csvFile.ToString(), headers, format); goto Label_00C0; }
public static void ConvertCSV2Binary(string csvFile, CSVFormat format, string binFile, int inputCount, int outputCount, bool headers, bool expectSignificance) { new FileInfo(binFile).Delete(); CSVMLDataSet set = new CSVMLDataSet(csvFile, inputCount, outputCount, false, format, expectSignificance); BufferedMLDataSet set2 = new BufferedMLDataSet(binFile); set2.BeginLoad(inputCount, outputCount); if ((((uint) inputCount) & 0) == 0) { foreach (IMLDataPair pair in set) { set2.Add(pair); } set2.EndLoad(); } }
public void RemoveAdditional(BufferedMLDataSet child) { lock (this) { this.xaa0d3e5126463e13.Remove(child); } }
/// <summary> /// Process the individual training file. /// </summary> /// <param name="file">The training file to process.</param> /// <param name="output">The data set to output to.</param> protected void ProcessFile(string file, BufferedMLDataSet output) { var inputData = new BasicMLData(output.InputSize); var idealData = new BasicMLData(output.IdealSize); var csv = new ReadCSV(file, true, CSVFormat.English); while (csv.Next()) { var a = new double[Config.InputWindow + 1]; double close = csv.GetDouble(1); const int fastIndex = 2; const int slowIndex = fastIndex + Config.InputWindow; a[0] = close; for (int i = 0; i < 3; i++) { double fast = csv.GetDouble(fastIndex + i); double slow = csv.GetDouble(slowIndex + i); double diff = _fieldDifference.Normalize((fast - slow)/Config.PipSize); a[i + 1] = diff; } _window.Add(a); if (_window.IsFull()) { double max = (_window.CalculateMax(0, Config.InputWindow) - close)/Config.PipSize; double min = (_window.CalculateMin(0, Config.InputWindow) - close)/Config.PipSize; double o = Math.Abs(max) > Math.Abs(min) ? max : min; a = _window.GetLast(); for (int i = 0; i < 3; i++) { inputData[i] = a[i + 1]; } o = _fieldOutcome.Normalize(o); idealData[0] = o; output.Add(inputData, idealData); } } }
/// <summary> /// Convert a CSV file to binary. /// </summary> /// <param name="csvFile">The CSV file to convert.</param> /// <param name="format">The format.</param> /// <param name="binFile">The binary file.</param> /// <param name="input">The input.</param> /// <param name="ideal">The ideal.</param> /// <param name="headers">True, if headers are present.</param> public static void ConvertCSV2Binary(FileInfo csvFile, CSVFormat format, FileInfo binFile, int[] input, int[] ideal, bool headers) { binFile.Delete(); var csv = new ReadCSV(csvFile.ToString(), headers, format); var buffer = new BufferedMLDataSet(binFile.ToString()); buffer.BeginLoad(input.Length, ideal.Length); while (csv.Next()) { var inputData = new BasicMLData(input.Length); var idealData = new BasicMLData(ideal.Length); // handle input data for (int i = 0; i < input.Length; i++) { inputData[i] = csv.GetDouble(input[i]); } // handle input data for (int i = 0; i < ideal.Length; i++) { idealData[i] = csv.GetDouble(ideal[i]); } // add to dataset buffer.Add(inputData, idealData); } buffer.EndLoad(); }
private void TrainCommand() { String methodFile = _cmd.Args[0]; String trainingFile = _cmd.Args[1]; String type = _cmd.PromptString("type", "rprop"); String args = _cmd.PromptString("args", ""); double maxError = _cmd.PromptDouble("maxError", 0.01); var dataSet = new BufferedMLDataSet(trainingFile); var method = (IMLMethod) EncogDirectoryPersistence.LoadObject(new FileInfo(methodFile)); var factory = new MLTrainFactory(); IMLTrain train = factory.Create(method, dataSet, type, args); _sw.Start(); EncogUtility.TrainToError(train, maxError); Console.WriteLine(@"Saving machine learning method"); EncogDirectoryPersistence.SaveObject(new FileInfo(methodFile), method); }