/// <summary>
/// Construct a Bayesian trainer. Use K2 to search, and the SimpleEstimator
/// to estimate probability. Init as Naive Bayes
/// </summary>
/// <param name="theNetwork">The network to train.</param>
/// <param name="theData">The data to train.</param>
/// <param name="theMaximumParents">The max number of parents.</param>
public TrainBayesian(BayesianNetwork theNetwork, IMLDataSet theData,
int theMaximumParents)
: this(theNetwork, theData, theMaximumParents,
BayesianInit.InitNaiveBayes, new SearchK2(),
new SimpleEstimator())
{
}
public TrainFlatNetworkBackPropagation(FlatNetwork network, IMLDataSet training, double theLearningRate, double theMomentum)
: base(network, training)
{
this._xef52c16be8e501c9 = theMomentum;
this._x9b481c22b6706459 = theLearningRate;
this._xe4def4d471bbc130 = new double[network.Weights.Length];
}
public PruneIncremental(IMLDataSet training, INeuralNetworkPattern pattern, int iterations, int weightTries, int numTopResults, IStatusReportable report)
: base(report)
{
goto Label_008E;
Label_0031:
this._x7890c8b3a33b26e2 = new double[numTopResults];
return;
Label_008E:
this._x0b03741e8f17a9f7 = false;
this._xab3ddaff42dd298a = new List<HiddenLayerParams>();
this._x823a2b9c8bf459c5 = training;
if ((((uint) numTopResults) - ((uint) weightTries)) < 0)
{
goto Label_0031;
}
this._x49d5b7c4ad0e0bdd = pattern;
if ((((uint) iterations) - ((uint) iterations)) <= uint.MaxValue)
{
this._xdbf51c857aeb8093 = iterations;
this._x64343a0786fb9a3f = report;
this._xe009ad1bd0a8245a = weightTries;
this._xc5f756e0b4a83af0 = new BasicNetwork[numTopResults];
goto Label_0031;
}
goto Label_008E;
}
/// <inheritdoc />
public override void Init(BasicNetwork theNetwork, IMLDataSet theTraining)
{
base.Init(theNetwork, theTraining);
int weightCount = theNetwork.Structure.Flat.Weights.Length;
_training = theTraining;
_network = theNetwork;
_hessianMatrix = new Matrix(weightCount, weightCount);
_hessian = _hessianMatrix.Data;
// create worker(s)
var determine = new DetermineWorkload(
ThreadCount, _training.Count);
_workers = new ChainRuleWorker[determine.ThreadCount];
int index = 0;
// handle CPU
foreach (IntRange r in determine.CalculateWorkers())
{
_workers[index++] = new ChainRuleWorker((FlatNetwork) _flat.Clone(),
_training.OpenAdditional(), r.Low,
r.High);
}
}
/// <summary>
/// Construct a training class.
/// </summary>
///
/// <param name="network">The network to train.</param>
/// <param name="training">The training data.</param>
public ScaledConjugateGradient(IContainsFlat network,
IMLDataSet training) : base(network, training)
{
var rpropFlat = new TrainFlatNetworkSCG(
network.Flat, Training);
FlatTraining = rpropFlat;
}
public static void evaluateNetwork(BasicNetwork network, IMLDataSet training)
{
double total = 0;
int seed = 0;
int completed = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
while (completed < SAMPLE_SIZE)
{
new ConsistentRandomizer(-1, 1, seed).Randomize(network);
int iter = Evaluate(network, training);
if (iter == -1)
{
seed++;
}
else
{
total += iter;
seed++;
completed++;
}
}
sw.Stop();
Console.WriteLine(network.GetActivation(1).GetType().Name + ": time="
+ Format.FormatInteger((int)sw.ElapsedMilliseconds)
+ "ms, Avg Iterations: "
+ Format.FormatInteger((int)(total / SAMPLE_SIZE)));
}
public Backpropagation(IContainsFlat network, IMLDataSet training, double learnRate, double momentum)
: base(network, training)
{
ValidateNetwork.ValidateMethodToData(network, training);
TrainFlatNetworkBackPropagation propagation = new TrainFlatNetworkBackPropagation(network.Flat, this.Training, learnRate, momentum);
base.FlatTraining = propagation;
}
/// <summary>
/// Construct a input field based on a NeuralDataSet.
/// </summary>
/// <param name="usedForNetworkInput">Is this field used for neural input.</param>
/// <param name="data">The data set to use.</param>
/// <param name="offset">The input or ideal index to use. This treats the input
/// and ideal as one long array, concatenated together.</param>
public InputFieldMLDataSet(bool usedForNetworkInput,
IMLDataSet data, int offset)
{
_data = data;
_offset = offset;
UsedForNetworkInput = usedForNetworkInput;
}
public SVMSearchTrain(SupportVectorMachine method, IMLDataSet training)
: base(TrainingImplementationType.Iterative)
{
this._x9425fdc2df7bcafc = 0;
this._x2350dfd8c7639ed6 = -5.0;
this._x38c942a9bdfcbac4 = 2.0;
while (true)
{
if (3 != 0)
{
this._xdee5cbd981b6d49e = 15.0;
if (0 == 0)
{
this._xec9380575da42aee = -10.0;
}
}
this._xd522fee165affb59 = 10.0;
this._x441f2c3a7d69c688 = 1.0;
this._x87a7fc6a72741c2e = method;
this.Training = training;
this._x9eeb587621db687c = false;
if (0 == 0)
{
if (0 == 0)
{
this._xab248fa87e95a7df = false;
this._x1e074b5762f8595b = new SVMTrain(this._x87a7fc6a72741c2e, training);
return;
}
return;
}
}
}
public JacobianChainRule(BasicNetwork network, IMLDataSet indexableTraining)
{
BasicMLData data;
BasicMLData data2;
if (0 == 0)
{
goto Label_0055;
}
Label_0009:
this._x61830ac74d65acc3 = new BasicMLDataPair(data, data2);
return;
Label_0055:
this._xb12276308f0fa6d9 = indexableTraining;
if (0 == 0)
{
}
this._x87a7fc6a72741c2e = network;
this._xabb126b401219ba2 = network.Structure.CalculateSize();
this._x530ae94d583e0ea1 = (int) this._xb12276308f0fa6d9.Count;
this._xbdeab667c25bbc32 = EngineArray.AllocateDouble2D(this._x530ae94d583e0ea1, this._xabb126b401219ba2);
this._xc8a462f994253347 = new double[this._x530ae94d583e0ea1];
data = new BasicMLData(this._xb12276308f0fa6d9.InputSize);
data2 = new BasicMLData(this._xb12276308f0fa6d9.IdealSize);
if (-2147483648 != 0)
{
goto Label_0009;
}
goto Label_0055;
}
/// <summary>
/// Saves an IMLDataset to a file.
/// </summary>
/// <param name="directory">The directory.</param>
/// <param name="file">The file.</param>
/// <param name="trainintoSave">The traininto save.</param>
public static void SaveTraining(string directory, string file, IMLDataSet trainintoSave)
{
FileInfo networkFile = FileUtil.CombinePath(new FileInfo(directory), file);
//save our training file.
EncogUtility.SaveEGB(networkFile, trainintoSave);
return;
}
public QuickPropagation(IContainsFlat network, IMLDataSet training, double learnRate)
: base(network, training)
{
ValidateNetwork.ValidateMethodToData(network, training);
TrainFlatNetworkQPROP kqprop = new TrainFlatNetworkQPROP(network.Flat, this.Training, learnRate);
base.FlatTraining = kqprop;
}
public override double[][][] EstimateXi(IMLDataSet sequence,
ForwardBackwardCalculator fbc, HiddenMarkovModel hmm)
{
if (sequence.Count <= 1)
{
throw new EncogError(
"Must have more than one observation");
}
double[][][] xi = EngineArray.AllocDouble3D((int)sequence.Count - 1, hmm
.StateCount, hmm.StateCount);
for (int t = 0; t < (sequence.Count - 1); t++)
{
IMLDataPair observation = sequence[t+1];
for (int i = 0; i < hmm.StateCount; i++)
{
for (int j = 0; j < hmm.StateCount; j++)
{
xi[t][i][j] = fbc.AlphaElement(t, i)
* hmm.TransitionProbability[i][j]
* hmm.StateDistributions[j].Probability(
observation) * fbc.BetaElement(t + 1, j);
}
}
}
return xi;
}
/// <summary>
/// Construct a Manhattan propagation training object.
/// </summary>
///
/// <param name="network">The network to train.</param>
/// <param name="training">The training data to use.</param>
/// <param name="learnRate">The learning rate.</param>
public ManhattanPropagation(BasicNetwork network,
IMLDataSet training, double learnRate)
: base(network, training)
{
_learningRate = learnRate;
_zeroTolerance = RPROPConst.DefaultZeroTolerance;
}
/// <summary>
/// Construct a QPROP trainer for flat networks.
/// </summary>
/// <param name="network">The network to train.</param>
/// <param name="training">The training data.</param>
/// <param name="learnRate">The learning rate. 2 is a good suggestion as
/// a learning rate to start with. If it fails to converge,
/// then drop it. Just like backprop, except QPROP can
/// take higher learning rates.</param>
public QuickPropagation(BasicNetwork network,
IMLDataSet training, double learnRate)
: base(network, training)
{
ValidateNetwork.ValidateMethodToData(network, training);
LearningRate = learnRate;
}
/// <summary>
/// Construct a new NeuralDataSet based on the parameters specified.
/// </summary>
/// <param name="inputCount">The input count.</param>
/// <param name="idealCount">The output count.</param>
public NormalizationStorageMLDataSet(int inputCount,
int idealCount)
{
_inputCount = inputCount;
_idealCount = idealCount;
_dataset = new BasicMLDataSet();
}
/// <summary>
/// Create an annealing trainer.
/// </summary>
/// <param name="method">The method to use.</param>
/// <param name="training">The training data to use.</param>
/// <param name="argsStr">The arguments to use.</param>
/// <returns>The newly created trainer.</returns>
public IMLTrain Create(IMLMethod method,
IMLDataSet training, String argsStr)
{
if (!(method is BasicNetwork))
{
throw new TrainingError(
"Invalid method type, requires BasicNetwork");
}
ICalculateScore score = new TrainingSetScore(training);
IDictionary<String, String> args = ArchitectureParse.ParseParams(argsStr);
var holder = new ParamsHolder(args);
int populationSize = holder.GetInt(
MLTrainFactory.PropertyPopulationSize, false, 5000);
IMLTrain train = new MLMethodGeneticAlgorithm( () => {
IMLMethod result = (IMLMethod) ObjectCloner.DeepCopy(method);
((IMLResettable)result).Reset();
return result;
}, score, populationSize);
return train;
}
/// <summary>
/// Create a SVM trainer.
/// </summary>
///
/// <param name="method">The method to use.</param>
/// <param name="training">The training data to use.</param>
/// <param name="argsStr">The arguments to use.</param>
/// <returns>The newly created trainer.</returns>
public IMLTrain Create(IMLMethod method,
IMLDataSet training, String argsStr)
{
if (!(method is SupportVectorMachine))
{
throw new EncogError(
"SVM Train training cannot be used on a method of type: "
+ method.GetType().FullName);
}
double defaultGamma = 1.0d/((SupportVectorMachine) method).InputCount;
double defaultC = 1.0d;
IDictionary<String, String> args = ArchitectureParse.ParseParams(argsStr);
var holder = new ParamsHolder(args);
double gamma = holder.GetDouble(MLTrainFactory.PropertyGamma,
false, defaultGamma);
double c = holder.GetDouble(MLTrainFactory.PropertyC, false,
defaultC);
var result = new SVMTrain((SupportVectorMachine) method, training);
result.Gamma = gamma;
result.C = c;
return result;
}
/// <summary>
/// Create an annealing trainer.
/// </summary>
///
/// <param name="method">The method to use.</param>
/// <param name="training">The training data to use.</param>
/// <param name="argsStr">The arguments to use.</param>
/// <returns>The newly created trainer.</returns>
public IMLTrain Create(IMLMethod method,
IMLDataSet training, String argsStr)
{
if (!(method is BasicNetwork))
{
throw new TrainingError(
"Invalid method type, requires BasicNetwork");
}
ICalculateScore score = new TrainingSetScore(training);
IDictionary<String, String> args = ArchitectureParse.ParseParams(argsStr);
var holder = new ParamsHolder(args);
double startTemp = holder.GetDouble(
MLTrainFactory.PropertyTemperatureStart, false, 10);
double stopTemp = holder.GetDouble(
MLTrainFactory.PropertyTemperatureStop, false, 2);
int cycles = holder.GetInt(MLTrainFactory.Cycles, false, 100);
IMLTrain train = new NeuralSimulatedAnnealing(
(BasicNetwork) method, score, startTemp, stopTemp, cycles);
return train;
}
/// <summary>
/// Don't use this constructor, it is for serialization only.
/// </summary>
public FreeformPropagationTraining()
: base(TrainingImplementationType.Iterative)
{
_network = null;
_training = null;
FixFlatSopt = true;
}
public TrainAdaline(BasicNetwork network, IMLDataSet training, double learningRate)
: base(TrainingImplementationType.Iterative)
{
if (((uint) learningRate) > uint.MaxValue)
{
goto Label_003B;
}
Label_0009:
if (network.LayerCount > 2)
{
goto Label_003B;
}
Label_0012:
this._x87a7fc6a72741c2e = network;
this._x823a2b9c8bf459c5 = training;
this._x9b481c22b6706459 = learningRate;
return;
Label_003B:
throw new NeuralNetworkError("An ADALINE network only has two layers.");
if (0x7fffffff == 0)
{
goto Label_0009;
}
goto Label_0012;
}
public IMLTrain Create(IMLMethod method, IMLDataSet training, string argsStr)
{
if (!(method is SupportVectorMachine))
{
throw new EncogError("SVM Train training cannot be used on a method of type: " + method.GetType().FullName);
}
double defaultValue = 1.0 / ((double) ((SupportVectorMachine) method).InputCount);
while (true)
{
double num4;
SVMTrain train;
double num2 = 1.0;
IDictionary<string, string> theParams = ArchitectureParse.ParseParams(argsStr);
ParamsHolder holder = new ParamsHolder(theParams);
double num3 = holder.GetDouble("GAMMA", false, defaultValue);
do
{
num4 = holder.GetDouble("C", false, num2);
train = new SVMTrain((SupportVectorMachine) method, training) {
Gamma = num3
};
}
while (((uint) defaultValue) > uint.MaxValue);
if ((((uint) num2) + ((uint) num3)) <= uint.MaxValue)
{
train.C = num4;
return train;
}
}
}
/// <summary>
/// Create an annealing trainer.
/// </summary>
///
/// <param name="method">The method to use.</param>
/// <param name="training">The training data to use.</param>
/// <param name="argsStr">The arguments to use.</param>
/// <returns>The newly created trainer.</returns>
public IMLTrain Create(IMLMethod method,
IMLDataSet training, String argsStr)
{
if (!(method is BasicNetwork))
{
throw new TrainingError(
"Invalid method type, requires BasicNetwork");
}
ICalculateScore score = new TrainingSetScore(training);
IDictionary<String, String> args = ArchitectureParse.ParseParams(argsStr);
var holder = new ParamsHolder(args);
int populationSize = holder.GetInt(
MLTrainFactory.PropertyPopulationSize, false, 5000);
double mutation = holder.GetDouble(
MLTrainFactory.PropertyMutation, false, 0.1d);
double mate = holder.GetDouble(MLTrainFactory.PropertyMate,
false, 0.25d);
IMLTrain train = new NeuralGeneticAlgorithm((BasicNetwork) method,
new RangeRandomizer(-1, 1), score, populationSize, mutation,
mate);
return train;
}
/// <summary>
/// Construct the outstar trainer.
/// </summary>
///
/// <param name="theNetwork">The network to train.</param>
/// <param name="theTraining">The training data, must provide ideal outputs.</param>
/// <param name="theLearningRate">The learning rate.</param>
public TrainOutstar(CPNNetwork theNetwork, IMLDataSet theTraining,
double theLearningRate) : base(TrainingImplementationType.Iterative)
{
_mustInit = true;
_network = theNetwork;
_training = theTraining;
_learningRate = theLearningRate;
}
/// <summary>
/// Construct a backprop trainer for flat networks.
/// </summary>
///
/// <param name="network">The network to train.</param>
/// <param name="training">The training data.</param>
/// <param name="theLearningRate">The learning rate.</param>
/// <param name="theMomentum">The momentum.</param>
public TrainFlatNetworkBackPropagation(FlatNetwork network,
IMLDataSet training, double theLearningRate,
double theMomentum) : base(network, training)
{
_momentum = theMomentum;
_learningRate = theLearningRate;
_lastDelta = new double[network.Weights.Length];
}
public TrainFlatNetworkQPROP(FlatNetwork network, IMLDataSet training, double theLearningRate)
: base(network, training)
{
this.LearningRate = theLearningRate;
this.LastDelta = new double[base.Network.Weights.Length];
this.Decay = 0.0001;
this.OutputEpsilon = 0.35;
}
/// <inheritdoc/>
public void Init(TrainBayesian theTrainer, BayesianNetwork theNetwork, IMLDataSet theData)
{
_network = theNetwork;
_data = theData;
_train = theTrainer;
OrderNodes();
_index = -1;
}
/**
*
*
* @param network
*
* @param training
*
* @param theLearningRate
*
*/
/// <summary>
/// Construct a QPROP trainer for flat networks.
/// </summary>
/// <param name="network">The network to train.</param>
/// <param name="training">The training data.</param>
/// <param name="learnRate">The learning rate. 2 is a good suggestion as
/// a learning rate to start with. If it fails to converge,
/// then drop it. Just like backprop, except QPROP can
/// take higher learning rates.</param>
public QuickPropagation(IContainsFlat network,
IMLDataSet training, double learnRate) : base(network, training)
{
ValidateNetwork.ValidateMethodToData(network, training);
var backFlat = new TrainFlatNetworkQPROP(
network.Flat, Training, learnRate);
FlatTraining = backFlat;
}
/// <summary>
/// Construct the instar training object.
/// </summary>
///
/// <param name="theNetwork">The network to be trained.</param>
/// <param name="theTraining">The training data.</param>
/// <param name="theLearningRate">The learning rate.</param>
/// <param name="theInitWeights">training elements as instar neurons.</param>
public TrainInstar(CPNNetwork theNetwork, IMLDataSet theTraining,
double theLearningRate, bool theInitWeights) : base(TrainingImplementationType.Iterative)
{
_network = theNetwork;
_training = theTraining;
_learningRate = theLearningRate;
_mustInit = theInitWeights;
}
public IMLTrain Create(IMLMethod method, IMLDataSet training, string args)
{
if (!(method is BasicPNN))
{
throw new EncogError("PNN training cannot be used on a method of type: " + method.GetType().FullName);
}
return new TrainBasicPNN((BasicPNN) method, training);
}
public void train(double[][] trainingInput, double[][] idealOutput)
{
// create training data
trainingSet = new BasicMLDataSet(trainingInput, idealOutput);
// train the neural network
IMLTrain train = new SVMSearchTrain(network, trainingSet);
int epoch = 1;
do
{
train.Iteration();
Console.WriteLine(@"Epoch #" + epoch + @" Error:" + train.Error);
epoch++;
} while (train.Error > 0.01);
train.FinishTraining();
}
/// <summary>
/// Create a SVM trainer.
/// </summary>
///
/// <param name="method">The method to use.</param>
/// <param name="training">The training data to use.</param>
/// <param name="argsStr">The arguments to use.</param>
/// <returns>The newly created trainer.</returns>
public IMLTrain Create(IMLMethod method,
IMLDataSet training, String argsStr)
{
if (!(method is SupportVectorMachine))
{
throw new EncogError(
"SVM Train training cannot be used on a method of type: "
+ method.GetType().FullName);
}
IDictionary <String, String> args = ArchitectureParse.ParseParams(argsStr);
new ParamsHolder(args);
var holder = new ParamsHolder(args);
double gammaStart = holder.GetDouble(
PropertyGamma1, false,
SVMSearchTrain.DefaultGammaBegin);
double cStart = holder.GetDouble(PropertyC1,
false, SVMSearchTrain.DefaultConstBegin);
double gammaStop = holder.GetDouble(
PropertyGamma2, false,
SVMSearchTrain.DefaultGammaEnd);
double cStop = holder.GetDouble(PropertyC2,
false, SVMSearchTrain.DefaultConstEnd);
double gammaStep = holder.GetDouble(
PropertyGammaStep, false,
SVMSearchTrain.DefaultGammaStep);
double cStep = holder.GetDouble(PropertyCStep,
false, SVMSearchTrain.DefaultConstStep);
var result = new SVMSearchTrain((SupportVectorMachine)method, training)
{
GammaBegin = gammaStart,
GammaEnd = gammaStop,
GammaStep = gammaStep,
ConstBegin = cStart,
ConstEnd = cStop,
ConstStep = cStep
};
return(result);
}
public double EvaluateMPROP(BasicNetwork network, IMLDataSet data)
{
var train = new ResilientPropagation(network, data);
long start = DateTime.Now.Ticks;
Console.WriteLine(@"Training 20 Iterations with MPROP");
for (int i = 1; i <= 20; i++)
{
train.Iteration();
Console.WriteLine("Iteration #" + i + " Error:" + train.Error);
}
//train.finishTraining();
long stop = DateTime.Now.Ticks;
double diff = new TimeSpan(stop - start).Seconds;
Console.WriteLine("MPROP Result:" + diff + " seconds.");
Console.WriteLine("Final MPROP error: " + network.CalculateError(data));
return(diff);
}
/// <summary>
/// Validate that the specified data can be used with the method.
/// </summary>
/// <param name="method">The method to validate.</param>
/// <param name="training">The training data.</param>
public static void ValidateMethodToData(IMLMethod method, IMLDataSet training)
{
if (!(method is IMLInput) || !(method is IMLOutput))
{
throw new EncogError(
"This machine learning method is not compatible with the provided data.");
}
int trainingInputCount = training.InputSize;
int trainingOutputCount = training.IdealSize;
int methodInputCount = 0;
int methodOutputCount = 0;
if (method is IMLInput)
{
methodInputCount = ((IMLInput)method).InputCount;
}
if (method is IMLOutput)
{
methodOutputCount = ((IMLOutput)method).OutputCount;
}
if (methodInputCount != trainingInputCount)
{
throw new EncogError(
"The machine learning method has an input length of "
+ methodInputCount + ", but the training data has "
+ trainingInputCount + ". They must be the same.");
}
if (!(method is BasicPNN))
{
if (trainingOutputCount > 0 && methodOutputCount != trainingOutputCount)
{
throw new EncogError(
"The machine learning method has an output length of "
+ methodOutputCount
+ ", but the training data has "
+ trainingOutputCount + ". They must be the same.");
}
}
}
/// <summary>
/// Encode the Encog dataset.
/// </summary>
///
/// <param name="training">The training data.</param>
/// <param name="outputIndex"></param>
/// <returns>The SVM problem.</returns>
public static svm_problem Encode(IMLDataSet training,
int outputIndex)
{
try
{
var result = new svm_problem {
l = (int)training.Count
};
result.y = new double[result.l];
result.x = new svm_node[result.l][];
for (int i = 0; i < result.l; i++)
{
result.x[i] = new svm_node[training.InputSize];
}
int elementIndex = 0;
foreach (IMLDataPair pair in training)
{
IMLData input = pair.Input;
IMLData output = pair.Ideal;
result.x[elementIndex] = new svm_node[input.Count];
for (int i = 0; i < input.Count; i++)
{
result.x[elementIndex][i] = new svm_node {
index = i + 1, value_Renamed = input[i]
};
}
result.y[elementIndex] = output[outputIndex];
elementIndex++;
}
return(result);
}
catch (OutOfMemoryException)
{
throw new EncogError("SVM Model - Out of Memory");
}
}
public void Execute(IExampleInterface app)
{
TemporalXOR temp = new TemporalXOR();
IMLDataSet trainingSet = temp.Generate(120);
FreeformNetwork elmanNetwork = FreeformNetwork.CreateElman(1, 6, 1, new ActivationSigmoid());
FreeformNetwork feedforwardNetwork = FreeformNetwork.CreateFeedforward(1, 6, 0, 1, new ActivationSigmoid());
double feedforwardError = TrainNetwork("feedforward", feedforwardNetwork, trainingSet);
double elmanError = TrainNetwork("elman", elmanNetwork, trainingSet);
Console.WriteLine(@"Best error rate with Elman Network: " + elmanError);
Console.WriteLine(@"Best error rate with Feedforward Network: "
+ feedforwardError);
Console.WriteLine(@"Elman should be able to get into the 10% range,\nfeedforward should not go below 25%.\nThe recurrent Elment net can learn better in this case.");
Console.WriteLine(@"If your results are not as good, try rerunning, or perhaps training longer.");
EncogFramework.Instance.Shutdown();
}
/// <summary>
/// Train the network, to a specific error, send the output to the console.
/// </summary>
/// <param name="method">The model to train.</param>
/// <param name="trainingSet">The training set to use.</param>
/// <param name="error">The error level to train to.</param>
public static void TrainToError(IMLMethod method,
IMLDataSet trainingSet, double error)
{
IMLTrain train;
if (method is SupportVectorMachine)
{
train = new SVMTrain((SupportVectorMachine)method, trainingSet);
}
if (method is FreeformNetwork)
{
train = new FreeformResilientPropagation((FreeformNetwork)method, trainingSet);
}
else
{
train = new ResilientPropagation((IContainsFlat)method, trainingSet);
}
TrainToError(train, error);
}
/// <summary>
/// Calculate an error for a method that uses regression.
/// </summary>
/// <param name="method">The method to evaluate.</param>
/// <param name="data">The training data to evaluate with.</param>
/// <returns>The error.</returns>
public static double CalculateError(IMLRegression method,
IMLDataSet data)
{
var errorCalculation = new ErrorCalculation();
// clear context
if (method is IMLContext)
{
((IMLContext)method).ClearContext();
}
// calculate error
foreach (IMLDataPair pair in data)
{
IMLData actual = method.Compute(pair.Input);
errorCalculation.UpdateError(actual, pair.Ideal, pair.Significance);
}
return(errorCalculation.Calculate());
}
/// <summary>
/// Convert a training set to an array.
/// </summary>
/// <param name="training"></param>
/// <returns></returns>
public static ObjectPair <double[][], double[][]> TrainingToArray(
IMLDataSet training)
{
var length = (int)training.Count;
double[][] a = EngineArray.AllocateDouble2D(length, training.InputSize);
double[][] b = EngineArray.AllocateDouble2D(length, training.IdealSize);
int index = 0;
foreach (IMLDataPair pair in training)
{
EngineArray.ArrayCopy(pair.InputArray, a[index]);
EngineArray.ArrayCopy(pair.IdealArray, b[index]);
index++;
}
return(new ObjectPair <double[][], double[][]>(a, b));
}
/// <summary>
/// Train to a specific error, using the specified training method, send the
/// output to the console.
/// </summary>
/// <param name="train">The training method.</param>
/// <param name="trainingSet">The training set to use.</param>
/// <param name="error">The desired error level.</param>
public static void TrainToError(IMLTrain train,
IMLDataSet trainingSet,
double error)
{
int epoch = 1;
Console.WriteLine(@"Beginning training...");
do
{
train.Iteration();
Console.WriteLine(@"Iteration #" + Format.FormatInteger(epoch)
+ @" Error:" + Format.FormatPercent(train.Error)
+ @" Target Error: " + Format.FormatPercent(error));
epoch++;
} while (train.Error > error && !train.TrainingDone);
train.FinishTraining();
}
/// <summary>
/// Calculate an error for a method that makes use of classification.
/// </summary>
/// <param name="method">The method to check.</param>
/// <param name="data">The data to check.</param>
/// <returns>The error.</returns>
public static double CalculateClassificationError(IMLClassification method,
IMLDataSet data)
{
int total = 0;
int correct = 0;
foreach (IMLDataPair pair in data)
{
var ideal = (int)pair.Ideal[0];
int actual = method.Classify(pair.Input);
if (actual == ideal)
{
correct++;
}
total++;
}
return((total - correct) / (double)total);
}
public void TestCluster()
{
var set = new BasicMLDataSet();
int i;
for (i = 0; i < Data.Length; i++)
{
set.Add(new BasicMLData(Data[i]));
}
var kmeans = new KMeansClustering(2, set);
kmeans.Iteration();
i = 1;
foreach (IMLCluster cluster in kmeans.Clusters)
{
IMLDataSet ds = cluster.CreateDataSet();
IMLDataPair pair;
pair = ds[0];
double t = pair.Input[0];
for (int j = 0; j < ds.Count; j++)
{
pair = ds[j];
for (j = 0; j < pair.Input.Count; j++)
{
if (t > 10)
{
Assert.IsTrue(pair.Input[j] > 10);
}
else
{
Assert.IsTrue(pair.Input[j] < 10);
}
}
}
i++;
}
}
static private void CalculateScore(IMLDataSet trainingSet, BasicNetwork network)
{
Results res = new Results(trainingSet.IdealSize);
CalculateProfit(trainingSet, network, res);
for (int i = 0; i < trainingSet.IdealSize; i++)
{
Console.Write($"{res.profit[i]:F0} ");
}
Console.WriteLine();
Console.Write("Win: ");
for (int i = 0; i < trainingSet.IdealSize; i++)
{
Console.Write($"{res.win[i]} ");
}
Console.WriteLine();
Console.Write("Loss: ");
for (int i = 0; i < trainingSet.IdealSize; i++)
{
Console.Write($"{res.loss[i]} ");
}
Console.WriteLine();
Console.Write("Good direction: ");
for (int i = 0; i < trainingSet.IdealSize; i++)
{
Console.Write($"{100.0 * res.goodDir[i] / (res.badDir[i] + res.goodDir[i]):F3} ");
}
Console.WriteLine();
Console.Write("Win rate: ");
for (int i = 0; i < trainingSet.IdealSize; i++)
{
Console.Write($"{100.0 * res.win[i] / (res.loss[i] + res.win[i]):F3} ");
}
Console.WriteLine();
Console.WriteLine($"Articles traded: {res.articlesTraded}");
Console.WriteLine("Total profit: " + res.profit.Sum());
}
/// <summary>
/// Create an instance of competitive training.
/// </summary>
///
/// <param name="network">The network to train.</param>
/// <param name="learningRate">The learning rate, how much to apply per iteration.</param>
/// <param name="training">The training set (unsupervised).</param>
/// <param name="neighborhood">The neighborhood function to use.</param>
public BasicTrainSOM(SOMNetwork network, double learningRate,
IMLDataSet training, INeighborhoodFunction neighborhood)
: base(TrainingImplementationType.Iterative)
{
_neighborhood = neighborhood;
Training = training;
_learningRate = learningRate;
_network = network;
_inputNeuronCount = network.InputCount;
_outputNeuronCount = network.OutputCount;
_forceWinner = false;
Error = 0;
// setup the correction matrix
_correctionMatrix = new Matrix(_inputNeuronCount,
_outputNeuronCount);
// create the BMU class
_bmuUtil = new BestMatchingUnit(network);
}
/// <summary>
/// Construct the object.
/// </summary>
/// <param name="oseq">The sequence.</param>
/// <param name="hmm">The hidden markov model to use.</param>
/// <param name="doAlpha">Do alpha?</param>
/// <param name="doBeta">Do beta?</param>
public ForwardBackwardCalculator(IMLDataSet oseq,
HiddenMarkovModel hmm, bool doAlpha, bool doBeta)
{
if (oseq.Count < 1)
{
throw new EncogError("Empty sequence");
}
if (doAlpha)
{
ComputeAlpha(hmm, oseq);
}
if (doBeta)
{
ComputeBeta(hmm, oseq);
}
ComputeProbability(oseq, hmm, doAlpha, doBeta);
}
public void Train(BasicNetwork network, IMLDataSet training)
{
IMLTrain trainMain = new LevenbergMarquardtTraining(network, training);
// train the neural network
var stop = new StopTrainingStrategy();
var score = new TrainingSetScore(trainMain.Training);
var trainAlt = new NeuralSimulatedAnnealing(network, score, 10, 2, 100);
trainMain.AddStrategy(new HybridStrategy(trainAlt));
trainMain.AddStrategy(stop);
var epoch = 0;
while (!stop.ShouldStop() && trainMain.IterationNumber < 1500)
{
trainMain.Iteration();
Console.WriteLine("Training " + ", Epoch #" + epoch + " Error:" + trainMain.Error);
epoch++;
}
}
public static double TrainNetwork(String what,
FreeformNetwork network, IMLDataSet trainingSet)
{
ICalculateScore score = new TrainingSetScore(trainingSet);
IMLTrain trainAlt = new NeuralSimulatedAnnealing(
network, score, 10, 2, 100);
IMLTrain trainMain = new FreeformBackPropagation(network, trainingSet, 0.00001, 0.0);
StopTrainingStrategy stop = new StopTrainingStrategy();
trainMain.AddStrategy(new Greedy());
trainMain.AddStrategy(new HybridStrategy(trainAlt));
trainMain.AddStrategy(stop);
EncogUtility.TrainToError(trainMain, 0.01);
return(trainMain.Error);
}
public StochasticGradientDescent(IContainsFlat network,
IMLDataSet training, IGenerateRandom theRandom) :
base(TrainingImplementationType.Iterative)
{
Training = training;
UpdateRule = new AdamUpdate();
if (!(training is BatchDataSet))
{
BatchSize = 25;
}
_method = network;
_flat = network.Flat;
_layerDelta = new double[_flat.LayerOutput.Length];
_gradients = new double[_flat.Weights.Length];
_errorCalculation = new ErrorCalculation();
_rnd = theRandom;
LearningRate = 0.001;
Momentum = 0.9;
}
/// <summary>
/// Validate a network for training.
/// </summary>
///
/// <param name="network">The network to validate.</param>
/// <param name="training">The training set to validate.</param>
public static void ValidateNetworkForTraining(IContainsFlat network,
IMLDataSet training)
{
int inputCount = network.Flat.InputCount;
int outputCount = network.Flat.OutputCount;
if (inputCount != training.InputSize)
{
throw new NeuralNetworkError("The input layer size of "
+ inputCount + " must match the training input size of "
+ training.InputSize + ".");
}
if ((training.IdealSize > 0) &&
(outputCount != training.IdealSize))
{
throw new NeuralNetworkError("The output layer size of "
+ outputCount + " must match the training input size of "
+ training.IdealSize + ".");
}
}
public static int Evaluate(BasicNetwork network, IMLDataSet training)
{
ResilientPropagation rprop = new ResilientPropagation(network, training);
int iterations = 0;
for (; ;)
{
rprop.Iteration();
iterations++;
if (rprop.Error < TARGET_ERROR)
{
return(iterations);
}
if (iterations > 1000)
{
iterations = 0;
return(-1);
}
}
}
/// <summary>
/// Construct the calculator.
/// </summary>
/// <param name="seq">The sequence.</param>
/// <param name="hmm">The HMM.</param>
/// <param name="doAlpha">Should alpha be calculated.</param>
/// <param name="doBeta">Should beta be calculated.</param>
public ForwardBackwardScaledCalculator(
IMLDataSet seq, HiddenMarkovModel hmm,
bool doAlpha, bool doBeta)
{
if (seq.Count < 1)
{
throw new EncogError("Count cannot be less than one.");
}
_ctFactors = new double[seq.Count];
EngineArray.Fill(_ctFactors, 0.0);
ComputeAlpha(hmm, seq);
if (doBeta)
{
ComputeBeta(hmm, seq);
}
ComputeProbability(seq, hmm, doAlpha, doBeta);
}
public static void TestXORDataSet(IMLDataSet set)
{
int row = 0;
foreach (IMLDataPair item in set)
{
for (int i = 0; i < XORInput[0].Length; i++)
{
Assert.AreEqual(item.Input[i],
XORInput[row][i]);
}
for (int i = 0; i < XORIdeal[0].Length; i++)
{
Assert.AreEqual(item.Ideal[i],
XORIdeal[row][i]);
}
row++;
}
}
/// <summary>
/// Perform the training.
/// </summary>
///
/// <param name="train">The training method.</param>
/// <param name="method">The ML method.</param>
/// <param name="trainingSet">The training set.</param>
private void PerformTraining(IMLTrain train, IMLMethod method,
IMLDataSet trainingSet)
{
ValidateNetwork.ValidateMethodToData(method, trainingSet);
double targetError = Prop.GetPropertyDouble(
ScriptProperties.MlTrainTargetError);
Analyst.ReportTrainingBegin();
int maxIteration = Analyst.MaxIteration;
do
{
train.Iteration();
Analyst.ReportTraining(train);
} while ((train.Error > targetError) &&
!Analyst.ShouldStopCommand() &&
!train.TrainingDone &&
((maxIteration == -1) || (train.IterationNumber < maxIteration)));
train.FinishTraining();
Analyst.ReportTrainingEnd();
}
/// <summary>
/// Compute the probability.
/// </summary>
/// <param name="oseq">The sequence.</param>
/// <param name="hmm">THe hidden markov model.</param>
/// <param name="doAlpha">Perform alpha step?</param>
/// <param name="doBeta">Perform beta step?</param>
private void ComputeProbability(IMLDataSet oseq,
HiddenMarkovModel hmm, bool doAlpha, bool doBeta)
{
probability = 0.0;
if (doAlpha)
{
for (int i = 0; i < hmm.StateCount; i++)
{
probability += Alpha[oseq.Count - 1][i];
}
}
else
{
for (int i = 0; i < hmm.StateCount; i++)
{
probability += hmm.GetPi(i)
* hmm.StateDistributions[i].Probability(oseq[0])
* Beta[0][i];
}
}
}
private void EmbedTraining(EncogProgramNode node)
{
var dataFile = (FileInfo)node.Args[0].Value;
IMLDataSet data = EncogUtility.LoadEGB2Memory(dataFile);
// generate the input data
IndentLine("public static final double[][] INPUT_DATA = {");
foreach (IMLDataPair pair in data)
{
IMLData item = pair.Input;
var line = new StringBuilder();
NumberList.ToList(CSVFormat.EgFormat, line, item);
line.Insert(0, "{ ");
line.Append(" },");
AddLine(line.ToString());
}
UnIndentLine("};");
AddBreak();
// generate the ideal data
IndentLine("public static final double[][] IDEAL_DATA = {");
foreach (IMLDataPair pair in data)
{
IMLData item = pair.Ideal;
var line = new StringBuilder();
NumberList.ToList(CSVFormat.EgFormat, line, item);
line.Insert(0, "{ ");
line.Append(" },");
AddLine(line.ToString());
}
UnIndentLine("};");
}
/// <summary>
/// Create a trainer, use cross validation if enabled.
/// </summary>
/// <param name="method">The method to use.</param>
/// <param name="trainingSet">The training set to use.</param>
/// <returns>The trainer.</returns>
private IMLTrain CreateTrainer(IMLMethod method,
IMLDataSet trainingSet)
{
var factory = new MLTrainFactory();
String type = Prop.GetPropertyString(
ScriptProperties.MlTrainType);
String args = Prop.GetPropertyString(
ScriptProperties.MlTrainArguments);
EncogLogging.Log(EncogLogging.LevelDebug, "training type:" + type);
EncogLogging.Log(EncogLogging.LevelDebug, "training args:" + args);
IMLTrain train = factory.Create(method, trainingSet, type, args);
if (_kfold > 0)
{
train = new CrossValidationKFold(train, _kfold);
}
return(train);
}
/// <summary>
/// </summary>
public override sealed bool ExecuteCommand(String args)
{
_kfold = ObtainCross();
IMLDataSet trainingSet = ObtainTrainingSet();
IMLMethod method = ObtainMethod();
IMLTrain trainer = CreateTrainer(method, trainingSet);
EncogLogging.Log(EncogLogging.LevelDebug, "Beginning training");
PerformTraining(trainer, method, trainingSet);
String resourceID = Prop.GetPropertyString(
ScriptProperties.MlConfigMachineLearningFile);
FileInfo resourceFile = Analyst.Script.ResolveFilename(
resourceID);
method = trainer.Method;
EncogDirectoryPersistence.SaveObject(resourceFile, method);
EncogLogging.Log(EncogLogging.LevelDebug, "save to:" + resourceID);
return(Analyst.ShouldStopCommand());
}
public void Execute(IExampleInterface app)
{
this.app = app;
var temp = new TemporalXOR();
IMLDataSet trainingSet = temp.Generate(100);
if (app.Args.Length > 0)
{
trainingSet = temp.Generate(Convert.ToInt16(app.Args[0]));
}
var elmanNetwork = (BasicNetwork)CreateElmanNetwork(trainingSet.InputSize);
var feedforwardNetwork = (BasicNetwork)CreateFeedforwardNetwork(trainingSet.InputSize);
double elmanError = TrainNetwork("Elman", elmanNetwork, trainingSet, "Leven");
double feedforwardError = TrainNetwork("Feedforward", feedforwardNetwork, trainingSet, "Leven");
app.WriteLine("Best error rate with Elman Network: " + elmanError);
app.WriteLine("Best error rate with Feedforward Network: " + feedforwardError);
app.WriteLine("(Elman should outperform feed forward)");
app.WriteLine("If your results are not as good, try rerunning, or perhaps training longer.");
}
/// <summary>
/// Create a RPROP trainer.
/// </summary>
///
/// <param name="method">The method to use.</param>
/// <param name="training">The training data to use.</param>
/// <param name="argsStr">The arguments to use.</param>
/// <returns>The newly created trainer.</returns>
public IMLTrain Create(IMLMethod method,
IMLDataSet training, String argsStr)
{
if (!(method is IContainsFlat))
{
throw new EncogError(
"RPROP training cannot be used on a method of type: "
+ method.GetType().FullName);
}
IDictionary <String, String> args = ArchitectureParse.ParseParams(argsStr);
var holder = new ParamsHolder(args);
double initialUpdate = holder.GetDouble(
MLTrainFactory.PropertyInitialUpdate, false,
RPROPConst.DefaultInitialUpdate);
double maxStep = holder.GetDouble(
MLTrainFactory.PropertyMaxStep, false,
RPROPConst.DefaultMaxStep);
return(new ResilientPropagation((IContainsFlat)method, training,
initialUpdate, maxStep));
}
public static double TrainNetworks(BasicNetwork network, IMLDataSet minis)
{
Backpropagation trainMain = new Backpropagation(network, minis, 0.0001, 0.6);
//set the number of threads below.
trainMain.ThreadCount = 0;
// train the neural network
ICalculateScore score = new TrainingSetScore(minis);
IMLTrain trainAlt = new NeuralSimulatedAnnealing(network, score, 10, 2, 100);
// IMLTrain trainMain = new Backpropagation(network, minis, 0.0001, 0.01);
StopTrainingStrategy stop = new StopTrainingStrategy(0.0001, 200);
trainMain.AddStrategy(new Greedy());
trainMain.AddStrategy(new HybridStrategy(trainAlt));
trainMain.AddStrategy(stop);
//prune strategy not in GIT!...Removing it.
//PruneStrategy strategypruning = new PruneStrategy(0.91d, 0.001d, 10, network,minis, 0, 20);
//trainMain.AddStrategy(strategypruning);
EncogUtility.TrainConsole(trainMain, network, minis, 15.2);
var sw = new Stopwatch();
sw.Start();
while (!stop.ShouldStop())
{
trainMain.Iteration();
Console.WriteLine(@"Iteration #:" + trainMain.IterationNumber + @" Error:" + trainMain.Error + @" Genetic Iteration:" + trainAlt.IterationNumber);
}
sw.Stop();
Console.WriteLine(@"Total elapsed time in seconds:" + TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds).Seconds);
return(trainMain.Error);
}
