/// <summary>
/// Clone a flat network.
/// </summary>
/// <param name="result">The cloned flat network.</param>
public void CloneFlatNetwork(FlatNetwork result)
{
result.inputCount = this.inputCount;
result.layerCounts = EngineArray.ArrayCopy(this.layerCounts);
result.layerIndex = EngineArray.ArrayCopy(this.layerIndex);
result.layerOutput = EngineArray.ArrayCopy(this.layerOutput);
result.layerFeedCounts = EngineArray.ArrayCopy(this.layerFeedCounts);
result.contextTargetOffset = EngineArray
.ArrayCopy(this.contextTargetOffset);
result.contextTargetSize = EngineArray
.ArrayCopy(this.contextTargetSize);
result.layerContextCount = EngineArray
.ArrayCopy(this.layerContextCount);
result.biasActivation = EngineArray.ArrayCopy(this.biasActivation);
result.outputCount = this.outputCount;
result.weightIndex = this.weightIndex;
result.weights = this.weights;
result.activationFunctions = new IActivationFunction[this.activationFunctions.Length];
for (int i = 0; i < result.activationFunctions.Length; i++)
{
result.activationFunctions[i] = (IActivationFunction)this.activationFunctions[i].Clone();
}
result.beginTraining = this.beginTraining;
result.endTraining = this.endTraining;
}
/// <summary>
/// Clone the network.
/// </summary>
///
/// <returns>A clone of the network.</returns>
public virtual Object Clone()
{
FlatNetwork result = new FlatNetwork();
CloneFlatNetwork(result);
return(result);
}
/// <summary>
/// Construct a trainer for flat networks to use the Manhattan update rule.
/// </summary>
///
/// <param name="network">The network to train.</param>
/// <param name="training">The training data to use.</param>
/// <param name="learningRate">The learning rate to use.</param>
public TrainFlatNetworkManhattan(FlatNetwork network,
IEngineDataSet training, double learningRate)
: base(network, training)
{
this.learningRate = learningRate;
this.zeroTolerance = RPROPConst.DEFAULT_ZERO_TOLERANCE;
}
/// <summary>
/// Construct a backprop trainer for flat networks.
/// </summary>
///
/// <param name="network">The network to train.</param>
/// <param name="training">The training data.</param>
/// <param name="learningRate">The learning rate.</param>
/// <param name="momentum">The momentum.</param>
public TrainFlatNetworkBackPropagation(FlatNetwork network,
IEngineDataSet training, double learningRate,
double momentum)
: base(network, training)
{
this.momentum = momentum;
this.learningRate = learningRate;
this.lastDelta = new double[network.Weights.Length];
}
/// <summary>
/// Construct a resilient trainer for flat networks.
/// </summary>
///
/// <param name="network">The network to train.</param>
/// <param name="training">The training data to use.</param>
/// <param name="zeroTolerance">How close a number should be to zero to be counted as zero.</param>
/// <param name="initialUpdate">The initial update value.</param>
/// <param name="maxStep">The maximum step value.</param>
public TrainFlatNetworkResilient(FlatNetwork network,
IEngineDataSet training, double zeroTolerance,
double initialUpdate, double maxStep)
: base(network, training)
{
this.updateValues = new double[network.Weights.Length];
this.zeroTolerance = zeroTolerance;
this.maxStep = maxStep;
for (int i = 0; i < this.updateValues.Length; i++)
{
this.updateValues[i] = initialUpdate;
}
}
/// <summary>
/// Construct a cross validation trainer.
/// </summary>
/// <param name="train">The training.</param>
/// <param name="k">The number of folds.</param>
public CrossValidationKFold(ITrain train, int k)
: base(train.Network, (FoldedDataSet)train.Training)
{
this.train = train;
Folded.Fold(k);
this.flatNetwork = train.Network.Structure.Flat;
this.networks = new NetworkFold[k];
for (int i = 0; i < networks.Length; i++)
{
this.networks[i] = new NetworkFold(flatNetwork);
}
}
/// <summary>
/// Determine if the network is valid for OpenCL.
/// </summary>
///
/// <param name="network">The network to check.</param>
/// <returns>The string indicating the error that prevents OpenCL from usingthe network, or null if the network is fine for OpenCL.</returns>
public override String IsValid(IEngineMachineLearning network)
{
if (!(network is FlatNetwork))
{
return("Only flat networks are valid to be used for OpenCL");
}
FlatNetwork flat = (FlatNetwork)network;
/* foreach */
foreach (IActivationFunction activation in flat.ActivationFunctions)
{
if (activation.GetOpenCLExpression(true) == null)
{
return("Can't use OpenCL if activation function does not have an OpenCL expression.");
}
}
if (flat.HasSameActivationFunction() == null)
{
return("Can't use OpenCL training on a neural network that uses multiple activation functions.");
}
bool hasContext = false;
for (int i = 0; i < flat.LayerCounts.Length; i++)
{
if (flat.ContextTargetOffset[i] != 0)
{
hasContext = true;
}
if (flat.ContextTargetSize[i] != 0)
{
hasContext = true;
}
}
if (hasContext)
{
return("Can't use OpenCL if context neurons are present.");
}
return(null);
}
/// <summary>
/// Compile the kernel.
/// </summary>
///
/// <param name="options">The options.</param>
/// <param name="network">The network to compile for.</param>
/// <param name="profile">The OpenCL training profile to use.</param>
public void Compile(IDictionary<String, String> options,
OpenCLTrainingProfile profile, FlatNetwork network)
{
IActivationFunction activation = network.ActivationFunctions[0];
StringBuilder source = new StringBuilder();
source.Append("#define ACTIVATION(x,slope)");
source.Append(activation.GetOpenCLExpression(false));
source.Append("\r\n");
source.Append("#define DERIVATIVE(x,slope)");
source.Append(activation.GetOpenCLExpression(true));
source.Append("\r\n");
source.Append(ResourceLoader.LoadString(SourceName));
CLSource = source.ToString();
Compile(options);
profile.CalculateKernelParams(this, training);
// setup
Init(profile);
}
/// <summary>
/// Construct a kernel to train the network.
/// </summary>
///
/// <param name="device">The OpenCL device to use.</param>
/// <param name="flat">The network to train.</param>
/// <param name="training">The training data.</param>
/// <param name="tempDataSize">How much temp data.</param>
public KernelNetworkTrain(EncogCLDevice device,
FlatNetwork flat, IEngineIndexableSet training,
int tempDataSize)
: base(device, "Encog.Engine.Resources.KernelNetTrain.txt", "NetworkTrain")
{
this.training = training;
this.trainingLength = (int)this.training.Count;
this.device = device;
this.flat = flat;
this.weightInArray = new float[flat.Weights.Length];
this.weightOutArray = new float[flat.Weights.Length];
this.tempDataArray = new float[tempDataSize];
this.gradients = new float[flat.Weights.Length];
this.layerDeltaSize = 0;
for (int i = 0; i < flat.LayerCounts.Length; i++)
{
this.layerDeltaSize += flat.LayerCounts[i];
}
int inputSize = flat.InputCount;
int idealSize = flat.OutputCount;
this.inputArray = new float[inputSize * this.trainingLength];
this.idealArray = new float[idealSize * this.trainingLength];
this.paramArray = new int[10];
IEngineData pair = BasicEngineData.CreatePair(
flat.InputCount, flat.OutputCount);
int inputIndex = 0;
int idealIndex = 0;
for (int i = 0; i < this.trainingLength; i++)
{
training.GetRecord(i, pair);
for (int col = 0; col < flat.InputCount; col++)
{
this.inputArray[inputIndex++] = (float)pair.InputArray[col];
}
for (int col = 0; col < flat.OutputCount; col++)
{
this.idealArray[idealIndex++] = (float)pair.IdealArray[col];
}
}
}
/// <summary>
/// Construct the training object.
/// </summary>
///
/// <param name="network">The network to train.</param>
/// <param name="training">The training data to use.</param>
public TrainFlatNetworkSCG(FlatNetwork network,
IEngineDataSet training)
: base(network, training)
{
this.success = true;
this.success = true;
this.delta = 0;
this.lambda2 = 0;
this.lambda = TrainFlatNetworkSCG.FIRST_LAMBDA;
this.oldError = 0;
this.magP = 0;
this.restart = false;
this.weights = EngineArray.ArrayCopy(network.Weights);
int numWeights = this.weights.Length;
// this.gradients = new double[numWeights];
this.oldWeights = new double[numWeights];
this.oldGradient = new double[numWeights];
this.p = new double[numWeights];
this.r = new double[numWeights];
this.shouldInit = true;
}
/// <summary>
/// Clone a flat network.
/// </summary>
/// <param name="result">The cloned flat network.</param>
public void CloneFlatNetwork(FlatNetwork result)
{
result.inputCount = this.inputCount;
result.layerCounts = EngineArray.ArrayCopy(this.layerCounts);
result.layerIndex = EngineArray.ArrayCopy(this.layerIndex);
result.layerOutput = EngineArray.ArrayCopy(this.layerOutput);
result.layerFeedCounts = EngineArray.ArrayCopy(this.layerFeedCounts);
result.contextTargetOffset = EngineArray
.ArrayCopy(this.contextTargetOffset);
result.contextTargetSize = EngineArray
.ArrayCopy(this.contextTargetSize);
result.layerContextCount = EngineArray
.ArrayCopy(this.layerContextCount);
result.biasActivation = EngineArray.ArrayCopy(this.biasActivation);
result.outputCount = this.outputCount;
result.weightIndex = this.weightIndex;
result.weights = this.weights;
result.activationFunctions = new IActivationFunction[this.activationFunctions.Length];
for (int i = 0; i < result.activationFunctions.Length; i++)
{
result.activationFunctions[i] = (IActivationFunction)this.activationFunctions[i].Clone();
}
result.beginTraining = this.beginTraining;
result.endTraining = this.endTraining;
}
/// <summary>
/// Clone the network.
/// </summary>
///
/// <returns>A clone of the network.</returns>
public virtual Object Clone()
{
FlatNetwork result = new FlatNetwork();
CloneFlatNetwork(result);
return result;
}
/// <summary>
/// Train a flat network multithreaded.
/// </summary>
///
/// <param name="network">The network to train.</param>
/// <param name="training">The training data to use.</param>
public TrainFlatNetworkProp(FlatNetwork network,
IEngineDataSet training)
{
if (!(training is IEngineIndexableSet))
{
throw new EncogEngineError(
"Training data must be Indexable for this training type.");
}
this.training = training;
this.network = network;
this.gradients = new double[this.network.Weights.Length];
this.lastGradient = new double[this.network.Weights.Length];
this.indexable = (IEngineIndexableSet)training;
this.numThreads = 0;
this.reportedException = null;
}
/// <summary>
/// Train a flat network multithreaded.
/// </summary>
///
/// <param name="network">The network to train.</param>
/// <param name="training">The training data to use.</param>
/// <param name="profile">The OpenCL training profile.</param>
public TrainFlatNetworkOpenCL(FlatNetwork network,
IEngineDataSet training, OpenCLTrainingProfile profile)
{
(new ValidateForOpenCL()).Validate(network);
if (!(training is IEngineIndexableSet))
{
throw new EncogEngineError(
"Training data must be Indexable for this training type.");
}
if (EncogEngine.Instance.CL == null)
{
throw new EncogEngineError(
"You must enable OpenCL before using this training type.");
}
this.profile = profile;
this.network = network;
this.training = (IEngineIndexableSet)training;
}
/// <summary>
/// Construct a gradient worker.
/// </summary>
///
/// <param name="network">The network to train.</param>
/// <param name="owner">The owner that is doing the training.</param>
/// <param name="training">The training data.</param>
/// <param name="low">The low index to use in the training data.</param>
/// <param name="high">The high index to use in the training data.</param>
public GradientWorkerCPU(FlatNetwork network,
TrainFlatNetworkProp owner,
IEngineIndexableSet training, int low, int high)
{
this.errorCalculation = new ErrorCalculation();
this.network = network;
this.training = training;
this.low = low;
this.high = high;
this.owner = owner;
this.stopwatch = new Stopwatch();
this.layerDelta = new double[network.LayerOutput.Length];
this.gradients = new double[network.Weights.Length];
this.actual = new double[network.OutputCount];
this.weights = network.Weights;
this.layerIndex = network.LayerIndex;
this.layerCounts = network.LayerCounts;
this.weightIndex = network.WeightIndex;
this.layerOutput = network.LayerOutput;
this.layerFeedCounts = network.LayerFeedCounts;
this.pair = BasicEngineData.CreatePair(network.InputCount,
network.OutputCount);
}
/// <summary>
/// Tran a network using RPROP.
/// </summary>
///
/// <param name="flat">The network to train.</param>
/// <param name="trainingSet">The training data to use.</param>
public TrainFlatNetworkResilient(FlatNetwork flat,
IEngineDataSet trainingSet)
: this(flat, trainingSet, RPROPConst.DEFAULT_ZERO_TOLERANCE, RPROPConst.DEFAULT_INITIAL_UPDATE, RPROPConst.DEFAULT_MAX_STEP)
{
}
/// <summary>
/// Create the flat neural network.
/// </summary>
public void Flatten()
{
bool isRBF = false;
IDictionary<ILayer, FlatLayer> regular2flat = new Dictionary<ILayer, FlatLayer>();
IDictionary<FlatLayer, ILayer> flat2regular = new Dictionary<FlatLayer, ILayer>();
IList<ObjectPair<ILayer, ILayer>> contexts = new List<ObjectPair<ILayer, ILayer>>();
this.flat = null;
ValidateForFlat val = new ValidateForFlat();
if (val.IsValid(this.network) == null)
{
if (this.layers.Count == 3
&& this.layers[1] is RadialBasisFunctionLayer)
{
RadialBasisFunctionLayer rbf = (RadialBasisFunctionLayer)this.layers[1];
this.flat = new FlatNetworkRBF(this.network.InputCount,
rbf.NeuronCount, this.network.OutputCount,
rbf.RadialBasisFunction);
FlattenWeights();
this.flatUpdate = FlatUpdateNeeded.None;
return;
}
int flatLayerCount = CountNonContext();
FlatLayer[] flatLayers = new FlatLayer[flatLayerCount];
int index = flatLayers.Length - 1;
foreach (ILayer layer in this.layers)
{
if (layer is ContextLayer)
{
ISynapse inboundSynapse = network.Structure
.FindPreviousSynapseByLayerType(layer,
typeof(BasicLayer));
ISynapse outboundSynapse = network
.Structure
.FindNextSynapseByLayerType(layer, typeof(BasicLayer));
if (inboundSynapse == null)
throw new NeuralNetworkError(
"Context layer must be connected to by one BasicLayer.");
if (outboundSynapse == null)
throw new NeuralNetworkError(
"Context layer must connect to by one BasicLayer.");
ILayer inbound = inboundSynapse.FromLayer;
ILayer outbound = outboundSynapse.ToLayer;
contexts
.Add(new ObjectPair<ILayer, ILayer>(inbound, outbound));
}
else
{
double bias = this.FindNextBias(layer);
IActivationFunction activationType;
double[] param = new double[1];
if (layer.ActivationFunction == null)
{
activationType = new ActivationLinear();
param = new double[1];
param[0] = 1;
}
else
{
activationType = layer.ActivationFunction;
param = layer.ActivationFunction.Params;
}
FlatLayer flatLayer = new FlatLayer(activationType, layer
.NeuronCount, bias, param);
regular2flat[layer] = flatLayer;
flat2regular[flatLayer] = layer;
flatLayers[index--] = flatLayer;
}
}
// now link up the context layers
foreach (ObjectPair<ILayer, ILayer> context in contexts)
{
// link the context layer on the FlatLayer
ILayer layer = context.B;
ISynapse synapse = this.network
.Structure
.FindPreviousSynapseByLayerType(layer, typeof(BasicLayer));
FlatLayer from = regular2flat[context.A];
FlatLayer to = regular2flat[synapse.FromLayer];
to.ContextFedBy = from;
}
this.flat = new FlatNetwork(flatLayers);
// update the context indexes on the non-flat network
for (int i = 0; i < flatLayerCount; i++)
{
FlatLayer fedBy = flatLayers[i].ContextFedBy;
if (fedBy != null)
{
ILayer fedBy2 = flat2regular[flatLayers[i + 1]];
ISynapse synapse = FindPreviousSynapseByLayerType(fedBy2, typeof(ContextLayer));
if (synapse == null)
throw new NeuralNetworkError("Can't find parent synapse to context layer.");
ContextLayer context = (ContextLayer)synapse.FromLayer;
// find fedby index
int fedByIndex = -1;
for(int j=0;j<flatLayerCount;j++)
{
if( flatLayers[j]==fedBy )
{
fedByIndex = j;
break;
}
}
if (fedByIndex == -1)
throw new NeuralNetworkError("Can't find layer feeding context.");
context.FlatContextIndex = this.flat.ContextTargetOffset[fedByIndex];
}
}
// RBF networks will not train every layer
if (isRBF)
{
this.flat.EndTraining = flatLayers.Length - 1;
}
FlattenWeights();
if (this.IsConnectionLimited)
{
}
this.flatUpdate = FlatUpdateNeeded.None;
}
else
this.flatUpdate = FlatUpdateNeeded.Never;
}
