public override void Optimize()
{
GradientClipper.Clip(this);
foreach (var data in Data)
{
if (data.Variable.Type == VariableType.Parameter)
{
if (data.Variable.DataType == typeof(float))
{
var a = _weights[data.Variable].Cast <float>();
var p = data.Tensor.Cast <float>();
var g = data.Gradient.Cast <float>();
Context.Assign(a, Rho.AsScalar <float>() * a + (1.0.AsScalar <float>() - Rho.AsScalar <float>()) * g * g);
Context.Assign(p, p - LearningRate.AsScalar <float>() * g / (Sqrt(a) + Epsilon.AsScalar <float>()));
}
else if (data.Variable.DataType == typeof(double))
{
var a = _weights[data.Variable].Cast <double>();
var p = data.Tensor.Cast <double>();
var g = data.Gradient.Cast <double>();
Context.Assign(a, Rho.AsScalar <double>() * a + (1.0.AsScalar <double>() - Rho.AsScalar <double>()) * g * g);
Context.Assign(p, p - LearningRate.AsScalar <double>() * g / (Sqrt(a) + Epsilon.AsScalar <double>()));
}
else
{
throw new InvalidOperationException();
}
}
}
}
public override string ToString()
{
var strValue = $"|Type:{LearnerType} |LRate:{LearningRate.ToString(CultureInfo.InvariantCulture)} " +
$"|Momentum:{Momentum.ToString(CultureInfo.InvariantCulture)} |Loss:{LossFunction}" +
$"|Eval:{EvaluationFunction}" + $"|L1:{L1Regularizer}" + $"|L2:{L2Regularizer}";
return(strValue);
}
public BackPropagation GetLearningMethod(object owner, IContextLookup globalVars, BasicNetwork theNetwork,
IList <BasicData> data)
{
double theLearningRate = LearningRate.GetValue(owner, globalVars);
double theMomentum = Momentum.GetValue(owner, globalVars);
var theL1 = L1.GetValue(owner, globalVars);
var theL2 = L2.GetValue(owner, globalVars);
var toReturn = new BackPropagation(theNetwork, data, theLearningRate, theMomentum);
toReturn.L1 = theL1;
toReturn.L2 = theL2;
return(toReturn);
}
public ProcessConfiguration GetConfiguration(PipelineContext context)
{
var pc = new ProcessConfiguration();
pc["LearningAlgorithm"] = LearningAlgorithm.ToString();
pc["Iterations"] = Iterations.ToString();
pc["Dimensions"] = Dimensions;
pc["LearningRate"] = LearningRate.ToString();
pc["TrainFile"] = context.GetAsString("TrainFile");
pc["TestFile"] = context.GetAsString("TestFile");
pc["AdditionalArgs"] = _additionalArgs;
return(pc);
}
public override int GetHashCode()
{
int hash = 1;
if (Engine.Length != 0)
{
hash ^= Engine.GetHashCode();
}
if (Normalization.Length != 0)
{
hash ^= Normalization.GetHashCode();
}
if (NInput != 0)
{
hash ^= NInput.GetHashCode();
}
if (NOutput != 0)
{
hash ^= NOutput.GetHashCode();
}
if (LearningRate != 0D)
{
hash ^= LearningRate.GetHashCode();
}
if (BatchSize != 0)
{
hash ^= BatchSize.GetHashCode();
}
if (EpochSize != 0)
{
hash ^= EpochSize.GetHashCode();
}
if (Optimizer.Length != 0)
{
hash ^= Optimizer.GetHashCode();
}
if (LossFunc.Length != 0)
{
hash ^= LossFunc.GetHashCode();
}
hash ^= maxV_.GetHashCode();
hash ^= minV_.GetHashCode();
hash ^= meanV_.GetHashCode();
hash ^= stdV_.GetHashCode();
hash ^= layers_.GetHashCode();
return(hash);
}
public void SaveNetworkAsXML(string path)
{
XmlWriterSettings setting = new XmlWriterSettings();
setting.Indent = true;
Stream stream;
path += ".xml";
stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write);
using (XmlWriter writer = XmlWriter.Create(stream, setting))
{
writer.WriteStartDocument();
writer.WriteStartElement("NeuralNetwork");
writer.WriteAttributeString("learningRate", LearningRate.ToString());
writer.WriteAttributeString("name", Name.ToString());
foreach (Layer layer in Layers)
{
writer.WriteStartElement("Layer");
writer.WriteAttributeString("activationFunction", layer.activationFunction.ToString());
foreach (Neuron neuron in layer.Neurons)
{
writer.WriteStartElement("Neuron");
writer.WriteAttributeString("bias", neuron.Bias.ToString());
foreach (Dendrite dendrite in neuron.Dendrites)
{
writer.WriteStartElement("InputDendrite");
writer.WriteAttributeString("weight", dendrite.Weight.ToString());
writer.WriteEndElement();
}
writer.WriteEndElement();
}
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteEndDocument();
}
}
public void SaveNetworkToFile(string path)
{
List <string> tmp = new List <string>();
for (int i = 1; i < Layers.Count; i++)
{
foreach (Neuron neuron in Layers[i].Neurons)
{
foreach (Synapse synapse in neuron.Inputs)
{
tmp.Add(synapse.Weight.ToString(CultureInfo.InvariantCulture));
}
}
}
string build = $"{LearningRate.ToString()} {Functions.Alpha.ToString()} {Synapse.MinInitWeight} {Synapse.MaxInitWeight}";
foreach (Layer layer in Layers)
{
build += " " + layer.Neurons.Count.ToString();
}
tmp.Insert(0, build);
File.WriteAllLines(path, tmp);
}
