public void Backward
(string s, double[] DValues,
AlphaMem am, NetworkMem ValMem, NetworkMem FocMem)
{
var LocDValues = am.DLocation(DValues);
DValues = am.DGlobalContext(DValues);
DValues = Activations.InverseSoftMax(DValues, am.GlobalContextOutputs);
Parallel.For(0, s.Length, j =>
{
var ldv = LocDValues[j];
double[] cdv = new double[1] {
DValues[j] / s.Length
};
for (int i = ValueNetwork.Layers.Count() - 1; i >= 0; i--)
{
ldv = ValMem.Layers[i].DActivation(ldv, am.LocationOutputs[j][i + 1]);
ValMem.Layers[i].DBiases(ldv, ValueNetwork.Layers[i], s.Length);
ValMem.Layers[i].DWeights(ldv, am.LocationOutputs[j][i], ValueNetwork.Layers[i], s.Length);
ldv = ValMem.Layers[i].DInputs(ldv, ValueNetwork.Layers[i]);
}
for (int i = AttentionNetwork.Layers.Count() - 1; i >= 0; i--)
{
try
{
cdv = FocMem.Layers[i].DActivation(cdv, am.LocalContextOutputs[j][i + 1]);
FocMem.Layers[i].DBiases(cdv, AttentionNetwork.Layers[i], s.Length);
FocMem.Layers[i].DWeights(cdv, am.LocalContextOutputs[j][i], AttentionNetwork.Layers[i], s.Length);
cdv = FocMem.Layers[i].DInputs(cdv, AttentionNetwork.Layers[i]);
}
catch (Exception e) { e.OutputError(); }
}
});
}
public double[] Forward(string s, AlphaMem am)
{
try
{
double[,] loc = new double[s.Length, Size];
Parallel.For(0, s.Length, j =>
{
am.LocalContextOutputs[j].Add(s.LocatePhrase(j, Radius));
for (int i = 0; i < AttentionNetwork.Layers.Count(); i++)
{
am.LocalContextOutputs[j].Add
(AttentionNetwork.Layers[i].Output(am.LocalContextOutputs[j].Last()));
}
am.LocationOutputs[j].Add(s.Locate(j, Radius));
for (int i = 0; i < ValueNetwork.Layers.Count(); i++)
{
am.LocationOutputs[j].Add
(ValueNetwork.Layers[i].Output(am.LocationOutputs[j].Last()));
}
loc.SetRank(am.LocationOutputs[j].Last(), j);
am.GlobalContextOutputs[j] = am.LocalContextOutputs[j].Last().First();
});
return(loc.Multiply(Activations.SoftMax(am.GlobalContextOutputs)));
}
catch (Exception e) { e.OutputError(); }
return(null);
}
public double Contextualize(string s, int c, AlphaMem am)
{
am.LocalContextOutputs[c].Add(CharSet.Locate(s, c, SearchRange));
for (int i = 0; i < Network.Layers.Count(); i++)
{
am.LocalContextOutputs[c].Add(Network.Layers[i].Output(am.LocalContextOutputs[c].Last()));
}
return(am.LocalContextOutputs[c].Last().First());
}
private double[] Locate(double[] values, int j, AlphaMem am)
{
for (int i = 0; i < ValueNetwork.Layers.Count(); i++)
{
values = ValueNetwork.Layers[i].Output(values);
am.LocationOutputs[j].Add(values);
}
return(values);
}
private double ScoreAttention(double[] values, int j, AlphaMem am)
{
for (int i = 0; i < AttentionNetwork.Layers.Count(); i++)
{
values = AttentionNetwork.Layers[i].Output(values);
am.LocalContextOutputs[j].Add(values);
}
return(values.First());
}
public static double Propogate
(Sample s, WriteToCMDLine write, bool tf = false)
{
double error = 0;
var Pred = Predict(s.TextInput, CMDLibrary.WriteNull);
if (s.DesiredOutput.ToList().IndexOf(s.DesiredOutput.Max()) != Pred.ToList().IndexOf(Pred.Max()) || tf)
{
NeuralNetwork net = GetNetwork(write);
var Samples = s.ReadSamples(24);
Alpha a = new Alpha(write);
AlphaContext ctxt = new AlphaContext(datatype, write);
NetworkMem NetMem = new NetworkMem(net);
NetworkMem AlphaMem = new NetworkMem(a.Network);
NetworkMem CtxtMem = new NetworkMem(ctxt.Network);
Parallel.For(0, Samples.Count(), j =>
{
AlphaMem am = new AlphaMem(Samples[j].TextInput.ToCharArray());
var output = a.Forward(Samples[j].TextInput, ctxt, am);
var F = net.Forward(output, dropout, write);
error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), Samples[j].DesiredOutput).Max();
var DValues = net.Backward(F, Samples[j].DesiredOutput, NetMem, write);
a.Backward(Samples[j].TextInput, DValues, ctxt, am, AlphaMem, CtxtMem);
});
NetMem.Update(Samples.Count(), 0.00001, net);
AlphaMem.Update(Samples.Count(), 0.00001, a.Network);
CtxtMem.Update(Samples.Count(), 0.00001, ctxt.Network);
write("Pre Training Error : " + error);
net.Save();
a.Network.Save();
ctxt.Network.Save(datatype);
error = 0;
Parallel.For(0, Samples.Count(), j =>
{
AlphaMem am = new AlphaMem(Samples[j].TextInput.ToCharArray());
var output = a.Forward(Samples[j].TextInput, ctxt, am);
var F = net.Forward(output, dropout, write);
error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), Samples[j].DesiredOutput).Max();
});
write("Post Training Error : " + error);
s.Save();
}
return(error);
}
public double[] Forward(string s, AlphaMem am)
{
double[] ctxt = new double[s.Length];
double[,] loc = new double[s.Length, Size];
Parallel.For(0, s.Length, j =>
{
var vals = s.Locate(j, Radius);
am.LocationOutputs[j].Add(vals);
am.LocalContextOutputs[j].Add(vals);
ctxt[j] = ScoreAttention(vals, j, am);
var output = Locate(vals, j, am);
loc.SetRank(output, j);
});
return(loc.Multiply(Activations.SoftMax(ctxt)));
}
public void Propogate
(Sample s, WriteToCMDLine write)
{
var check = Predict(s);
if (s.DesiredOutput.ToList().IndexOf(s.DesiredOutput.Max()) != check.ToList().IndexOf(check.Max()))
{
Alpha a = new Alpha(write);
AlphaContext ctxt1 = new AlphaContext(datatype, write);
AlphaContext ctxt2 = new AlphaContext(datatype, write, 1);
var Samples = s.ReadSamples();
List <string> lines = new List <string>();
for (int i = 0; i < 5; i++)
{
NetworkMem ObjMem = new NetworkMem(Network);
NetworkMem AlphaMem = new NetworkMem(a.Network);
NetworkMem CtxtMem1 = new NetworkMem(ctxt1.Network);
NetworkMem CtxtMem2 = new NetworkMem(ctxt2.Network);
Parallel.For(0, Samples.Count(), j =>
{
AlphaMem am = new AlphaMem(Samples[j].TextInput.ToCharArray());
Samples[j].TextOutput = a.Forward(Samples[j].TextInput, ctxt1, am);
AlphaMem am2 = new AlphaMem(Samples[j].SecondaryText.ToCharArray());
Samples[j].SecondaryTextOutput = a.Forward(Samples[j].SecondaryText, ctxt2, am2);
var F = Forward(Samples[j]);
lines.AddRange(Samples[j].OutputError(CategoricalCrossEntropy.Forward(F.Last(), Samples[j].DesiredOutput)));
var DValues = Backward(Samples[j], F, ObjMem);
var DV1 = DValues.ToList().Take(Alpha.DictSize).ToArray();
var DV2 = Enumerable.Reverse(DValues).Take(Alpha.DictSize).Reverse().ToArray();
a.Backward(Samples[j].TextInput, DV1, ctxt1, am, AlphaMem, CtxtMem1);
a.Backward(Samples[j].SecondaryText, DV2, ctxt2, am2, AlphaMem, CtxtMem2);
});
ObjMem.Update(1, 0.0001, Network);
AlphaMem.Update(1, 0.00001, a.Network);
CtxtMem1.Update(1, 0.0001, ctxt1.Network);
CtxtMem2.Update(1, 0.0001, ctxt2.Network);
}
lines.ShowErrorOutput();
Network.Save();
a.Network.Save();
ctxt1.Save();
ctxt2.Save();
s.Save();
}
}
public static double Propogate
(WriteToCMDLine write)
{
double error = 0;
NeuralNetwork net = GetNetwork(write);
var Samples = ReadVals(24);
Alpha a = new Alpha(write);
AlphaContext ctxt = new AlphaContext(datatype, write);
NetworkMem OLFMem = new NetworkMem(net);
NetworkMem AlphaMem = new NetworkMem(a.Network);
NetworkMem CtxtMem = new NetworkMem(ctxt.Network);
Parallel.For(0, Samples.Count(), j =>
{
AlphaMem am = new AlphaMem(Samples.Keys.ToList()[j].ToCharArray());
var output = a.Forward(Samples.Keys.ToList()[j], ctxt, am);
var F = net.Forward(output, dropout, write);
var desired = new double[Enum.GetNames(typeof(Command)).Length];
desired[Samples.Values.ToList()[j]] = 1;
error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), desired).Max();
var DValues = net.Backward(F, desired, OLFMem, write);
a.Backward(Samples.Keys.ToList()[j], DValues, ctxt, am, AlphaMem, CtxtMem);
});
OLFMem.Update(Samples.Count(), 0.0001, net);
AlphaMem.Update(Samples.Count(), 0.0001, a.Network);
CtxtMem.Update(Samples.Count(), 0.0001, ctxt.Network);
write("Pre Training Error : " + error);
net.Save();
a.Network.Save();
ctxt.Network.Save(datatype);
error = 0;
Parallel.For(0, Samples.Count(), j =>
{
AlphaMem am = new AlphaMem(Samples.Keys.ToList()[j].ToCharArray());
var output = a.Forward(Samples.Keys.ToList()[j], ctxt, am);
var F = net.Forward(output, dropout, write);
var desired = new double[Enum.GetNames(typeof(Command)).Length];
desired[Samples.Values.ToList()[j]] = 1;
error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), desired).Max();
});
write("Post Training Error : " + error);
return(error);
}
public double[] Forward(string s, AlphaContext context, AlphaMem am)
{
double[,] loc = new double[s.Length, DictSize];
Parallel.For(0, s.Length, j =>
{
double[] a = s.Locate(j, SearchRange);
am.LocationOutputs[j].Add(a);
for (int i = 0; i < Network.Layers.Count(); i++)
{
a = Network.Layers[i].Output(a);
am.LocationOutputs[j].Add(a);
}
loc.SetRank(a, j);
am.GlobalContextOutputs[j] = context.Contextualize(s, j, am);
});
return(loc.Multiply(Activations.SoftMax(am.GlobalContextOutputs)));
}
public void Backward(string s, double[] DValues, AlphaContext context, AlphaMem am, NetworkMem mem, NetworkMem CtxtMem)
{
var LocDValues = am.DLocation(DValues);
DValues = am.DGlobalContext(DValues);
DValues = Activations.InverseSoftMax(DValues, am.GlobalContextOutputs);
context.Backward(DValues, s.Length, am, CtxtMem);
Parallel.For(0, s.Length, j =>
{
var ldv = LocDValues[j];
for (int i = Network.Layers.Count() - 1; i >= 0; i--)
{
ldv = mem.Layers[i].DActivation(ldv, am.LocationOutputs[j][i + 1]);
mem.Layers[i].DBiases(ldv, Network.Layers[i], s.Length);
mem.Layers[i].DWeights(ldv, am.LocationOutputs[j][i], Network.Layers[i], s.Length);
ldv = mem.Layers[i].DInputs(ldv, Network.Layers[i]);
}
});
}
public static double Propogate
(WriteToCMDLine write, bool tf = false)
{
double error = 0;
NeuralNetwork net = GetNetwork(write);
var Samples = datatype.ReadSamples(24);
Alpha a = new Alpha(write);
AlphaContext ctxt = new AlphaContext(datatype, write);
NetworkMem OLFMem = new NetworkMem(net);
NetworkMem AlphaMem = new NetworkMem(a.Network);
NetworkMem CtxtMem = new NetworkMem(ctxt.Network);
Parallel.For(0, Samples.Count(), j =>
{
AlphaMem am = new AlphaMem(Samples[j].TextInput.ToCharArray());
var output = a.Forward(Samples[j].TextInput, ctxt, am);
var F = net.Forward(output, dropout, write);
error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), Samples[j].DesiredOutput).Max();
var DValues = net.Backward(F, Samples[j].DesiredOutput, OLFMem, write);
a.Backward(Samples[j].TextInput, DValues, ctxt, am, AlphaMem, CtxtMem);
});
OLFMem.Update(Samples.Count(), 0.0001, net);
AlphaMem.Update(Samples.Count(), 0.0001, a.Network);
CtxtMem.Update(Samples.Count(), 0.0001, ctxt.Network);
write("Pre Training Error : " + error);
net.Save();
a.Network.Save();
ctxt.Network.Save(Datatype.OccupantLoadFactor);
error = 0;
Parallel.For(0, Samples.Count(), j =>
{
AlphaMem am = new AlphaMem(Samples[j].TextInput.ToCharArray());
var output = a.Forward(Samples[j].TextInput, ctxt, am);
var F = net.Forward(output, dropout, write);
error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), Samples[j].DesiredOutput).Max();
});
write("Post Training Error : " + error);
return(error);
}
public void Backward(double[] DValues, int runs, AlphaMem am, NetworkMem mem)
{
Parallel.For(0, runs, j =>
{
double[] cdv = new double[1] {
DValues[j] / runs
};
for (int i = Network.Layers.Count() - 1; i >= 0; i--)
{
try
{
cdv = mem.Layers[i].DActivation(cdv, am.LocalContextOutputs[j][i + 1]);
mem.Layers[i].DBiases(cdv, Network.Layers[i], runs);
mem.Layers[i].DWeights(cdv, am.LocalContextOutputs[j][i], Network.Layers[i], runs);
cdv = mem.Layers[i].DInputs(cdv, Network.Layers[i]);
}
catch (Exception e) { e.OutputError(); }
}
});
}
public AlphaMem[] CreateAlphaMemory(string s)
{
AlphaMem[] mem = new AlphaMem[Filters.Count];
Parallel.For(0, Filters.Count, j => mem[j] = new AlphaMem(s.ToCharArray()));
return(mem);
}
