public void Backward (string s, double[] DValues, AlphaMem am, NetworkMem ValMem, NetworkMem FocMem) { var LocDValues = am.DLocation(DValues); DValues = am.DGlobalContext(DValues); DValues = Activations.InverseSoftMax(DValues, am.GlobalContextOutputs); Parallel.For(0, s.Length, j => { var ldv = LocDValues[j]; double[] cdv = new double[1] { DValues[j] / s.Length }; for (int i = ValueNetwork.Layers.Count() - 1; i >= 0; i--) { ldv = ValMem.Layers[i].DActivation(ldv, am.LocationOutputs[j][i + 1]); ValMem.Layers[i].DBiases(ldv, ValueNetwork.Layers[i], s.Length); ValMem.Layers[i].DWeights(ldv, am.LocationOutputs[j][i], ValueNetwork.Layers[i], s.Length); ldv = ValMem.Layers[i].DInputs(ldv, ValueNetwork.Layers[i]); } for (int i = AttentionNetwork.Layers.Count() - 1; i >= 0; i--) { try { cdv = FocMem.Layers[i].DActivation(cdv, am.LocalContextOutputs[j][i + 1]); FocMem.Layers[i].DBiases(cdv, AttentionNetwork.Layers[i], s.Length); FocMem.Layers[i].DWeights(cdv, am.LocalContextOutputs[j][i], AttentionNetwork.Layers[i], s.Length); cdv = FocMem.Layers[i].DInputs(cdv, AttentionNetwork.Layers[i]); } catch (Exception e) { e.OutputError(); } } }); }
public NetworkMem[,] CreateMemory() { NetworkMem[,] mem = new NetworkMem[Filters.Count(), 2]; try { Parallel.For(0, Filters.Count, j => { mem[j, 0] = new NetworkMem(Filters[j].ValueNetwork); mem[j, 1] = new NetworkMem(Filters[j].AttentionNetwork); }); } catch (Exception e) { e.OutputError(); } return(mem); }
public static double Propogate (Sample s, WriteToCMDLine write, bool tf = false) { double error = 0; var Pred = Predict(s.TextInput, CMDLibrary.WriteNull); if (s.DesiredOutput.ToList().IndexOf(s.DesiredOutput.Max()) != Pred.ToList().IndexOf(Pred.Max()) || tf) { NeuralNetwork net = GetNetwork(write); var Samples = s.ReadSamples(24); Alpha a = new Alpha(write); AlphaContext ctxt = new AlphaContext(datatype, write); NetworkMem NetMem = new NetworkMem(net); NetworkMem AlphaMem = new NetworkMem(a.Network); NetworkMem CtxtMem = new NetworkMem(ctxt.Network); Parallel.For(0, Samples.Count(), j => { AlphaMem am = new AlphaMem(Samples[j].TextInput.ToCharArray()); var output = a.Forward(Samples[j].TextInput, ctxt, am); var F = net.Forward(output, dropout, write); error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), Samples[j].DesiredOutput).Max(); var DValues = net.Backward(F, Samples[j].DesiredOutput, NetMem, write); a.Backward(Samples[j].TextInput, DValues, ctxt, am, AlphaMem, CtxtMem); }); NetMem.Update(Samples.Count(), 0.00001, net); AlphaMem.Update(Samples.Count(), 0.00001, a.Network); CtxtMem.Update(Samples.Count(), 0.00001, ctxt.Network); write("Pre Training Error : " + error); net.Save(); a.Network.Save(); ctxt.Network.Save(datatype); error = 0; Parallel.For(0, Samples.Count(), j => { AlphaMem am = new AlphaMem(Samples[j].TextInput.ToCharArray()); var output = a.Forward(Samples[j].TextInput, ctxt, am); var F = net.Forward(output, dropout, write); error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), Samples[j].DesiredOutput).Max(); }); write("Post Training Error : " + error); s.Save(); } return(error); }
public void Propogate (Sample s, WriteToCMDLine write) { var check = Predict(s); if (s.DesiredOutput.ToList().IndexOf(s.DesiredOutput.Max()) != check.ToList().IndexOf(check.Max())) { Alpha a = new Alpha(write); AlphaContext ctxt1 = new AlphaContext(datatype, write); AlphaContext ctxt2 = new AlphaContext(datatype, write, 1); var Samples = s.ReadSamples(); List <string> lines = new List <string>(); for (int i = 0; i < 5; i++) { NetworkMem ObjMem = new NetworkMem(Network); NetworkMem AlphaMem = new NetworkMem(a.Network); NetworkMem CtxtMem1 = new NetworkMem(ctxt1.Network); NetworkMem CtxtMem2 = new NetworkMem(ctxt2.Network); Parallel.For(0, Samples.Count(), j => { AlphaMem am = new AlphaMem(Samples[j].TextInput.ToCharArray()); Samples[j].TextOutput = a.Forward(Samples[j].TextInput, ctxt1, am); AlphaMem am2 = new AlphaMem(Samples[j].SecondaryText.ToCharArray()); Samples[j].SecondaryTextOutput = a.Forward(Samples[j].SecondaryText, ctxt2, am2); var F = Forward(Samples[j]); lines.AddRange(Samples[j].OutputError(CategoricalCrossEntropy.Forward(F.Last(), Samples[j].DesiredOutput))); var DValues = Backward(Samples[j], F, ObjMem); var DV1 = DValues.ToList().Take(Alpha.DictSize).ToArray(); var DV2 = Enumerable.Reverse(DValues).Take(Alpha.DictSize).Reverse().ToArray(); a.Backward(Samples[j].TextInput, DV1, ctxt1, am, AlphaMem, CtxtMem1); a.Backward(Samples[j].SecondaryText, DV2, ctxt2, am2, AlphaMem, CtxtMem2); }); ObjMem.Update(1, 0.0001, Network); AlphaMem.Update(1, 0.00001, a.Network); CtxtMem1.Update(1, 0.0001, ctxt1.Network); CtxtMem2.Update(1, 0.0001, ctxt2.Network); } lines.ShowErrorOutput(); Network.Save(); a.Network.Save(); ctxt1.Save(); ctxt2.Save(); s.Save(); } }
public double[] Backward (Sample s, List <double[]> Results, NetworkMem mem) { var DValues = s.DesiredOutput; for (int l = Network.Layers.Count() - 1; l >= 0; l--) { DValues = mem.Layers[l].DActivation(DValues, Results[l + 1]); mem.Layers[l].DBiases(DValues, Network.Layers[l]); mem.Layers[l].DWeights(DValues, Results[l], Network.Layers[l]); DValues = mem.Layers[l].DInputs(DValues, Network.Layers[l]); } return(DValues.ToList().Take(2 * Alpha.DictSize).ToArray()); }
public static double Propogate (WriteToCMDLine write) { double error = 0; NeuralNetwork net = GetNetwork(write); var Samples = ReadVals(24); Alpha a = new Alpha(write); AlphaContext ctxt = new AlphaContext(datatype, write); NetworkMem OLFMem = new NetworkMem(net); NetworkMem AlphaMem = new NetworkMem(a.Network); NetworkMem CtxtMem = new NetworkMem(ctxt.Network); Parallel.For(0, Samples.Count(), j => { AlphaMem am = new AlphaMem(Samples.Keys.ToList()[j].ToCharArray()); var output = a.Forward(Samples.Keys.ToList()[j], ctxt, am); var F = net.Forward(output, dropout, write); var desired = new double[Enum.GetNames(typeof(Command)).Length]; desired[Samples.Values.ToList()[j]] = 1; error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), desired).Max(); var DValues = net.Backward(F, desired, OLFMem, write); a.Backward(Samples.Keys.ToList()[j], DValues, ctxt, am, AlphaMem, CtxtMem); }); OLFMem.Update(Samples.Count(), 0.0001, net); AlphaMem.Update(Samples.Count(), 0.0001, a.Network); CtxtMem.Update(Samples.Count(), 0.0001, ctxt.Network); write("Pre Training Error : " + error); net.Save(); a.Network.Save(); ctxt.Network.Save(datatype); error = 0; Parallel.For(0, Samples.Count(), j => { AlphaMem am = new AlphaMem(Samples.Keys.ToList()[j].ToCharArray()); var output = a.Forward(Samples.Keys.ToList()[j], ctxt, am); var F = net.Forward(output, dropout, write); var desired = new double[Enum.GetNames(typeof(Command)).Length]; desired[Samples.Values.ToList()[j]] = 1; error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), desired).Max(); }); write("Post Training Error : " + error); return(error); }
public void Propogate (ValueSet val, WriteToCMDLine write) { Stonk stk = new Stonk(); StonkContext ctxt = new StonkContext(datatype); var vals = val.ReadValues(Datatypes.Datatype.AAPL, 24); NetworkMem AAPLMem = new NetworkMem(Network); NetworkMem StkMem = new NetworkMem(stk.Network); NetworkMem CtxtMem = new NetworkMem(ctxt.Network); double e = 0; Parallel.For(0, vals.Count(), j => { try { List <Comparison> comps = Comparison.GenerateComparisons(vals[j]); if (j == 0 || j == 1) { write("Comparisons : " + comps.Count()); } StonkMem sm = new StonkMem(comps.Count()); var MktOutput = stk.Forward(comps, ctxt, sm); var F = Network.Forward(MktOutput, dropout, write); var output = new double[2]; int opnumb = vals[j].Increase ? 1 : 0; output[opnumb] = 1; var Error = CategoricalCrossEntropy.Forward(F.Last().GetRank(0), output); e += Error.Max(); var D = Network.Backward(F, output, AAPLMem, write); stk.Backward(D, ctxt, sm, StkMem, CtxtMem); } catch { } }); write("Samples : " + vals.Count()); write("Loss : " + e); AAPLMem.Update(vals.Count(), 1e-4, Network); StkMem.Update(vals.Count(), 1e-4, stk.Network); CtxtMem.Update(vals.Count(), 1e-4, ctxt.Network); Network.Save(); stk.Network.Save(); ctxt.Save(); }
public void Backward(double[] DValues, StonkContext context, StonkMem sm, NetworkMem mem, NetworkMem CtxtMem) { var LocDValues = sm.DLocation(DValues); DValues = sm.DGlobalContext(DValues); DValues = Activations.InverseSoftMax(DValues, sm.GlobalOutputs.ToArray()); context.Backward(DValues, sm.LocationOutputs.Count(), sm, CtxtMem); Parallel.For(0, sm.GlobalOutputs.Count(), j => { var ldv = LocDValues[j]; for (int i = Network.Layers.Count() - 1; i >= 0; i--) { ldv = mem.Layers[i].DActivation(ldv, sm.LocationOutputs[j][i + 1]); mem.Layers[i].DBiases(ldv, Network.Layers[i], sm.GlobalOutputs.Count()); mem.Layers[i].DWeights(ldv, sm.LocationOutputs[j][i], Network.Layers[i], sm.GlobalOutputs.Count()); ldv = mem.Layers[i].DInputs(ldv, Network.Layers[i]); } }); }
public static double Propogate (WriteToCMDLine write, bool tf = false) { double error = 0; NeuralNetwork net = GetNetwork(write); var Samples = datatype.ReadSamples(24); Alpha a = new Alpha(write); AlphaContext ctxt = new AlphaContext(datatype, write); NetworkMem OLFMem = new NetworkMem(net); NetworkMem AlphaMem = new NetworkMem(a.Network); NetworkMem CtxtMem = new NetworkMem(ctxt.Network); Parallel.For(0, Samples.Count(), j => { AlphaMem am = new AlphaMem(Samples[j].TextInput.ToCharArray()); var output = a.Forward(Samples[j].TextInput, ctxt, am); var F = net.Forward(output, dropout, write); error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), Samples[j].DesiredOutput).Max(); var DValues = net.Backward(F, Samples[j].DesiredOutput, OLFMem, write); a.Backward(Samples[j].TextInput, DValues, ctxt, am, AlphaMem, CtxtMem); }); OLFMem.Update(Samples.Count(), 0.0001, net); AlphaMem.Update(Samples.Count(), 0.0001, a.Network); CtxtMem.Update(Samples.Count(), 0.0001, ctxt.Network); write("Pre Training Error : " + error); net.Save(); a.Network.Save(); ctxt.Network.Save(Datatype.OccupantLoadFactor); error = 0; Parallel.For(0, Samples.Count(), j => { AlphaMem am = new AlphaMem(Samples[j].TextInput.ToCharArray()); var output = a.Forward(Samples[j].TextInput, ctxt, am); var F = net.Forward(output, dropout, write); error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), Samples[j].DesiredOutput).Max(); }); write("Post Training Error : " + error); return(error); }
public void Backward(double[] DValues, int runs, AlphaMem am, NetworkMem mem) { Parallel.For(0, runs, j => { double[] cdv = new double[1] { DValues[j] / runs }; for (int i = Network.Layers.Count() - 1; i >= 0; i--) { try { cdv = mem.Layers[i].DActivation(cdv, am.LocalContextOutputs[j][i + 1]); mem.Layers[i].DBiases(cdv, Network.Layers[i], runs); mem.Layers[i].DWeights(cdv, am.LocalContextOutputs[j][i], Network.Layers[i], runs); cdv = mem.Layers[i].DInputs(cdv, Network.Layers[i]); } catch (Exception e) { e.OutputError(); } } }); }
public double[] Backward(List<double[,]> Results, double[] desired, NetworkMem mem, WriteToCMDLine write) { var DValues = desired; for (int l = Layers.Count() - 1; l >= 0; l--) { try { if (l != Layers.Count() - 1) DValues = InverseDropOut(DValues, Results[l].GetRank(1)); DValues = mem.Layers[l].DActivation(DValues, Results[l + 1].GetRank(0)); mem.Layers[l].DBiases(DValues, Layers[l]); mem.Layers[l].DWeights(DValues, Results[l].GetRank(0), Layers[l]); DValues = mem.Layers[l].DInputs(DValues, Layers[l]); } catch (Exception e) { write("Failed at Layer : " + l); e.OutputError(); } } return DValues; }
public void Backward(string s, double[] DValues, AlphaContext context, AlphaMem am, NetworkMem mem, NetworkMem CtxtMem) { var LocDValues = am.DLocation(DValues); DValues = am.DGlobalContext(DValues); DValues = Activations.InverseSoftMax(DValues, am.GlobalContextOutputs); context.Backward(DValues, s.Length, am, CtxtMem); Parallel.For(0, s.Length, j => { var ldv = LocDValues[j]; for (int i = Network.Layers.Count() - 1; i >= 0; i--) { ldv = mem.Layers[i].DActivation(ldv, am.LocationOutputs[j][i + 1]); mem.Layers[i].DBiases(ldv, Network.Layers[i], s.Length); mem.Layers[i].DWeights(ldv, am.LocationOutputs[j][i], Network.Layers[i], s.Length); ldv = mem.Layers[i].DInputs(ldv, Network.Layers[i]); } }); }
public static double Propogate (Sample s, WriteToCMDLine write, bool tf = false) { double error = 0; //var Pred = Predict(s.TextInput, new WriteToCMDLine(CMDLibrary.WriteNull)); //if (s.DesiredOutput.ToList().IndexOf(s.DesiredOutput.Max()) != Pred.ToList().IndexOf(Pred.Max()) || tf) { NeuralNetwork net = GetNetwork(write); var Samples = s.ReadSamples(24); Alpha2 a = datatype.LoadAlpha(write); var am = a.CreateMemory(); //Alpha a = new Alpha(write); //AlphaContext ctxt = new AlphaContext(datatype, write); NetworkMem MFMem = new NetworkMem(net); //NetworkMem AlphaMem = new NetworkMem(a.Network); //NetworkMem CtxtMem = new NetworkMem(ctxt.Network); try { Parallel.For(0, Samples.Count(), j => { //AlphaMem am = new AlphaMem(Samples[j].TextInput.ToCharArray()); //var output = a.Forward(Samples[j].TextInput, ctxt, am); var AMem = a.CreateAlphaMemory(Samples[j].TextInput); var output = a.Forward(Samples[j].TextInput, AMem, write); var F = net.Forward(output, dropout, write); error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), Samples[j].DesiredOutput).Max(); var DValues = net.Backward(F, Samples[j].DesiredOutput, MFMem, write); a.Backward(Samples[j].TextInput, DValues, AMem, am, write); //a.Backward(Samples[j].TextInput, DValues, ctxt, am, AlphaMem, CtxtMem); }); } catch (Exception e) { e.OutputError(); } MFMem.Update(Samples.Count(), 0.00001, net); a.Update(am, Samples.Count()); //AlphaMem.Update(Samples.Count(), 0.00001, a.Network); //CtxtMem.Update(Samples.Count(), 0.00001, ctxt.Network); write("Pre Training Error : " + error); net.Save(); a.Save(); //a.Network.Save(); //ctxt.Network.Save(Datatype.Masterformat); error = 0; Parallel.For(0, Samples.Count(), j => { var AMem = a.CreateAlphaMemory(Samples[j].TextInput); var output = a.Forward(Samples[j].TextInput, AMem, write); var F = net.Forward(output, 0, write); error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), Samples[j].DesiredOutput).Max(); //AlphaMem am = new AlphaMem(Samples[j].TextInput.ToCharArray()); //var output = a.Forward(Samples[j].TextInput, ctxt, am); //var F = net.Forward(output, dropout, write); //error += CategoricalCrossEntropy.Forward(F.Last().GetRank(0), Samples[j].DesiredOutput).Max(); }); write("Post Training Error : " + error); //s.Save(); } return(error); }