public static JobHandle UpdateParameters(FCNetwork net, FCGradients gradients, float rate, JobHandle handle = new JobHandle())
{
// Biases -= DCDZ * rate
// Weights -= DCDW * rate
// Todo: Find a nice way to fold the multiply by learning rate and addition together in one pass over the data
// Also, parallelize over all the arrays
for (int l = 0; l < net.Layers.Length; l++)
{
var m = new MultiplyAssignJob();
m.Data = gradients.Layers[l].DCDZ;
m.Value = rate;
handle = m.Schedule(handle);
var s = new SubtractAssignJob();
s.Data = gradients.Layers[l].DCDZ;
s.From = net.Layers[l].Biases;
handle = s.Schedule(handle);
m = new MultiplyAssignJob();
m.Data = gradients.Layers[l].DCDW;
m.Value = rate;
handle = m.Schedule(handle);
s = new SubtractAssignJob();
s.Data = gradients.Layers[l].DCDW;
s.From = net.Layers[l].Weights;
handle = s.Schedule(handle);
}
return(handle);
}
public static JobHandle ForwardPass(FCNetwork net, JobHandle handle = new JobHandle())
{
NativeArray <float> last = net.Inputs;
for (int l = 0; l < net.Layers.Length; l++)
{
var layer = net.Layers[l];
const int numThreads = 8;
var b = new CopyParallelJob();
b.From = layer.Biases;
b.To = layer.Outputs;
handle = b.Schedule(layer.Outputs.Length, layer.Outputs.Length / numThreads, handle);
var d = new DotParallelJob();
d.Input = last;
d.Weights = layer.Weights;
d.Output = layer.Outputs;
handle = d.Schedule(layer.Outputs.Length, layer.Outputs.Length / numThreads, handle);
var s = new SigmoidAssignParallelJob();
s.Data = layer.Outputs;
handle = s.Schedule(layer.Outputs.Length, layer.Outputs.Length / numThreads, handle);
last = layer.Outputs;
}
return(handle);
}
public static void Initialize(FCNetwork net, ref Rng rng)
{
// Todo: init as jobs too. Needs Burst-compatible RNG.
for (int l = 0; l < net.Layers.Length; l++)
{
NeuralMath.RandomGaussian(ref rng, net.Layers[l].Weights, 0f, 1f);
NeuralMath.RandomGaussian(ref rng, net.Layers[l].Biases, 0f, 1f);
}
}
public static JobHandle BackwardsPass(FCNetwork net, FCGradients gradients, NativeArray <float> target, JobHandle handle = new JobHandle())
{
JobHandle h = handle;
// Todo:
// DCDW can make good use of ParallelFor, depends on DCDZ
// Oh, but then, while DCDW for layer L is being calculated, DCDZ for L-1 can calculate while DCDW of L is calculating
// DCDW computation is an orphan. Other computation doesn't have to wait for it.
var sj = new SubtractJob();
sj.A = net.Last.Outputs;
sj.B = target;
sj.Output = gradients.DCDO;
h = sj.Schedule(h);
var bfj = new BackPropSingleOutputJob();
bfj.DCDZNext = gradients.DCDO;
bfj.DCDZ = gradients.Last.DCDZ;
bfj.Outputs = net.Last.Outputs;
h = bfj.Schedule(h);
var bwj = new BackPropWeightsJob();
bwj.DCDZ = gradients.Last.DCDZ;
bwj.DCDW = gradients.Last.DCDW;
bwj.Outputs = net.Last.Outputs;
bwj.OutputsPrev = net.Layers[net.Layers.Length - 2].Outputs;
h = bwj.Schedule(h);
// Note, indexing using net.layers.length here is misleading, since that count is one less than if you include input layer
for (int l = net.Layers.Length - 2; l >= 0; l--)
{
var bej = new BackPropMultiOutputJob();
bej.DCDZNext = gradients.Layers[l + 1].DCDZ;
bej.WeightsNext = net.Layers[l + 1].Weights;
bej.DCDZ = gradients.Layers[l].DCDZ;
bej.Outputs = net.Layers[l].Outputs;
h = bej.Schedule(h);
bwj = new BackPropWeightsJob();
bwj.DCDZ = gradients.Layers[l].DCDZ;
bwj.DCDW = gradients.Layers[l].DCDW;
bwj.Outputs = net.Layers[l].Outputs;
bwj.OutputsPrev = l == 0 ? net.Inputs : net.Layers[l - 1].Outputs;
h = bwj.Schedule(h);
}
return(h);
}
private void Awake()
{
Application.runInBackground = true;
DataManager.Load();
_rng = new Rng(1234);
var config = new FCNetworkConfig();
config.Layers.Add(new FCLayerConfig {
NumNeurons = DataManager.ImgDims * DataManager.Channels
});
config.Layers.Add(new FCLayerConfig {
NumNeurons = 40
});
config.Layers.Add(new FCLayerConfig {
NumNeurons = 20
});
config.Layers.Add(new FCLayerConfig {
NumNeurons = 10
});
_net = new FCNetwork(config);
NeuralUtils.Initialize(_net, ref _rng);
_gradients = new FCGradients(config);
_gradientsAvg = new FCGradients(config);
_batch = new NativeArray <int>(BatchSize, Allocator.Persistent, NativeArrayOptions.ClearMemory);
_targetOutputs = new NativeArray <float>(OutputClassCount, Allocator.Persistent, NativeArrayOptions.UninitializedMemory);
_dCdO = new NativeArray <float>(OutputClassCount, Allocator.Persistent, NativeArrayOptions.UninitializedMemory);
_watch = System.Diagnostics.Stopwatch.StartNew();
int testImgIdx = 8392;
_lbl = DataManager.Test.Labels[testImgIdx];
_img = new Texture2D(32, 32, TextureFormat.ARGB32, false, true);
DataManager.ToTexture(DataManager.Test, testImgIdx, _img);
Test();
}
private void Awake()
{
Application.runInBackground = true;
DataManager.LoadFloatData();
_rng = new Rng(1234);
var config = new FCNetworkConfig();
config.Layers.Add(new FCLayerConfig {
NumNeurons = DataManager.ImgDims
});
config.Layers.Add(new FCLayerConfig {
NumNeurons = 30
});
config.Layers.Add(new FCLayerConfig {
NumNeurons = 10
});
_net = new FCNetwork(config);
NeuralUtils.Initialize(_net, ref _rng);
_gradients = new FCGradients(config);
_gradientsAvg = new FCGradients(config);
_batch = new NativeArray <int>(BatchSize, Allocator.Persistent, NativeArrayOptions.ClearMemory);
_targetOutputs = new NativeArray <float>(OutputClassCount, Allocator.Persistent, NativeArrayOptions.UninitializedMemory);
_dCdO = new NativeArray <float>(OutputClassCount, Allocator.Persistent, NativeArrayOptions.UninitializedMemory);
_watch = System.Diagnostics.Stopwatch.StartNew();
// Visual test of data
const int testImgIdx = 18;
_label = DataManager.TrainFloats.Labels[testImgIdx];
_tex = new Texture2D(DataManager.Width, DataManager.Height, TextureFormat.ARGB32, false, true);
_tex.filterMode = FilterMode.Point;
DataManager.ToTexture(DataManager.TrainFloats, testImgIdx, _tex);
Test();
}