public static JobHandle UpdateParameters(FCNetwork net, FCGradients gradients, float rate, JobHandle handle = new JobHandle())
{
// Biases -= DCDZ * rate
// Weights -= DCDW * rate
// Todo: Find a nice way to fold the multiply by learning rate and addition together in one pass over the data
// Also, parallelize over all the arrays
for (int l = 0; l < net.Layers.Length; l++)
{
var m = new MultiplyAssignJob();
m.Data = gradients.Layers[l].DCDZ;
m.Value = rate;
handle = m.Schedule(handle);
var s = new SubtractAssignJob();
s.Data = gradients.Layers[l].DCDZ;
s.From = net.Layers[l].Biases;
handle = s.Schedule(handle);
m = new MultiplyAssignJob();
m.Data = gradients.Layers[l].DCDW;
m.Value = rate;
handle = m.Schedule(handle);
s = new SubtractAssignJob();
s.Data = gradients.Layers[l].DCDW;
s.From = net.Layers[l].Weights;
handle = s.Schedule(handle);
}
return(handle);
}
public static JobHandle BackwardsPass(FCNetwork net, FCGradients gradients, NativeArray <float> target, JobHandle handle = new JobHandle())
{
JobHandle h = handle;
// Todo:
// DCDW can make good use of ParallelFor, depends on DCDZ
// Oh, but then, while DCDW for layer L is being calculated, DCDZ for L-1 can calculate while DCDW of L is calculating
// DCDW computation is an orphan. Other computation doesn't have to wait for it.
var sj = new SubtractJob();
sj.A = net.Last.Outputs;
sj.B = target;
sj.Output = gradients.DCDO;
h = sj.Schedule(h);
var bfj = new BackPropSingleOutputJob();
bfj.DCDZNext = gradients.DCDO;
bfj.DCDZ = gradients.Last.DCDZ;
bfj.Outputs = net.Last.Outputs;
h = bfj.Schedule(h);
var bwj = new BackPropWeightsJob();
bwj.DCDZ = gradients.Last.DCDZ;
bwj.DCDW = gradients.Last.DCDW;
bwj.Outputs = net.Last.Outputs;
bwj.OutputsPrev = net.Layers[net.Layers.Length - 2].Outputs;
h = bwj.Schedule(h);
// Note, indexing using net.layers.length here is misleading, since that count is one less than if you include input layer
for (int l = net.Layers.Length - 2; l >= 0; l--)
{
var bej = new BackPropMultiOutputJob();
bej.DCDZNext = gradients.Layers[l + 1].DCDZ;
bej.WeightsNext = net.Layers[l + 1].Weights;
bej.DCDZ = gradients.Layers[l].DCDZ;
bej.Outputs = net.Layers[l].Outputs;
h = bej.Schedule(h);
bwj = new BackPropWeightsJob();
bwj.DCDZ = gradients.Layers[l].DCDZ;
bwj.DCDW = gradients.Layers[l].DCDW;
bwj.Outputs = net.Layers[l].Outputs;
bwj.OutputsPrev = l == 0 ? net.Inputs : net.Layers[l - 1].Outputs;
h = bwj.Schedule(h);
}
return(h);
}
private void Awake()
{
Application.runInBackground = true;
DataManager.Load();
_rng = new Rng(1234);
var config = new FCNetworkConfig();
config.Layers.Add(new FCLayerConfig {
NumNeurons = DataManager.ImgDims * DataManager.Channels
});
config.Layers.Add(new FCLayerConfig {
NumNeurons = 40
});
config.Layers.Add(new FCLayerConfig {
NumNeurons = 20
});
config.Layers.Add(new FCLayerConfig {
NumNeurons = 10
});
_net = new FCNetwork(config);
NeuralUtils.Initialize(_net, ref _rng);
_gradients = new FCGradients(config);
_gradientsAvg = new FCGradients(config);
_batch = new NativeArray <int>(BatchSize, Allocator.Persistent, NativeArrayOptions.ClearMemory);
_targetOutputs = new NativeArray <float>(OutputClassCount, Allocator.Persistent, NativeArrayOptions.UninitializedMemory);
_dCdO = new NativeArray <float>(OutputClassCount, Allocator.Persistent, NativeArrayOptions.UninitializedMemory);
_watch = System.Diagnostics.Stopwatch.StartNew();
int testImgIdx = 8392;
_lbl = DataManager.Test.Labels[testImgIdx];
_img = new Texture2D(32, 32, TextureFormat.ARGB32, false, true);
DataManager.ToTexture(DataManager.Test, testImgIdx, _img);
Test();
}
private void Awake()
{
Application.runInBackground = true;
DataManager.LoadFloatData();
_rng = new Rng(1234);
var config = new FCNetworkConfig();
config.Layers.Add(new FCLayerConfig {
NumNeurons = DataManager.ImgDims
});
config.Layers.Add(new FCLayerConfig {
NumNeurons = 30
});
config.Layers.Add(new FCLayerConfig {
NumNeurons = 10
});
_net = new FCNetwork(config);
NeuralUtils.Initialize(_net, ref _rng);
_gradients = new FCGradients(config);
_gradientsAvg = new FCGradients(config);
_batch = new NativeArray <int>(BatchSize, Allocator.Persistent, NativeArrayOptions.ClearMemory);
_targetOutputs = new NativeArray <float>(OutputClassCount, Allocator.Persistent, NativeArrayOptions.UninitializedMemory);
_dCdO = new NativeArray <float>(OutputClassCount, Allocator.Persistent, NativeArrayOptions.UninitializedMemory);
_watch = System.Diagnostics.Stopwatch.StartNew();
// Visual test of data
const int testImgIdx = 18;
_label = DataManager.TrainFloats.Labels[testImgIdx];
_tex = new Texture2D(DataManager.Width, DataManager.Height, TextureFormat.ARGB32, false, true);
_tex.filterMode = FilterMode.Point;
DataManager.ToTexture(DataManager.TrainFloats, testImgIdx, _tex);
Test();
}
public static JobHandle AddGradients(FCGradients from, FCGradients to, JobHandle handle = new JobHandle())
{
// to.Bias += from.Bias
// to.Weights += from.Weights
// Todo: parallelize over layers and/or biases/weights
for (int l = 0; l < from.Layers.Length; l++)
{
var addBiasJob = new AddAssignJob();
addBiasJob.Data = from.Layers[l].DCDZ;
addBiasJob.To = to.Layers[l].DCDZ;
handle = addBiasJob.Schedule(handle);
var addWeightsJob = new AddAssignJob();
addWeightsJob.Data = from.Layers[l].DCDW;
addWeightsJob.To = to.Layers[l].DCDW;
handle = addWeightsJob.Schedule(handle);
}
return(handle);
}
public static JobHandle ZeroGradients(FCGradients gradients, JobHandle handle = new JobHandle())
{
// gradients = 0;
// Todo: parallelize over all these independent calculations
for (int l = 0; l < gradients.Layers.Length; l++)
{
var setBiasJob = new SetValueJob();
setBiasJob.Data = gradients.Layers[l].DCDZ;
setBiasJob.Value = 0f;
var j0 = setBiasJob.Schedule(handle);
var setWeightsJob = new SetValueJob();
setWeightsJob.Data = gradients.Layers[l].DCDW;
setWeightsJob.Value = 0f;
var j1 = setWeightsJob.Schedule(handle);
handle = JobHandle.CombineDependencies(j0, j1);
}
return(handle);
}
