private static void Main(string[] args)
{
// species a 2-layer neural network with one hidden layer of 20 neurons
var net = new Net();
// input layer declares size of input. here: 2-D data
// ConvNetJS works on 3-Dimensional volumes (width, height, depth), but if you're not dealing with images
// then the first two dimensions (width, height) will always be kept at size 1
net.AddLayer(new InputLayer(1, 1, 2));
// declare 20 neurons, followed by ReLU (rectified linear unit non-linearity)
net.AddLayer(new FullyConnLayer(20, Activation.Relu));
// declare the linear classifier on top of the previous hidden layer
net.AddLayer(new SoftmaxLayer(10));
// forward a random data point through the network
var x = new Volume(new[] {0.3, -0.5});
var prob = net.Forward(x);
// prob is a Volume. Volumes have a property Weights that stores the raw data, and WeightGradients that stores gradients
Console.WriteLine("probability that x is class 0: " + prob.Weights[0]); // prints e.g. 0.50101
var trainer = new Trainer(net) {LearningRate = 0.01, L2Decay = 0.001};
trainer.Train(x, 0); // train the network, specifying that x is class zero
var prob2 = net.Forward(x);
Console.WriteLine("probability that x is class 0: " + prob2.Weights[0]);
// now prints 0.50374, slightly higher than previous 0.50101: the networks
// weights have been adjusted by the Trainer to give a higher probability to
// the class we trained the network with (zero)
}
public override Volume Forward(Volume input, bool isTraining = false)
{
this.InputActivation = input;
var output = input.Clone();
var length = input.Weights.Length;
if (isTraining)
{
// do dropout
for (var i = 0; i < length; i++)
{
if (Random.NextDouble() < this.DropProb.Value)
{
output.Weights[i] = 0;
this.dropped[i] = true;
} // drop!
else
{
this.dropped[i] = false;
}
}
}
else
{
// scale the activations during prediction
for (var i = 0; i < length; i++)
{
output.Weights[i] *= this.DropProb.Value;
}
}
this.OutputActivation = output;
return this.OutputActivation; // dummy identity function for now
}
public override Volume Forward(Volume input, bool isTraining = false)
{
this.InputActivation = input;
var outputActivation = new Volume(1, 1, this.OutputDepth, 0.0);
double[] vw = input.Weights;
#if PARALLEL
Parallel.For(0, this.OutputDepth, i =>
#else
for (var i = 0; i < this.OutputDepth; i++)
#endif
{
var a = 0.0;
double[] wi = this.Filters[i].Weights;
for (var d = 0; d < this.inputCount; d++)
{
a += vw[d] * wi[d]; // for efficiency use Vols directly for now
}
a += this.Biases.Weights[i];
outputActivation.Weights[i] = a;
}
#if PARALLEL
);
#endif
this.OutputActivation = outputActivation;
return this.OutputActivation;
}
public override Volume Forward(Volume input, bool isTraining = false)
{
// optimized code by @mdda that achieves 2x speedup over previous version
this.InputActivation = input;
var outputActivation = new Volume(this.OutputWidth, this.OutputHeight, this.OutputDepth, 0.0);
var volumeWidth = input.Width;
var volumeHeight = input.Height;
var xyStride = this.Stride;
#if PARALLEL
Parallel.For(0, this.OutputDepth, depth =>
#else
for (var depth = 0; depth < this.OutputDepth; depth++)
#endif
{
var filter = this.Filters[depth];
var y = -this.Pad;
for (var ay = 0; ay < this.OutputHeight; y += xyStride, ay++)
{
// xyStride
var x = -this.Pad;
for (var ax = 0; ax < this.OutputWidth; x += xyStride, ax++)
{
// xyStride
// convolve centered at this particular location
var a = 0.0;
for (var fy = 0; fy < filter.Height; fy++)
{
var oy = y + fy; // coordinates in the original input array coordinates
for (var fx = 0; fx < filter.Width; fx++)
{
var ox = x + fx;
if (oy >= 0 && oy < volumeHeight && ox >= 0 && ox < volumeWidth)
{
for (var fd = 0; fd < filter.Depth; fd++)
{
// avoid function call overhead (x2) for efficiency, compromise modularity :(
a += filter.Weights[((filter.Width * fy) + fx) * filter.Depth + fd] *
input.Weights[((volumeWidth * oy) + ox) * input.Depth + fd];
}
}
}
}
a += this.Biases.Weights[depth];
outputActivation.Set(ax, ay, depth, a);
}
}
}
#if PARALLEL
);
#endif
this.OutputActivation = outputActivation;
return this.OutputActivation;
}
private static void Regression1DDemo()
{
var net = new Net();
net.AddLayer(new InputLayer(1, 1, 1));
net.AddLayer(new FullyConnLayer(20, Activation.Relu));
net.AddLayer(new FullyConnLayer(20, Activation.Sigmoid));
net.AddLayer(new RegressionLayer(1));
var trainer = new Trainer(net) { LearningRate = 0.01, Momentum = 0.0, BatchSize = 1, L2Decay = 0.001 };
// Function we want to learn
double[] x = { 0.0, 0.5, 1.0 };
double[] y = { 0.0, 0.1, 0.2 };
var n = x.Length;
// Training
do
{
RegressionUpdate(n, x, trainer, y);
} while (!Console.KeyAvailable);
// Testing
var netx = new Volume(1, 1, 1);
for (var ix = 0; ix < n; ix++)
{
netx.Weights = new[] { x[ix] };
var result = net.Forward(netx);
}
}
public void Train(Volume x, double[] y)
{
this.Forward(x);
this.Backward(y);
this.TrainImplem();
}
/// <summary>
/// Intended for use with data augmentation
/// </summary>
/// <param name="volume">Input volume</param>
/// <param name="crop">Size of output</param>
/// <param name="dx">Offset wrt incoming volume, of the shift</param>
/// <param name="dy">Offset wrt incoming volume, of the shift</param>
/// <param name="flipLeftRight">flip left/right</param>
/// <returns></returns>
public static Volume Augment(this Volume volume, int crop, int dx = -1, int dy = -1, bool flipLeftRight = false)
{
if (dx == -1)
{
dx = Random.Next(volume.Width - crop);
}
if (dy == -1)
{
dy = Random.Next(volume.Height - crop);
}
// randomly sample a crop in the input volume
Volume w;
if (crop != volume.Width || dx != 0 || dy != 0)
{
w = new Volume(crop, crop, volume.Depth, 0.0);
for (var x = 0; x < crop; x++)
{
for (var y = 0; y < crop; y++)
{
if (x + dx < 0 || x + dx >= volume.Width || y + dy < 0 || y + dy >= volume.Width)
{
continue; // oob
}
for (var depth = 0; depth < volume.Depth; depth++)
{
w.Set(x, y, depth, volume.Get(x + dx, y + dy, depth)); // copy data over
}
}
}
}
else
{
w = volume;
}
if (flipLeftRight)
{
// flip volume horziontally
var w2 = w.CloneAndZero();
for (var x = 0; x < w.Width; x++)
{
for (var y = 0; y < w.Height; y++)
{
for (var depth = 0; depth < w.Depth; depth++)
{
w2.Set(x, y, depth, w.Get(w.Width - x - 1, y, depth)); // copy data over
}
}
}
w = w2; //swap
}
return w;
}
private static void Classify2DDemo()
{
var net = new Net();
net.AddLayer(new InputLayer(1, 1, 2));
net.AddLayer(new FullyConnLayer(6, Activation.Tanh));
net.AddLayer(new FullyConnLayer(2, Activation.Tanh));
net.AddLayer(new SoftmaxLayer(2));
var trainer = new Trainer(net) { LearningRate = 0.01, Momentum = 0.0, BatchSize = 10, L2Decay = 0.001 };
// Data
var data = new List<double[]>();
var labels = new List<int>();
data.Add(new[] { -0.4326, 1.1909 });
labels.Add(1);
data.Add(new[] { 3.0, 4.0 });
labels.Add(1);
data.Add(new[] { 0.1253, -0.0376 });
labels.Add(1);
data.Add(new[] { 0.2877, 0.3273 });
labels.Add(1);
data.Add(new[] { -1.1465, 0.1746 });
labels.Add(1);
data.Add(new[] { 1.8133, 1.0139 });
labels.Add(0);
data.Add(new[] { 2.7258, 1.0668 });
labels.Add(0);
data.Add(new[] { 1.4117, 0.5593 });
labels.Add(0);
data.Add(new[] { 4.1832, 0.3044 });
labels.Add(0);
data.Add(new[] { 1.8636, 0.1677 });
labels.Add(0);
data.Add(new[] { 0.5, 3.2 });
labels.Add(1);
data.Add(new[] { 0.8, 3.2 });
labels.Add(1);
data.Add(new[] { 1.0, -2.2 });
labels.Add(1);
var n = labels.Count;
// Training
do
{
Classify2DUpdate(n, data, trainer, labels);
} while (!Console.KeyAvailable);
// Testing
var netx = new Volume(1, 1, 1);
for (var ix = 0; ix < n; ix++)
{
netx.Weights = data[ix];
var result = net.Forward(netx);
var c = net.GetPrediction();
bool accurate = c == labels[ix];
}
}
public override Volume Forward(Volume input, bool isTraining = false)
{
this.InputActivation = input;
var volume2 = input.CloneAndZero();
var length = input.Weights.Length;
double[] v2w = volume2.Weights;
double[] vw = input.Weights;
for (var i = 0; i < length; i++)
{
v2w[i] = 1.0 / (1.0 + Math.Exp(-vw[i]));
}
this.OutputActivation = volume2;
return this.OutputActivation;
}
private static void Classify2DUpdate(int n, List<double[]> data, Trainer trainer, List<int> labels)
{
var netx = new Volume(1, 1, 1);
var avloss = 0.0;
for (var iters = 0; iters < 50; iters++)
{
for (var ix = 0; ix < n; ix++)
{
netx.Weights = data[ix];
trainer.Train(netx, labels[ix]);
avloss += trainer.Loss;
}
}
avloss /= n * 50.0;
Console.WriteLine("Loss:" + avloss);
}
private static void RegressionUpdate(int n, double[] x, Trainer trainer, double[] y)
{
var netx = new Volume(1, 1, 1);
var avloss = 0.0;
for (var iters = 0; iters < 50; iters++)
{
for (var ix = 0; ix < n; ix++)
{
netx.Weights = new[] { x[ix] };
trainer.Train(netx, y[ix]);
avloss += trainer.Loss;
}
}
avloss /= n * 50.0;
Console.WriteLine("Loss:" + avloss);
}
public override Volume Forward(Volume input, bool isTraining = false)
{
this.InputActivation = input;
var outputActivation = input.CloneAndZero();
var length = input.Weights.Length;
#if PARALLEL
Parallel.For(0, length, i =>
#else
for (var i = 0; i < length; i++)
#endif
{ outputActivation.Weights[i] = Math.Tanh(input.Weights[i]); }
#if PARALLEL
);
#endif
this.OutputActivation = outputActivation;
return this.OutputActivation;
}
public override Volume Forward(Volume input, bool isTraining = false)
{
this.InputActivation = input;
var outputActivation = new Volume(1, 1, this.OutputDepth, 0.0);
// compute max activation
double[] temp = input.Weights;
var amax = input.Weights[0];
for (var i = 1; i < this.OutputDepth; i++)
{
if (temp[i] > amax)
{
amax = temp[i];
}
}
// compute exponentials (carefully to not blow up)
var es = new double[this.OutputDepth];
var esum = 0.0;
for (var i = 0; i < this.OutputDepth; i++)
{
var e = Math.Exp(temp[i] - amax);
esum += e;
es[i] = e;
}
// normalize and output to sum to one
for (var i = 0; i < this.OutputDepth; i++)
{
es[i] /= esum;
outputActivation.Weights[i] = es[i];
}
this.es = es; // save these for backprop
this.OutputActivation = outputActivation;
return this.OutputActivation;
}
public override Volume Forward(Volume input, bool isTraining = false)
{
this.InputActivation = input;
var output = input.Clone();
var length = input.Weights.Length;
double[] outputWeights = output.Weights;
#if PARALLEL
Parallel.For(0, length, i =>
#else
for (var i = 0; i < length; i++)
#endif
{
if (outputWeights[i] < 0)
{
outputWeights[i] = 0; // threshold at 0
}
}
#if PARALLEL
);
#endif
this.OutputActivation = output;
return this.OutputActivation;
}
public override Volume Forward(Volume input, bool isTraining = false)
{
this.InputActivation = input;
var depth = this.OutputDepth;
var outputActivation = new Volume(this.OutputWidth, this.OutputHeight, this.OutputDepth, 0.0);
// optimization branch. If we're operating on 1D arrays we dont have
// to worry about keeping track of x,y,d coordinates inside
// input volumes. In convnets we do :(
if (this.OutputWidth == 1 && this.OutputHeight == 1)
{
for (var i = 0; i < depth; i++)
{
var ix = i * this.GroupSize; // base index offset
var a = input.Weights[ix];
var ai = 0;
for (var j = 1; j < this.GroupSize; j++)
{
var a2 = input.Weights[ix + j];
if (a2 > a)
{
a = a2;
ai = j;
}
}
outputActivation.Weights[i] = a;
this.switches[i] = ix + ai;
}
}
else
{
var n = 0; // counter for switches
for (var x = 0; x < input.Width; x++)
{
for (var y = 0; y < input.Height; y++)
{
for (var i = 0; i < depth; i++)
{
var ix = i * this.GroupSize;
var a = input.Get(x, y, ix);
var ai = 0;
for (var j = 1; j < this.GroupSize; j++)
{
var a2 = input.Get(x, y, ix + j);
if (a2 > a)
{
a = a2;
ai = j;
}
}
outputActivation.Set(x, y, i, a);
this.switches[n] = ix + ai;
n++;
}
}
}
}
this.OutputActivation = outputActivation;
return this.OutputActivation;
}
private void Forward(Volume x)
{
var chrono = Stopwatch.StartNew();
this.net.Forward(x, true); // also set the flag that lets the net know we're just training
this.ForwardTime = chrono.Elapsed;
}
public Volume Clone()
{
var volume = new Volume(this.Width, this.Height, this.Depth, 0.0);
var n = this.Weights.Length;
for (var i = 0; i < n; i++)
{
volume.Weights[i] = this.Weights[i];
}
return volume;
}
public override Volume Forward(Volume input, bool isTraining = false)
{
this.InputActivation = input;
this.OutputActivation = input;
return this.OutputActivation; // simply identity function for now
}
public void AddGradientFrom(Volume volume)
{
for (var i = 0; i < this.WeightGradients.Length; i++)
{
this.WeightGradients[i] += volume.WeightGradients[i];
}
}
public void AddFromScaled(Volume volume, double a)
{
for (var i = 0; i < this.Weights.Length; i++)
{
this.Weights[i] += a * volume.Weights[i];
}
}
private List<Item> SampleTestingInstance()
{
var result = new List<Item>();
var n = this.random.Next(this.testing.Count);
var entry = this.testing[n];
// Create volume from image data
var x = new Volume(28, 28, 1, 0.0);
for (var i = 0; i < 28; i++)
{
for (var j = 0; j < 28; j++)
{
x.Weights[j + i * 28] = entry.Image[j + i * 28] / 255.0;
}
}
for (var i = 0; i < 4; i++)
{
result.Add(new Item { Volume = x.Augment(24), Label = entry.Label });
}
return result;
}
public Volume Forward(Volume volume, bool isTraining = false)
{
var activation = this.layers[0].Forward(volume, isTraining);
for (var i = 1; i < this.layers.Count; i++)
{
var layerBase = this.layers[i];
activation = layerBase.Forward(activation, isTraining);
}
return activation;
}
public override Volume Forward(Volume input, bool isTraining = false)
{
this.InputActivation = input;
this.OutputActivation = input; // nothing to do, output raw scores
return input;
}
private void TestPredict()
{
for (var i = 0; i < 50; i++)
{
List<Item> sample = this.SampleTestingInstance();
var y = sample[0].Label; // ground truth label
// forward prop it through the network
var average = new Volume(1, 1, 10, 0.0);
var n = sample.Count;
for (var j = 0; j < n; j++)
{
var a = this.net.Forward(sample[j].Volume);
average.AddFrom(a);
}
var predictions = average.Weights.Select((w, k) => new { Label = k, Weight = w }).OrderBy(o => -o.Weight);
}
}
public override Volume Forward(Volume input, bool isTraining = false)
{
this.InputActivation = input;
var outputActivation = new Volume(this.OutputWidth, this.OutputHeight, this.OutputDepth, 0.0);
#if PARALLEL
Parallel.For(0, this.OutputDepth, depth =>
#else
for (var depth = 0; depth < this.OutputDepth; depth++)
#endif
{
var n = depth * this.OutputWidth * this.OutputHeight; // a counter for switches
var x = -this.Pad;
var y = -this.Pad;
for (var ax = 0; ax < this.OutputWidth; x += this.Stride, ax++)
{
y = -this.Pad;
for (var ay = 0; ay < this.OutputHeight; y += this.Stride, ay++)
{
// convolve centered at this particular location
var a = double.MinValue;
int winx = -1, winy = -1;
for (var fx = 0; fx < this.Width; fx++)
{
for (var fy = 0; fy < this.Height; fy++)
{
var oy = y + fy;
var ox = x + fx;
if (oy >= 0 && oy < input.Height && ox >= 0 && ox < input.Width)
{
var v = input.Get(ox, oy, depth);
// perform max pooling and store pointers to where
// the max came from. This will speed up backprop
// and can help make nice visualizations in future
if (v > a)
{
a = v;
winx = ox;
winy = oy;
}
}
}
}
this.switchx[n] = winx;
this.switchy[n] = winy;
n++;
outputActivation.Set(ax, ay, depth, a);
}
}
}
#if PARALLEL
);
#endif
this.OutputActivation = outputActivation;
return this.OutputActivation;
}
public double GetCostLoss(Volume volume, double[] y)
{
this.Forward(volume);
var lastLayer = this.layers[this.layers.Count - 1] as ILastLayer;
if (lastLayer != null)
{
var loss = lastLayer.Backward(y);
return loss;
}
throw new Exception("Last layer doesnt implement ILastLayer interface");
}
public abstract Volume Forward(Volume input, bool isTraining = false);
private Item SampleTrainingInstance()
{
var n = this.random.Next(this.trainingCount);
var entry = this.training[n];
// load more batches over time
if (this.stepCount % 5000 == 0 && this.stepCount > 0)
{
this.trainingCount = Math.Min(this.trainingCount + BatchSize, this.training.Count);
}
// Create volume from image data
var x = new Volume(28, 28, 1, 0.0);
for (var i = 0; i < 28; i++)
{
for (var j = 0; j < 28; j++)
{
x.Weights[j + i * 28] = entry.Image[j + i * 28] / 255.0;
}
}
x = x.Augment(24);
return new Item { Volume = x, Label = entry.Label, IsValidation = n % 10 == 0 };
}