public static void Run()
{
// Create
var trainX = new NDArray(new float[] { 0, 0, 0, 1, 1, 0, 1, 1 }).Reshape(4, 2);
var trainY = new NDArray(new float[] { 0, 1, 1, 0 });
var batch_size = 2;
var train_data = new NDArrayIter(trainX, trainY, batch_size);
var val_data = new NDArrayIter(trainX, trainY, batch_size);
var net = new Sequential();
net.Add(new Dense(64, ActivationType.Relu));
net.Add(new Dense(1));
var gpus = TestUtils.ListGpus();
var ctxList = gpus.Count > 0 ? gpus.Select(x => Context.Gpu(x)).ToArray() : new[] { Context.Cpu() };
net.Initialize(new Uniform(), ctxList.ToArray());
var trainer = new Trainer(net.CollectParams(), new Adam());
var epoch = 1000;
var metric = new BinaryAccuracy();
var binary_crossentropy = new LogisticLoss();
float lossVal = 0;
for (var iter = 0; iter < epoch; iter++)
{
train_data.Reset();
lossVal = 0;
while (!train_data.End())
{
var batch = train_data.Next();
var data = Utils.SplitAndLoad(batch.Data[0], ctxList);
var label = Utils.SplitAndLoad(batch.Label[0], ctxList);
NDArrayList outputs = null;
using (var ag = Autograd.Record())
{
outputs = Enumerable.Zip(data, label, (x, y) =>
{
var z = net.Call(x);
NDArray loss = binary_crossentropy.Call(z, y);
loss.Backward();
lossVal += loss.Mean();
return(z);
}).ToList();
}
metric.Update(label, outputs.ToArray());
trainer.Step(batch.Data[0].Shape[0]);
}
var(name, acc) = metric.Get();
metric.Reset();
Console.WriteLine($"Loss: {lossVal}");
Console.WriteLine($"Training acc at epoch {iter}: {name}={acc * 100}%");
}
}
public static void GetStarted()
{
var ctx = mx.Cpu();
var net = new Sequential();
// Similar to Dense, it is not necessary to specify the input channels
// by the argument `in_channels`, which will be automatically inferred
// in the first forward pass. Also, we apply a relu activation on the
// output. In addition, we can use a tuple to specify a non-square
// kernel size, such as `kernel_size=(2,4)`
net.Add(new Conv2D(channels: 6, kernel_size: (5, 5), activation: ActivationType.Relu),
// One can also use a tuple to specify non-symmetric pool and stride sizes
new MaxPool2D(pool_size: (2, 2), strides: (2, 2)),
new Conv2D(channels: 16, kernel_size: (3, 3), activation: ActivationType.Relu),
new MaxPool2D(pool_size: (2, 2), strides: (2, 2)),
// The dense layer will automatically reshape the 4-D output of last
// max pooling layer into the 2-D shape: (x.shape[0], x.size/x.shape[0])
new Dense(120, activation: ActivationType.Relu),
new Dense(84, activation: ActivationType.Relu),
new Dense(10)
);
net.Initialize();
// Input shape is (batch_size, color_channels, height, width)
var x = nd.Random.Uniform(shape: new Shape(4, 1, 28, 28));
NDArray y = net.Call(x);
Console.WriteLine(y.Shape);
Console.WriteLine(net[0].Params["weight"].Data().Shape);
Console.WriteLine(net[5].Params["bias"].Data().Shape);
//var net = new MixMLP();
//net.Initialize();
//var x = nd.Random.Uniform(shape: new Shape(2, 2));
//var y = net.Call(x);
//net.blk[1].Params["weight"].Data();
}
public static void RunSimple()
{
var mnist = TestUtils.GetMNIST(); //Get the MNIST dataset, it will download if not found
var batch_size = 100; //Set training batch size
var train_data = new NDArrayIter(mnist["train_data"], mnist["train_label"], batch_size);
var val_data = new NDArrayIter(mnist["test_data"], mnist["test_label"], batch_size);
// Define simple network with dense layers
var net = new Sequential();
net.Add(new Dense(128, ActivationType.Relu));
net.Add(new Dense(64, ActivationType.Relu));
net.Add(new Dense(10));
//Set context, multi-gpu supported
var gpus = TestUtils.ListGpus();
var ctx = gpus.Count > 0 ? gpus.Select(x => Context.Gpu(x)).ToArray() : new[] { Context.Cpu(0) };
//Initialize the weights
net.Initialize(new Xavier(magnitude: 2.24f), ctx);
//Create the trainer with all the network parameters and set the optimizer
var trainer = new Trainer(net.CollectParams(), new Adam());
var epoch = 10;
var metric = new Accuracy(); //Use Accuracy as the evaluation metric.
var softmax_cross_entropy_loss = new SoftmaxCrossEntropyLoss();
float lossVal = 0; //For loss calculation
for (var iter = 0; iter < epoch; iter++)
{
var tic = DateTime.Now;
// Reset the train data iterator.
train_data.Reset();
lossVal = 0;
// Loop over the train data iterator.
while (!train_data.End())
{
var batch = train_data.Next();
// Splits train data into multiple slices along batch_axis
// and copy each slice into a context.
var data = Utils.SplitAndLoad(batch.Data[0], ctx, batch_axis: 0);
// Splits train labels into multiple slices along batch_axis
// and copy each slice into a context.
var label = Utils.SplitAndLoad(batch.Label[0], ctx, batch_axis: 0);
var outputs = new NDArrayList();
// Inside training scope
NDArray loss = null;
for (int i = 0; i < data.Length; i++)
{
using (var ag = Autograd.Record())
{
var x = data[i];
var y = label[i];
var z = net.Call(x);
// Computes softmax cross entropy loss.
loss = softmax_cross_entropy_loss.Call(z, y);
outputs.Add(z);
}
// Backpropagate the error for one iteration.
loss.Backward();
lossVal += loss.Mean();
}
// Updates internal evaluation
metric.Update(label, outputs.ToArray());
// Make one step of parameter update. Trainer needs to know the
// batch size of data to normalize the gradient by 1/batch_size.
trainer.Step(batch.Data[0].Shape[0]);
}
var toc = DateTime.Now;
// Gets the evaluation result.
var(name, acc) = metric.Get();
// Reset evaluation result to initial state.
metric.Reset();
Console.Write($"Loss: {lossVal} ");
Console.WriteLine($"Training acc at epoch {iter}: {name}={(acc * 100).ToString("0.##")}%, Duration: {(toc - tic).TotalSeconds.ToString("0.#")}s");
}
}
public static void RunConv()
{
var mnist = TestUtils.GetMNIST();
var batch_size = 128;
var train_data = new NDArrayIter(mnist["train_data"], mnist["train_label"], batch_size, true);
var val_data = new NDArrayIter(mnist["test_data"], mnist["test_label"], batch_size);
var net = new Sequential();
net.Add(new Conv2D(20, kernel_size: (5, 5), activation: ActivationType.Tanh));
net.Add(new MaxPool2D(pool_size: (2, 2), strides: (2, 2)));
net.Add(new Conv2D(50, kernel_size: (5, 5), activation: ActivationType.Tanh));
net.Add(new MaxPool2D(pool_size: (2, 2), strides: (2, 2)));
net.Add(new Flatten());
net.Add(new Dense(500, ActivationType.Tanh));
net.Add(new Dense(10));
var gpus = TestUtils.ListGpus();
var ctx = gpus.Count > 0 ? gpus.Select(x => Context.Gpu(x)).ToArray() : new[] { Context.Cpu(0) };
net.Initialize(new Xavier(magnitude: 2.24f), ctx);
var trainer = new Trainer(net.CollectParams(), new SGD(learning_rate: 0.02f));
var epoch = 10;
var metric = new Accuracy();
var softmax_cross_entropy_loss = new SoftmaxCELoss();
float lossVal = 0;
for (var iter = 0; iter < epoch; iter++)
{
var tic = DateTime.Now;
train_data.Reset();
lossVal = 0;
while (!train_data.End())
{
var batch = train_data.Next();
var data = Utils.SplitAndLoad(batch.Data[0], ctx, batch_axis: 0);
var label = Utils.SplitAndLoad(batch.Label[0], ctx, batch_axis: 0);
var outputs = new NDArrayList();
using (var ag = Autograd.Record())
{
for (var i = 0; i < data.Length; i++)
{
var x = data[i];
var y = label[i];
var z = net.Call(x);
NDArray loss = softmax_cross_entropy_loss.Call(z, y);
loss.Backward();
lossVal += loss.Mean();
outputs.Add(z);
}
//outputs = Enumerable.Zip(data, label, (x, y) =>
//{
// var z = net.Call(x);
// NDArray loss = softmax_cross_entropy_loss.Call(z, y);
// loss.Backward();
// lossVal += loss.Mean();
// return z;
//}).ToList();
}
metric.Update(label, outputs.ToArray());
trainer.Step(batch.Data[0].Shape[0]);
}
var toc = DateTime.Now;
var(name, acc) = metric.Get();
metric.Reset();
Console.Write($"Loss: {lossVal} ");
Console.WriteLine($"Training acc at epoch {iter}: {name}={(acc * 100).ToString("0.##")}%, Duration: {(toc - tic).TotalSeconds.ToString("0.#")}s");
}
}
public static void Run()
{
var mnist_train = new FashionMNIST(train: true);
var(x, y) = mnist_train[0];
Console.WriteLine($"X shape: {x.Shape}, X dtype: {x.DataType}, Y shape: {y.Shape}, Y dtype: {y.DataType}");
var transformer = new Compose(
new ToTensor(),
new Normalize(new MxNet.Tuple <float>(0.13f, 0.31f))
);
var train = mnist_train.TransformFirst(transformer);
int batch_size = 256;
var train_data = new DataLoader(train, batch_size: batch_size, shuffle: true);
foreach (var(data, label) in train_data)
{
Console.WriteLine(data.Shape + ", " + label.Shape);
break;
}
var mnist_valid = new FashionMNIST(train: false);
var valid_data = new DataLoader(mnist_valid, batch_size: batch_size, shuffle: true);
var net = new Sequential();
net.Add(new Conv2D(channels: 6, kernel_size: (5, 5), activation: ActivationType.Relu),
new MaxPool2D(pool_size: (2, 2), strides: (2, 2)),
new Conv2D(channels: 16, kernel_size: (3, 3), activation: ActivationType.Relu),
new MaxPool2D(pool_size: (2, 2), strides: (2, 2)),
new Flatten(),
new Dense(120, activation: ActivationType.Relu),
new Dense(84, activation: ActivationType.Relu),
new Dense(10));
net.Initialize(new Xavier());
var softmax_cross_entropy = new SoftmaxCrossEntropyLoss();
var trainer = new Trainer(net.CollectParams(), new SGD(learning_rate: 0.1f));
for (int epoch = 0; epoch < 10; epoch++)
{
var tic = DateTime.Now;
float train_loss = 0;
float train_acc = 0;
float valid_acc = 0;
foreach (var(data, label) in train_data)
{
NDArray loss = null;
NDArray output = null;
// forward + backward
using (Autograd.Record())
{
output = net.Call(data);
loss = softmax_cross_entropy.Call(output, label);
}
loss.Backward();
//update parameters
trainer.Step(batch_size);
//calculate training metrics
train_loss += loss.Mean();
train_acc += Acc(output, label);
}
// calculate validation accuracy
foreach (var(data, label) in valid_data)
{
valid_acc += Acc(net.Call(data), label);
}
Console.WriteLine($"Epoch {epoch}: loss {train_loss / train_data.Length}," +
$" train acc {train_acc / train_data.Length}, " +
$"test acc {train_acc / train_data.Length} " +
$"in {(DateTime.Now - tic).TotalMilliseconds} ms");
}
net.SaveParameters("net.params");
}