public ResNetV1(string block, int[] layers, int[] channels, int classes = 1000, bool thumbnail = false,
string prefix = "", ParameterDict @params = null) : base(prefix, @params)
{
if (layers.Length != channels.Length - 1)
{
throw new Exception("layers.length should be equal to channels.length - 1");
}
Features = new HybridSequential();
if (thumbnail)
{
Features.Add(ResNet.Conv3x3(channels[0], 1, 0));
}
else
{
Features.Add(new Conv2D(channels[0], (7, 7), (2, 2), (3, 3), use_bias: false));
Features.Add(new BatchNorm());
Features.Add(new Activation(ActivationType.Relu));
Features.Add(new MaxPool2D((3, 3), (2, 2), (1, 1)));
}
for (var i = 0; i < layers.Length; i++)
{
var stride = i == 0 ? 1 : 2;
var num_layer = layers[i];
Features.Add(MakeLayer(block, num_layer, channels[i + 1], stride, i + 1, channels[i]));
}
Features.Add(new GlobalAvgPool2D());
Output = new Dense(classes, in_units: channels.Last());
RegisterChild(Features, "features");
RegisterChild(Output, "output");
}
public MobileNetV2(float multiplier = 1, int classes = 1000, string prefix = null, ParameterDict @params = null)
: base(prefix, @params)
{
Features = new HybridSequential("");
MobileNet.AddConv(Features, Convert.ToInt32(32 * multiplier), 3, pad: 1, stride: 2);
for (var i = 0; i < in_channels_group.Length; i++)
{
var in_c = Convert.ToInt32(multiplier * in_channels_group[i]);
var c = Convert.ToInt32(multiplier * channels_group[i]);
var s = strides[i];
var t = ts[i];
Features.Add(new LinearBottleneck(in_c, c, t, s));
}
var last_channel = multiplier > 1 ? Convert.ToInt32(1280 * multiplier) : 1280;
MobileNet.AddConv(Features, last_channel, relu6: true);
Features.Add(new GlobalAvgPool2D());
Output = new HybridSequential("output_");
Output.Add(new Conv2D(classes, (1, 1), use_bias: false, prefix: "pred_"));
Output.Add(new Flatten());
}
internal static HybridSequential Conv2d(int channel, int kernel, int padding, int stride, string norm_layer = "BatchNorm", FuncArgs norm_kwargs = null)
{
var cell = new HybridSequential(prefix: "");
cell.Add(new Conv2D(channel, kernel_size: (kernel, kernel), strides: (stride, stride), padding: (padding, stride), use_bias: false));
cell.Add(LayerUtils.NormLayer(norm_layer, norm_kwargs));
cell.Add(new LeakyReLU(0.1f));
return(cell);
}
public HybridSequential BNRelu(int channels, string norm_layer = "BatchNorm", FuncArgs norm_kwargs = null)
{
var @out = new HybridSequential(prefix: "");
var bn = LayerUtils.NormLayer(norm_layer, norm_kwargs);
@out.Add(LayerUtils.NormLayer(norm_layer, norm_kwargs));
@out.Add(new Activation(ActivationType.Relu));
return(@out);
}
public SqueezeNet(string version, int classes = 1000, string prefix = "", ParameterDict @params = null) :
base(prefix, @params)
{
if (version != "1.0" && version != "1.1")
{
throw new NotSupportedException("Unsupported version");
}
Features = new HybridSequential();
if (version == "1.0")
{
Features.Add(new Conv2D(96, (7, 7), (2, 2)));
Features.Add(new Activation(ActivationType.Relu));
Features.Add(new MaxPool2D((3, 3), (2, 2), ceil_mode: true));
Features.Add(MakeFire(16, 64, 64));
Features.Add(MakeFire(16, 64, 64));
Features.Add(MakeFire(32, 128, 128));
Features.Add(new MaxPool2D((3, 3), (2, 2), ceil_mode: true));
Features.Add(MakeFire(32, 128, 128));
Features.Add(MakeFire(48, 192, 192));
Features.Add(MakeFire(48, 192, 192));
Features.Add(MakeFire(64, 256, 256));
Features.Add(new MaxPool2D((3, 3), (2, 2), ceil_mode: true));
Features.Add(MakeFire(64, 256, 256));
}
else if (version == "1.1")
{
Features.Add(new Conv2D(64, (3, 3), (2, 2)));
Features.Add(new Activation(ActivationType.Relu));
Features.Add(new MaxPool2D((3, 3), (2, 2), ceil_mode: true));
Features.Add(MakeFire(16, 64, 64));
Features.Add(MakeFire(16, 64, 64));
Features.Add(new MaxPool2D((3, 3), (2, 2), ceil_mode: true));
Features.Add(MakeFire(32, 128, 128));
Features.Add(MakeFire(32, 128, 128));
Features.Add(new MaxPool2D((3, 3), (2, 2), ceil_mode: true));
Features.Add(MakeFire(48, 192, 192));
Features.Add(MakeFire(48, 192, 192));
Features.Add(MakeFire(64, 256, 256));
Features.Add(MakeFire(64, 256, 256));
}
Features.Add(new Dropout(0.5f));
RegisterChild(Features, "features");
Output = new HybridSequential();
Output.Add(new Conv2D(classes, (1, 1)));
Output.Add(new Activation(ActivationType.Relu));
Output.Add(new AvgPool2D((13, 13)));
Output.Add(new Flatten());
RegisterChild(Output, "output");
}
internal static HybridSequential Conv2d(int channel, int kernel, int padding, int stride, string norm_layer = "BatchNorm", FuncArgs norm_kwargs = null)
{
var cell = new HybridSequential();
cell.Add(new Conv2D(channel, kernel_size: (kernel, kernel), strides: (stride, stride), padding: (padding, padding), use_bias: false));
if (norm_kwargs == null)
{
norm_kwargs = new FuncArgs();
}
norm_kwargs["epsilon"] = 1e-5f;
norm_kwargs["momentum"] = 0.9f;
cell.Add(LayerUtils.NormLayer(norm_layer, norm_kwargs));
cell.Add(new LeakyReLU(0.1f));
return(cell);
}
public Compose(HybridBlock[] transforms)
{
var hybrid = new List <HybridBlock>();
foreach (var item in transforms)
{
if (item.GetType().Name == "HybridBlock")
{
hybrid.Add(item);
continue;
}
if (hybrid.Count == 1)
{
Add(hybrid[0]);
hybrid.Clear();
}
else if (hybrid.Count > 1)
{
var hblock = new HybridSequential();
foreach (var j in hybrid)
{
hblock.Add(j);
}
hblock.Hybridize();
Add(hblock);
hybrid.Clear();
}
Add(item);
}
}
public BottleneckV1(int channels, int stride, bool downsample = false, int in_channels = 0,
string prefix = "", ParameterDict @params = null) : base(prefix, @params)
{
var channel_one_fourth = Convert.ToInt32(channels / 4);
body = new HybridSequential("");
body.Add(new Conv2D(channel_one_fourth, (1, 1), (stride, stride)));
body.Add(new BatchNorm());
body.Add(new Activation(ActivationType.Relu));
body.Add(ResNet.Conv3x3(channel_one_fourth, 1, channel_one_fourth));
body.Add(new BatchNorm());
body.Add(new Activation(ActivationType.Relu));
body.Add(new Conv2D(channels, (1, 1), (1, 1)));
body.Add(new BatchNorm());
if (downsample)
{
ds = new HybridSequential();
ds.Add(new Conv2D(channels, (1, 1), (stride, stride), use_bias: false, in_channels: in_channels));
ds.Add(new BatchNorm());
RegisterChild(ds, "downsample");
}
else
{
ds = null;
}
RegisterChild(body, "body");
}
public new HybridSequential this[string key]
{
get
{
var net = new HybridSequential(Prefix);
net.Add((HybridBlock)_childrens[key]);
return(net);
}
}
public static void Run()
{
//Logistic Regression is one of the first models newcomers to Deep Learning are implementing.
//The focus of this tutorial is to show how to do logistic regression using Gluon API.
var ctx = mx.Cpu();
int train_data_size = 1000;
int val_data_size = 100;
var(train_x, train_ground_truth_class) = GetRandomData(train_data_size, ctx);
var train_dataset = new ArrayDataset((train_x, train_ground_truth_class));
train_dataloader = new DataLoader(train_dataset, batch_size: batch_size, shuffle: true);
var(val_x, val_ground_truth_class) = GetRandomData(val_data_size, ctx);
var val_dataset = new ArrayDataset((val_x, val_ground_truth_class));
val_dataloader = new DataLoader(val_dataset, batch_size: batch_size, shuffle: true);
net = new HybridSequential();
net.Add(new Dense(units: 10, activation: ActivationType.Relu));
net.Add(new Dense(units: 10, activation: ActivationType.Relu));
net.Add(new Dense(units: 10, activation: ActivationType.Relu));
net.Add(new Dense(units: 1));
net.Initialize(new Xavier());
loss = new SigmoidBinaryCrossEntropyLoss();
trainer = new Trainer(net.CollectParams(), new SGD(learning_rate: 0.1f));
accuracy = new Accuracy();
f1 = new F1();
int epochs = 10;
float threshold = 0.5f;
foreach (var e in Enumerable.Range(0, epochs))
{
var avg_train_loss = TrainModel() / train_data_size;
var avg_val_loss = ValidateModel(threshold) / val_data_size;
Console.WriteLine($"Epoch: {e}, Training loss: {avg_train_loss}, Validation loss: {avg_val_loss}, Validation accuracy: {accuracy.Get().Item2}, F1 score: {f1.Get().Item2}");
accuracy.Reset();
}
}
public MobileNet(float multiplier = 1, int classes = 1000, string prefix = null, ParameterDict @params = null) :
base(prefix, @params)
{
Features = new HybridSequential("");
AddConv(Features, Convert.ToInt32(32 * multiplier), 3, pad: 1, stride: 2);
for (var i = 0; i < dw_channels.Length; i++)
{
var dwc = Convert.ToInt32(multiplier * dw_channels[i]);
var c = Convert.ToInt32(multiplier * channels[i]);
var s = strides[i];
AddConvDW(Features, dwc, c, s);
}
Features.Add(new GlobalAvgPool2D());
Features.Add(new Flatten());
Output = new Dense(classes);
}
internal static void AddConv(HybridSequential @out, int channels = 1, int kernel = 1, int stride = 1,
int pad = 0,
int num_group = 1, bool active = true, bool relu6 = false)
{
@out.Add(new Conv2D(channels, (kernel, kernel), (stride, stride), (pad, pad), groups: num_group,
use_bias: false));
@out.Add(new BatchNorm());
if (active)
{
if (relu6)
{
@out.Add(new RELU6());
}
else
{
@out.Add(new Activation(ActivationType.Relu));
}
}
}
public BasicBlockV1(int channels, int stride, bool downsample = false, int in_channels = 0,
string prefix = null, ParameterDict @params = null) : base(prefix, @params)
{
body = new HybridSequential("");
body.Add(ResNet.Conv3x3(channels, stride, in_channels));
body.Add(new BatchNorm());
body.Add(new Activation(ActivationType.Relu));
body.Add(ResNet.Conv3x3(channels, 1, in_channels));
body.Add(new BatchNorm());
if (downsample)
{
ds = new HybridSequential();
ds.Add(new Conv2D(channels, (1, 1), (stride, stride), use_bias: false, in_channels: in_channels));
ds.Add(new BatchNorm());
}
else
{
ds = null;
}
}
public static void Run()
{
//Logistic Regression is one of the first models newcomers to Deep Learning are implementing.
//The focus of this tutorial is to show how to do logistic regression using Gluon API.
var ctx = mx.Cpu();
int train_data_size = 1000;
int val_data_size = 100;
int batch_size = 10;
var(train_x, train_ground_truth_class) = GetRandomState(train_data_size, ctx);
var train_dataset = new ArrayDataset(train_x, train_ground_truth_class);
var train_dataloader = new DataLoader(train_dataset, batch_size: batch_size, shuffle: true);
var(val_x, val_ground_truth_class) = GetRandomState(val_data_size, ctx);
var val_dataset = new ArrayDataset(val_x, val_ground_truth_class);
var val_dataloader = new DataLoader(val_dataset, batch_size: batch_size, shuffle: true);
var net = new HybridSequential();
net.Add(new Dense(units: 10, activation: ActivationType.Relu));
net.Add(new Dense(units: 10, activation: ActivationType.Relu));
net.Add(new Dense(units: 10, activation: ActivationType.Relu));
net.Add(new Dense(units: 1));
net.Initialize(new Xavier());
var loss = new SigmoidBinaryCrossEntropyLoss();
var trainer = new Trainer(net.CollectParams(), new SGD(learning_rate: 0.1f));
var accuracy = new Accuracy();
var f1 = new F1();
float cumulative_train_loss = 0;
int i = 0;
foreach (var item in train_dataloader)
{
var data = item[0];
var label = item[1];
}
}
private HybridSequential MakeFire(int squeeze_channels, int expand1x1_channels, int expand3x3_channels)
{
var output = new HybridSequential("");
output.Add(MakeFireConv(squeeze_channels, (1, 1)));
var paths = new HybridConcurrent(1, "");
paths.Add(MakeFireConv(expand1x1_channels, (1, 1)));
paths.Add(MakeFireConv(expand3x3_channels, (3, 3), (1, 1)));
output.Add(paths);
return(output);
}
public AlexNet(int classes = 1000, string prefix = "", ParameterDict @params = null) : base(prefix, @params)
{
Features = new HybridSequential(prefix);
Features.Add(new Conv2D(64, (11, 11), (4, 4), (2, 2), activation: ActivationType.Relu));
Features.Add(new MaxPool2D((3, 3), (2, 2)));
Features.Add(new Conv2D(192, (5, 5), padding: (2, 2), activation: ActivationType.Relu));
Features.Add(new MaxPool2D((3, 3), (2, 2)));
Features.Add(new Conv2D(384, (3, 3), padding: (1, 1), activation: ActivationType.Relu));
Features.Add(new Conv2D(256, (3, 3), padding: (1, 1), activation: ActivationType.Relu));
Features.Add(new Conv2D(256, (3, 3), padding: (1, 1), activation: ActivationType.Relu));
Features.Add(new MaxPool2D((3, 3), (2, 2)));
Features.Add(new Flatten());
Features.Add(new Dense(4096, ActivationType.Relu));
Features.Add(new Dropout(0.5f));
Features.Add(new Dense(4096, ActivationType.Relu));
Features.Add(new Dropout(0.5f));
Output = new Dense(classes);
RegisterChild(Features);
RegisterChild(Output);
}
private HybridSequential MakeLayer(string block, int layers, int channels, int stride, int stage_index,
int in_channels = 0)
{
var layer = new HybridSequential($"stage{stage_index}_");
if (block == "basic_block")
{
layer.Add(new BasicBlockV2(channels, stride, channels != in_channels, in_channels, ""));
for (var i = 0; i < layers - 1; i++)
{
layer.Add(new BasicBlockV2(channels, 1, false, channels, ""));
}
}
else if (block == "bottle_neck")
{
layer.Add(new BottleneckV2(channels, stride, channels != in_channels, in_channels, ""));
for (var i = 0; i < layers - 1; i++)
{
layer.Add(new BottleneckV2(channels, 1, false, channels, ""));
}
}
return(layer);
}
public static void ExportBlock(string path, HybridBlock block, Shape data_shape = null, int epoch = 0, HybridBlock preprocess = null, string layout = "HWC", Context ctx = null)
{
if (ctx == null)
{
ctx = mx.Cpu();
}
NDArray x;
int t;
int c;
int w;
int h;
HybridSequential wrapper_block;
List <Shape> data_shapes = new List <Shape>();
// input image layout
layout = layout.ToUpper();
if (data_shape == null)
{
if (layout == "HWC")
{
data_shapes = (from s in new int[] { 224, 256, 299, 300, 320, 416, 512, 600 }
select new Shape(s, s, 3)).ToList();
}
else if (layout == "CHW")
{
data_shapes = (from s in new int[] { 224, 256, 299, 300, 320, 416, 512, 600 }
select new Shape(3, s, s)).ToList();
}
else
{
throw new Exception("Unable to predict data_shape, please specify.");
}
}
else
{
data_shapes.Add(data_shape);
}
if (preprocess != null)
{
wrapper_block = new HybridSequential();
preprocess.Initialize(ctx: new Context[] { ctx });
wrapper_block.Add(preprocess);
wrapper_block.Add(block);
}
else
{
wrapper_block = new HybridSequential();
preprocess.Initialize(ctx: new Context[] { ctx });
wrapper_block.Add(block);
Debug.Assert(new string[] { "CHW", "CTHW" }.Contains(layout), $"Default layout is CHW for 2D models and CTHW for 3D models if preprocess is None, provided {layout}.");
}
wrapper_block.CollectParams().ResetCtx(ctx);
// try different data_shape if possible, until one fits the network
object last_exception = null;
foreach (var dshape in data_shapes)
{
if (layout == "HWC")
{
(h, w, c) = dshape;
x = nd.Zeros(new Shape(1, h, w, c), ctx: ctx);
}
else if (layout == "CHW")
{
(c, h, w) = dshape;
x = nd.Zeros(new Shape(1, c, h, w), ctx: ctx);
}
else if (layout == "THWC")
{
(t, h, w, c) = dshape;
x = nd.Zeros(new Shape(1, t, h, w, c), ctx: ctx);
}
else if (layout == "CTHW")
{
(c, t, h, w) = dshape;
x = nd.Zeros(new Shape(1, c, t, h, w), ctx: ctx);
}
else
{
throw new Exception(String.Format("Input layout {0} is not supported yet.", layout));
}
// hybridize and forward once
wrapper_block.Hybridize();
try
{
wrapper_block.Call(x);
wrapper_block.Export(path, epoch);
last_exception = null;
break;
}
catch (MXNetException ex)
{
last_exception = ex;
}
}
if (last_exception != null)
{
throw new Exception(last_exception.ToString().Split('\n')[0]);
}
}
public Inception3(int classes = 1000, string prefix = "", ParameterDict @params = null) : base(prefix,
@params)
{
Features = new HybridSequential("");
Features.Add(Inception.MakeBasicConv(32, (3, 3), (2, 2)));
Features.Add(Inception.MakeBasicConv(32, (3, 3)));
Features.Add(Inception.MakeBasicConv(64, (3, 3), padding: (1, 1)));
Features.Add(new MaxPool2D((3, 3), (2, 2)));
Features.Add(Inception.MakeBasicConv(80, (1, 1)));
Features.Add(Inception.MakeBasicConv(192, (3, 3)));
Features.Add(new MaxPool2D((3, 3), (2, 2)));
Features.Add(Inception.MakeA(32, "A1_"));
Features.Add(Inception.MakeA(64, "A2_"));
Features.Add(Inception.MakeA(64, "A3_"));
Features.Add(Inception.MakeB("B_"));
Features.Add(Inception.MakeC(128, "C1_"));
Features.Add(Inception.MakeC(160, "C2_"));
Features.Add(Inception.MakeC(160, "C3_"));
Features.Add(Inception.MakeC(192, "C4_"));
Features.Add(Inception.MakeD("D_"));
Features.Add(Inception.MakeE("E1_"));
Features.Add(Inception.MakeE("E2_"));
Features.Add(new AvgPool2D((8, 8)));
Features.Add(new Dropout(0.5f));
RegisterChild(Features, "features");
Output = new Dense(classes);
RegisterChild(Output, "output");
}