public DarknetV3(int[] layers, int[] channels, int classes = 1000, string norm_layer = "BatchNorm", FuncArgs norm_kwargs = null, string prefix = null, ParameterDict @params = null) : base(prefix, @params)
{
Debug.Assert(layers.Length == channels.Length - 1, $"len(channels) should equal to len(layers) + 1, given {channels.Length} vs {layers.Length}");
this.features = new HybridSequential();
// first 3x3 conv
this.features.Add(Conv2d(channels[0], 3, 1, 1, norm_layer: norm_layer, norm_kwargs: norm_kwargs));
for (int i = 0; i < layers.Length; i++)
{
int nlayer = layers[i];
int channel = channels[i];
Debug.Assert(channel % 2 == 0, $"channel {channel} cannot be divided by 2");
// add downsample conv with stride=2
this.features.Add(Conv2d(channel, 3, 1, 2, norm_layer: norm_layer, norm_kwargs: norm_kwargs));
// add nlayer basic blocks
foreach (var _ in Enumerable.Range(0, nlayer))
{
this.features.Add(new DarknetBasicBlockV3(channel / 2, norm_layer: norm_layer, norm_kwargs: norm_kwargs));
}
}
// output
this.output = new Dense(classes);
RegisterChild(features);
RegisterChild(output);
}
public BottleneckV1(int channels, int stride, bool downsample = false, int in_channels = 0,
string prefix = "", ParameterDict @params = null) : base(prefix, @params)
{
var channel_one_fourth = Convert.ToInt32(channels / 4);
body = new HybridSequential("");
body.Add(new Conv2D(channel_one_fourth, (1, 1), (stride, stride)));
body.Add(new BatchNorm());
body.Add(new Activation(ActivationType.Relu));
body.Add(ResNet.Conv3x3(channel_one_fourth, 1, channel_one_fourth));
body.Add(new BatchNorm());
body.Add(new Activation(ActivationType.Relu));
body.Add(new Conv2D(channels, (1, 1), (1, 1)));
body.Add(new BatchNorm());
if (downsample)
{
ds = new HybridSequential();
ds.Add(new Conv2D(channels, (1, 1), (stride, stride), use_bias: false, in_channels: in_channels));
ds.Add(new BatchNorm());
RegisterChild(ds, "downsample");
}
else
{
ds = null;
}
RegisterChild(body, "body");
}
public Compose(HybridBlock[] transforms)
{
var hybrid = new List <HybridBlock>();
foreach (var item in transforms)
{
if (item.GetType().Name == "HybridBlock")
{
hybrid.Add(item);
continue;
}
if (hybrid.Count == 1)
{
Add(hybrid[0]);
hybrid.Clear();
}
else if (hybrid.Count > 1)
{
var hblock = new HybridSequential();
foreach (var j in hybrid)
{
hblock.Add(j);
}
hblock.Hybridize();
Add(hblock);
hybrid.Clear();
}
Add(item);
}
}
public MobileNetV2(float multiplier = 1, int classes = 1000, string prefix = null, ParameterDict @params = null)
: base(prefix, @params)
{
Features = new HybridSequential("");
MobileNet.AddConv(Features, Convert.ToInt32(32 * multiplier), 3, pad: 1, stride: 2);
for (var i = 0; i < in_channels_group.Length; i++)
{
var in_c = Convert.ToInt32(multiplier * in_channels_group[i]);
var c = Convert.ToInt32(multiplier * channels_group[i]);
var s = strides[i];
var t = ts[i];
Features.Add(new LinearBottleneck(in_c, c, t, s));
}
var last_channel = multiplier > 1 ? Convert.ToInt32(1280 * multiplier) : 1280;
MobileNet.AddConv(Features, last_channel, relu6: true);
Features.Add(new GlobalAvgPool2D());
Output = new HybridSequential("output_");
Output.Add(new Conv2D(classes, (1, 1), use_bias: false, prefix: "pred_"));
Output.Add(new Flatten());
}
public Inception3(int classes = 1000, string prefix = "", ParameterDict @params = null) : base(prefix,
@params)
{
Features = new HybridSequential("");
Features.Add(Inception.MakeBasicConv(32, (3, 3), (2, 2)));
Features.Add(Inception.MakeBasicConv(32, (3, 3)));
Features.Add(Inception.MakeBasicConv(64, (3, 3), padding: (1, 1)));
Features.Add(new MaxPool2D((3, 3), (2, 2)));
Features.Add(Inception.MakeBasicConv(80, (1, 1)));
Features.Add(Inception.MakeBasicConv(192, (3, 3)));
Features.Add(new MaxPool2D((3, 3), (2, 2)));
Features.Add(Inception.MakeA(32, "A1_"));
Features.Add(Inception.MakeA(64, "A2_"));
Features.Add(Inception.MakeA(64, "A3_"));
Features.Add(Inception.MakeB("B_"));
Features.Add(Inception.MakeC(128, "C1_"));
Features.Add(Inception.MakeC(160, "C2_"));
Features.Add(Inception.MakeC(160, "C3_"));
Features.Add(Inception.MakeC(192, "C4_"));
Features.Add(Inception.MakeD("D_"));
Features.Add(Inception.MakeE("E1_"));
Features.Add(Inception.MakeE("E2_"));
Features.Add(new AvgPool2D((8, 8)));
Features.Add(new Dropout(0.5f));
RegisterChild(Features, "features");
Output = new Dense(classes);
RegisterChild(Output, "output");
}
public ResNetV1(string block, int[] layers, int[] channels, int classes = 1000, bool thumbnail = false,
string prefix = "", ParameterDict @params = null) : base(prefix, @params)
{
if (layers.Length != channels.Length - 1)
{
throw new Exception("layers.length should be equal to channels.length - 1");
}
Features = new HybridSequential();
if (thumbnail)
{
Features.Add(ResNet.Conv3x3(channels[0], 1, 0));
}
else
{
Features.Add(new Conv2D(channels[0], (7, 7), (2, 2), (3, 3), use_bias: false));
Features.Add(new BatchNorm());
Features.Add(new Activation(ActivationType.Relu));
Features.Add(new MaxPool2D((3, 3), (2, 2), (1, 1)));
}
for (var i = 0; i < layers.Length; i++)
{
var stride = i == 0 ? 1 : 2;
var num_layer = layers[i];
Features.Add(MakeLayer(block, num_layer, channels[i + 1], stride, i + 1, channels[i]));
}
Features.Add(new GlobalAvgPool2D());
Output = new Dense(classes, in_units: channels.Last());
RegisterChild(Features, "features");
RegisterChild(Output, "output");
}
public CompositeMatcher(HybridBlock[] matchers, string prefix = null, ParameterDict @params = null) : base(prefix, @params)
{
this._matchers = new HybridSequential();
foreach (var m in matchers)
{
this._matchers.Add(m);
}
}
public DarknetBasicBlockV3(int channel, string norm_layer = "BatchNorm", FuncArgs norm_kwargs = null, string prefix = null, ParameterDict @params = null) : base(prefix, @params)
{
this.body = new HybridSequential(prefix: "");
// 1x1 reduce
this.body.Add(DarknetV3.Conv2d(channel, 1, 0, 1, norm_layer: norm_layer, norm_kwargs: norm_kwargs));
// 3x3 conv expand
this.body.Add(DarknetV3.Conv2d(channel * 2, 3, 1, 1, norm_layer: norm_layer, norm_kwargs: norm_kwargs));
}
internal static HybridSequential Conv2d(int channel, int kernel, int padding, int stride, string norm_layer = "BatchNorm", FuncArgs norm_kwargs = null)
{
var cell = new HybridSequential(prefix: "");
cell.Add(new Conv2D(channel, kernel_size: (kernel, kernel), strides: (stride, stride), padding: (padding, stride), use_bias: false));
cell.Add(LayerUtils.NormLayer(norm_layer, norm_kwargs));
cell.Add(new LeakyReLU(0.1f));
return(cell);
}
public new HybridSequential this[string key]
{
get
{
var net = new HybridSequential(Prefix);
net.Add((HybridBlock)_childrens[key]);
return(net);
}
}
public HybridSequential BNRelu(int channels, string norm_layer = "BatchNorm", FuncArgs norm_kwargs = null)
{
var @out = new HybridSequential(prefix: "");
var bn = LayerUtils.NormLayer(norm_layer, norm_kwargs);
@out.Add(LayerUtils.NormLayer(norm_layer, norm_kwargs));
@out.Add(new Activation(ActivationType.Relu));
return(@out);
}
public LinearBottleneck(int in_channels, int channels, int t, int stride, string prefix = null,
ParameterDict @params = null) : base(prefix, @params)
{
use_shortcut = stride == 1 && in_channels == channels;
output = new HybridSequential();
MobileNet.AddConv(output, in_channels * t, relu6: true);
MobileNet.AddConv(output, in_channels * t, 3, stride, 1, in_channels, relu6: true);
MobileNet.AddConv(output, channels, active: false, relu6: true);
}
public SqueezeNet(string version, int classes = 1000, string prefix = "", ParameterDict @params = null) :
base(prefix, @params)
{
if (version != "1.0" && version != "1.1")
{
throw new NotSupportedException("Unsupported version");
}
Features = new HybridSequential();
if (version == "1.0")
{
Features.Add(new Conv2D(96, (7, 7), (2, 2)));
Features.Add(new Activation(ActivationType.Relu));
Features.Add(new MaxPool2D((3, 3), (2, 2), ceil_mode: true));
Features.Add(MakeFire(16, 64, 64));
Features.Add(MakeFire(16, 64, 64));
Features.Add(MakeFire(32, 128, 128));
Features.Add(new MaxPool2D((3, 3), (2, 2), ceil_mode: true));
Features.Add(MakeFire(32, 128, 128));
Features.Add(MakeFire(48, 192, 192));
Features.Add(MakeFire(48, 192, 192));
Features.Add(MakeFire(64, 256, 256));
Features.Add(new MaxPool2D((3, 3), (2, 2), ceil_mode: true));
Features.Add(MakeFire(64, 256, 256));
}
else if (version == "1.1")
{
Features.Add(new Conv2D(64, (3, 3), (2, 2)));
Features.Add(new Activation(ActivationType.Relu));
Features.Add(new MaxPool2D((3, 3), (2, 2), ceil_mode: true));
Features.Add(MakeFire(16, 64, 64));
Features.Add(MakeFire(16, 64, 64));
Features.Add(new MaxPool2D((3, 3), (2, 2), ceil_mode: true));
Features.Add(MakeFire(32, 128, 128));
Features.Add(MakeFire(32, 128, 128));
Features.Add(new MaxPool2D((3, 3), (2, 2), ceil_mode: true));
Features.Add(MakeFire(48, 192, 192));
Features.Add(MakeFire(48, 192, 192));
Features.Add(MakeFire(64, 256, 256));
Features.Add(MakeFire(64, 256, 256));
}
Features.Add(new Dropout(0.5f));
RegisterChild(Features, "features");
Output = new HybridSequential();
Output.Add(new Conv2D(classes, (1, 1)));
Output.Add(new Activation(ActivationType.Relu));
Output.Add(new AvgPool2D((13, 13)));
Output.Add(new Flatten());
RegisterChild(Output, "output");
}
public YOLODetectionBlockV3(int channel, string norm_layer = "BatchNorm", FuncArgs norm_kwargs = null, string prefix = null, ParameterDict @params = null) : base(prefix, @params)
{
Debug.Assert(channel % 2 == 0, $"channel {channel} cannot be divided by 2");
this.body = new HybridSequential(prefix: "");
foreach (var _ in Enumerable.Range(0, 2))
{
// 1x1 reduce
this.body.Add(DarknetV3.Conv2d(channel, 1, 0, 1, norm_layer: norm_layer, norm_kwargs: norm_kwargs));
// 3x3 expand
this.body.Add(DarknetV3.Conv2d(channel * 2, 3, 1, 1, norm_layer: norm_layer, norm_kwargs: norm_kwargs));
}
this.body.Add(DarknetV3.Conv2d(channel, 1, 0, 1, norm_layer: norm_layer, norm_kwargs: norm_kwargs));
this.tip = DarknetV3.Conv2d(channel * 2, 3, 1, 1, norm_layer: norm_layer, norm_kwargs: norm_kwargs);
}
internal static HybridSequential Conv2d(int channel, int kernel, int padding, int stride, string norm_layer = "BatchNorm", FuncArgs norm_kwargs = null)
{
var cell = new HybridSequential();
cell.Add(new Conv2D(channel, kernel_size: (kernel, kernel), strides: (stride, stride), padding: (padding, padding), use_bias: false));
if (norm_kwargs == null)
{
norm_kwargs = new FuncArgs();
}
norm_kwargs["epsilon"] = 1e-5f;
norm_kwargs["momentum"] = 0.9f;
cell.Add(LayerUtils.NormLayer(norm_layer, norm_kwargs));
cell.Add(new LeakyReLU(0.1f));
return(cell);
}
private HybridSequential MakeFire(int squeeze_channels, int expand1x1_channels, int expand3x3_channels)
{
var output = new HybridSequential("");
output.Add(MakeFireConv(squeeze_channels, (1, 1)));
var paths = new HybridConcurrent(1, "");
paths.Add(MakeFireConv(expand1x1_channels, (1, 1)));
paths.Add(MakeFireConv(expand3x3_channels, (3, 3), (1, 1)));
output.Add(paths);
return(output);
}
public static void Run()
{
//Logistic Regression is one of the first models newcomers to Deep Learning are implementing.
//The focus of this tutorial is to show how to do logistic regression using Gluon API.
var ctx = mx.Cpu();
int train_data_size = 1000;
int val_data_size = 100;
var(train_x, train_ground_truth_class) = GetRandomData(train_data_size, ctx);
var train_dataset = new ArrayDataset((train_x, train_ground_truth_class));
train_dataloader = new DataLoader(train_dataset, batch_size: batch_size, shuffle: true);
var(val_x, val_ground_truth_class) = GetRandomData(val_data_size, ctx);
var val_dataset = new ArrayDataset((val_x, val_ground_truth_class));
val_dataloader = new DataLoader(val_dataset, batch_size: batch_size, shuffle: true);
net = new HybridSequential();
net.Add(new Dense(units: 10, activation: ActivationType.Relu));
net.Add(new Dense(units: 10, activation: ActivationType.Relu));
net.Add(new Dense(units: 10, activation: ActivationType.Relu));
net.Add(new Dense(units: 1));
net.Initialize(new Xavier());
loss = new SigmoidBinaryCrossEntropyLoss();
trainer = new Trainer(net.CollectParams(), new SGD(learning_rate: 0.1f));
accuracy = new Accuracy();
f1 = new F1();
int epochs = 10;
float threshold = 0.5f;
foreach (var e in Enumerable.Range(0, epochs))
{
var avg_train_loss = TrainModel() / train_data_size;
var avg_val_loss = ValidateModel(threshold) / val_data_size;
Console.WriteLine($"Epoch: {e}, Training loss: {avg_train_loss}, Validation loss: {avg_val_loss}, Validation accuracy: {accuracy.Get().Item2}, F1 score: {f1.Get().Item2}");
accuracy.Reset();
}
}
internal static void AddConv(HybridSequential @out, int channels = 1, int kernel = 1, int stride = 1,
int pad = 0,
int num_group = 1, bool active = true, bool relu6 = false)
{
@out.Add(new Conv2D(channels, (kernel, kernel), (stride, stride), (pad, pad), groups: num_group,
use_bias: false));
@out.Add(new BatchNorm());
if (active)
{
if (relu6)
{
@out.Add(new RELU6());
}
else
{
@out.Add(new Activation(ActivationType.Relu));
}
}
}
public MobileNet(float multiplier = 1, int classes = 1000, string prefix = null, ParameterDict @params = null) :
base(prefix, @params)
{
Features = new HybridSequential("");
AddConv(Features, Convert.ToInt32(32 * multiplier), 3, pad: 1, stride: 2);
for (var i = 0; i < dw_channels.Length; i++)
{
var dwc = Convert.ToInt32(multiplier * dw_channels[i]);
var c = Convert.ToInt32(multiplier * channels[i]);
var s = strides[i];
AddConvDW(Features, dwc, c, s);
}
Features.Add(new GlobalAvgPool2D());
Features.Add(new Flatten());
Output = new Dense(classes);
}
public BasicBlockV1(int channels, int stride, bool downsample = false, int in_channels = 0,
string prefix = null, ParameterDict @params = null) : base(prefix, @params)
{
body = new HybridSequential("");
body.Add(ResNet.Conv3x3(channels, stride, in_channels));
body.Add(new BatchNorm());
body.Add(new Activation(ActivationType.Relu));
body.Add(ResNet.Conv3x3(channels, 1, in_channels));
body.Add(new BatchNorm());
if (downsample)
{
ds = new HybridSequential();
ds.Add(new Conv2D(channels, (1, 1), (stride, stride), use_bias: false, in_channels: in_channels));
ds.Add(new BatchNorm());
}
else
{
ds = null;
}
}
public static void Run()
{
//Logistic Regression is one of the first models newcomers to Deep Learning are implementing.
//The focus of this tutorial is to show how to do logistic regression using Gluon API.
var ctx = mx.Cpu();
int train_data_size = 1000;
int val_data_size = 100;
int batch_size = 10;
var(train_x, train_ground_truth_class) = GetRandomState(train_data_size, ctx);
var train_dataset = new ArrayDataset(train_x, train_ground_truth_class);
var train_dataloader = new DataLoader(train_dataset, batch_size: batch_size, shuffle: true);
var(val_x, val_ground_truth_class) = GetRandomState(val_data_size, ctx);
var val_dataset = new ArrayDataset(val_x, val_ground_truth_class);
var val_dataloader = new DataLoader(val_dataset, batch_size: batch_size, shuffle: true);
var net = new HybridSequential();
net.Add(new Dense(units: 10, activation: ActivationType.Relu));
net.Add(new Dense(units: 10, activation: ActivationType.Relu));
net.Add(new Dense(units: 10, activation: ActivationType.Relu));
net.Add(new Dense(units: 1));
net.Initialize(new Xavier());
var loss = new SigmoidBinaryCrossEntropyLoss();
var trainer = new Trainer(net.CollectParams(), new SGD(learning_rate: 0.1f));
var accuracy = new Accuracy();
var f1 = new F1();
float cumulative_train_loss = 0;
int i = 0;
foreach (var item in train_dataloader)
{
var data = item[0];
var label = item[1];
}
}
public AlexNet(int classes = 1000, string prefix = "", ParameterDict @params = null) : base(prefix, @params)
{
Features = new HybridSequential(prefix);
Features.Add(new Conv2D(64, (11, 11), (4, 4), (2, 2), activation: ActivationType.Relu));
Features.Add(new MaxPool2D((3, 3), (2, 2)));
Features.Add(new Conv2D(192, (5, 5), padding: (2, 2), activation: ActivationType.Relu));
Features.Add(new MaxPool2D((3, 3), (2, 2)));
Features.Add(new Conv2D(384, (3, 3), padding: (1, 1), activation: ActivationType.Relu));
Features.Add(new Conv2D(256, (3, 3), padding: (1, 1), activation: ActivationType.Relu));
Features.Add(new Conv2D(256, (3, 3), padding: (1, 1), activation: ActivationType.Relu));
Features.Add(new MaxPool2D((3, 3), (2, 2)));
Features.Add(new Flatten());
Features.Add(new Dense(4096, ActivationType.Relu));
Features.Add(new Dropout(0.5f));
Features.Add(new Dense(4096, ActivationType.Relu));
Features.Add(new Dropout(0.5f));
Output = new Dense(classes);
RegisterChild(Features);
RegisterChild(Output);
}
private HybridSequential MakeLayer(string block, int layers, int channels, int stride, int stage_index,
int in_channels = 0)
{
var layer = new HybridSequential($"stage{stage_index}_");
if (block == "basic_block")
{
layer.Add(new BasicBlockV2(channels, stride, channels != in_channels, in_channels, ""));
for (var i = 0; i < layers - 1; i++)
{
layer.Add(new BasicBlockV2(channels, 1, false, channels, ""));
}
}
else if (block == "bottle_neck")
{
layer.Add(new BottleneckV2(channels, stride, channels != in_channels, in_channels, ""));
for (var i = 0; i < layers - 1; i++)
{
layer.Add(new BottleneckV2(channels, 1, false, channels, ""));
}
}
return(layer);
}
public IdentityResidualBlock(int in_channels, int channels, int strides = 1, int dilation = 1, int groups = 1, HybridSequential norm_act = null,
float?dropout = null, bool dist_bn = false, string prefix = "", ParameterDict @params = null) : base(prefix, @params)
{
throw new NotImplementedException();
}
public static HybridSequential AddSigmoidLayer(HybridSequential @out, int channels, string norm_layer, FuncArgs norm_kwargs)
{
throw new NotImplementedException();
}
public WiderResNetA2(int[] structure, HybridSequential norm_act = null, int classes = 0, bool dilation = false, bool dist_bn = false, string prefix = "", ParameterDict @params = null) : base(prefix, @params)
{
throw new NotImplementedException();
}
public static WiderResNetA2 GetWiderResNetA2(int[] structure, bool pretrained = false, Context ctx = null, string root = "", HybridSequential norm_act = null, bool dilation = false, bool dist_bn = false)
{
throw new NotImplementedException();
}
public static WiderResNetA2 WiderResNetA2_38(bool pretrained = false, Context ctx = null, string root = "", HybridSequential norm_act = null, bool dilation = false, bool dist_bn = false)
{
return(GetWiderResNetA2(new int[] { 3, 3, 6, 3, 1, 1 }, pretrained, ctx, root, norm_act, dilation, dist_bn));
}
public static HybridSequential AddBlock(HybridSequential @out, HybridBlock block)
{
throw new NotImplementedException();
}
internal static void AddConvDW(HybridSequential @out, int dw_channels, int channels = 1, int stride = 1,
bool relu6 = false)
{
AddConv(@out, dw_channels, 3, stride, 1, dw_channels, relu6);
AddConv(@out, channels, relu6: relu6);
}
