public override void Process(TransformContext context)
{
var space = (SpaceToBatchNd)context.MatchedLayers[0];
var dwConv2d = (DepthwiseConv2d)context.MatchedLayers[1];
var conv2d = (Conv2d)context.MatchedLayers[2];
var input = space.Input.Connection.From;
var output = conv2d.Output;
space.Input.ClearConnection();
var newDwConv2d = new DepthwiseConv2d(input.Dimensions, dwConv2d.Weights, dwConv2d.Bias, Padding.Same, 1, 1, dwConv2d.FusedActivationFunction);
var quantize = new Quantize(newDwConv2d.Output.Dimensions);
var upload = new K210Upload(quantize.Output.Dimensions);
var newConv2d = new K210Conv2d(upload.Output.Dimensions, K210Conv2dType.Conv2d, conv2d.Weights, conv2d.Bias, K210PoolType.LeftTop, conv2d.FusedActivationFunction, null);
var dequantize = new Dequantize(newConv2d.Output.Dimensions);
newDwConv2d.Input.SetConnection(input);
quantize.Input.SetConnection(newDwConv2d.Output);
upload.Input.SetConnection(quantize.Output);
newConv2d.Input.SetConnection(upload.Output);
dequantize.Input.SetConnection(newConv2d.Output);
var oldOuts = output.Connections.Select(o => o.To).ToList();
foreach (var oldOut in oldOuts)
{
oldOut.SetConnection(dequantize.Output);
}
}
public override void Process(TransformContext context)
{
K210Conv2d newLayer;
OutputConnector output;
OutputConnector input;
var conv = context.MatchedLayers[0];
if (conv is Conv2d conv2d)
{
newLayer = new K210Conv2d(conv2d.Input.Dimensions, K210Conv2dType.Conv2d, conv2d.Weights, conv2d.Bias, K210PoolType.LeftTop, conv2d.FusedActivationFunction, null);
input = conv2d.Input.Connection.From;
output = conv2d.Output;
}
else
{
throw new InvalidOperationException();
}
var quantize = new Quantize(input.Dimensions);
var upload = new K210Upload(input.Dimensions);
var dequantize = new Dequantize(newLayer.Output.Dimensions);
quantize.Input.SetConnection(input);
upload.Input.SetConnection(quantize.Output);
newLayer.Input.SetConnection(upload.Output);
dequantize.Input.SetConnection(newLayer.Output);
var oldOuts = output.Connections.Select(o => o.To).ToList();
foreach (var oldOut in oldOuts)
{
oldOut.SetConnection(dequantize.Output);
}
}
private void ConvertK210Conv2d(K210Conv2d layer, ConvertContext context)
{
if (layer.Conv2dType != K210Conv2dType.Conv2d)
{
throw new NotSupportedException("Depthwise conv2d is not supported.");
}
context.Layers.Add(new ScriptConv2dLayerConfig
{
KernelSize = layer.KernelWidth,
Stride = GetStride(layer.PoolType),
Filters = layer.OutputChannels,
Activation = layer.FusedActivationFunction,
Input = context.Outputs[layer.Input.Connection.From],
Output = context.AddOutput(layer.Output)
});
}
public override void Process(TransformContext context)
{
var space = (SpaceToBatchNd)context.MatchedLayers[0];
var input = space.Input.Connection.From;
space.Input.ClearConnection();
K210Conv2d newLayer;
OutputConnector output;
var conv = context.MatchedLayers[1];
if (conv is Conv2d conv2d)
{
newLayer = new K210Conv2d(input.Dimensions, K210Conv2dType.Conv2d, conv2d.Weights, conv2d.Bias, conv2d.StrideWidth == 2 ? K210PoolType.LeftTop : K210PoolType.None, conv2d.FusedActivationFunction, null);
output = conv2d.Output;
}
else if (conv is DepthwiseConv2d dwConv2d)
{
newLayer = new K210Conv2d(input.Dimensions, K210Conv2dType.DepthwiseConv2d, dwConv2d.Weights, dwConv2d.Bias, K210PoolType.None, dwConv2d.FusedActivationFunction, null);
output = dwConv2d.Output;
}
else
{
throw new InvalidOperationException();
}
var quantize = new Quantize(input.Dimensions);
var upload = new K210Upload(input.Dimensions);
var dequantize = new Dequantize(newLayer.Output.Dimensions);
quantize.Input.SetConnection(input);
upload.Input.SetConnection(quantize.Output);
newLayer.Input.SetConnection(upload.Output);
dequantize.Input.SetConnection(newLayer.Output);
var oldOuts = output.Connections.Select(o => o.To).ToList();
foreach (var oldOut in oldOuts)
{
oldOut.SetConnection(dequantize.Output);
}
}
public void Infer(K210Conv2d layer, K210Conv2dLayerArgument argument, InferenceContext context)
{
var inputAlloc = context.KPUMemoryMap[layer.Input.Connection.From];
MemoryAllocation outputAlloc;
argument.Config.InputAddress = inputAlloc.GetAddress();
if (context.MainMemoryMap.TryGetValue(layer.Output, out var mainAlloc))
{
argument.Flags = K210LayerFlags.MainMemoryOutput;
argument.MainMemoryOutputAddress = mainAlloc.GetAddress();
outputAlloc = context.GetOrAllocateKPUMemory(layer.Output);
}
else
{
argument.Flags = K210LayerFlags.None;
outputAlloc = context.KPUMemoryMap[layer.Output];
}
argument.Config.OutputAddress = outputAlloc.GetAddress();
}
public override void Process(TransformContext context)
{
var conv2d = (K210Conv2d)context.MatchedLayers[0];
var act = context.MatchedLayers[2];
var input = conv2d.Input.Connection.From;
var output = act.OutputConnectors[0];
conv2d.Input.ClearConnection();
var newConv2d = new K210Conv2d(conv2d.Input.Dimensions, conv2d.Conv2dType, conv2d.Weights, conv2d.Bias, conv2d.PoolType, conv2d.FusedActivationFunction, act);
newConv2d.Input.SetConnection(input);
var newDequantize = new Dequantize(newConv2d.Output.Dimensions);
newDequantize.Input.SetConnection(newConv2d.Output);
var oldOuts = output.Connections.Select(o => o.To).ToList();
foreach (var oldOut in oldOuts)
{
oldOut.SetConnection(newDequantize.Output);
}
}
public K210Conv2dLayerArgument Convert(K210Conv2d layer, ConvertContext context)
{
var config = new K210ConvLayerConfig {
BNConfigs = new K210LayerBNConfig[layer.OutputChannels], ActConfigs = new K210LayerActConfig[16]
};
(var sw, var bw) = QuantizeWeights(layer.Conv2dType == K210Conv2dType.Conv2d, layer.Weights, config, context.WeightsBits);
(var sx, var bx) = QuantizeInput(context.Quantization.Distributions[layer.Input.Connection.From].Global, config);
config.ArgAdd = (long)Math.Round(bw * bx * layer.KernelWidth * layer.KernelHeight);
var scale = new double[layer.OutputChannels];
for (int i = 0; i < scale.Length; i++)
{
scale[i] = sw[i] * sx;
}
QuantizeBiasAndOutput(layer, layer.Bias, context.Quantization.Distributions[layer.Output], context.Quantization.AdditionalDistributions[layer.OutputBeforeActivation], scale, config);
config.InputChannels = layer.InputChannels;
config.OutputChannels = layer.OutputChannels;
config.InputWidth = layer.Input.Dimensions[3];
config.InputHeight = layer.Input.Dimensions[2];
(config.InputGroups, config.InputRowLength) = K210Helper.GetRowLayout(config.InputWidth);
config.OutputWidth = layer.Output.Dimensions[3];
config.OutputHeight = layer.Output.Dimensions[2];
(config.OutputGroups, config.OutputRowLength) = K210Helper.GetRowLayout(config.OutputWidth);
config.KernelType = layer.KernelWidth == 3 ? 1 : 0;
config.IsDepthwise = layer.Conv2dType == K210Conv2dType.DepthwiseConv2d;
config.PoolType = (int)layer.PoolType;
config.PadValue = (int)Math.Round(-bx);
if (layer.Conv2dType == K210Conv2dType.Conv2d)
{
var kernelSize = (int)layer.Weights.Length * context.WeightsBits / 8;
var oneChannelSize = layer.KernelWidth * layer.KernelHeight * layer.InputChannels * context.WeightsBits / 8;
var sizeLimit = context.WeightsBits == 8 ? 30 : 60;
var oneLoadChannels = Math.Min(layer.OutputChannels, (int)Math.Floor(sizeLimit * 1024.0 / oneChannelSize));
config.OneLoadKernelsSize = oneChannelSize * oneLoadChannels;
config.LoadTimes = (int)Math.Ceiling(layer.OutputChannels / (double)oneLoadChannels);
config.OutputChannelsOnTime = oneLoadChannels;
}
else
{
config.OneLoadKernelsSize = (int)layer.Weights.Length * context.WeightsBits / 8;
config.LoadTimes = 1;
config.OutputChannelsOnTime = layer.OutputChannels;
}
var inputOneLineChannels = Math.Min(layer.InputChannels, config.InputGroups);
config.InputSize = config.InputRowLength * config.InputHeight * config.InputChannels / inputOneLineChannels;
var outputOneLineChannels = Math.Min(layer.OutputChannels, config.OutputGroups);
config.OutputSize = config.OutputRowLength * config.OutputHeight * config.OutputChannels / outputOneLineChannels;
return(new K210Conv2dLayerArgument
{
Config = config,
ParamAddress = new K210Conv2dParamAddress()
});
}
private static void QuantizeActivation(K210Conv2d layer, double postMul, QuantizationRange range, QuantizationRange beforeActRange, K210ConvLayerConfig config)
{
if (layer.NonTrivialActivation == null)
{
switch (layer.FusedActivationFunction)
{
case ActivationFunctionType.Linear:
case ActivationFunctionType.Relu:
case ActivationFunctionType.Relu6:
break;
default:
throw new NotSupportedException($"Activation of {layer.FusedActivationFunction} is not supported.");
}
var starts = new ulong[]
{
0x800000000, 0xf7d4cf4b8, 0xf8ed5a20c, 0xfa05e4f60,
0xfb2e05baa, 0xfc46908fe, 0xfd5f1b652, 0xfe77a63a6,
0xff9fc6ff0, 0xfffd4a9b7, 0, 0x7FFFFFFF0,
0x7FFFFFFF1, 0x7FFFFFFF2, 0x7FFFFFFF3, 0x7FFFFFFF4
};
for (int i = 0; i < starts.Length; i++)
{
var param = config.ActConfigs[i] = new K210LayerActConfig();
param.StartX = starts[i];
if (i == 10)
{
(var mul, var shift) = Quantizer.ExtractValueAndShift(1 / postMul, 16, 20);
param.Mul = (int)Math.Round(mul);
param.Shift = shift;
}
}
}
else if (layer.NonTrivialActivation is LeakyRelu leakyRelu)
{
(var scale, var bias) = range.GetScaleBias(8);
var zero = (long)(Quantizer.Quantize(0, scale, bias) * postMul);
var yTable = Generator.IntegerStep(0, (int)-bias, 15).Take(14).ToArray();
for (int i = 0; i < 16; i++)
{
var param = config.ActConfigs[i] = new K210LayerActConfig();
if (i == 0)
{
param.StartX = 0x800000000;
}
else if (i == 15)
{
(var mul, var shift) = Quantizer.ExtractValueAndShift(1 / postMul, 16, 20);
param.StartX = (ulong)zero;
param.Mul = (int)Math.Round(mul);
param.Shift = shift;
param.Add = (byte)(-bias);
}
else
{
// f(x) = (1 - slope) * zero + x * slope
// f(x1) - f(x0) = (x1 - x0) * slope
// x0 = zero - (zero - y0) / slope
var add = (byte)yTable[i - 1];
var y0 = add * postMul;
var x0 = zero - (zero - y0) / leakyRelu.Slope;
(var mul, var shift) = Quantizer.ExtractValueAndShift(1 / postMul * leakyRelu.Slope, 16, 20);
param.StartX = (ulong)(long)Math.Floor(x0);
param.Mul = (int)Math.Round(mul);
param.Shift = shift;
param.Add = add;
}
}
}
else
{
throw new NotSupportedException($"Activation of {layer.NonTrivialActivation.GetType().Name} is not supported.");
}
}
private static void QuantizeBiasAndOutput(K210Conv2d layer, Tensor <float> bias, ChannelwiseRange range, ChannelwiseRange beforeActRange, double[] scale, K210ConvLayerConfig config)
{
var upshift = 10;
var postMul = Math.Pow(2, upshift);
if (layer.IsChannelwiseOutput)
{
for (int i = 0; i < config.BNConfigs.Length; i++)
{
(var so, var bo) = range.Channels[i].GetScaleBias(8);
var b = bias[i];
var scomb = so * postMul / scale[i];
(var mul, var shift) = Quantizer.ExtractValueAndShift(scomb, 22, 15);
config.BNConfigs[i] = new K210LayerBNConfig
{
Mul = (int)Math.Round(mul),
Shift = shift,
Add = (int)Math.Round((b * so - bo) * postMul)
};
}
}
else
{
(var so, var bo) = range.Global.GetScaleBias(8);
#if CHANNEL_WISE
for (int i = 0; i < config.BNConfigs.Length; i++)
{
var b = bias[i];
var scomb = so * postMul / scale[i];
(var mul, var shift) = Quantizer.ExtractValueAndShift(scomb, 22, 15);
config.BNConfigs[i] = new K210LayerBNConfig
{
Mul = (int)Math.Round(mul),
Shift = shift,
Add = (int)Math.Round((b * so - bo) * postMul)
};
}
#else
var scomb = so / scale[0];
(var mul, var shift) = ExtractValueAndShift(scomb, 22, 255);
var upscale = shift - 15;
Debug.Assert(upscale >= 0);
var postMul = Math.Round(mul) / mul * Math.Pow(2, upscale);
for (int i = 0; i < config.BNConfigs.Length; i++)
{
var b = bias[i];
config.BNConfigs[i] = new K210LayerBNConfig
{
Mul = (int)Math.Round(mul),
Shift = 15,
Add = (int)Math.Round((b * so - bo) * postMul)
};
}
#endif
}
QuantizeActivation(layer, postMul, range.Global, beforeActRange.Global, config);
}
public override void Process(TransformContext context)
{
var space = context.MatchedLayers[0];
var dwConv2d = (DepthwiseConv2d)context.MatchedLayers[1];
var conv2d = (Conv2d)context.MatchedLayers.Last();
var input = space.InputConnectors[0].Connection.From;
var output = conv2d.Output;
space.InputConnectors[0].ClearConnection();
var newDwConv2d = new DepthwiseConv2d(input.Dimensions, dwConv2d.Weights, dwConv2d.Bias, Padding.Same, 1, 1, dwConv2d.FusedActivationFunction);
var quantize = new Quantize(newDwConv2d.Output.Dimensions);
var upload = new K210Upload(quantize.Output.Dimensions);
var newConv2d = new K210Conv2d(upload.Output.Dimensions, K210Conv2dType.Conv2d, conv2d.Weights, conv2d.Bias, K210PoolType.LeftTop, conv2d.FusedActivationFunction, null);
var dequantize = new Dequantize(newConv2d.Output.Dimensions);
newDwConv2d.Input.SetConnection(input);
quantize.Input.SetConnection(newDwConv2d.Output);
upload.Input.SetConnection(quantize.Output);
newConv2d.Input.SetConnection(upload.Output);
dequantize.Input.SetConnection(newConv2d.Output);
var oldOuts = output.Connections.Select(o => o.To).ToList();
if (context.MatchedLayers.Count == 3)
{
foreach (var oldOut in oldOuts)
{
oldOut.SetConnection(dequantize.Output);
}
}
else
{
var newOutput = dequantize.Output;
foreach (var middleLayer in context.MatchedLayers.Skip(2).Take(context.MatchedLayers.Count - 3))
{
Layer newLayer;
switch (middleLayer)
{
case Quantize _:
newLayer = new Quantize(newOutput.Dimensions);
break;
case Dequantize _:
newLayer = new Dequantize(newOutput.Dimensions);
break;
case LeakyRelu l:
newLayer = new LeakyRelu(newOutput.Dimensions, l.Slope);
break;
default:
throw new NotImplementedException();
}
newLayer.InputConnectors[0].SetConnection(newOutput);
newOutput = newLayer.OutputConnectors[0];
}
foreach (var oldOut in oldOuts)
{
oldOut.SetConnection(newOutput);
}
}
}
public override void Process(TransformContext context)
{
var conv2d = (K210Conv2d)context.MatchedLayers[0];
var input = conv2d.Input.Connection.From;
OutputConnector output;
conv2d.Input.ClearConnection();
K210PoolType poolType;
if (context.MatchedLayers[2] is MaxPool2d maxPool)
{
if (maxPool.FilterWidth == 2 && maxPool.StrideWidth == 2)
{
poolType = K210PoolType.MaxPool2x2;
}
else if (maxPool.FilterWidth == 2 && maxPool.StrideWidth == 1)
{
poolType = K210PoolType.MaxPool2x2Stride1;
}
else if (maxPool.FilterWidth == 4 && maxPool.StrideWidth == 4)
{
poolType = K210PoolType.MaxPool4x4;
}
else
{
throw new NotSupportedException("Unsupported max pool.");
}
output = maxPool.Output;
}
else
{
var avgPool = (AveragePool2d)context.MatchedLayers[2];
if (avgPool.FilterWidth == 2 && avgPool.StrideWidth == 2)
{
poolType = K210PoolType.AveragePool2x2;
}
else if (avgPool.FilterWidth == 2 && avgPool.StrideWidth == 1)
{
poolType = K210PoolType.AveragePool2x2Stride1;
}
else if (avgPool.FilterWidth == 4 && avgPool.StrideWidth == 4)
{
poolType = K210PoolType.AveragePool4x4;
}
else
{
throw new NotSupportedException("Unsupported average pool.");
}
output = avgPool.Output;
}
var newConv2d = new K210Conv2d(conv2d.Input.Dimensions, conv2d.Conv2dType, conv2d.Weights, conv2d.Bias, poolType, conv2d.FusedActivationFunction, conv2d.NonTrivialActivation);
newConv2d.Input.SetConnection(input);
var newDequantize = new Dequantize(newConv2d.Output.Dimensions);
newDequantize.Input.SetConnection(newConv2d.Output);
var oldOuts = output.Connections.Select(o => o.To).ToList();
foreach (var oldOut in oldOuts)
{
oldOut.SetConnection(newDequantize.Output);
}
}