private static async Task ExportK210Code(string modelPath, string datasetDir, string codePath)
{
var file = File.ReadAllBytes(modelPath);
var model = tflite.Model.GetRootAsModel(new FlatBuffers.ByteBuffer(file));
var tfc = new TfLiteToGraphConverter(model, model.Subgraphs(0).Value);
tfc.Convert();
var graph = tfc.Graph;
Transform.Process(graph, new Transform[] {
new K210SeparableConv2dTransform(),
new K210SpaceToBatchNdAndValidConv2dTransform(),
new K210SameConv2dTransform(),
new K210Stride2Conv2dTransform(),
new GlobalAveragePoolTransform(),
new K210FullyConnectedTransform(),
new K210Conv2dWithMaxAvgPoolTransform(),
new Conv2d1x1ToFullyConnectedTransform()
});
var ctx = new GraphPlanContext();
graph.Plan(ctx);
var dim = graph.Inputs.First().Output.Dimensions.ToArray();
var k210c = new GraphToK210Converter(graph, 16);
await k210c.ConvertAsync(new ImageDataset(
datasetDir,
new[] { dim[1], dim[2], dim[3] },
1,
PreprocessMethods.None,
PostprocessMethods.Normalize0To1),
ctx,
Path.GetDirectoryName(codePath),
Path.GetFileNameWithoutExtension(codePath));
}
protected override void OnPlanning(GraphPlanContext context)
{
var input = context.TFOutputs[Input.Connection.From];
var graph = context.TFGraph;
context.TFOutputs[Output] = graph.Reshape(input, graph.Const(Output.Dimensions.ToArray()));
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var inputs = Inputs.Select(x => context.TFOutputs[x.Connection.From]).ToArray();
context.TFOutputs[Output] = graph.Concat(graph.Const(3), inputs);
}
protected override void OnPlanning(GraphPlanContext context)
{
var y = context.TFOutputs[Input.Connection.From];
var graph = context.TFGraph;
// To NCH
if (Input.Dimensions.Length == 3)
{
y = graph.Transpose(y, graph.Const(new[] { 0, 2, 1 }));
}
// To NCHW
else if (Input.Dimensions.Length == 4)
{
y = graph.Transpose(y, graph.Const(new[] { 0, 3, 1, 2 }));
}
y = context.TFGraph.Reshape(y, context.TFGraph.Const(NewShape.ToArray()));
// To NHC
if (NewShape.Length == 3)
{
y = graph.Transpose(y, graph.Const(new[] { 0, 2, 1 }));
}
// To NHWC
else if (NewShape.Length == 4)
{
y = graph.Transpose(y, graph.Const(new[] { 0, 2, 3, 1 }));
}
context.TFOutputs[Output] = y;
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var input = context.TFOutputs[Input.Connection.From];
context.TFOutputs[Output] = graph.Mean(input, graph.Const(new[] { 1, 2 }));
}
protected override void OnPlanning(GraphPlanContext context)
{
var y = context.TFOutputs[Input.Connection.From];
var graph = context.TFGraph;
context.TFOutputs[Output] = context.TFGraph.Reshape(y, context.TFGraph.Const(NewShape.ToNHWC()));
}
protected override void OnPlanning(GraphPlanContext context)
{
var input = context.TFOutputs[Input.Connection.From];
var graph = context.TFGraph;
context.TFOutputs[Output] = graph.Transpose(input, graph.Const(Axes.ToTFAxes()));
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var input = context.TFOutputs[Input.Connection.From];
context.TFOutputs[Output] = graph.ResizeBilinear(input, graph.Const(new long[] { Output.Dimensions[2], Output.Dimensions[3] }), AlignCorners);
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var x = context.TFOutputs[Input.Connection.From];
context.TFOutputs[Output] = graph.Maximum(x, graph.Mul(x, graph.Const(Slope)));
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var x = context.TFOutputs[Input.Connection.From];
context.TFOutputs[Output] = graph.Softmax(x);
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var input = context.TFOutputs[Input.Connection.From];
context.TFOutputs[Output] = graph.Pad(input, graph.Const(Paddings.ToTFTensor()));
}
private static async Task ConvertToTFLite(Graph graph, string tflitePath)
{
var ctx = new GraphPlanContext();
graph.Plan(ctx);
var dim = graph.Inputs.First().Output.Dimensions.ToArray();
var input = graph.Inputs.First().Name;
var output = string.Join(',', graph.Outputs.Select(x => x.Name));
var tmpPb = Path.GetTempFileName();
using (var f = File.Open(tmpPb, FileMode.Create, FileAccess.Write))
await ctx.SaveAsync(f);
var binPath = Path.Combine(Path.GetDirectoryName(typeof(Program).Assembly.Location), "bin");
var args = $" --input_file={tmpPb} --input_format=TENSORFLOW_GRAPHDEF --output_file={tflitePath} --output_format=TFLITE --input_shape=1,{dim[2]},{dim[3]},{dim[1]} --input_array={input} --output_arrays={output} --inference_type=FLOAT";
using (var toco = Process.Start(new ProcessStartInfo(Path.Combine(binPath, "toco"), args)
{
WorkingDirectory = binPath
}))
{
toco.WaitForExit();
if (toco.ExitCode != 0)
{
throw new InvalidOperationException("Convert to tflite failed.");
}
}
File.Delete(tmpPb);
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var input = context.TFOutputs[Input.Connection.From];
context.TFOutputs[Output] = graph.BiasAdd(input, graph.Const(Bias.ToNHWC()));
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var a = context.TFOutputs[InputA.Connection.From];
var b = context.TFOutputs[InputB.Connection.From];
context.TFOutputs[Output] = graph.Add(a, b);
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var input = context.TFOutputs[Input.Connection.From];
context.TFOutputs[Output] = graph.AvgPool(input, new long[] { 1, Input.Dimensions[2], Input.Dimensions[3], 1 },
new long[] { 1, 1, 1, 1 }, "VALID");
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var input = context.TFOutputs[Input.Connection.From];
context.TFOutputs[Output] = graph.AddActivation(graph.AvgPool(input, new long[] { 1, FilterHeight, FilterWidth, 1 },
new long[] { 1, StrideHeight, StrideWidth, 1 }, Padding.ToString().ToUpperInvariant()), FusedActivationFunction);
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var input = context.TFOutputs[Input.Connection.From];
var y = graph.MaxPool(input, new long[] { 1, 1, 1, 1 }, new long[] { 1, 4, 4, 1 }, "VALID");
context.TFOutputs[Output] = graph.Reshape(y, graph.Const(new[] { -1, Input.Dimensions[1] }));
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var input = context.TFOutputs[Input.Connection.From];
input = graph.Reshape(input, graph.Const(new[] { Input.Dimensions[0], 1, 1, Input.Dimensions[1] }));
context.TFOutputs[Output] = graph.Pad(input, graph.Const(new[, ] {
{ 0, 0 }, { 0, 3 }, { 0, 3 }, { 0, 0 }
}));
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var x = context.TFOutputs[Input.Connection.From];
var y = graph.Sum(graph.Square(x), graph.Const(Input.Dimensions.Length - 1), keep_dims: true);
y = graph.Rsqrt(graph.Maximum(y, graph.Const(1e-10f)));
context.TFOutputs[Output] = graph.Mul(x, y);
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var x = context.TFOutputs[Input.Connection.From];
var alpha = graph.Const(Slope.ToNHWC());
var zero = graph.Const(0.0f, TensorFlow.TFDataType.Float);
var pos = graph.Maximum(zero, x);
var neg = graph.Mul(alpha, graph.Minimum(zero, x));
context.TFOutputs[Output] = graph.Add(pos, neg);
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var input = context.TFOutputs[Input.Connection.From];
if (Scale is Tensor <float> tensorScale)
{
context.TFOutputs[Output] = graph.Mul(input, graph.Const(tensorScale.ToNHWC()));
}
else if (Scale is float scalarScale)
{
context.TFOutputs[Output] = graph.Mul(input, graph.Const(scalarScale));
}
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var input = context.TFOutputs[Input.Connection.From];
var weights = Weights.ToHWIO();
var bias = Bias?.ToNHWC();
var y = graph.Conv2D(input, graph.Const(weights),
new long[] { 1, StrideHeight, StrideWidth, 1 }, Padding.ToString().ToUpperInvariant());
if (bias != null)
{
y = graph.BiasAdd(y, graph.Const(bias));
}
context.TFOutputs[Output] = graph.AddActivation(y, FusedActivationFunction);
}
public static async Task<QuantizationContext> QuantizeAsync(Dataset dataset, GraphPlanContext planContext, bool channelwiseOutput)
{
using (var session = new TFSession(planContext.TFGraph))
{
var connectors = new List<OutputConnector>();
var additionalOutputs = new List<Guid>();
var toFetches = new List<TFOutput>();
foreach (var output in planContext.TFOutputs)
{
connectors.Add(output.Key);
if (!(output.Key.Owner is InputLayer))
toFetches.Add(output.Value);
}
foreach (var additional in planContext.AdditionalTFOutputs)
{
additionalOutputs.Add(additional.Key);
toFetches.Add(additional.Value);
}
var quantizationContext = new QuantizationContext
{
Outputs = connectors,
AdditionalOutputs = additionalOutputs,
PlanContext = planContext,
Mean = dataset.Mean,
Std = dataset.Std,
ChannelwiseOutput = channelwiseOutput
};
#if NET471
await dataset.GetBatchesAsync().ForEachAsync(async batch =>
#else
await foreach (var batch in dataset.GetBatchesAsync())
#endif
{
var input = batch.ToNHWC();
var runner = session.GetRunner();
runner.AddInput(planContext.Inputs.Values.First(), input);
foreach (var fetch in toFetches)
runner.Fetch(fetch);
var outputs = runner.Run();
RecordOutputs(new[] { input }.Concat(outputs).ToList(), quantizationContext);
}
public async Task ConvertAsync(Dataset dataset, GraphPlanContext planContext, string outputDir, string prefix, bool channelwiseOutput)
{
_graph.Plan(planContext);
var quantize = await Stages.Quantize.Quantizer.QuantizeAsync(dataset, planContext, channelwiseOutput);
var convert = Stages.Convert.Converter.Convert(_graph, quantize, _weightsBits);
var infer = Stages.Inference.InferExecutor.Infer(_graph, convert);
Console.WriteLine($"KPU memory usage: {infer.KPUMemoryAllocator.MaxUsage * 64} B");
Console.WriteLine($"Main memory usage: {infer.MainMemoryAllocator.MaxEnd} B");
using (var bin = File.Open(Path.Combine(outputDir, $"{prefix}.kmodel"), FileMode.Create, FileAccess.Write))
{
Stages.Generate.Generator.GenerateBin(_graph, bin, _weightsBits, prefix, infer);
}
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var dummyBoxes = new DenseTensor <float>(Boxes.Dimensions);
var variances = new DenseTensor <float>(VariancesOutput.Dimensions);
for (int i = 0; i < variances.Length / 4; i++)
{
for (int j = 0; j < 4; j++)
{
variances[i * 4 + j] = Variances[j];
}
}
context.TFOutputs[Boxes] = graph.Const(dummyBoxes.ToTFTensor());
context.TFOutputs[VariancesOutput] = graph.Const(variances.ToTFTensor());
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var x = context.TFOutputs[Input.Connection.From];
var scale = Scale.ToNHWC();
var offset = Offset.ToNHWC();
var mean = Mean.ToNHWC();
var variance = Variance.ToNHWC();
if (Input.Dimensions.Length == 4)
{
context.TFOutputs[Output] = graph.FusedBatchNorm(x, graph.Const(scale), graph.Const(offset), graph.Const(mean), graph.Const(variance), Epsilon, is_training: false).y;
}
else
{
throw new NotImplementedException();
}
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var y = context.TFOutputs[Input.Connection.From];
var weights = Weights.ToHWIO();
if (Input.Dimensions.Length == 4 && Input.Dimensions[2] == 1 && Input.Dimensions[3] == 1)
{
y = graph.Reshape(y, graph.Const(new[] { Input.Dimensions[0], Input.Dimensions[1] }));
}
y = graph.MatMul(y, graph.Const(weights));
if (Bias != null)
{
y = graph.BiasAdd(y, graph.Const(Bias.ToNHWC()));
}
context.TFOutputs[Output] = graph.AddActivation(y, FusedActivationFunction);
}
protected override void OnPlanning(GraphPlanContext context)
{
var graph = context.TFGraph;
var x = context.TFOutputs[Input.Connection.From];
var scale = Scale.ToNHWC();
var offset = Offset.ToNHWC();
var mean = Mean.ToNHWC();
var variance = Variance.ToNHWC();
if (Input.Dimensions.Length == 4)
{
context.TFOutputs[Output] = graph.FusedBatchNorm(x, graph.Const(scale), graph.Const(offset), graph.Const(mean), graph.Const(variance), Epsilon, is_training: false).y;
}
else
{
var y = graph.Sub(x, graph.Const(mean));
var v = graph.Sqrt(graph.Add(graph.Const(variance), graph.Const(Epsilon)));
y = graph.RealDiv(y, v);
y = graph.Mul(y, graph.Const(scale));
context.TFOutputs[Output] = graph.Add(y, graph.Const(offset));
}
}
protected override void OnPlanning(GraphPlanContext context)
{
context.Inputs[this] = context.TFOutputs[Output] = context.TFGraph.Placeholder(TFDataType.Float, Output.Dimensions.ToNHWC(), Name);
}
protected override void OnPlanning(GraphPlanContext context)
{
context.TFOutputs[Output] = context.TFGraph.Const(Value.ToNHWC(), TFDataType.Float);
}
