public void Run(ref Model model)
{
var warnings = new List <Model.ImporterWarning>();
var shapeInferencePass = new IRShapeInferenceAndConstantFusing();
shapeInferencePass.Run(ref model, warnings);
if (Optimize)
{
// Optimization
var linearLayerFusingPass = new Optimization.FuseLinearLayersPass();
linearLayerFusingPass.Run(ref model);
var activationFusingPass = new Optimization.FuseActivationPass();
activationFusingPass.Run(ref model);
// Cleanup
var removeUnusedPass = new Cleanup.RemoveUnusedLayersPass();
removeUnusedPass.Run(ref model);
var removeNoOpPass = new Cleanup.RemoveNoOpsPass();
removeNoOpPass.Run(ref model);
}
// TODO, put asserts in ImporterWarning?
var validateNCHWPass = new ValidateNCHWPass();
validateNCHWPass.Run(model, ref warnings);
// to runnable NHWC
var nhwcPass = new NCHWToNHWCPass();
nhwcPass.Run(ref model);
// optimizations
if (Optimize)
{
var contractToSimplerLayerPass = new Optimization.ContractToSimplerLayerPass();
contractToSimplerLayerPass.Run(ref model);
var concatenateTransposesPass = new Optimization.ConcatenateTransposesPass();
concatenateTransposesPass.Run(ref model);
var dense3FusingPass = new Optimization.FuseDense3Pass();
dense3FusingPass.Run(ref model);
}
var validateNHWCPass = new ValidateNHWCPass();
validateNHWCPass.Run(model, ref warnings);
model.Warnings.AddRange(warnings);
}
// works on IRModel
public bool InferAllLayersChannelOrder(Model model, out Dictionary <string, ChannelsOrder> layerChannelOrder)
{
layerChannelOrder = new Dictionary <string, ChannelsOrder>();
IDictionary <string, TensorShape?> shapesByName = new Dictionary <string, TensorShape?>();
IDictionary <string, int?> ranksByName = new Dictionary <string, int?>();
foreach (var i in model.inputs)
{
ranksByName[i.name] = i.rank;
if (!ModelAnalyzer.IsInputShapeAcceptablyKnowForShapeInference(i))
{
continue;
}
shapesByName[i.name] = new TensorShape(i.shape);
}
IRShapeInferenceAndConstantFusing shapeInferencePass = new IRShapeInferenceAndConstantFusing();
shapeInferencePass.InferAllShapes(model, ref shapesByName, ref ranksByName);
// flood-fill approach: NCHW layout is propagated from NCHW ops
// * onnx-nchw ops are flagged as being native nchw
// * nchw layout is propagated to upstream and downstream nodes
// foreach node:
// take layout being propagated to
// if T or T-1 flip layout depending on upstream/downstream direction
// - stop if layout is the same as previously propagated
// - native nchw layout has priority
Queue <(string, ChannelsOrder, FlowDirection)> layersToInferLayout = new Queue <(string, ChannelsOrder, FlowDirection)>();
for (int l = 0; l < model.layers.Count; l++)
{
var layer = model.layers[l];
if (!IsLayerNecessarilyNCHWOnnx(layer))
{
continue;
}
layersToInferLayout.Enqueue((layer.name, ChannelsOrder.NativeNCHW, FlowDirection.Seed));
}
while (layersToInferLayout.Any())
{
(string, ChannelsOrder, FlowDirection)layerData = layersToInferLayout.Dequeue();
string name = layerData.Item1;
ChannelsOrder deducedChannelOrder = layerData.Item2;
// 0: in-place native
// 1: downstream
// 2: upstream
FlowDirection flowDirection = layerData.Item3;
if (!layerChannelOrder.ContainsKey(name))
{
layerChannelOrder[name] = deducedChannelOrder;
}
else if (deducedChannelOrder == layerChannelOrder[name])
{
continue;
}
else if (layerChannelOrder[name] == ChannelsOrder.NativeNCHW)
{
continue;
}
// heuristic to stop ping-pong loop, prioritize NHWC over NCHW as it implies less transposes
// if incoming is NativeNCHW always propagate that
// TODO: count # of transpose swaps
else if (layerChannelOrder[name] == ChannelsOrder.NHWC && deducedChannelOrder != ChannelsOrder.NativeNCHW)
{
continue;
}
Layer layer;
bool found = ModelAnalyzer.FindLayerByName(model, name, out layer);
if (IsLayerChangingLayoutToNHWC(layer, shapesByName, ranksByName))
{
// NCHW -> T -> NHWC
if (((deducedChannelOrder == ChannelsOrder.NCHW) || (deducedChannelOrder == ChannelsOrder.NativeNCHW)) && (flowDirection == FlowDirection.Downstream))
{
deducedChannelOrder = ChannelsOrder.TransposeToNHWC;
}
// NCHW <- T <- NHWC
else if ((deducedChannelOrder == ChannelsOrder.NHWC) && (flowDirection == FlowDirection.Upstream))
{
deducedChannelOrder = ChannelsOrder.TransposeToNHWC;
}
}
else if (IsLayerChangingLayoutToNCHW(layer, shapesByName, ranksByName))
{
// NHWC -> T-1 -> NCHW
if ((deducedChannelOrder == ChannelsOrder.NHWC) && (flowDirection == FlowDirection.Downstream))
{
deducedChannelOrder = ChannelsOrder.TransposeToNCHW;
}
// NHWC <- T-1 <- NCHW
else if (((deducedChannelOrder == ChannelsOrder.NCHW) || (deducedChannelOrder == ChannelsOrder.NativeNCHW)) && (flowDirection == FlowDirection.Upstream))
{
deducedChannelOrder = ChannelsOrder.TransposeToNCHW;
}
}
if ((deducedChannelOrder == ChannelsOrder.TransposeToNCHW || deducedChannelOrder == ChannelsOrder.TransposeToNHWC) && (deducedChannelOrder == layerChannelOrder[name]))
{
continue;
}
layerChannelOrder[name] = deducedChannelOrder;
foreach (var input in layer.inputs)
{
if (deducedChannelOrder == ChannelsOrder.TransposeToNCHW)
{
layersToInferLayout.Enqueue((input, ChannelsOrder.NHWC, FlowDirection.Upstream));
}
else if (deducedChannelOrder == ChannelsOrder.TransposeToNHWC)
{
layersToInferLayout.Enqueue((input, ChannelsOrder.NCHW, FlowDirection.Upstream));
}
else
{
layersToInferLayout.Enqueue((input, deducedChannelOrder, FlowDirection.Upstream));
}
}
var outputs = ModelAnalyzer.FindLayerOutputs(model, layer.name);
foreach (var output in outputs)
{
if (deducedChannelOrder == ChannelsOrder.TransposeToNCHW)
{
layersToInferLayout.Enqueue((output, ChannelsOrder.NCHW, FlowDirection.Downstream));
}
else if (deducedChannelOrder == ChannelsOrder.TransposeToNHWC)
{
layersToInferLayout.Enqueue((output, ChannelsOrder.NHWC, FlowDirection.Downstream));
}
else
{
layersToInferLayout.Enqueue((output, deducedChannelOrder, FlowDirection.Downstream));
}
}
}
bool modelExportedASNHWC = false;
foreach (string key in layerChannelOrder.Keys.ToList())
{
var value = layerChannelOrder[key];
if (value == ChannelsOrder.NativeNCHW)
{
layerChannelOrder[key] = ChannelsOrder.NCHW;
}
if (value == ChannelsOrder.NHWC)
{
modelExportedASNHWC = true;
}
}
return(modelExportedASNHWC);
}
public void Run(Model model, ref List <Model.ImporterWarning> warnings)
{
var modelTemp = model.ShallowCopy();
IDictionary <string, TensorShape> inputShapes = new Dictionary <string, TensorShape>();
// force batch to 1
for (int i = 0; i < modelTemp.inputs.Count; i++)
{
var input = modelTemp.inputs[i];
var shape = input.shape.ToArray();
if (shape[TensorShape.DataBatch] <= 0)
{
shape[TensorShape.DataBatch] = 1;
}
input.shape = shape;
modelTemp.inputs[i] = input;
if (!ModelAnalyzer.IsInputShapeAcceptablyKnowForShapeInference(input))
{
continue;
}
inputShapes[input.name] = new TensorShape(input.shape);
}
ValidationHelper.AppendWarning(inputShapes.Count == modelTemp.inputs.Count, "model", "Input Shape: unkown non batch dimension", ref warnings);
IRShapeInferenceAndConstantFusing shapeInferencePass = new IRShapeInferenceAndConstantFusing();
shapeInferencePass.Run(ref modelTemp);
IDictionary <string, int?> ranksByName;
IRShapeInferenceHelper.RankInference.ListTemporaryTensorRanks(modelTemp, out ranksByName);
IDictionary <string, TensorShape?> shapesByName;
IRShapeInferenceHelper.ShapeInference.ListTemporaryTensorShapesNCHW(modelTemp, inputShapes, ranksByName, out shapesByName);
int negativeRanks = ranksByName.Values.Count(x => x < 0);
ValidationHelper.AppendWarning(negativeRanks == 0, "model", $"StaticRankInference: {negativeRanks} negative rank(s) found!", ref warnings, MessageType.Warning);
int knowRanks = ranksByName.Count(x => x.Value != null);
int knowShapes = shapesByName.Count(x => x.Value != null);
ValidationHelper.AppendWarning(knowRanks == knowShapes, "model", "StaticShape/RankInference: known ranks # != known shape #", ref warnings);
foreach (var i in modelTemp.inputs)
{
var name = i.name;
ValidationHelper.AppendWarning(ranksByName.ContainsKey(name), name, "StaticRankInference: did not find input", ref warnings);
if (ranksByName.ContainsKey(name))
{
ValidationHelper.AppendWarning(ranksByName[name] != null, name, "StaticRankInference: unknown input rank at compile time", ref warnings);
}
ValidationHelper.AppendWarning(shapesByName.ContainsKey(name), name, "StaticShapeInference: did not find input", ref warnings);
if (shapesByName.ContainsKey(name))
{
ValidationHelper.AppendWarning(shapesByName[name] != null, name, "StaticShapeInference: unknown input shape for at compile time", ref warnings);
}
}
foreach (var l in modelTemp.layers)
{
var name = l.name;
ValidationHelper.AppendWarning(ranksByName.ContainsKey(name), name, "StaticRankInference: did not find layer", ref warnings);
if (ranksByName.ContainsKey(name))
{
ValidationHelper.AppendWarning(ranksByName[name] != null, name, "StaticRankInference: unknown layer rank at compile time", ref warnings);
}
ValidationHelper.AppendWarning(shapesByName.ContainsKey(name), name, "StaticShapeInference: did not find layer", ref warnings);
if (shapesByName.ContainsKey(name))
{
ValidationHelper.AppendWarning(shapesByName[name] != null, name, "StaticShapeInference: unknown layer shape at compile time", ref warnings);
}
}
}
// works on IRModel
public bool InferAllLayersChannelOrder(Model model, out Dictionary <string, ChannelsOrder> layerChannelOrder)
{
// TF2Onnx : pattern T (.* Conv .*) T-1
// * being transpose commutative layer
layerChannelOrder = new Dictionary <string, ChannelsOrder>();
IDictionary <string, TensorShape?> shapesByName = new Dictionary <string, TensorShape?>();
IDictionary <string, int?> ranksByName = new Dictionary <string, int?>();
foreach (var i in model.inputs)
{
ranksByName[i.name] = i.rank;
if (!ModelAnalyzer.IsInputShapeAcceptablyKnowForShapeInference(i))
{
continue;
}
shapesByName[i.name] = new TensorShape(i.shape);
}
IRShapeInferenceAndConstantFusing shapeInferencePass = new IRShapeInferenceAndConstantFusing();
shapeInferencePass.InferAllShapes(model, ref shapesByName, ref ranksByName);
bool inputsNHWC = false;
bool inputsNHWCExportedInputsAsNCHW = false;
bool patternMatchStart = false;
bool patternMatchConv = false;
// tf to onnx does not swizzle axis, need to match * Conv * T-1 ...
bool patternMatchStartInputsAsNCHWConv = false;
for (int l = 0; l < model.layers.Count; l++)
{
var layer = model.layers[l];
if (!patternMatchStart &&
IsLayerTranpose(layer) && Enumerable.SequenceEqual(layer.pool, new[] { 0, 3, 1, 2 }) ||
IsLayerReshape(layer) && (shapesByName[layer.inputs[0]] != null) && IsReshapeTransposeToNCHW(layer, shapesByName[layer.inputs[0]].Value))
{
patternMatchStart = true;
}
else if (patternMatchStart && patternMatchConv &&
((IsLayerTranpose(layer) && Enumerable.SequenceEqual(layer.pool, new[] { 0, 2, 3, 1 })) ||
(IsLayerReshape(layer) && (shapesByName[layer.inputs[0]] != null) && IsReshapeTransposeToNHWC(layer, shapesByName[layer.inputs[0]].Value)) ||
(IsLayerSqueeze(layer) && (ranksByName[layer.inputs[0]] != null) && IsSqueezeTransposeToNHWC(layer, ranksByName[layer.inputs[0]].Value)) ||
(IsLayerFlatten(layer) && (ranksByName[layer.inputs[0]] != null) && IsFlattenTransposeToNHWC(layer, ranksByName[layer.inputs[0]].Value))))
{
inputsNHWC = true;
}
else if (patternMatchStart && IsLayerConv(layer))
{
patternMatchConv = true;
}
if (!inputsNHWCExportedInputsAsNCHW && patternMatchStartInputsAsNCHWConv &&
((IsLayerTranpose(layer) && Enumerable.SequenceEqual(layer.pool, new[] { 0, 2, 3, 1 })) ||
(IsLayerReshape(layer) && (shapesByName[layer.inputs[0]] != null) && IsReshapeTransposeToNHWC(layer, shapesByName[layer.inputs[0]].Value))))
{
inputsNHWCExportedInputsAsNCHW = true;
}
else if (!patternMatchStartInputsAsNCHWConv && !patternMatchStart && IsLayerConv(layer))
{
patternMatchStartInputsAsNCHWConv = true;
}
}
// flag each layer as being NHWC or NCHW
for (int i = 0; i < model.inputs.Count; i++)
{
Model.Input input = model.inputs[i];
if (!inputsNHWCExportedInputsAsNCHW)
{
layerChannelOrder[input.name] = inputsNHWC ? ChannelsOrder.NHWC : ChannelsOrder.NCHW;
}
else
{
layerChannelOrder[input.name] = ChannelsOrder.NCHW;
}
}
for (int l = 0; l < model.layers.Count; l++)
{
var layer = model.layers[l];
if (IsLayerTranpose(layer) && Enumerable.SequenceEqual(layer.pool, new[] { 0, 3, 1, 2 }) ||
IsLayerReshape(layer) && (shapesByName[layer.inputs[0]] != null) && IsReshapeTransposeToNCHW(layer, shapesByName[layer.inputs[0]].Value) &&
layerChannelOrder[layer.inputs[0]] == ChannelsOrder.NHWC)
{
layerChannelOrder[layer.name] = ChannelsOrder.TransposeToNCHW;
}
else if (IsLayerTranpose(layer) && Enumerable.SequenceEqual(layer.pool, new[] { 0, 2, 3, 1 }) ||
IsLayerReshape(layer) && (shapesByName[layer.inputs[0]] != null) && IsReshapeTransposeToNHWC(layer, shapesByName[layer.inputs[0]].Value) &&
layerChannelOrder[layer.inputs[0]] == ChannelsOrder.NCHW)
{
layerChannelOrder[layer.name] = ChannelsOrder.TransposeToNHWC;
}
else
{
string inputWithKnownOrder = null;
for (int i = 0; i < layer.inputs.Length; i++)
{
var input = layer.inputs[i];
if (layerChannelOrder.ContainsKey(input))
{
inputWithKnownOrder = input;
break;
}
}
if (inputWithKnownOrder == null)
{
continue;
}
Assert.IsNotNull(inputWithKnownOrder);
ChannelsOrder inputOrder = layerChannelOrder[inputWithKnownOrder];
if (inputOrder == ChannelsOrder.TransposeToNCHW)
{
inputOrder = ChannelsOrder.NCHW;
}
else if (inputOrder == ChannelsOrder.TransposeToNHWC)
{
inputOrder = ChannelsOrder.NHWC;
}
// all layers with unknown layout are const
for (int i = 0; i < layer.inputs.Length; i++)
{
var input = layer.inputs[i];
if (!layerChannelOrder.ContainsKey(input))
{
layerChannelOrder[input] = inputOrder;
}
}
layerChannelOrder[layer.name] = inputOrder;
}
}
// TODO Assert that all layers have a channel order
// Assert that all layers are NHWC if inputsNHWC
return(inputsNHWC || inputsNHWCExportedInputsAsNCHW);
}
