public void Run(ref Model model) { var warnings = new List <Model.ImporterWarning>(); var shapeInferencePass = new IRShapeInferenceAndConstantFusing(); shapeInferencePass.Run(ref model, warnings); if (Optimize) { // Optimization var linearLayerFusingPass = new Optimization.FuseLinearLayersPass(); linearLayerFusingPass.Run(ref model); var activationFusingPass = new Optimization.FuseActivationPass(); activationFusingPass.Run(ref model); // Cleanup var removeUnusedPass = new Cleanup.RemoveUnusedLayersPass(); removeUnusedPass.Run(ref model); var removeNoOpPass = new Cleanup.RemoveNoOpsPass(); removeNoOpPass.Run(ref model); } // TODO, put asserts in ImporterWarning? var validateNCHWPass = new ValidateNCHWPass(); validateNCHWPass.Run(model, ref warnings); // to runnable NHWC var nhwcPass = new NCHWToNHWCPass(); nhwcPass.Run(ref model); // optimizations if (Optimize) { var contractToSimplerLayerPass = new Optimization.ContractToSimplerLayerPass(); contractToSimplerLayerPass.Run(ref model); var concatenateTransposesPass = new Optimization.ConcatenateTransposesPass(); concatenateTransposesPass.Run(ref model); var dense3FusingPass = new Optimization.FuseDense3Pass(); dense3FusingPass.Run(ref model); } var validateNHWCPass = new ValidateNHWCPass(); validateNHWCPass.Run(model, ref warnings); model.Warnings.AddRange(warnings); }
// works on IRModel public bool InferAllLayersChannelOrder(Model model, out Dictionary <string, ChannelsOrder> layerChannelOrder) { layerChannelOrder = new Dictionary <string, ChannelsOrder>(); IDictionary <string, TensorShape?> shapesByName = new Dictionary <string, TensorShape?>(); IDictionary <string, int?> ranksByName = new Dictionary <string, int?>(); foreach (var i in model.inputs) { ranksByName[i.name] = i.rank; if (!ModelAnalyzer.IsInputShapeAcceptablyKnowForShapeInference(i)) { continue; } shapesByName[i.name] = new TensorShape(i.shape); } IRShapeInferenceAndConstantFusing shapeInferencePass = new IRShapeInferenceAndConstantFusing(); shapeInferencePass.InferAllShapes(model, ref shapesByName, ref ranksByName); // flood-fill approach: NCHW layout is propagated from NCHW ops // * onnx-nchw ops are flagged as being native nchw // * nchw layout is propagated to upstream and downstream nodes // foreach node: // take layout being propagated to // if T or T-1 flip layout depending on upstream/downstream direction // - stop if layout is the same as previously propagated // - native nchw layout has priority Queue <(string, ChannelsOrder, FlowDirection)> layersToInferLayout = new Queue <(string, ChannelsOrder, FlowDirection)>(); for (int l = 0; l < model.layers.Count; l++) { var layer = model.layers[l]; if (!IsLayerNecessarilyNCHWOnnx(layer)) { continue; } layersToInferLayout.Enqueue((layer.name, ChannelsOrder.NativeNCHW, FlowDirection.Seed)); } while (layersToInferLayout.Any()) { (string, ChannelsOrder, FlowDirection)layerData = layersToInferLayout.Dequeue(); string name = layerData.Item1; ChannelsOrder deducedChannelOrder = layerData.Item2; // 0: in-place native // 1: downstream // 2: upstream FlowDirection flowDirection = layerData.Item3; if (!layerChannelOrder.ContainsKey(name)) { layerChannelOrder[name] = deducedChannelOrder; } else if (deducedChannelOrder == layerChannelOrder[name]) { continue; } else if (layerChannelOrder[name] == ChannelsOrder.NativeNCHW) { continue; } // heuristic to stop ping-pong loop, prioritize NHWC over NCHW as it implies less transposes // if incoming is NativeNCHW always propagate that // TODO: count # of transpose swaps else if (layerChannelOrder[name] == ChannelsOrder.NHWC && deducedChannelOrder != ChannelsOrder.NativeNCHW) { continue; } Layer layer; bool found = ModelAnalyzer.FindLayerByName(model, name, out layer); if (IsLayerChangingLayoutToNHWC(layer, shapesByName, ranksByName)) { // NCHW -> T -> NHWC if (((deducedChannelOrder == ChannelsOrder.NCHW) || (deducedChannelOrder == ChannelsOrder.NativeNCHW)) && (flowDirection == FlowDirection.Downstream)) { deducedChannelOrder = ChannelsOrder.TransposeToNHWC; } // NCHW <- T <- NHWC else if ((deducedChannelOrder == ChannelsOrder.NHWC) && (flowDirection == FlowDirection.Upstream)) { deducedChannelOrder = ChannelsOrder.TransposeToNHWC; } } else if (IsLayerChangingLayoutToNCHW(layer, shapesByName, ranksByName)) { // NHWC -> T-1 -> NCHW if ((deducedChannelOrder == ChannelsOrder.NHWC) && (flowDirection == FlowDirection.Downstream)) { deducedChannelOrder = ChannelsOrder.TransposeToNCHW; } // NHWC <- T-1 <- NCHW else if (((deducedChannelOrder == ChannelsOrder.NCHW) || (deducedChannelOrder == ChannelsOrder.NativeNCHW)) && (flowDirection == FlowDirection.Upstream)) { deducedChannelOrder = ChannelsOrder.TransposeToNCHW; } } if ((deducedChannelOrder == ChannelsOrder.TransposeToNCHW || deducedChannelOrder == ChannelsOrder.TransposeToNHWC) && (deducedChannelOrder == layerChannelOrder[name])) { continue; } layerChannelOrder[name] = deducedChannelOrder; foreach (var input in layer.inputs) { if (deducedChannelOrder == ChannelsOrder.TransposeToNCHW) { layersToInferLayout.Enqueue((input, ChannelsOrder.NHWC, FlowDirection.Upstream)); } else if (deducedChannelOrder == ChannelsOrder.TransposeToNHWC) { layersToInferLayout.Enqueue((input, ChannelsOrder.NCHW, FlowDirection.Upstream)); } else { layersToInferLayout.Enqueue((input, deducedChannelOrder, FlowDirection.Upstream)); } } var outputs = ModelAnalyzer.FindLayerOutputs(model, layer.name); foreach (var output in outputs) { if (deducedChannelOrder == ChannelsOrder.TransposeToNCHW) { layersToInferLayout.Enqueue((output, ChannelsOrder.NCHW, FlowDirection.Downstream)); } else if (deducedChannelOrder == ChannelsOrder.TransposeToNHWC) { layersToInferLayout.Enqueue((output, ChannelsOrder.NHWC, FlowDirection.Downstream)); } else { layersToInferLayout.Enqueue((output, deducedChannelOrder, FlowDirection.Downstream)); } } } bool modelExportedASNHWC = false; foreach (string key in layerChannelOrder.Keys.ToList()) { var value = layerChannelOrder[key]; if (value == ChannelsOrder.NativeNCHW) { layerChannelOrder[key] = ChannelsOrder.NCHW; } if (value == ChannelsOrder.NHWC) { modelExportedASNHWC = true; } } return(modelExportedASNHWC); }
public void Run(Model model, ref List <Model.ImporterWarning> warnings) { var modelTemp = model.ShallowCopy(); IDictionary <string, TensorShape> inputShapes = new Dictionary <string, TensorShape>(); // force batch to 1 for (int i = 0; i < modelTemp.inputs.Count; i++) { var input = modelTemp.inputs[i]; var shape = input.shape.ToArray(); if (shape[TensorShape.DataBatch] <= 0) { shape[TensorShape.DataBatch] = 1; } input.shape = shape; modelTemp.inputs[i] = input; if (!ModelAnalyzer.IsInputShapeAcceptablyKnowForShapeInference(input)) { continue; } inputShapes[input.name] = new TensorShape(input.shape); } ValidationHelper.AppendWarning(inputShapes.Count == modelTemp.inputs.Count, "model", "Input Shape: unkown non batch dimension", ref warnings); IRShapeInferenceAndConstantFusing shapeInferencePass = new IRShapeInferenceAndConstantFusing(); shapeInferencePass.Run(ref modelTemp); IDictionary <string, int?> ranksByName; IRShapeInferenceHelper.RankInference.ListTemporaryTensorRanks(modelTemp, out ranksByName); IDictionary <string, TensorShape?> shapesByName; IRShapeInferenceHelper.ShapeInference.ListTemporaryTensorShapesNCHW(modelTemp, inputShapes, ranksByName, out shapesByName); int negativeRanks = ranksByName.Values.Count(x => x < 0); ValidationHelper.AppendWarning(negativeRanks == 0, "model", $"StaticRankInference: {negativeRanks} negative rank(s) found!", ref warnings, MessageType.Warning); int knowRanks = ranksByName.Count(x => x.Value != null); int knowShapes = shapesByName.Count(x => x.Value != null); ValidationHelper.AppendWarning(knowRanks == knowShapes, "model", "StaticShape/RankInference: known ranks # != known shape #", ref warnings); foreach (var i in modelTemp.inputs) { var name = i.name; ValidationHelper.AppendWarning(ranksByName.ContainsKey(name), name, "StaticRankInference: did not find input", ref warnings); if (ranksByName.ContainsKey(name)) { ValidationHelper.AppendWarning(ranksByName[name] != null, name, "StaticRankInference: unknown input rank at compile time", ref warnings); } ValidationHelper.AppendWarning(shapesByName.ContainsKey(name), name, "StaticShapeInference: did not find input", ref warnings); if (shapesByName.ContainsKey(name)) { ValidationHelper.AppendWarning(shapesByName[name] != null, name, "StaticShapeInference: unknown input shape for at compile time", ref warnings); } } foreach (var l in modelTemp.layers) { var name = l.name; ValidationHelper.AppendWarning(ranksByName.ContainsKey(name), name, "StaticRankInference: did not find layer", ref warnings); if (ranksByName.ContainsKey(name)) { ValidationHelper.AppendWarning(ranksByName[name] != null, name, "StaticRankInference: unknown layer rank at compile time", ref warnings); } ValidationHelper.AppendWarning(shapesByName.ContainsKey(name), name, "StaticShapeInference: did not find layer", ref warnings); if (shapesByName.ContainsKey(name)) { ValidationHelper.AppendWarning(shapesByName[name] != null, name, "StaticShapeInference: unknown layer shape at compile time", ref warnings); } } }
// works on IRModel public bool InferAllLayersChannelOrder(Model model, out Dictionary <string, ChannelsOrder> layerChannelOrder) { // TF2Onnx : pattern T (.* Conv .*) T-1 // * being transpose commutative layer layerChannelOrder = new Dictionary <string, ChannelsOrder>(); IDictionary <string, TensorShape?> shapesByName = new Dictionary <string, TensorShape?>(); IDictionary <string, int?> ranksByName = new Dictionary <string, int?>(); foreach (var i in model.inputs) { ranksByName[i.name] = i.rank; if (!ModelAnalyzer.IsInputShapeAcceptablyKnowForShapeInference(i)) { continue; } shapesByName[i.name] = new TensorShape(i.shape); } IRShapeInferenceAndConstantFusing shapeInferencePass = new IRShapeInferenceAndConstantFusing(); shapeInferencePass.InferAllShapes(model, ref shapesByName, ref ranksByName); bool inputsNHWC = false; bool inputsNHWCExportedInputsAsNCHW = false; bool patternMatchStart = false; bool patternMatchConv = false; // tf to onnx does not swizzle axis, need to match * Conv * T-1 ... bool patternMatchStartInputsAsNCHWConv = false; for (int l = 0; l < model.layers.Count; l++) { var layer = model.layers[l]; if (!patternMatchStart && IsLayerTranpose(layer) && Enumerable.SequenceEqual(layer.pool, new[] { 0, 3, 1, 2 }) || IsLayerReshape(layer) && (shapesByName[layer.inputs[0]] != null) && IsReshapeTransposeToNCHW(layer, shapesByName[layer.inputs[0]].Value)) { patternMatchStart = true; } else if (patternMatchStart && patternMatchConv && ((IsLayerTranpose(layer) && Enumerable.SequenceEqual(layer.pool, new[] { 0, 2, 3, 1 })) || (IsLayerReshape(layer) && (shapesByName[layer.inputs[0]] != null) && IsReshapeTransposeToNHWC(layer, shapesByName[layer.inputs[0]].Value)) || (IsLayerSqueeze(layer) && (ranksByName[layer.inputs[0]] != null) && IsSqueezeTransposeToNHWC(layer, ranksByName[layer.inputs[0]].Value)) || (IsLayerFlatten(layer) && (ranksByName[layer.inputs[0]] != null) && IsFlattenTransposeToNHWC(layer, ranksByName[layer.inputs[0]].Value)))) { inputsNHWC = true; } else if (patternMatchStart && IsLayerConv(layer)) { patternMatchConv = true; } if (!inputsNHWCExportedInputsAsNCHW && patternMatchStartInputsAsNCHWConv && ((IsLayerTranpose(layer) && Enumerable.SequenceEqual(layer.pool, new[] { 0, 2, 3, 1 })) || (IsLayerReshape(layer) && (shapesByName[layer.inputs[0]] != null) && IsReshapeTransposeToNHWC(layer, shapesByName[layer.inputs[0]].Value)))) { inputsNHWCExportedInputsAsNCHW = true; } else if (!patternMatchStartInputsAsNCHWConv && !patternMatchStart && IsLayerConv(layer)) { patternMatchStartInputsAsNCHWConv = true; } } // flag each layer as being NHWC or NCHW for (int i = 0; i < model.inputs.Count; i++) { Model.Input input = model.inputs[i]; if (!inputsNHWCExportedInputsAsNCHW) { layerChannelOrder[input.name] = inputsNHWC ? ChannelsOrder.NHWC : ChannelsOrder.NCHW; } else { layerChannelOrder[input.name] = ChannelsOrder.NCHW; } } for (int l = 0; l < model.layers.Count; l++) { var layer = model.layers[l]; if (IsLayerTranpose(layer) && Enumerable.SequenceEqual(layer.pool, new[] { 0, 3, 1, 2 }) || IsLayerReshape(layer) && (shapesByName[layer.inputs[0]] != null) && IsReshapeTransposeToNCHW(layer, shapesByName[layer.inputs[0]].Value) && layerChannelOrder[layer.inputs[0]] == ChannelsOrder.NHWC) { layerChannelOrder[layer.name] = ChannelsOrder.TransposeToNCHW; } else if (IsLayerTranpose(layer) && Enumerable.SequenceEqual(layer.pool, new[] { 0, 2, 3, 1 }) || IsLayerReshape(layer) && (shapesByName[layer.inputs[0]] != null) && IsReshapeTransposeToNHWC(layer, shapesByName[layer.inputs[0]].Value) && layerChannelOrder[layer.inputs[0]] == ChannelsOrder.NCHW) { layerChannelOrder[layer.name] = ChannelsOrder.TransposeToNHWC; } else { string inputWithKnownOrder = null; for (int i = 0; i < layer.inputs.Length; i++) { var input = layer.inputs[i]; if (layerChannelOrder.ContainsKey(input)) { inputWithKnownOrder = input; break; } } if (inputWithKnownOrder == null) { continue; } Assert.IsNotNull(inputWithKnownOrder); ChannelsOrder inputOrder = layerChannelOrder[inputWithKnownOrder]; if (inputOrder == ChannelsOrder.TransposeToNCHW) { inputOrder = ChannelsOrder.NCHW; } else if (inputOrder == ChannelsOrder.TransposeToNHWC) { inputOrder = ChannelsOrder.NHWC; } // all layers with unknown layout are const for (int i = 0; i < layer.inputs.Length; i++) { var input = layer.inputs[i]; if (!layerChannelOrder.ContainsKey(input)) { layerChannelOrder[input] = inputOrder; } } layerChannelOrder[layer.name] = inputOrder; } } // TODO Assert that all layers have a channel order // Assert that all layers are NHWC if inputsNHWC return(inputsNHWC || inputsNHWCExportedInputsAsNCHW); }