private void FuseShapesIntoConstants(ref Model model, IDictionary <string, TensorShape?> shapesByName, IDictionary <string, int?> ranksByName, ref List <Model.ImporterWarning> warnings) { var toRunnableNCHW = new IntermediateToRunnableNCHWPass(); var knownLayersValue = new Dictionary <string, Tensor>(); var newKnownLayers = new HashSet <string>(); var keepLayers = new HashSet <string>(); for (int l = 0; l < model.layers.Count; ++l) { var layer = model.layers[l]; if (layer.flags == Layer.Flags.Preserve) { keepLayers.Add(layer.name); } // NN is a directed graph, if we just fused constants + shapes, update following nodes // re-evaluate shapes FuseInputsIntoLayer(ref layer, knownLayersValue, ranksByName, warnings); // TODO optimization, pass in index, or add shape IRShapeInferenceHelper.RankInference.UpdateKnownTensorRanks(model, ranksByName); IRShapeInferenceHelper.ShapeInference.UpdateKnownTensorShapesNCHW(model, ranksByName, ref shapesByName); if (ModelOptimizer.IsLayerConstant(layer)) { knownLayersValue[layer.name] = new Tensor(layer.datasets[0].shape, layer.weights); } else if (layer.type == Layer.Type.Shape) { // assert inputs.Lenght == 1 var input = layer.inputs[0]; if (shapesByName.ContainsKey(input) && shapesByName[input] != null && ranksByName.ContainsKey(input) && ranksByName[input] != null ) { var shape = shapesByName[input].Value; var rank = ranksByName[input].Value; knownLayersValue[layer.name] = ShapeToNCHWTensor(shape, rank); newKnownLayers.Add(layer.name); continue; } } bool allInputsAreKnown = layer.inputs.Length > 0 ? knownLayersValue.ContainsKey(layer.inputs[0]) : false; for (int i = 1; i < layer.inputs.Length; i++) { allInputsAreKnown &= knownLayersValue.ContainsKey(layer.inputs[i]); } // if all inputs are known, execute layer if (!allInputsAreKnown) { continue; } var layerInputs = new Dictionary <string, Tensor>(); var opsModel = new Model(); opsModel.layout = "iNCHW"; for (int i = 0; i < layer.inputs.Length; i++) { Model.Input input; input.name = layer.inputs[i]; input.shape = shapesByName[input.name].Value.ToArray(); input.rank = ranksByName[input.name].Value; opsModel.inputs.Add(input); layerInputs[input.name] = knownLayersValue[input.name]; } Layer newLayer = new Layer(layer.name.ToString(), layer.activation); newLayer.type = layer.type; newLayer.activation = layer.activation; newLayer.pad = layer.pad.ToArray(); newLayer.stride = layer.stride.ToArray(); newLayer.pool = layer.pool.ToArray(); newLayer.axis = layer.axis; newLayer.alpha = layer.alpha; newLayer.beta = layer.beta; newLayer.inputs = layer.inputs.ToArray(); newLayer.datasets = layer.datasets; newLayer.weights = layer.weights; if (layer.outputs != null) { newLayer.outputs = layer.outputs.ToArray(); } if (layer.axes != null) { newLayer.axes = layer.axes.ToArray(); } opsModel.layers.Add(newLayer); opsModel.outputs.Add(newLayer.name); toRunnableNCHW.Run(ref opsModel); // bake var useCPUforBaking = WorkerFactory.Device.CPU; using (var worker = WorkerFactory.CreateWorker(opsModel, useCPUforBaking)) { var bakedConstant = worker.Execute(layerInputs).CopyOutput(); knownLayersValue[layer.name] = bakedConstant; newKnownLayers.Add(layer.name); } } // remove new baked layers since we will insert constants for those model.layers.RemoveAll(x => newKnownLayers.Contains(x.name) && !keepLayers.Contains(x.name)); // TODO use ModelBuilder? foreach (var l in newKnownLayers) { if (keepLayers.Contains(l)) { continue; } var name = l; var tensor = knownLayersValue[name]; Layer c = new Layer(name, Layer.Type.Load); c.datasets = new Layer.DataSet[1]; c.datasets[0].name = name; c.datasets[0].shape = tensor.shape; c.datasets[0].itemSizeInBytes = 4; c.datasets[0].length = tensor.shape.length; c.datasets[0].offset = 0; c.axis = ranksByName[c.name].Value; c.weights = new BarracudaArray(tensor.length); BarracudaArray.Copy(tensor.ToReadOnlyArray(), c.weights, tensor.length); model.layers.Insert(0, c); } foreach (var l in knownLayersValue) { l.Value.Dispose(); } // TODO remove? // remove unused constants var removeUnusedLayersPass = new Cleanup.RemoveUnusedLayersPass(); removeUnusedLayersPass.Run(ref model); }
// TODO: refactor with FuseShapesIntoConstants public void InferAllShapes(Model model, ref IDictionary <string, TensorShape?> shapesByName, ref IDictionary <string, int?> ranksByName) { var toRunnableNCHW = new IntermediateToRunnableNCHWPass(); var knownLayersValue = new Dictionary <string, Tensor>(); var newKnownLayers = new HashSet <string>(); var keepLayers = new HashSet <string>(); for (int l = 0; l < model.layers.Count; ++l) { var layer = model.layers[l]; if (layer.flags == Layer.Flags.Preserve) { keepLayers.Add(layer.name); } // NN is a directed graph, if we just fused constants + shapes, update following nodes // re-evaluate shapes FuseInputsIntoLayer(ref layer, knownLayersValue, ranksByName, null);//TODO handle potential folding errors/warnings // TODO optimization, pass in index, or add shape IRShapeInferenceHelper.RankInference.UpdateKnownTensorRanks(model, ranksByName); IRShapeInferenceHelper.ShapeInference.UpdateKnownTensorShapesNCHW(model, ranksByName, ref shapesByName); if (ModelOptimizer.IsLayerConstant(layer)) { knownLayersValue[layer.name] = new Tensor(layer.datasets[0].shape, layer.weights); } else if (layer.type == Layer.Type.Shape) { // assert inputs.Lenght == 1 var input = layer.inputs[0]; if (shapesByName.ContainsKey(input) && shapesByName[input] != null && ranksByName.ContainsKey(input) && ranksByName[input] != null ) { var shape = shapesByName[input].Value; var rank = ranksByName[input].Value; knownLayersValue[layer.name] = ShapeToNCHWTensor(shape, rank); newKnownLayers.Add(layer.name); continue; } } bool allInputsAreKnown = layer.inputs.Length > 0 ? knownLayersValue.ContainsKey(layer.inputs[0]) : false; for (int i = 1; i < layer.inputs.Length; i++) { allInputsAreKnown &= knownLayersValue.ContainsKey(layer.inputs[i]); } // if all inputs are known, execute layer if (!allInputsAreKnown) { continue; } var layerInputs = new Dictionary <string, Tensor>(); var opsModel = new Model(); opsModel.layout = "iNCHW"; for (int i = 0; i < layer.inputs.Length; i++) { Model.Input input; input.name = layer.inputs[i]; input.shape = shapesByName[input.name].Value.ToArray(); input.rank = ranksByName[input.name].Value; opsModel.inputs.Add(input); layerInputs[input.name] = knownLayersValue[input.name]; } Layer newLayer = new Layer(layer.name.ToString(), layer.activation); newLayer.type = layer.type; newLayer.activation = layer.activation; newLayer.pad = layer.pad.ToArray(); newLayer.stride = layer.stride.ToArray(); newLayer.pool = layer.pool.ToArray(); newLayer.axis = layer.axis; newLayer.alpha = layer.alpha; newLayer.beta = layer.beta; newLayer.inputs = layer.inputs.ToArray(); newLayer.datasets = layer.datasets; newLayer.weights = layer.weights; if (layer.outputs != null) { newLayer.outputs = layer.outputs.ToArray(); } if (layer.axes != null) { newLayer.axes = layer.axes.ToArray(); } opsModel.layers.Add(newLayer); opsModel.outputs.Add(newLayer.name); toRunnableNCHW.Run(ref opsModel); toRunnableNCHW.Run(ref opsModel); // bake var useCPUforBaking = WorkerFactory.Device.CPU; using (var worker = WorkerFactory.CreateWorker(opsModel, useCPUforBaking)) { var bakedConstant = worker.Execute(layerInputs).PeekOutput(); bakedConstant.TakeOwnership(); knownLayersValue[layer.name] = bakedConstant; newKnownLayers.Add(layer.name); } } // clear allocated tensors foreach (var l in knownLayersValue) { l.Value.Dispose(); } // remove unused constants var removeUnusedLayersPass = new Cleanup.RemoveUnusedLayersPass(); removeUnusedLayersPass.Run(ref model); }