private static void PackConstantsForMathOps(Model model, Dictionary <string, Layer> constantLayers)
        {
            for (int l = 0; l < model.layers.Count; ++l)
            {
                var layer = model.layers[l];

                if (!LinearLayerFusing.IsLayerLinearMathOp(layer))
                {
                    continue;
                }
                var constInputs = layer.inputs.Count(x => constantLayers.ContainsKey(x));
                // @TODO fuse multi const inputs here
                if (!(layer.inputs.Length == 2 && constInputs == 1))
                {
                    continue;
                }

                var constInput = layer.inputs.ToList().Find(x => constantLayers.ContainsKey(x));

                layer.datasets = new Layer.DataSet[constantLayers[constInput].datasets.Length];
                Array.Copy(constantLayers[constInput].datasets, layer.datasets, constantLayers[constInput].datasets.Length);
                layer.weights = new BarracudaArray(constantLayers[constInput].weights.Length);
                BarracudaArray.Copy(constantLayers[constInput].weights, layer.weights, constantLayers[constInput].weights.Length);

                layer.axis = constantLayers[constInput].axis; // rank TODO name correctly

                model.layers[l].inputs = layer.inputs.Where(x => x != constInput).ToArray();
            }
        }
        private static void UnpackConstantsForMathOps(Model model)
        {
            List <Layer> newConstants = new List <Layer>();

            for (int l = 0; l < model.layers.Count; ++l)
            {
                var layer = model.layers[l];
                if (!LinearLayerFusing.IsLayerLinearMathOp(layer))
                {
                    continue;
                }

                if (layer.datasets == null || layer.datasets.Length != 1)
                {
                    continue;
                }

                var   name       = "c" + layer.name;
                Layer constInput = new Layer(name, Layer.Type.Load);

                constInput.datasets = new Layer.DataSet[layer.datasets.Length];
                Array.Copy(layer.datasets, constInput.datasets, layer.datasets.Length);
                for (int d = 0; d < constInput.datasets.Length; ++d)
                {
                    constInput.datasets[d].name = name;
                }

                constInput.weights = new BarracudaArray(layer.weights.Length);
                BarracudaArray.Copy(layer.weights, constInput.weights, layer.weights.Length);

                constInput.axis = layer.axis; // rank TODO rename

                Array.Resize(ref layer.inputs, layer.inputs.Length + 1);
                layer.inputs[layer.inputs.Length - 1] = constInput.name;

                newConstants.Add(constInput);

                layer.datasets = new Layer.DataSet[0];
                layer.weights  = new BarracudaArray(0);//TODO fp16
            }
            newConstants.AddRange(model.layers);
            model.layers = newConstants;
        }
        private void FuseShapesIntoConstants(ref Model model, IDictionary <string, TensorShape?> shapesByName, IDictionary <string, int?> ranksByName, ref List <Model.ImporterWarning> warnings)
        {
            var toRunnableNCHW = new IntermediateToRunnableNCHWPass();

            var knownLayersValue = new Dictionary <string, Tensor>();
            var newKnownLayers   = new HashSet <string>();
            var keepLayers       = new HashSet <string>();

            for (int l = 0; l < model.layers.Count; ++l)
            {
                var layer = model.layers[l];
                if (layer.flags == Layer.Flags.Preserve)
                {
                    keepLayers.Add(layer.name);
                }

                // NN is a directed graph, if we just fused constants + shapes, update following nodes
                // re-evaluate shapes
                FuseInputsIntoLayer(ref layer, knownLayersValue, ranksByName, warnings);
                // TODO optimization, pass in index, or add shape
                IRShapeInferenceHelper.RankInference.UpdateKnownTensorRanks(model, ranksByName);
                IRShapeInferenceHelper.ShapeInference.UpdateKnownTensorShapesNCHW(model, ranksByName, ref shapesByName);

                if (ModelOptimizer.IsLayerConstant(layer))
                {
                    knownLayersValue[layer.name] = new Tensor(layer.datasets[0].shape, layer.weights);
                }
                else if (layer.type == Layer.Type.Shape)
                {
                    // assert inputs.Lenght == 1
                    var input = layer.inputs[0];
                    if (shapesByName.ContainsKey(input) && shapesByName[input] != null &&
                        ranksByName.ContainsKey(input) && ranksByName[input] != null
                        )
                    {
                        var shape = shapesByName[input].Value;
                        var rank  = ranksByName[input].Value;
                        knownLayersValue[layer.name] = ShapeToNCHWTensor(shape, rank);
                        newKnownLayers.Add(layer.name);
                        continue;
                    }
                }

                bool allInputsAreKnown = layer.inputs.Length > 0 ? knownLayersValue.ContainsKey(layer.inputs[0]) : false;
                for (int i = 1; i < layer.inputs.Length; i++)
                {
                    allInputsAreKnown &= knownLayersValue.ContainsKey(layer.inputs[i]);
                }

                // if all inputs are known, execute layer
                if (!allInputsAreKnown)
                {
                    continue;
                }

                var layerInputs = new Dictionary <string, Tensor>();
                var opsModel    = new Model();
                opsModel.layout = "iNCHW";
                for (int i = 0; i < layer.inputs.Length; i++)
                {
                    Model.Input input;
                    input.name  = layer.inputs[i];
                    input.shape = shapesByName[input.name].Value.ToArray();
                    input.rank  = ranksByName[input.name].Value;

                    opsModel.inputs.Add(input);
                    layerInputs[input.name] = knownLayersValue[input.name];
                }
                Layer newLayer = new Layer(layer.name.ToString(), layer.activation);
                newLayer.type       = layer.type;
                newLayer.activation = layer.activation;
                newLayer.pad        = layer.pad.ToArray();
                newLayer.stride     = layer.stride.ToArray();
                newLayer.pool       = layer.pool.ToArray();
                newLayer.axis       = layer.axis;
                newLayer.alpha      = layer.alpha;
                newLayer.beta       = layer.beta;
                newLayer.inputs     = layer.inputs.ToArray();
                newLayer.datasets   = layer.datasets;
                newLayer.weights    = layer.weights;
                if (layer.outputs != null)
                {
                    newLayer.outputs = layer.outputs.ToArray();
                }
                if (layer.axes != null)
                {
                    newLayer.axes = layer.axes.ToArray();
                }


                opsModel.layers.Add(newLayer);
                opsModel.outputs.Add(newLayer.name);

                toRunnableNCHW.Run(ref opsModel);

                // bake
                var useCPUforBaking = WorkerFactory.Device.CPU;
                using (var worker = WorkerFactory.CreateWorker(opsModel, useCPUforBaking))
                {
                    var bakedConstant = worker.Execute(layerInputs).CopyOutput();
                    knownLayersValue[layer.name] = bakedConstant;
                    newKnownLayers.Add(layer.name);
                }
            }

            // remove new baked layers since we will insert constants for those
            model.layers.RemoveAll(x => newKnownLayers.Contains(x.name) && !keepLayers.Contains(x.name));

            // TODO use ModelBuilder?
            foreach (var l in newKnownLayers)
            {
                if (keepLayers.Contains(l))
                {
                    continue;
                }

                var   name   = l;
                var   tensor = knownLayersValue[name];
                Layer c      = new Layer(name, Layer.Type.Load);

                c.datasets                    = new Layer.DataSet[1];
                c.datasets[0].name            = name;
                c.datasets[0].shape           = tensor.shape;
                c.datasets[0].itemSizeInBytes = 4;
                c.datasets[0].length          = tensor.shape.length;
                c.datasets[0].offset          = 0;

                c.axis = ranksByName[c.name].Value;

                c.weights = new BarracudaArray(tensor.length);
                BarracudaArray.Copy(tensor.ToReadOnlyArray(), c.weights, tensor.length);
                model.layers.Insert(0, c);
            }

            foreach (var l in knownLayersValue)
            {
                l.Value.Dispose();
            }

            // TODO remove?
            // remove unused constants
            var removeUnusedLayersPass = new Cleanup.RemoveUnusedLayersPass();

            removeUnusedLayersPass.Run(ref model);
        }
        public static void FuseConstants(ref Model model)
        {
            var knownLayersValue = new Dictionary <string, Tensor>();
            var newKnownLayers   = new HashSet <string>();
            var keepLayers       = new HashSet <string>();

            for (int l = 0; l < model.layers.Count; ++l)
            {
                var layer = model.layers[l];
                if (layer.flags == Layer.Flags.Preserve)
                {
                    keepLayers.Add(layer.name);
                }

                // NN is a directed graph, if we just fused constants + shapes, update following nodes
                // TODO optimization, pass in index, or add shape
                if (ModelOptimizer.IsLayerConstant(layer))
                {
                    knownLayersValue[layer.name] = new Tensor(layer.datasets[0].shape, layer.weights);
                }

                bool allInputsAreKnown = layer.inputs.Length > 0 ? knownLayersValue.ContainsKey(layer.inputs[0]) : false;
                for (int i = 1; i < layer.inputs.Length; i++)
                {
                    allInputsAreKnown &= knownLayersValue.ContainsKey(layer.inputs[i]);
                }

                // if all inputs are known, execute layer
                if (!allInputsAreKnown)
                {
                    continue;
                }

                var layerInputs = new Dictionary <string, Tensor>();
                var opsModel    = new Model();
                for (int i = 0; i < layer.inputs.Length; i++)
                {
                    Model.Input input;
                    input.name  = layer.inputs[i];
                    input.shape = knownLayersValue[input.name].shape.ToArray();
                    input.rank  = knownLayersValue[input.name].shape.dimensions;

                    opsModel.inputs.Add(input);
                    layerInputs[input.name] = knownLayersValue[input.name];
                }
                opsModel.layers.Add(layer);
                opsModel.outputs.Add(layer.name);

                // bake
                var useCPUforBaking = WorkerFactory.Device.CPU;
                using (var worker = WorkerFactory.CreateWorker(opsModel, useCPUforBaking))
                {
                    // TODO use ModelIR2RunnableNCHWPass
                    var bakedConstant = worker.Execute(layerInputs).PeekOutput();
                    bakedConstant.TakeOwnership();
                    knownLayersValue[layer.name] = bakedConstant;
                    newKnownLayers.Add(layer.name);
                }
            }

            // remove new baked layers since we will insert constants for those
            model.layers.RemoveAll(x => newKnownLayers.Contains(x.name) && !keepLayers.Contains(x.name));

            // TODO use ModelBuilder?
            foreach (var l in newKnownLayers)
            {
                if (keepLayers.Contains(l))
                {
                    continue;
                }

                var   name   = l;
                var   tensor = knownLayersValue[name];
                Layer c      = new Layer(name, Layer.Type.Load);

                c.datasets                    = new Layer.DataSet[1];
                c.datasets[0].name            = name;
                c.datasets[0].shape           = tensor.shape;
                c.datasets[0].itemSizeInBytes = 4;
                c.datasets[0].length          = tensor.shape.length;
                c.datasets[0].offset          = 0;

                c.axis = tensor.shape.dimensions;

                c.weights = new BarracudaArray(tensor.length);
                BarracudaArray.Copy(tensor.ToReadOnlyArray(), c.weights, tensor.length);
                model.layers.Insert(0, c);
            }

            // clear allocated tensors
            foreach (var l in knownLayersValue)
            {
                l.Value.Dispose();
            }

            // remove unused constants
            var removeUnusedLayersPass = new Cleanup.RemoveUnusedLayersPass();

            removeUnusedLayersPass.Run(ref model);
        }
Exemple #5
0
 public static void CopyToBarracudaArray(this float[] sourceArray, BarracudaArray destinationArray, long destinationIndex)
 {
     BarracudaArray.Copy(sourceArray, 0, destinationArray, (int)destinationIndex, sourceArray.Length);
 }