public static void FuseLinear(Model model, HashSet <string> keepLayers = null)
        {
            // outputs and memories can be queried by the user, make sure they are not removed
            var preserve = new HashSet <string>(
                model.memories.Select(mem => mem.input).Concat(
                    model.memories.Select(mem => mem.output)).Concat(
                    model.outputs));

            var constantLayers = new Dictionary <string, Layer>();

            foreach (var l in model.layers)
            {
                if (IsLayerConstant(l))
                {
                    constantLayers[l.name] = l;
                }
            }

            // pack constants into layer database
            PackConstants(model, constantLayers);

            var remap        = new Dictionary <string, string>();
            var mergedLayers = new HashSet <Layer>();

            for (int l = 0; l < model.layers.Count; ++l)
            {
                var layer = model.layers[l];

                bool isLayerLinear      = LinearLayerFusing.IsLayerLinear(layer, constantLayers);
                bool isLayerPreserved   = preserve.Contains(layer.name);
                bool layerHasActivation = IsLayerFusedActivation(layer);

                if (!isLayerLinear)
                {
                    continue;
                }

                // if layer has an activation, we fuse it, but treat it as non linear for future children
                if (!layerHasActivation)
                {
                    remap[layer.name] = layer.name;
                }

                // Multi input nodes can only fuse constants and same inputs
                // only merge constants. @TODO: fuse equal input nodes
                var nonLinearInputs = layer.inputs.Where(x => !remap.ContainsKey(x) && !constantLayers.ContainsKey(x)).ToList();
                var linearInputs    = layer.inputs.Where(x => remap.ContainsKey(x)).ToList();

                // merge layer with one linearInput and eventual constants
                if (nonLinearInputs.Count > 0 || linearInputs.Count > 1)
                {
                    continue;
                }

                var input = linearInputs[0];

                // input is a linear layer, fuse it
                int   inputLayerIndex = model.layers.FindIndex(x => x.name == remap[input]);
                Layer inputLayer      = model.layers[inputLayerIndex];

                if (!AreLayersFusable(inputLayer, layer))
                {
                    continue;
                }

                // convention: layer will be fused into inputLayer
                // => fused layer will have the same inputs as inputLayer
                Layer fusedLayer = FuseConsecutiveLayers(inputLayer, layer);

                if (LayerComplextity(fusedLayer) > LayerComplextity(inputLayer) + LayerComplextity(layer))
                {
                    continue;
                }

                if (layerHasActivation)
                {
                    fusedLayer.activation = layer.activation;
                }

                bool hasNoSkipConnection = (model.GetDownStreamLayersCount(input) == 1);
                //  if input has more than 1 child, we can't override input with fused result
                //  same if input is preserved
                if (!hasNoSkipConnection || preserve.Contains(input))
                {
                    fusedLayer.name = layer.name;
                    model.layers[l] = fusedLayer;
                    continue;
                }

                // preserve layer if output/memory
                if (isLayerPreserved)
                {
                    // cannot merge layer into input:
                    // remove input, no need to remap as inputs == input.inputs
                    fusedLayer.name = layer.name;
                    mergedLayers.Add(inputLayer);
                    model.layers[l] = fusedLayer;
                }
                else
                {
                    // merge layer into input
                    // remove current and remap input names
                    mergedLayers.Add(layer);
                    remap[layer.name]             = fusedLayer.name;
                    model.layers[inputLayerIndex] = fusedLayer;
                }
            }

            // remove merged layers
            model.layers.RemoveAll(x => mergedLayers.Contains(x));

            // update remapped inputs
            for (int l = 0; l < model.layers.Count; ++l)
            {
                Layer layer = model.layers[l];
                for (int i = 0; i < layer.inputs.Length; ++i)
                {
                    var input = layer.inputs[i];
                    if (remap.ContainsKey(input))
                    {
                        model.layers[l].inputs[i] = remap[input];
                    }
                }
            }

            // unpack constants
            UnpackConstants(model);

            // remove unused constants
            foreach (var l in model.layers)
            {
                foreach (var i in l.inputs)
                {
                    if (constantLayers.ContainsKey(i))
                    {
                        constantLayers.Remove(i);
                    }
                }
            }
            model.layers.RemoveAll(x => constantLayers.ContainsKey(x.name) &&
                                   !preserve.Contains(x.name) &&
                                   (keepLayers == null ? true : !keepLayers.Contains(x.name)));
        }
 static Layer FuseConsecutiveLayers(Layer previous, Layer current)
 {
     return(linearLayerFuser.FuseLayers(previous, current));
 }
 static bool AreLayersFusable(Layer l0, Layer l1)
 {
     // can't fuse if input has a fused activation or if fusing code not implemented
     return(!IsLayerFusedActivation(l0) && linearLayerFuser.AreLayersFusable(l0, l1));
 }
 static bool IsLayerFusedActivation(Layer layer)
 {
     return(layer.activation != Layer.Activation.None);
 }
 static long LayerComplextity(Layer l)
 {
     return(m_LayerComplexity.LayerComplextity(l));
 }
 public static bool IsLayerConstant(Layer layer)
 {
     return(layer.type == Layer.Type.Load);
 }
 public override void PrepareStorage(Layer forLayer)
 {
     base.PrepareStorage(forLayer);
     m_LayerRequiresStorage = m_LayersWithStorage.Contains(forLayer);
 }
 public void LayerExecutionStarted(Layer layer)
 {
     Assert.IsNotNull(CurrentModelExecutionReport);
     CurrentModelExecutionReport.LayerExecutionStarted(layer);
 }
 public void TakeMemorySnapshot(IOps ops, IVars vars, string context, Layer layer)
 {
     MemorySnapshotsReport.TakeMemorySnapshot(ops, vars, context, layer);
 }
 internal void LayerExecutionStarted(Layer layer)
 {
     Assert.IsNull(CurrentLayerExecutionReport);
     CurrentLayerExecutionReport = new LayerExecutionReport(layer);
 }