private static void InitializeReLUs(NetworkGraph graph) { // relus like a bit of positive bias to get gradients early // otherwise it's technically possible that a relu unit will never turn on (by chance) // and will never get any gradient and never contribute any computation. Dead relu. foreach (Layer layer in graph.Vertices.Where(x => x is ReLULayer).ToList()) { Layer target = layer; if (graph.InDegree(target) == 1) { Edge <Layer> edge = graph.InEdges(target)[0]; if (edge.Target is MaxPoolingLayer) { target = edge.Source; } } if (graph.InDegree(target) == 1) { Edge <Layer> edge = graph.InEdges(target)[0]; if (edge.Source is StochasticLayer stochasticLayer) { stochasticLayer.B.Set(0.1f); } } } }
/// <summary> /// Initializes a new instance of the <see cref="Network"/> class. /// </summary> /// <param name="graph">The network graph.</param> internal Network(NetworkGraph graph) { this.Graph = graph ?? throw new ArgumentNullException(nameof(graph)); // the graph source must be input layer if (graph.Sources.Count() != 1 || !(graph.Sources.FirstOrDefault() is InputLayer)) { throw new ArgumentException(Properties.Resources.E_InvalidNetArchitecture_MissingInputLayer); } }
private static void AddLossLayers(NetworkGraph graph) { foreach (Layer layer in graph.Sinks.ToList()) { if (!(layer is LossLayer)) { graph.AddEdge(layer, new SoftMaxLayer(layer.OutputShape)); } } }
/// <summary> /// Initializes a new instance of the <see cref="ClassificationNetwork"/> class. /// </summary> /// <param name="graph">The network graph.</param> /// <param name="classes">The classes the network should be able to classify.</param> /// <param name="allowedClasses">The classes the network is allowed to classify.</param> /// <param name="blankClass">The blank class that represents none of the real classes.</param> private ClassificationNetwork(NetworkGraph graph, IList <string> classes, IList <string> allowedClasses, string blankClass) : base(graph) { if (classes == null) { throw new ArgumentNullException(nameof(classes)); } if (classes.Count == 0) { throw new ArgumentException(Properties.Resources.E_InvalidNetArchitecture_NoClasses, nameof(classes)); } if (!string.IsNullOrEmpty(blankClass) && !classes.Contains(blankClass)) { throw new ArgumentException("The blank class must be included in the list of classes.", nameof(classes)); } this.classes.AddRange(classes); if (allowedClasses != null) { this.allowedClasses.UnionWith(allowedClasses); this.allowedClasses.IntersectWith(classes); } else { this.allowedClasses.UnionWith(classes); } this.BlankClass = blankClass; // the graph sinks must be loss layers if (!graph.Sinks.All(x => x is LossLayer)) { throw new ArgumentException(Properties.Resources.E_InvalidNetArchitecture_MissingLossLayer); } // the number of classes in loss layers must match the number of network classes if (graph.Sinks.OfType <LossLayer>().Any(x => x.NumberOfClasses != classes.Count)) { throw new ArgumentException(Properties.Resources.E_InvalidNetArchitecture_InvalidClassCountInLossLayer); } this.ConfigureAllowedClasses(); }
private static void AddActivationLayers(NetworkGraph graph) { foreach (Layer layer in graph.Vertices.Where(x => ((x as TrainableLayer)?.NeedsActivation).GetValueOrDefault()).ToList()) { Layer source = layer; if (graph.OutDegree(source) == 1) { // optimization - add activation layer after max pooling layer that follows stochastic Edge <Layer> edge = graph.OutEdges(source)[0]; if (edge.Target is MaxPoolingLayer) { source = edge.Target; } } if (graph.OutDegree(source) == 1) { Edge <Layer> edge = graph.OutEdges(source)[0]; if (!(edge.Target is ActivationLayer) && !(edge.Target is LossLayer)) { Layer activationLayer = new TanhLayer(edge.Source.OutputShape); graph.AddVertex(activationLayer); Edge <Layer> newEdge = new Edge <Layer>(edge.Source, activationLayer); graph.OutEdges(source)[0] = newEdge; graph.InEdges(activationLayer).Add(newEdge); if (edge.Target != null) { IList <Edge <Layer> > inedges = graph.InEdges(edge.Target); int index = inedges.IndexOf(edge); newEdge = new Edge <Layer>(activationLayer, edge.Target); inedges[index] = newEdge; graph.OutEdges(activationLayer).Add(newEdge); } } } } }
public static NetworkGraph CreateNetworkGraph(string architecture, bool addActivationLayers, bool addLossLayer) { if (architecture == null) { throw new ArgumentNullException(nameof(architecture)); } if (string.IsNullOrEmpty(architecture)) { throw new ArgumentException(Properties.Resources.E_InvalidNetArchitecture_NoLayers, nameof(architecture)); } // 1. parse architecture string and build preliminary graph ComponentGraph componentGraph = NetworkGraphBuilder.ParseArchitecture(architecture, false); // 2. create layers in the preliminary graph RandomNumberGenerator <float> random = null; //// new Random(0); foreach (ComponentVertex sink in componentGraph.Sinks) { NetworkGraphBuilder.CreateLayerInGraph(componentGraph, sink, random); } // 3. convert to network graph NetworkGraph graph = new NetworkGraph(); foreach (Edge <ComponentVertex> edge in componentGraph.Edges) { /*NetworkGraph sourceGraph = (edge.Source.Layer as RNNLayer)?.Graph; * NetworkGraph targetGraph = (edge.Target.Layer as RNNLayer)?.Graph; * * if (sourceGraph != null) * { * graph.AddGraph(sourceGraph); * if (targetGraph != null) * { * graph.AddEdges(sourceGraph.Sinks, targetGraph.Sources); * graph.AddGraph(targetGraph); * } * else * { * graph.AddEdges(sourceGraph.Sinks, edge.Target.Layer); * } * } * else if (targetGraph != null) * { * graph.AddEdges(edge.Source.Layer, targetGraph.Sources); * graph.AddGraph(targetGraph); * } * else*/ { graph.AddEdge(edge.Source.Layer, edge.Target.Layer); } } // 4. add missing loss layers if (addLossLayer) { NetworkGraphBuilder.AddLossLayers(graph); } // 5. add missing activation layers if (addActivationLayers) { NetworkGraphBuilder.AddActivationLayers(graph); } // 6. initialize stochastic biases with ReLU activations NetworkGraphBuilder.InitializeReLUs(graph); return(graph); }
/// <summary> /// Creates a classification neural network from a string that contains network architecture. /// </summary> /// <param name="architecture">The network architecture.</param> /// <returns>The <see cref="Network"/> object this method creates.</returns> public static Network FromArchitecture(string architecture) { NetworkGraph graph = NetworkGraphBuilder.CreateNetworkGraph(architecture, true, false); return(new Network(graph)); }
/// <summary> /// Creates a classification neural network from a string that contains network architecture. /// </summary> /// <param name="architecture">The network architecture.</param> /// <param name="classes">The classes the network should able to classify into.</param> /// <param name="allowedClasses">The classes the network is allowed to classify.</param> /// <param name="blankClass">The blank class that represents none of the real classes.</param> /// <returns> /// The <see cref="ClassificationNetwork"/> object this method creates. /// </returns> public static ClassificationNetwork FromArchitecture(string architecture, IList <string> classes, IList <string> allowedClasses, string blankClass) { NetworkGraph graph = NetworkGraphBuilder.CreateNetworkGraph(architecture, true, true); return(new ClassificationNetwork(graph, classes, allowedClasses, blankClass)); }
/// <summary> /// Creates a graph from the specified byte array. /// </summary> /// <param name="buffer">The buffer to read the <see cref="NetworkGraph"/> from.</param> /// <returns>The <see cref="NetworkGraph"/> this method creates.</returns> public static NetworkGraph FromMemory(byte[] buffer) { return(NetworkGraph.FromString(UTF8Encoding.UTF8.GetString(buffer))); }
/// <summary> /// Creates a graph from the specified file. /// </summary> /// <param name="fileName">A string that contains the name of the file from which to create the <see cref="NetworkGraph"/>.</param> /// <returns>The <see cref="NetworkGraph"/> this method creates.</returns> public static NetworkGraph FromFile(string fileName) { return(NetworkGraph.FromString(File.ReadAllText(fileName, Encoding.UTF8))); }
/// <summary> /// Initializes a new instance of the <see cref="NetworkGraph"/> class /// using existing graph as a source. /// </summary> /// <param name="other">The existing <see cref="NetworkGraph"/> to create this graph from.</param> /// <param name="cloneLayers">The value indicating whether the layers should be cloned.</param> public NetworkGraph(NetworkGraph other, bool cloneLayers) : base(other, cloneLayers) { }