private static void InitializeReLUs(NetworkGraph graph)
{
// relus like a bit of positive bias to get gradients early
// otherwise it's technically possible that a relu unit will never turn on (by chance)
// and will never get any gradient and never contribute any computation. Dead relu.
foreach (Layer layer in graph.Vertices.Where(x => x is ReLULayer).ToList())
{
Layer target = layer;
if (graph.InDegree(target) == 1)
{
Edge <Layer> edge = graph.InEdges(target)[0];
if (edge.Target is MaxPoolingLayer)
{
target = edge.Source;
}
}
if (graph.InDegree(target) == 1)
{
Edge <Layer> edge = graph.InEdges(target)[0];
if (edge.Source is StochasticLayer stochasticLayer)
{
stochasticLayer.B.Set(0.1f);
}
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="Network"/> class.
/// </summary>
/// <param name="graph">The network graph.</param>
internal Network(NetworkGraph graph)
{
this.Graph = graph ?? throw new ArgumentNullException(nameof(graph));
// the graph source must be input layer
if (graph.Sources.Count() != 1 || !(graph.Sources.FirstOrDefault() is InputLayer))
{
throw new ArgumentException(Properties.Resources.E_InvalidNetArchitecture_MissingInputLayer);
}
}
private static void AddLossLayers(NetworkGraph graph)
{
foreach (Layer layer in graph.Sinks.ToList())
{
if (!(layer is LossLayer))
{
graph.AddEdge(layer, new SoftMaxLayer(layer.OutputShape));
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ClassificationNetwork"/> class.
/// </summary>
/// <param name="graph">The network graph.</param>
/// <param name="classes">The classes the network should be able to classify.</param>
/// <param name="allowedClasses">The classes the network is allowed to classify.</param>
/// <param name="blankClass">The blank class that represents none of the real classes.</param>
private ClassificationNetwork(NetworkGraph graph, IList <string> classes, IList <string> allowedClasses, string blankClass)
: base(graph)
{
if (classes == null)
{
throw new ArgumentNullException(nameof(classes));
}
if (classes.Count == 0)
{
throw new ArgumentException(Properties.Resources.E_InvalidNetArchitecture_NoClasses, nameof(classes));
}
if (!string.IsNullOrEmpty(blankClass) && !classes.Contains(blankClass))
{
throw new ArgumentException("The blank class must be included in the list of classes.", nameof(classes));
}
this.classes.AddRange(classes);
if (allowedClasses != null)
{
this.allowedClasses.UnionWith(allowedClasses);
this.allowedClasses.IntersectWith(classes);
}
else
{
this.allowedClasses.UnionWith(classes);
}
this.BlankClass = blankClass;
// the graph sinks must be loss layers
if (!graph.Sinks.All(x => x is LossLayer))
{
throw new ArgumentException(Properties.Resources.E_InvalidNetArchitecture_MissingLossLayer);
}
// the number of classes in loss layers must match the number of network classes
if (graph.Sinks.OfType <LossLayer>().Any(x => x.NumberOfClasses != classes.Count))
{
throw new ArgumentException(Properties.Resources.E_InvalidNetArchitecture_InvalidClassCountInLossLayer);
}
this.ConfigureAllowedClasses();
}
private static void AddActivationLayers(NetworkGraph graph)
{
foreach (Layer layer in graph.Vertices.Where(x => ((x as TrainableLayer)?.NeedsActivation).GetValueOrDefault()).ToList())
{
Layer source = layer;
if (graph.OutDegree(source) == 1)
{
// optimization - add activation layer after max pooling layer that follows stochastic
Edge <Layer> edge = graph.OutEdges(source)[0];
if (edge.Target is MaxPoolingLayer)
{
source = edge.Target;
}
}
if (graph.OutDegree(source) == 1)
{
Edge <Layer> edge = graph.OutEdges(source)[0];
if (!(edge.Target is ActivationLayer) && !(edge.Target is LossLayer))
{
Layer activationLayer = new TanhLayer(edge.Source.OutputShape);
graph.AddVertex(activationLayer);
Edge <Layer> newEdge = new Edge <Layer>(edge.Source, activationLayer);
graph.OutEdges(source)[0] = newEdge;
graph.InEdges(activationLayer).Add(newEdge);
if (edge.Target != null)
{
IList <Edge <Layer> > inedges = graph.InEdges(edge.Target);
int index = inedges.IndexOf(edge);
newEdge = new Edge <Layer>(activationLayer, edge.Target);
inedges[index] = newEdge;
graph.OutEdges(activationLayer).Add(newEdge);
}
}
}
}
}
public static NetworkGraph CreateNetworkGraph(string architecture, bool addActivationLayers, bool addLossLayer)
{
if (architecture == null)
{
throw new ArgumentNullException(nameof(architecture));
}
if (string.IsNullOrEmpty(architecture))
{
throw new ArgumentException(Properties.Resources.E_InvalidNetArchitecture_NoLayers, nameof(architecture));
}
// 1. parse architecture string and build preliminary graph
ComponentGraph componentGraph = NetworkGraphBuilder.ParseArchitecture(architecture, false);
// 2. create layers in the preliminary graph
RandomNumberGenerator <float> random = null; //// new Random(0);
foreach (ComponentVertex sink in componentGraph.Sinks)
{
NetworkGraphBuilder.CreateLayerInGraph(componentGraph, sink, random);
}
// 3. convert to network graph
NetworkGraph graph = new NetworkGraph();
foreach (Edge <ComponentVertex> edge in componentGraph.Edges)
{
/*NetworkGraph sourceGraph = (edge.Source.Layer as RNNLayer)?.Graph;
* NetworkGraph targetGraph = (edge.Target.Layer as RNNLayer)?.Graph;
*
* if (sourceGraph != null)
* {
* graph.AddGraph(sourceGraph);
* if (targetGraph != null)
* {
* graph.AddEdges(sourceGraph.Sinks, targetGraph.Sources);
* graph.AddGraph(targetGraph);
* }
* else
* {
* graph.AddEdges(sourceGraph.Sinks, edge.Target.Layer);
* }
* }
* else if (targetGraph != null)
* {
* graph.AddEdges(edge.Source.Layer, targetGraph.Sources);
* graph.AddGraph(targetGraph);
* }
* else*/
{
graph.AddEdge(edge.Source.Layer, edge.Target.Layer);
}
}
// 4. add missing loss layers
if (addLossLayer)
{
NetworkGraphBuilder.AddLossLayers(graph);
}
// 5. add missing activation layers
if (addActivationLayers)
{
NetworkGraphBuilder.AddActivationLayers(graph);
}
// 6. initialize stochastic biases with ReLU activations
NetworkGraphBuilder.InitializeReLUs(graph);
return(graph);
}
/// <summary>
/// Creates a classification neural network from a string that contains network architecture.
/// </summary>
/// <param name="architecture">The network architecture.</param>
/// <returns>The <see cref="Network"/> object this method creates.</returns>
public static Network FromArchitecture(string architecture)
{
NetworkGraph graph = NetworkGraphBuilder.CreateNetworkGraph(architecture, true, false);
return(new Network(graph));
}
/// <summary>
/// Creates a classification neural network from a string that contains network architecture.
/// </summary>
/// <param name="architecture">The network architecture.</param>
/// <param name="classes">The classes the network should able to classify into.</param>
/// <param name="allowedClasses">The classes the network is allowed to classify.</param>
/// <param name="blankClass">The blank class that represents none of the real classes.</param>
/// <returns>
/// The <see cref="ClassificationNetwork"/> object this method creates.
/// </returns>
public static ClassificationNetwork FromArchitecture(string architecture, IList <string> classes, IList <string> allowedClasses, string blankClass)
{
NetworkGraph graph = NetworkGraphBuilder.CreateNetworkGraph(architecture, true, true);
return(new ClassificationNetwork(graph, classes, allowedClasses, blankClass));
}
/// <summary>
/// Creates a graph from the specified byte array.
/// </summary>
/// <param name="buffer">The buffer to read the <see cref="NetworkGraph"/> from.</param>
/// <returns>The <see cref="NetworkGraph"/> this method creates.</returns>
public static NetworkGraph FromMemory(byte[] buffer)
{
return(NetworkGraph.FromString(UTF8Encoding.UTF8.GetString(buffer)));
}
/// <summary>
/// Creates a graph from the specified file.
/// </summary>
/// <param name="fileName">A string that contains the name of the file from which to create the <see cref="NetworkGraph"/>.</param>
/// <returns>The <see cref="NetworkGraph"/> this method creates.</returns>
public static NetworkGraph FromFile(string fileName)
{
return(NetworkGraph.FromString(File.ReadAllText(fileName, Encoding.UTF8)));
}
/// <summary>
/// Initializes a new instance of the <see cref="NetworkGraph"/> class
/// using existing graph as a source.
/// </summary>
/// <param name="other">The existing <see cref="NetworkGraph"/> to create this graph from.</param>
/// <param name="cloneLayers">The value indicating whether the layers should be cloned.</param>
public NetworkGraph(NetworkGraph other, bool cloneLayers)
: base(other, cloneLayers)
{
}
