private List <VectorBundle> DivideSamplesForForecastTask(List <double[]> predictorsCollection,
List <double[]> idealValueCollection,
int bundleSize
)
{
int numOfBundles = idealValueCollection.Count / bundleSize;
List <VectorBundle> bundleCollection = new List <VectorBundle>(numOfBundles);
//Bundles creation
int samplesPos = 0;
for (int bundleNum = 0; bundleNum < numOfBundles; bundleNum++)
{
VectorBundle bundle = new VectorBundle();
for (int i = 0; i < bundleSize && samplesPos < idealValueCollection.Count; i++)
{
bundle.InputVectorCollection.Add(predictorsCollection[samplesPos]);
bundle.OutputVectorCollection.Add(idealValueCollection[samplesPos]);
++samplesPos;
}
bundleCollection.Add(bundle);
}
//Remaining samples
for (int i = 0; i < idealValueCollection.Count - samplesPos; i++)
{
int bundleIdx = i % bundleCollection.Count;
bundleCollection[bundleIdx].InputVectorCollection.Add(predictorsCollection[samplesPos + i]);
bundleCollection[bundleIdx].OutputVectorCollection.Add(idealValueCollection[samplesPos + i]);
}
return(bundleCollection);
}
/// <summary>
/// Prepares input for Readout Layer training.
/// All input patterns are processed by internal reservoirs and the corresponding network predictors are recorded.
/// </summary>
/// <param name="patternBundle">
/// The bundle containing known sample input patterns and desired output vectors
/// </param>
/// <param name="informativeCallback">
/// Function to be called after each processed input.
/// </param>
/// <param name="userObject">
/// The user object to be passed to informativeCallback.
/// </param>
public VectorBundle InitializeAndPreprocessBundle(PatternBundle patternBundle,
PredictorsCollectionCallbackDelegate informativeCallback = null,
Object userObject = null
)
{
//Check correctness
if (_settings.InputConfig.FeedingType == CommonEnums.InputFeedingType.Continuous)
{
throw new Exception("Called incorrect version of InitializeAndPreprocessBundle function for continuous input feeding.");
}
//Reset the internal states and also statistics
Reset(true);
//Initialize normalizers and normalize input data
List <List <double[]> > nrmInputPatternCollection = NormalizeInputPatternCollection(patternBundle.InputPatternCollection);
//Allocate output bundle
VectorBundle outputBundle = new VectorBundle(patternBundle.InputPatternCollection.Count);
//Collection
for (int dataSetIdx = 0; dataSetIdx < nrmInputPatternCollection.Count; dataSetIdx++)
{
//Push input data into the network
double[] predictors = PushInput(nrmInputPatternCollection[dataSetIdx], true);
outputBundle.InputVectorCollection.Add(predictors);
//Add desired outputs
outputBundle.OutputVectorCollection.Add(patternBundle.OutputVectorCollection[dataSetIdx]);
//Informative callback
informativeCallback?.Invoke(patternBundle.InputPatternCollection.Count, dataSetIdx + 1, userObject);
}
return(outputBundle);
}
//Constructor
/// <summary>
/// Creates an instance ready to start building trained non-recurrent network
/// </summary>
/// <param name="networkName">Name of the network to be built</param>
/// <param name="networkSettings">Network configuration (FeedForwardNetworkSettings or ParallelPerceptronSettings object)</param>
/// <param name="foldNum">Current fold number</param>
/// <param name="numOfFolds">Total number of the folds</param>
/// <param name="foldNetworkNum">Current fold network number</param>
/// <param name="numOfFoldNetworks">Total number of the fold networks</param>
/// <param name="trainingBundle">Bundle of predictors and ideal values to be used for training purposes</param>
/// <param name="testingBundle">Bundle of predictors and ideal values to be used for testing purposes</param>
/// <param name="binBorder">If specified, it indicates that the whole network output is binary and specifies numeric border where GE network output is decided as a 1 and LT output as a 0.</param>
/// <param name="rand">Random generator to be used (optional)</param>
/// <param name="controller">Regression controller (optional)</param>
public TrainedNetworkBuilder(string networkName,
INonRecurrentNetworkSettings networkSettings,
int foldNum,
int numOfFolds,
int foldNetworkNum,
int numOfFoldNetworks,
VectorBundle trainingBundle,
VectorBundle testingBundle,
double binBorder = double.NaN,
Random rand = null,
RegressionControllerDelegate controller = null
)
{
_networkName = networkName;
_networkSettings = networkSettings;
_foldNum = foldNum;
_numOfFolds = numOfFolds;
_foldNetworkNum = foldNetworkNum;
_numOfFoldNetworks = numOfFoldNetworks;
//Check num of output values is 1
if (trainingBundle.OutputVectorCollection[0].Length != 1)
{
throw new InvalidOperationException($"Only single output value is allowed.");
}
_trainingBundle = trainingBundle;
_testingBundle = testingBundle;
_binBorder = binBorder;
_rand = rand ?? new Random(0);
_controller = controller ?? DefaultRegressionController;
return;
}
/// <summary>
/// Builds the internal probabilistic cluster chain and makes the "One Takes All" group operable.
/// </summary>
/// <param name="readoutUnitsResultsCollection">The collection of the collections of all readout units composite results.</param>
/// <param name="readoutUnitsIdealValuesCollection">The collection of the collections of all readout units ideal values.</param>
/// <param name="filters">The feature filters to be used to denormalize output data.</param>
/// <param name="rand">The random object to be used.</param>
/// <param name="controller">The build process controller (optional).</param>
public void Build(List <CompositeResult[]> readoutUnitsResultsCollection,
List <double[]> readoutUnitsIdealValuesCollection,
BinFeatureFilter[] filters,
Random rand,
TNRNetBuilder.BuildControllerDelegate controller = null
)
{
if (DecisionMethod != OneTakesAllDecisionMethod.ClusterChain)
{
throw new InvalidOperationException("Wrong call of the Build method.");
}
OneTakesAllClusterChainDecisionSettings decisionCfg = (OneTakesAllClusterChainDecisionSettings)_groupCfg.DecisionCfg;
//Prepare the training data bundle for the cluster chain
VectorBundle trainingDataBundle = new VectorBundle(readoutUnitsIdealValuesCollection.Count);
for (int sampleIdx = 0; sampleIdx < readoutUnitsIdealValuesCollection.Count; sampleIdx++)
{
double[] inputVector = CreateInputVector(readoutUnitsResultsCollection[sampleIdx]);
double[] outputVector = CreateOutputVector(readoutUnitsIdealValuesCollection[sampleIdx], filters);
trainingDataBundle.AddPair(inputVector, outputVector);
}
//Cluster chain builder
TNRNetClusterChainBuilder builder = new TNRNetClusterChainBuilder(Name,
decisionCfg.ClusterChainCfg,
rand,
controller
);
builder.ChainBuildProgressChanged += OnChainBuildProgressChanged;
ProbabilisticClusterChain = builder.Build(trainingDataBundle, filters);
return;
}
//Methods
/// <summary>
/// Adds a new member network and updates the cluster error statistics.
/// </summary>
/// <param name="newMemberNet">The new member network.</param>
/// <param name="scopeID">The ID of a network's scope.</param>
/// <param name="testData">The testing data bundle (unseen by the network to be added).</param>
/// <param name="filters">The filters to be used to denormalize outputs.</param>
public void AddMember(TNRNet newMemberNet, int scopeID, VectorBundle testData, FeatureFilterBase[] filters)
{
//Check the network output
if (Output != newMemberNet.Output)
{
throw new ArgumentException("Inconsistent output type of the network to be added.", "newMemberNet");
}
//Check number of outputs consistency
if (_memberNetCollection.Count > 0)
{
if (newMemberNet.Network.NumOfOutputValues != NumOfOutputs)
{
throw new ArgumentException("Number of outputs of the network differs from already clustered networks.", "newMemberNet");
}
}
//Add member to inner collection
_memberNetCollection.Add(newMemberNet);
_memberNetScopeIDCollection.Add(scopeID);
//Update cluster error statistics
for (int sampleIdx = 0; sampleIdx < testData.OutputVectorCollection.Count; sampleIdx++)
{
double[] nrmComputedValues = newMemberNet.Network.Compute(testData.InputVectorCollection[sampleIdx]);
for (int outIdx = 0; outIdx < nrmComputedValues.Length; outIdx++)
{
double naturalComputedValue = filters != null ? filters[outIdx].ApplyReverse(nrmComputedValues[outIdx]) : nrmComputedValues[outIdx];
double naturalIdealValue = filters != null ? filters[outIdx].ApplyReverse(testData.OutputVectorCollection[sampleIdx][outIdx]) : testData.OutputVectorCollection[sampleIdx][outIdx];
ErrorStats.Update(nrmComputedValues[outIdx],
testData.OutputVectorCollection[sampleIdx][outIdx],
naturalComputedValue,
naturalIdealValue
);
} //outIdx
} //sampleIdx
return;
}
/// <summary>
/// Standardizes input vectors data of given vector bundle using specified set of filters.
/// </summary>
/// <param name="bundle">A bundle of the input and output data vectors.</param>
/// <param name="filters">A set of filters to be used.</param>
protected void StandardizeInputVectors(VectorBundle bundle, FeatureFilterBase[] filters)
{
foreach (double[] vector in bundle.InputVectorCollection)
{
//Standardize all data values in the vector.
StandardizeVectorData(vector, filters);
}
return;
}
/// <summary>
/// Initializes the input encoder and its feature filters from the specified samples data bundle.
/// </summary>
/// <param name="inputBundle">Sample input data</param>
public void Initialize(VectorBundle inputBundle)
{
//Full reset
Reset();
if (_encoderCfg.FeedingCfg.FeedingType == InputFeedingType.Continuous)
{
//Continuous feeding
//Fixed lengths
_varyingDataVectorLength = inputBundle.InputVectorCollection[0].Length;
NumOfTimePoints = 1;
//Add internal inputs and initialize feature filters
UpdateFeatureFilters(AddInternalInputs(inputBundle.InputVectorCollection));
}
else
{
//Patterned feeding
FeedingPatternedSettings feedingPatternedCfg = (FeedingPatternedSettings)_encoderCfg.FeedingCfg;
//Input pattern length constraint and number of time-points
NumOfTimePoints = feedingPatternedCfg.UnificationCfg.ResamplingCfg.TargetTimePoints != ResamplingSettings.AutoTargetTimePointsNum ? feedingPatternedCfg.UnificationCfg.ResamplingCfg.TargetTimePoints : VariableNumOfTimePoints;
if (NumOfTimePoints == VariableNumOfTimePoints && (_routedVaryingFieldCollection.Count > 0 || feedingPatternedCfg.Slices > 1))
{
//because no resampling, length of external input vector must be fixed to keep consistent data
_varyingDataVectorLength = inputBundle.InputVectorCollection[0].Length - _numOfSteadyFields;
//Number of time-points must be also fixed
NumOfTimePoints = _varyingDataVectorLength / _varyingFields.Count;
}
//Convert input vectors to InputPatterns
foreach (double[] orgVector in inputBundle.InputVectorCollection)
{
//Split steady and varying data
SplitSteadyAndVaryingInputData(orgVector, out double[] steadyVector, out double[] varyingVector);
//Check length of the external varying data vector
ValidateExtInputVectorLength(varyingVector.Length);
//Convert external vector to pattern
FeedingPatternedSettings feedingCfg = (FeedingPatternedSettings)_encoderCfg.FeedingCfg;
InputPattern inputPattern = new InputPattern(varyingVector,
_encoderCfg.VaryingFieldsCfg.ExternalFieldsCfg.FieldCfgCollection.Count,
feedingCfg.VarSchema,
feedingCfg.UnificationCfg.Detrend,
feedingCfg.UnificationCfg.UnifyAmplitude,
feedingCfg.UnificationCfg.ResamplingCfg.SignalBeginThreshold,
feedingCfg.UnificationCfg.ResamplingCfg.SignalEndThreshold,
feedingCfg.UnificationCfg.ResamplingCfg.UniformTimeScale,
NumOfTimePoints == VariableNumOfTimePoints ? ResamplingSettings.AutoTargetTimePointsNum : NumOfTimePoints
);
List <double[]> inputPatternVectors = CompleteInputPattern(inputPattern);
UpdateFeatureFilters(inputPatternVectors);
}
}
//Number of routed input values
NumOfRoutedValues = _routedSteadyFieldIndexCollection.Count;
if (_routedVaryingFieldCollection.Count > 0)
{
NumOfRoutedValues += _routedVaryingFieldCollection.Count * NumOfTimePoints;
}
return;
}
/// <summary>
/// Creates training data.
/// Input vector contains 0/1 combination and output vector contains appropriate results of the AND, OR and XOR operation
/// </summary>
private VectorBundle CreateTrainingData()
{
VectorBundle trainingData = new VectorBundle();
trainingData.AddPair(new double[] { 0, 0 }, new double[] { 0, 0, 0 });
trainingData.AddPair(new double[] { 0, 1 }, new double[] { 0, 1, 1 });
trainingData.AddPair(new double[] { 1, 0 }, new double[] { 0, 1, 1 });
trainingData.AddPair(new double[] { 1, 1 }, new double[] { 1, 1, 0 });
return(trainingData);
}
/// <summary>
/// Prepares input for regression stage of State Machine training.
/// All input vectors are processed by internal reservoirs and the corresponding network predictors are recorded.
/// </summary>
/// <param name="vectorBundle">
/// The bundle containing known sample input and desired output vectors (in time order)
/// </param>
/// <param name="informativeCallback">
/// Function to be called after each processed input.
/// </param>
/// <param name="userObject">
/// The user object to be passed to informativeCallback.
/// </param>
public RegressionInput PrepareRegressionData(VectorBundle vectorBundle,
NeuralPreprocessor.PredictorsCollectionCallbackDelegate informativeCallback = null,
Object userObject = null
)
{
return(new RegressionInput(NP.InitializeAndPreprocessBundle(vectorBundle, informativeCallback, userObject),
NP.CollectStatatistics(),
NP.NumOfNeurons,
NP.NumOfInternalSynapses
));
}
//Constructor
/// <summary>
/// Creates an initialized instance
/// </summary>
/// <param name="preprocessedData">Bundle of the NeuralPreprocessor's predictors and desired ideal outputs</param>
/// <param name="reservoirStatCollection">Collection of statistics of NeuralPreprocessor's internal reservoirs</param>
/// <param name="totalNumOfNeurons">Total number of NeuralPreprocessor's neurons</param>
/// <param name="totalNumOfInternalSynapses">Total number of NeuralPreprocessor's internal synapses</param>
public RegressionInput(VectorBundle preprocessedData,
List <ReservoirStat> reservoirStatCollection,
int totalNumOfNeurons,
int totalNumOfInternalSynapses
)
{
PreprocessedData = preprocessedData;
ReservoirStatCollection = reservoirStatCollection;
TotalNumOfNeurons = totalNumOfNeurons;
TotalNumOfInternalSynapses = totalNumOfInternalSynapses;
return;
}
/// <summary>
/// Loads the specified file and executes the StateMachine training.
/// </summary>
/// <param name="stateMachine">An instance of StateMachine to be trained.</param>
/// <param name="trainingDataFileName">The name of the csv file containing the training data.</param>
/// <param name="predictionInputVector">The vector to be used for next prediction (relevant only in case of continuous feeding of the input).</param>
protected void TrainStateMachine(StateMachine stateMachine, string trainingDataFileName, out double[] predictionInputVector)
{
//Register to EpochDone event
stateMachine.RL.RLBuildProgressChanged += OnRLBuildProgressChanged;
//Load csv data
CsvDataHolder trainingCsvData = new CsvDataHolder(trainingDataFileName);
//Convert csv data to VectorBundle useable for StateMachine training
VectorBundle trainingData;
if (stateMachine.Config.NeuralPreprocessorCfg != null)
{
//Neural preprocessing is enabled
if (stateMachine.Config.NeuralPreprocessorCfg.InputEncoderCfg.FeedingCfg.FeedingType == InputEncoder.InputFeedingType.Continuous)
{
//Continuous feeding data format
trainingData = VectorBundle.Load(trainingCsvData,
stateMachine.Config.NeuralPreprocessorCfg.InputEncoderCfg.VaryingFieldsCfg.ExternalFieldsCfg.GetFieldNames(),
stateMachine.Config.ReadoutLayerCfg.OutputFieldNameCollection,
out predictionInputVector
);
}
else
{
//Patterned feeding data format
predictionInputVector = null;
trainingData = VectorBundle.Load(trainingCsvData,
stateMachine.Config.ReadoutLayerCfg.OutputFieldNameCollection.Count
);
}
//Register to PreprocessingProgressChanged event
stateMachine.NP.PreprocessingProgressChanged += OnPreprocessingProgressChanged;
}
else
{
//Neural preprocessing is bypassed
predictionInputVector = null;
trainingData = VectorBundle.Load(trainingCsvData,
stateMachine.Config.ReadoutLayerCfg.OutputFieldNameCollection.Count
);
}
//StateMachine training
StateMachine.TrainingResults trainingResults = stateMachine.Train(trainingData);
_log.Write(string.Empty);
//Report training results
_log.Write(" Training results", false);
string trainingReport = trainingResults.RegressionResults.GetTrainingResultsReport(6);
_log.Write(trainingReport);
_log.Write(string.Empty);
//Finished
return;
}
/// <summary>
/// Loads the specified file and executes the StateMachine verification.
/// </summary>
/// <param name="stateMachine">An instance of StateMachine to be verified.</param>
/// <param name="verificationDataFileName">The name of the csv file containing the verification data.</param>
/// <param name="omittedInputVector">Remaining input vector from training phase (relevant only in case of continuous feeding of the input).</param>
/// <param name="predictionInputVector">The vector to be used for next prediction (relevant only in case of continuous feeding of the input).</param>
protected void VerifyStateMachine(StateMachine stateMachine, string verificationDataFileName, double[] omittedInputVector, out double[] predictionInputVector)
{
//Load csv data
CsvDataHolder verificationCsvData = new CsvDataHolder(verificationDataFileName);
//Convert csv data to VectorBundle useable for StateMachine verification
VectorBundle verificationData;
//Check NeuralPreprocessor is configured
if (stateMachine.Config.NeuralPreprocessorCfg != null)
{
//Neural preprocessing is enabled
if (stateMachine.Config.NeuralPreprocessorCfg.InputEncoderCfg.FeedingCfg.FeedingType == InputEncoder.InputFeedingType.Continuous)
{
//Continuous input feeding
//Last known input values from training (predictionInputVector) must be pushed into the reservoirs to keep time series continuity
//(first input data in verification.csv is output of the last data in training.csv)
double[] tmp = stateMachine.Compute(omittedInputVector, out ReadoutLayer.ReadoutData readoutData);
//Load verification data and get new predictionInputVector for final prediction
verificationData = VectorBundle.Load(verificationCsvData,
stateMachine.Config.NeuralPreprocessorCfg.InputEncoderCfg.VaryingFieldsCfg.ExternalFieldsCfg.GetFieldNames(),
stateMachine.Config.ReadoutLayerCfg.OutputFieldNameCollection,
out predictionInputVector
);
}
else
{
predictionInputVector = null;
//Patterned feeding data format
verificationData = VectorBundle.Load(verificationCsvData, stateMachine.Config.ReadoutLayerCfg.OutputFieldNameCollection.Count);
}
}
else
{
//Neural preprocessing is bypassed
predictionInputVector = null;
verificationData = VectorBundle.Load(verificationCsvData, stateMachine.Config.ReadoutLayerCfg.OutputFieldNameCollection.Count);
}
//StateMachine verification
//Register to VerificationProgressChanged event
stateMachine.VerificationProgressChanged += OnVerificationProgressChanged;
StateMachine.VerificationResults verificationResults = stateMachine.Verify(verificationData);
_log.Write(string.Empty);
//Report verification results
_log.Write(" Verification results", false);
_log.Write(verificationResults.GetReport(6));
_log.Write(string.Empty);
//Finished
return;
}
/// <summary>
/// Prepares input for regression stage of State Machine training.
/// All input vectors are processed by internal reservoirs and the corresponding network predictors are recorded.
/// </summary>
/// <param name="vectorBundle">
/// The bundle containing known sample input and desired output vectors (in time order)
/// </param>
/// <param name="informativeCallback">
/// Function to be called after each processed input.
/// </param>
/// <param name="userObject">
/// The user object to be passed to informativeCallback.
/// </param>
public RegressionInput PrepareRegressionData(VectorBundle vectorBundle,
NeuralPreprocessor.PredictorsCollectionCallbackDelegate informativeCallback = null,
Object userObject = null
)
{
VectorBundle preprocessedData = NP.InitializeAndPreprocessBundle(vectorBundle, informativeCallback, userObject);
InitPredictorsGeneralSwitches(preprocessedData.InputVectorCollection);
return(new RegressionInput(preprocessedData,
NP.CollectStatatistics(),
NP.NumOfNeurons,
NP.NumOfInternalSynapses,
NumOfUnusedPredictors
));
}
/// <summary>
/// Runs the example code.
/// </summary>
public void Run()
{
//Create configuration of the feed forward network having Identity output layer and two LeakyReLU hidden layers
//with associated resilient back propagation trainer configuration
const int HiddenLayerSize = 3;
HiddenLayerSettings hiddenLayerCfg = new HiddenLayerSettings(HiddenLayerSize, new LeakyReLUSettings());
FeedForwardNetworkSettings ffNetCfg = new FeedForwardNetworkSettings(new IdentitySettings(),
new HiddenLayersSettings(hiddenLayerCfg, hiddenLayerCfg),
new RPropTrainerSettings(2, 200)
);
//Collect training data
VectorBundle trainingData = CreateTrainingData();
//Create network instance
//We specify 2 input values, 3 output values and previously prepared network structure configuration
FeedForwardNetwork ffNet = new FeedForwardNetwork(2, 3, ffNetCfg);
//Training
_log.Write("Training");
_log.Write("--------");
//Create trainer instance
RPropTrainer trainer = new RPropTrainer(ffNet,
trainingData.InputVectorCollection,
trainingData.OutputVectorCollection,
(RPropTrainerSettings)ffNetCfg.TrainerCfg,
new Random(0)
);
//Training loop
while (trainer.Iteration() && trainer.MSE > 1e-6)
{
_log.Write($" Attempt {trainer.Attempt} / Epoch {trainer.AttemptEpoch,3} Mean Squared Error = {Math.Round(trainer.MSE, 8).ToString(CultureInfo.InvariantCulture)}", false);
}
_log.Write(string.Empty);
//Training is done
//Display network computation results
_log.Write("Trained network computations:");
_log.Write("-----------------------------");
foreach (double[] input in trainingData.InputVectorCollection)
{
double[] results = ffNet.Compute(input);
_log.Write($" Input {input[0]} {input[1]} Results: AND={Math.Round(results[0])} OR={Math.Round(results[1])} XOR={Math.Round(results[2])}");
}
_log.Write(string.Empty);
//Finished
return;
} //Run
/// <summary>
/// Initializes the preprocessor, preprocess the specified data bundle and returns the predictors together with the ideal values.
/// </summary>
/// <param name="inputBundle">The data bundle to be preprocessed.</param>
/// <param name="preprocessingOverview">The statistics and other important information related to data preprocessing.</param>
public VectorBundle InitializeAndPreprocessBundle(VectorBundle inputBundle, out PreprocessingOverview preprocessingOverview)
{
//Check amount of input data
if (BootCycles > 0 && inputBundle.InputVectorCollection.Count <= BootCycles)
{
throw new InvalidOperationException($"Insufficient number of input data instances. The number of instances must be greater than the number of boot cycles ({BootCycles.ToString(CultureInfo.InvariantCulture)}).");
}
//Reset reservoirs
ResetReservoirs(true);
//Reset input encoder and initialize its feature filters
_inputEncoder.Initialize(inputBundle);
//Initialize output features descriptors
InitPredictorsDescriptors();
//Allocate output bundle
VectorBundle outputBundle = new VectorBundle(inputBundle.InputVectorCollection.Count);
//Process data
//Collect predictors
for (int dataSetIdx = 0; dataSetIdx < inputBundle.InputVectorCollection.Count; dataSetIdx++)
{
bool readyToCollect = dataSetIdx >= BootCycles || _preprocessorCfg.InputEncoderCfg.FeedingCfg.FeedingType == InputEncoder.InputFeedingType.Patterned;
//Push input data into the network
double[] outputFeatures = PushExtInputVector(inputBundle.InputVectorCollection[dataSetIdx], readyToCollect);
//Collect output features?
if (readyToCollect)
{
//Predictors
outputBundle.InputVectorCollection.Add(outputFeatures);
//Desired outputs
outputBundle.OutputVectorCollection.Add(inputBundle.OutputVectorCollection[dataSetIdx]);
}
//Raise informative event
PreprocessingProgressChanged?.Invoke(inputBundle.InputVectorCollection.Count, dataSetIdx + 1, null);
}
//Initialize output features switches
InitOutputFeaturesGeneralSwitches(outputBundle.InputVectorCollection);
//Buld preprocessing overview
preprocessingOverview = new PreprocessingOverview(CollectStatatistics(),
TotalNumOfHiddenNeurons,
PredictorDescriptorCollection.Count,
NumOfSuppressedPredictors,
NumOfActivePredictors
);
//Raise final informative event
PreprocessingProgressChanged(inputBundle.InputVectorCollection.Count, inputBundle.InputVectorCollection.Count, preprocessingOverview);
//Return output
return(outputBundle);
}
/// <summary>
/// Trains FF network to solve boolean algebra. It shows how to do it on the lowest level,
/// without use of TNRNetBuilder.
/// </summary>
private void FullyManualLearning()
{
_log.Write("Example of a FF network low level training:");
//Create FF network configuration.
FeedForwardNetworkSettings ffNetCfg = CreateFFNetConfig();
_log.Write($"Network configuration xml:");
_log.Write(ffNetCfg.GetXml(true).ToString());
//Collect training data
VectorBundle trainingData = CreateTrainingData();
//Create network instance
//We specify 2 input values, 3 output values and previously prepared network structure configuration
FeedForwardNetwork ffNet = new FeedForwardNetwork(2, //The number of input values
3, //The number of output values
ffNetCfg //Network structure and a trainer
);
//Training
_log.Write(string.Empty);
_log.Write(" Training");
_log.Write(string.Empty);
//Create the trainer instance
RPropTrainer trainer = new RPropTrainer(ffNet,
trainingData.InputVectorCollection,
trainingData.OutputVectorCollection,
(RPropTrainerSettings)ffNetCfg.TrainerCfg,
new Random(0)
);
//Training loop
while (trainer.Iteration())
{
_log.Write($" Attempt {trainer.Attempt} / Epoch {trainer.AttemptEpoch,3} Mean Squared Error = {Math.Round(trainer.MSE, 8).ToString(CultureInfo.InvariantCulture)}", true);
//Check training exit condition
if (trainer.MSE < 1e-7)
{
break;
}
}
_log.Write(string.Empty);
//Training is done
//Display the network computation results
DisplayNetworkComputations(ffNet);
//Finished
return;
}
/// <summary>
/// Builds the inner cluster chain.
/// </summary>
/// <param name="dataBundle">The data to be used for training.</param>
/// <param name="filter">The feature filter to be used to denormalize output.</param>
/// <param name="rand">The random object to be used (optional).</param>
/// <param name="controller">The build process controller (optional).</param>
public void Build(VectorBundle dataBundle,
FeatureFilterBase filter,
Random rand = null,
TNRNetBuilder.BuildControllerDelegate controller = null
)
{
rand = rand ?? new Random(0);
TNRNetClusterChainBuilder builder = new TNRNetClusterChainBuilder(Name,
_clusterChainCfg,
rand,
controller
);
builder.ChainBuildProgressChanged += OnChainBuildProgressChanged;
_clusterChain = builder.Build(dataBundle, new FeatureFilterBase[] { filter });
return;
}
private void TestDataBundleFolderization(string dataFile, int numOfClasses)
{
//Load csv data
CsvDataHolder csvData = new CsvDataHolder(dataFile);
//Convert csv data to a VectorBundle
VectorBundle vectorData = VectorBundle.Load(csvData, numOfClasses);
double binBorder = 0.5d;
double[] foldDataRatios = { -1d, 0d, 0.1d, 0.5d, 0.75d, 1d, 2d };
Console.WriteLine($"Folderization test of {dataFile}. NumOfSamples={vectorData.InputVectorCollection.Count.ToString(CultureInfo.InvariantCulture)}, NumOfFoldDataRatios={foldDataRatios.Length.ToString(CultureInfo.InvariantCulture)}");
foreach (double foldDataRatio in foldDataRatios)
{
Console.WriteLine($" Testing fold data ratio = {foldDataRatio.ToString(CultureInfo.InvariantCulture)}");
List <VectorBundle> folds = vectorData.Folderize(foldDataRatio, binBorder);
Console.WriteLine($" Number of resulting folds = {folds.Count.ToString(CultureInfo.InvariantCulture)}");
for (int foldIdx = 0; foldIdx < folds.Count; foldIdx++)
{
int numOfFoldSamples = folds[foldIdx].InputVectorCollection.Count;
Console.WriteLine($" FoldIdx={foldIdx.ToString(CultureInfo.InvariantCulture),-4} FoldSize={numOfFoldSamples.ToString(CultureInfo.InvariantCulture),-4}");
int[] classesBin1Counts = new int[numOfClasses];
classesBin1Counts.Populate(0);
for (int sampleIdx = 0; sampleIdx < numOfFoldSamples; sampleIdx++)
{
for (int classIdx = 0; classIdx < numOfClasses; classIdx++)
{
if (folds[foldIdx].OutputVectorCollection[sampleIdx][classIdx] >= binBorder)
{
++classesBin1Counts[classIdx];
}
}
}
Console.WriteLine($" Number of positive samples per class");
for (int classIdx = 0; classIdx < numOfClasses; classIdx++)
{
Console.WriteLine($" ClassID={classIdx.ToString(CultureInfo.InvariantCulture),-3}, Bin1Samples={classesBin1Counts[classIdx].ToString(CultureInfo.InvariantCulture)}");
}
}
Console.ReadLine();
}
return;
}
//Constructor
/// <summary>
/// Creates an initialized instance.
/// </summary>
/// <param name="networkName">The name of the network to be built.</param>
/// <param name="networkCfg">The configuration of the network to be built.</param>
/// <param name="networkOutput">The type of output of the network to be built.</param>
/// <param name="trainingBundle">The bundle of input and ideal vectors to be used for the network training.</param>
/// <param name="testingBundle">The bundle of input and ideal vectors to be used for the network testing.</param>
/// <param name="rand">The random generator to be used (optional).</param>
/// <param name="controller">The build process controller (optional).</param>
public TNRNetBuilder(string networkName,
INonRecurrentNetworkSettings networkCfg,
TNRNet.OutputType networkOutput,
VectorBundle trainingBundle,
VectorBundle testingBundle,
Random rand = null,
BuildControllerDelegate controller = null
)
{
_networkName = networkName;
NonRecurrentNetUtils.CheckNetCfg(networkOutput, networkCfg);
_networkCfg = networkCfg;
_networkOutput = networkOutput;
NonRecurrentNetUtils.CheckData(_networkOutput, trainingBundle);
_trainingBundle = trainingBundle;
NonRecurrentNetUtils.CheckData(_networkOutput, testingBundle);
_testingBundle = testingBundle;
_rand = rand ?? new Random(0);
_controller = controller ?? DefaultNetworkBuildController;
return;
}
/// <summary>
/// Performs the training of the state machine.
/// </summary>
/// <param name="trainingData">The training data bundle.</param>
/// <param name="controller">The build process controller (optional).</param>
/// <returns>The training results.</returns>
public TrainingResults Train(VectorBundle trainingData, TNRNetBuilder.BuildControllerDelegate controller = null)
{
//StateMachine reset
Reset();
VectorBundle readoutTrainingData;
NeuralPreprocessor.PreprocessingOverview preprocessingOverview = null;
if (NP == null)
{
//Neural preprocessor is bypassed
readoutTrainingData = trainingData;
}
else
{
//Neural preprocessing
readoutTrainingData = NP.InitializeAndPreprocessBundle(trainingData, out preprocessingOverview);
}
//Training of the readout layer
ReadoutLayer.RegressionOverview regressionOverview = RL.Build(readoutTrainingData, BuildPredictorsMapper(), controller, Config.RandomizerSeek);
//Return the training results
return(new TrainingResults(preprocessingOverview, regressionOverview));
}
/// <summary>
/// Performs training of the StateMachine
/// </summary>
/// <param name="vectorBundle">Training data bundle (input vectors and desired output vectors)</param>
/// <param name="regressionController">Optional regression controller.</param>
/// <returns>Output of the regression stage</returns>
public TrainingResults Train(VectorBundle vectorBundle, TrainedNetworkBuilder.RegressionControllerDelegate regressionController = null)
{
//StateMachine reset
Reset();
VectorBundle readoutInput;
NeuralPreprocessor.PreprocessingOverview preprocessingOverview = null;
if (NP == null)
{
//Neural preprocessor is bypassed
readoutInput = vectorBundle;
}
else
{
//Neural preprocessing
readoutInput = NP.InitializeAndPreprocessBundle(vectorBundle, out preprocessingOverview);
}
//Training of the readout layer
ReadoutLayer.RegressionOverview regressionOverview = RL.Build(readoutInput, BuildPredictorsMapper(), regressionController);
//Return compact results
return(new TrainingResults(preprocessingOverview, regressionOverview));
}
/// <summary>
/// Tests and displays network's computations.
/// </summary>
/// <param name="network">A network to be tested.</param>
private void DisplayNetworkComputations(INonRecurrentNetwork network)
{
VectorBundle verificationData = CreateTrainingData();
_log.Write(" Trained network computations:");
int sampleIdx = 0;
foreach (double[] input in verificationData.InputVectorCollection)
{
double[] results = network.Compute(input);
//Compute absolute errors
double[] absError = new double[results.Length];
for (int i = 0; i < results.Length; i++)
{
absError[i] = Math.Abs(results[i] - verificationData.OutputVectorCollection[sampleIdx][i]);
}
_log.Write($" Input {input[0]} {input[1]} Results: AND={Math.Abs(Math.Round(results[0])).ToString(CultureInfo.InvariantCulture)} OR={Math.Abs(Math.Round(results[1])).ToString(CultureInfo.InvariantCulture)} XOR={Math.Abs(Math.Round(results[2])).ToString(CultureInfo.InvariantCulture)}, Absolute Errors: {absError[0].ToString("E3", CultureInfo.InvariantCulture)} {absError[1].ToString("E3", CultureInfo.InvariantCulture)} {absError[2].ToString("E3", CultureInfo.InvariantCulture)}");
++sampleIdx;
}
_log.Write(string.Empty);
return;
}
/// <summary>
/// Verifies the state machine's accuracy.
/// </summary>
/// <remarks>
/// Evaluates the computed data against the ideal data.
/// </remarks>
/// <param name="verificationData">The verification data bundle.</param>
/// <returns>The verification results.</returns>
public VerificationResults Verify(VectorBundle verificationData)
{
VerificationResults verificationResults = new VerificationResults(Config.ReadoutLayerCfg);
for (int sampleIdx = 0; sampleIdx < verificationData.InputVectorCollection.Count; sampleIdx++)
{
double[] predictors;
if (NP == null)
{
//Neural preprocessor is bypassed
predictors = verificationData.InputVectorCollection[sampleIdx];
}
else
{
//Neural preprocessing
predictors = NP.Preprocess(verificationData.InputVectorCollection[sampleIdx]);
}
double[] outputVector = RL.Compute(predictors, out ReadoutLayer.ReadoutData readoutData);
verificationResults.Update(predictors, readoutData, verificationData.OutputVectorCollection[sampleIdx]);
VerificationProgressChanged?.Invoke(verificationData.InputVectorCollection.Count, sampleIdx + 1);
}
return(verificationResults);
}
/// <summary>
/// Trains FF network to solve boolean algebra.
/// It shows how to use the TNRNetBuilder (TNRNetBuilder is using its default build controller).
/// </summary>
private void TNRNetBuilderLearning_DefaultController()
{
_log.Write("Example of a FF network build using the TNRNetBuilder component with default build controller:");
//Create FF network configuration.
FeedForwardNetworkSettings ffNetCfg = CreateFFNetConfig();
_log.Write($"Network configuration xml:");
_log.Write(ffNetCfg.GetXml(true).ToString());
//Collect training data
VectorBundle trainingData = CreateTrainingData();
//In our case, testing data is the same as training data
VectorBundle testingData = trainingData;
//Training
//Create builder instance
TNRNetBuilder builder = new TNRNetBuilder("Boolean Algebra", //Network name
ffNetCfg, //Network configuration
TNRNet.OutputType.Real, //Network output is one or more real numbers
trainingData, //Training data
testingData, //Testing data
null, //No specific random generator object to be used
null //No specific build controller -> use default
);
//Register notification event handler
builder.NetworkBuildProgressChanged += OnNetworkBuildProgressChanged;
//Build the network
_log.Write(string.Empty);
_log.Write(" Training");
TNRNet ffNet = builder.Build();
//Training is done
_log.Write(string.Empty);
//Display the network computation results
DisplayNetworkComputations(ffNet.Network);
//Finished
return;
}
/// <summary>
/// Prepares input for Readout Layer training.
/// All input vectors are processed by internal reservoirs and the corresponding network predictors are recorded.
/// </summary>
/// <param name="vectorBundle">
/// The bundle containing known sample input and desired output vectors (in time order)
/// </param>
/// <param name="informativeCallback">
/// Function to be called after each processed input.
/// </param>
/// <param name="userObject">
/// The user object to be passed to informativeCallback.
/// </param>
public VectorBundle InitializeAndPreprocessBundle(VectorBundle vectorBundle,
PredictorsCollectionCallbackDelegate informativeCallback = null,
Object userObject = null
)
{
//Check correctness
if (_settings.InputConfig.FeedingType == CommonEnums.InputFeedingType.Patterned)
{
throw new Exception("Called incorrect version of InitializeAndPreprocessBundle function for patterned input feeding.");
}
//Reset the internal states and also statistics
Reset(true);
//Initialize normalizers and normalize input data
List <double[]> nrmInputVectorCollection = NormalizeInputVectorCollection(vectorBundle.InputVectorCollection);
//Allocate output bundle
VectorBundle outputBundle = new VectorBundle(vectorBundle.InputVectorCollection.Count - _settings.InputConfig.BootCycles);
//Collect predictors
for (int dataSetIdx = 0; dataSetIdx < vectorBundle.InputVectorCollection.Count; dataSetIdx++)
{
bool afterBoot = (dataSetIdx >= _settings.InputConfig.BootCycles);
//Push input data into the network
double[] predictors = PushInput(nrmInputVectorCollection[dataSetIdx], afterBoot);
//Is boot sequence passed? Collect predictors?
if (afterBoot)
{
//YES
//Predictors
outputBundle.InputVectorCollection.Add(predictors);
//Desired outputs
outputBundle.OutputVectorCollection.Add(vectorBundle.OutputVectorCollection[dataSetIdx]);
}
//An informative callback
informativeCallback?.Invoke(vectorBundle.InputVectorCollection.Count, dataSetIdx + 1, userObject);
}
return(outputBundle);
}
//Methods
/// <summary>
/// Prepares a set of filters for the input data standardization.
/// </summary>
/// <param name="bundle">A bundle of the input and output data vectors.</param>
protected FeatureFilterBase[] PrepareInputFeatureFilters(VectorBundle bundle)
{
//Allocation of the feature filters
FeatureFilterBase[] inputFeatureFilters = new FeatureFilterBase[bundle.InputVectorCollection[0].Length];
for (int i = 0; i < inputFeatureFilters.Length; i++)
{
//Input data is always considered as the real numbers.
inputFeatureFilters[i] = new RealFeatureFilter(Interval.IntN1P1, //Data to be standardized between -1 and 1
new RealFeatureFilterSettings(true, //We want to standardize data
true //We want to keep a range reserve for unseen data
)
);
}
//Update feature filters
foreach (double[] vector in bundle.InputVectorCollection)
{
for (int i = 0; i < vector.Length; i++)
{
//Update filter by the next known data value.
inputFeatureFilters[i].Update(vector[i]);
}
}
return(inputFeatureFilters);
}
//Methods
/// <summary>
/// Performs specified demo case.
/// </summary>
/// <param name="demoCaseParams">An instance of DemoSettings.CaseSettings to be performed</param>
public void PerformDemoCase(SMDemoSettings.CaseSettings demoCaseParams)
{
bool continuousFeedingDataFormat = false;
//Prediction input vector (relevant only for input continuous feeding)
double[] predictionInputVector = null;
//Log start
_log.Write(" Performing demo case " + demoCaseParams.Name, false);
_log.Write(" ", false);
//Instantiate the StateMachine
StateMachine stateMachine = new StateMachine(demoCaseParams.StateMachineCfg);
//////////////////////////////////////////////////////////////////////////////////////
//Train StateMachine
//Register to RegressionEpochDone event
stateMachine.RL.RegressionEpochDone += OnRegressionEpochDone;
StateMachine.TrainingResults trainingResults;
CsvDataHolder trainingCsvData = new CsvDataHolder(demoCaseParams.TrainingDataFileName);
VectorBundle trainingData;
if (trainingCsvData.ColNameCollection.NumOfStringValues > 0)
{
//Continuous feeding data format
continuousFeedingDataFormat = true;
//Check NeuralPreprocessor is not bypassed
if (stateMachine.NP == null)
{
throw new InvalidOperationException($"Incorrect file format. When NeuralPreprocessor is bypassed, only patterned data are allowed.");
}
trainingData = VectorBundle.Load(trainingCsvData,
demoCaseParams.StateMachineCfg.NeuralPreprocessorCfg.InputEncoderCfg.VaryingFieldsCfg.ExternalFieldsCfg.GetFieldNames(),
demoCaseParams.StateMachineCfg.ReadoutLayerCfg.OutputFieldNameCollection,
out predictionInputVector
);
}
else
{
//Patterned feeding data format
trainingData = VectorBundle.Load(trainingCsvData, demoCaseParams.StateMachineCfg.ReadoutLayerCfg.OutputFieldNameCollection.Count);
}
if (stateMachine.NP != null)
{
//Register to PreprocessingProgressChanged event
stateMachine.NP.PreprocessingProgressChanged += OnPreprocessingProgressChanged;
}
//Training
trainingResults = stateMachine.Train(trainingData);
_log.Write(string.Empty);
//Report training (regression) results
_log.Write(" Training results", false);
string trainingReport = trainingResults.RegressionResults.GetTrainingResultsReport(6);
_log.Write(trainingReport);
_log.Write(string.Empty);
//////////////////////////////////////////////////////////////////////////////////////
//Verification of training quality on verification data
if (demoCaseParams.VerificationDataFileName.Length > 0)
{
stateMachine.VerificationProgressChanged += OnVerificationProgressChanged;
StateMachine.VerificationResults verificationResults;
CsvDataHolder verificationCsvData = new CsvDataHolder(demoCaseParams.VerificationDataFileName);
VectorBundle verificationData;
if (continuousFeedingDataFormat)
{
//Continuous input feeding
//Last known input values from training (predictionInputVector) must be pushed into the reservoirs to keep time series continuity
//(first input data in verification.csv is output of the last data in training.csv)
double[] tmp = stateMachine.Compute(predictionInputVector);
//Load verification data and get new predictionInputVector for final prediction
verificationData = VectorBundle.Load(verificationCsvData,
demoCaseParams.StateMachineCfg.NeuralPreprocessorCfg.InputEncoderCfg.VaryingFieldsCfg.ExternalFieldsCfg.GetFieldNames(),
demoCaseParams.StateMachineCfg.ReadoutLayerCfg.OutputFieldNameCollection,
out predictionInputVector
);
}
else
{
//Patterned feeding data format
verificationData = VectorBundle.Load(verificationCsvData, demoCaseParams.StateMachineCfg.ReadoutLayerCfg.OutputFieldNameCollection.Count);
}
verificationResults = stateMachine.Verify(verificationData);
_log.Write(string.Empty);
//Report verification results
_log.Write(" Verification results", false);
_log.Write(verificationResults.GetReport(6));
_log.Write(string.Empty);
}
//Perform prediction in case the input feeding is continuous (we know the input but we don't know the ideal output)
if (continuousFeedingDataFormat)
{
double[] predictionOutputVector = stateMachine.Compute(predictionInputVector);
string predictionReport = stateMachine.RL.GetForecastReport(predictionOutputVector, 6);
_log.Write(" Forecasts", false);
_log.Write(predictionReport);
_log.Write(string.Empty);
}
return;
}
/// <summary>
/// Builds the cluster chain.
/// </summary>
/// <param name="dataBundle">The data bundle for training.</param>
/// <param name="filters">The filters to be used to denormalize outputs.</param>
public TNRNetClusterChain Build(VectorBundle dataBundle, FeatureFilterBase[] filters)
{
//The chain to be built
TNRNetClusterChain chain = new TNRNetClusterChain(_chainName, _clusterChainCfg.Output);
//Instantiate chained clusters
List <TNRNetCluster> chainClusters = new List <TNRNetCluster>(_clusterChainCfg.ClusterCfgCollection.Count);
for (int clusterIdx = 0; clusterIdx < _clusterChainCfg.ClusterCfgCollection.Count; clusterIdx++)
{
//Cluster
chainClusters.Add(new TNRNetCluster(_chainName,
_clusterChainCfg.ClusterCfgCollection[clusterIdx].Output,
_clusterChainCfg.ClusterCfgCollection[clusterIdx].TrainingGroupWeight,
_clusterChainCfg.ClusterCfgCollection[clusterIdx].TestingGroupWeight,
_clusterChainCfg.ClusterCfgCollection[clusterIdx].SamplesWeight,
_clusterChainCfg.ClusterCfgCollection[clusterIdx].NumericalPrecisionWeight,
_clusterChainCfg.ClusterCfgCollection[clusterIdx].MisrecognizedFalseWeight,
_clusterChainCfg.ClusterCfgCollection[clusterIdx].UnrecognizedTrueWeight
)
);
}
//Common crossvalidation configuration
double boolBorder = _clusterChainCfg.Output == TNRNet.OutputType.Real ? double.NaN : chain.OutputDataRange.Mid;
VectorBundle localDataBundle = dataBundle.CreateShallowCopy();
//Member's training
ResetProgressTracking();
for (_repetitionIdx = 0; _repetitionIdx < _clusterChainCfg.CrossvalidationCfg.Repetitions; _repetitionIdx++)
{
//Split data to folds
List <VectorBundle> foldCollection = localDataBundle.Folderize(_clusterChainCfg.CrossvalidationCfg.FoldDataRatio, boolBorder);
_numOfFoldsPerRepetition = Math.Min(_clusterChainCfg.CrossvalidationCfg.Folds <= 0 ? foldCollection.Count : _clusterChainCfg.CrossvalidationCfg.Folds, foldCollection.Count);
List <VectorBundle> currentClusterFoldCollection = CopyFolds(foldCollection);
List <VectorBundle> nextClusterFoldCollection = new List <VectorBundle>(foldCollection.Count);
//For each cluster
for (_clusterIdx = 0; _clusterIdx < chainClusters.Count; _clusterIdx++)
{
//Train networks for each testing fold.
for (_testingFoldIdx = 0; _testingFoldIdx < _numOfFoldsPerRepetition; _testingFoldIdx++)
{
//Prepare training data bundle
VectorBundle trainingData = new VectorBundle();
for (int foldIdx = 0; foldIdx < currentClusterFoldCollection.Count; foldIdx++)
{
if (foldIdx != _testingFoldIdx)
{
trainingData.Add(currentClusterFoldCollection[foldIdx]);
}
}
VectorBundle nextClusterUpdatedDataFold = foldCollection[_testingFoldIdx].CreateShallowCopy();
for (_netCfgIdx = 0; _netCfgIdx < _clusterChainCfg.ClusterCfgCollection[_clusterIdx].ClusterNetConfigurations.Count; _netCfgIdx++)
{
TNRNetBuilder netBuilder = new TNRNetBuilder(_chainName,
_clusterChainCfg.ClusterCfgCollection[_clusterIdx].ClusterNetConfigurations[_netCfgIdx],
_clusterChainCfg.ClusterCfgCollection[_clusterIdx].Output,
trainingData,
currentClusterFoldCollection[_testingFoldIdx],
_rand,
_controller
);
//Register notification
netBuilder.NetworkBuildProgressChanged += OnNetworkBuildProgressChanged;
//Build trained network. Trained network becomes to be the cluster member
TNRNet tn = netBuilder.Build();
int netScopeID = _repetitionIdx * NetScopeDelimiterCoeff + _testingFoldIdx;
chainClusters[_clusterIdx].AddMember(tn, netScopeID, currentClusterFoldCollection[_testingFoldIdx], filters);
//Update input data in the data fold for the next cluster
for (int sampleIdx = 0; sampleIdx < currentClusterFoldCollection[_testingFoldIdx].InputVectorCollection.Count; sampleIdx++)
{
double[] computedNetData = tn.Network.Compute(currentClusterFoldCollection[_testingFoldIdx].InputVectorCollection[sampleIdx]);
nextClusterUpdatedDataFold.InputVectorCollection[sampleIdx] = nextClusterUpdatedDataFold.InputVectorCollection[sampleIdx].Concat(computedNetData);
}
}//netCfgIdx
//Add updated data fold for the next cluster
nextClusterFoldCollection.Add(nextClusterUpdatedDataFold);
}//testingFoldIdx
//Switch fold collection
currentClusterFoldCollection = nextClusterFoldCollection;
nextClusterFoldCollection = new List <VectorBundle>(currentClusterFoldCollection.Count);
}//clusterIdx
if (_repetitionIdx < _clusterChainCfg.CrossvalidationCfg.Repetitions - 1)
{
//Reshuffle the data
localDataBundle.Shuffle(_rand);
}
}//repetitionIdx
//Make the clusters operable and add them into the chain
for (int clusterIdx = 0; clusterIdx < chainClusters.Count; clusterIdx++)
{
chainClusters[clusterIdx].FinalizeCluster();
chain.AddCluster(chainClusters[clusterIdx]);
}
//Return the built chain
return(chain);
}
/// <summary>
/// Checks whether the data is in accordance with the specified type of the network output.
/// </summary>
/// <param name="outputType">The type of the network output.</param>
/// <param name="data">The data to be checked.</param>
public static void CheckData(TNRNet.OutputType outputType, VectorBundle data)
{
//General checks
if (data.InputVectorCollection.Count != data.OutputVectorCollection.Count)
{
throw new ArgumentException($"Incorrect data. Different number of input and output vectors.", "data");
}
if (data.OutputVectorCollection.Count < 2)
{
throw new ArgumentException($"Too few samples.", "data");
}
int outputVectorLength = data.OutputVectorCollection[0].Length;
//Output vector length checks
if (outputType == TNRNet.OutputType.Probabilistic)
{
if (outputVectorLength <= 1)
{
throw new ArgumentException($"Number of output vector values must be GT 1.", "data");
}
}
else if (outputType == TNRNet.OutputType.SingleBool)
{
if (outputVectorLength != 1)
{
throw new ArgumentException($"Number of output vector values must be ET 1.", "data");
}
}
//Data scan
foreach (double[] outputVector in data.OutputVectorCollection)
{
if (outputVectorLength != outputVector.Length)
{
throw new ArgumentException($"Inconsistent length of output vectors.", "data");
}
switch (outputType)
{
case TNRNet.OutputType.Probabilistic:
{
int bin1Counter = 0;
int bin0Counter = 0;
for (int i = 0; i < outputVector.Length; i++)
{
if (outputVector[i] == 0d)
{
++bin0Counter;
}
else if (outputVector[i] == 1d)
{
++bin1Counter;
}
else
{
throw new ArgumentException($"Output data vectors contain different values than 0 or 1.", "data");
}
}
if (bin1Counter != 1)
{
throw new ArgumentException($"Output data vector contains more than one 1.", "data");
}
}
break;
case TNRNet.OutputType.SingleBool:
{
}
break;
default:
break;
}
}
return;
}
/// <summary>
/// Trains the network cluster to perform classification task and then verifies its performance.
/// </summary>
/// <param name="name">The name of a classification task.</param>
/// <param name="trainDataFile">The name of a csv datafile containing the training data.</param>
/// <param name="verifyDataFile">The name of a csv datafile containing the verification data.</param>
/// <param name="numOfClasses">The number of classes.</param>
/// <param name="foldDataRatio">Specifies what part of training data is reserved for testing. It determines the size of data fold and also number of networks within the cluster.</param>
private void PerformClassification(string name, string trainDataFile, string verifyDataFile, int numOfClasses, double foldDataRatio)
{
_log.Write($"{name} classification performed by the Probabilistic cluster chain ({numOfClasses.ToString(CultureInfo.InvariantCulture)} classes).");
//Load csv data and create vector bundles
_log.Write($"Loading {trainDataFile}...");
CsvDataHolder trainCsvData = new CsvDataHolder(trainDataFile);
VectorBundle trainData = VectorBundle.Load(trainCsvData, numOfClasses);
_log.Write($"Loading {verifyDataFile}...");
CsvDataHolder verifyCsvData = new CsvDataHolder(verifyDataFile);
VectorBundle verifyData = VectorBundle.Load(verifyCsvData, numOfClasses);
//Input data standardization
//Allocation and preparation of the input feature filters
FeatureFilterBase[] inputFeatureFilters = PrepareInputFeatureFilters(trainData);
//Standardize training input data
StandardizeInputVectors(trainData, inputFeatureFilters);
//Standardize verification input data
StandardizeInputVectors(verifyData, inputFeatureFilters);
//Output data
//Output data is already in the 0/1 form requested by the SoftMax activation so we don't
//need to modify it. We only allocate the binary feature filters requested by the cluster chain builder.
FeatureFilterBase[] outputFeatureFilters = new BinFeatureFilter[numOfClasses];
for (int i = 0; i < numOfClasses; i++)
{
outputFeatureFilters[i] = new BinFeatureFilter(Interval.IntZP1);
}
//Cluster chain configuration (we will have two chained clusters)
//Configuration of the first cluster in the chain
//End-networks configuration for the first cluster in the chain. For every testing fold will be trained two end-networks with different structure.
List <FeedForwardNetworkSettings> netCfgs1 = new List <FeedForwardNetworkSettings>
{
//The first FF network will have two hidden layers of 30 TanH activated neurons.
//Output layer will have the SoftMax activation (it must be SoftMax because we will use the Probabilistic cluster).
new FeedForwardNetworkSettings(new AFAnalogSoftMaxSettings(),
new HiddenLayersSettings(new HiddenLayerSettings(30, new AFAnalogTanHSettings()),
new HiddenLayerSettings(30, new AFAnalogTanHSettings())
),
new RPropTrainerSettings(3, 200)
),
//The second FF network will have two hidden layers of 30 LeakyReLU activated neurons.
//Output layer will have the SoftMax activation (it must be SoftMax because we will use the Probabilistic cluster).
new FeedForwardNetworkSettings(new AFAnalogSoftMaxSettings(),
new HiddenLayersSettings(new HiddenLayerSettings(30, new AFAnalogLeakyReLUSettings()),
new HiddenLayerSettings(30, new AFAnalogLeakyReLUSettings())
),
new RPropTrainerSettings(3, 200)
)
};
//The first probabilistic network cluster configuration instance
TNRNetClusterProbabilisticSettings clusterCfg1 =
new TNRNetClusterProbabilisticSettings(new TNRNetClusterProbabilisticNetworksSettings(netCfgs1),
new TNRNetClusterProbabilisticWeightsSettings()
);
//Configuration of the second cluster in the chain
//End-network configuration for the second cluster in the chain. For every testing fold will be trained one end-network.
List <FeedForwardNetworkSettings> netCfgs2 = new List <FeedForwardNetworkSettings>
{
//FF network will have two hidden layers of 30 Elliot activated neurons.
//Output layer will have the SoftMax activation (it must be SoftMax because we will use the Probabilistic cluster chain).
new FeedForwardNetworkSettings(new AFAnalogSoftMaxSettings(),
new HiddenLayersSettings(new HiddenLayerSettings(30, new AFAnalogElliotSettings()),
new HiddenLayerSettings(30, new AFAnalogElliotSettings())
),
new RPropTrainerSettings(3, 200)
)
};
//The second probabilistic network cluster configuration instance
TNRNetClusterProbabilisticSettings clusterCfg2 =
new TNRNetClusterProbabilisticSettings(new TNRNetClusterProbabilisticNetworksSettings(netCfgs2),
new TNRNetClusterProbabilisticWeightsSettings()
);
//Probabilistic network cluster chain configuration instance
ITNRNetClusterChainSettings chainCfg =
new TNRNetClusterChainProbabilisticSettings(new CrossvalidationSettings(foldDataRatio),
new TNRNetClustersProbabilisticSettings(clusterCfg1,
clusterCfg2
)
);
_log.Write($"Cluster configuration xml:");
_log.Write(chainCfg.GetXml(true).ToString());
//Training
_log.Write($"Cluster chain training on {trainDataFile}...");
//An instance of network cluster chain builder.
TNRNetClusterChainBuilder builder =
new TNRNetClusterChainBuilder("Probabilistic Cluster Chain", chainCfg);
//Register progress event handler
builder.ChainBuildProgressChanged += OnClusterChainBuildProgressChanged;
//Build the trained network cluster chain.
TNRNetClusterChain trainedClusterChain = builder.Build(trainData, outputFeatureFilters);
//Verification
_log.Write(string.Empty);
_log.Write(string.Empty);
_log.Write($"Cluster chain verification on {verifyDataFile}...");
_log.Write(string.Empty);
int numOfErrors = 0;
for (int i = 0; i < verifyData.InputVectorCollection.Count; i++)
{
double[] computed = trainedClusterChain.Compute(verifyData.InputVectorCollection[i], out _);
//Cluster result
int computedWinnerIdx = computed.MaxIdx();
//Real result
int realWinnerIdx = verifyData.OutputVectorCollection[i].MaxIdx();
if (computedWinnerIdx != realWinnerIdx)
{
++numOfErrors;
}
_log.Write($"({i + 1}/{verifyData.InputVectorCollection.Count}) Errors: {numOfErrors}", true);
}
_log.Write(string.Empty);
_log.Write($"Accuracy {(1d - (double)numOfErrors / (double)verifyData.InputVectorCollection.Count).ToString(CultureInfo.InvariantCulture)}");
_log.Write(string.Empty);
return;
}