/// <summary>
/// For outputting the final cup results
/// </summary>
private static void TrainAndPRoduceFinalResult()
{
AA1_MLP.DataManagers.CupDataManager dm = new AA1_MLP.DataManagers.CupDataManager();
DataSet trainDS = dm.LoadData(@"D:\dropbox\Dropbox\Master Course\SEM-3\ML\CM_CUP_Datasets\ML-17-PRJ lecture package-20171225\ML-CUP17-TR.csv", 10, 2, skip: 1, standardize: true);
DataSet FinalTestDS = dm.LoadData(@"D:\dropbox\Dropbox\Master Course\SEM-3\ML\CM_CUP_Datasets\ML-17-PRJ lecture package-20171225\ML-CUP17-TS.csv", 10, skip: 1, reportOsutput: false, standardize: true);
/*AdamParams passedParams = new AdamParams();
* IOptimizer trainer = new Adam();*/
GradientDescentParams passedParams = new GradientDescentParams();
Gradientdescent trainer = new Gradientdescent();
passedParams.numberOfEpochs = 5000;
passedParams.batchSize = 10;
passedParams.trainingSet = trainDS;
passedParams.learningRate = 0.001;
passedParams.regularization = Regularizations.L2;
passedParams.regularizationRate = 0.001;
passedParams.nestrov = true;
passedParams.resilient = false;
passedParams.resilientUpdateAccelerationRate = 2;
passedParams.resilientUpdateSlowDownRate = 0.5;
passedParams.momentum = 0.5;
passedParams.NumberOfHiddenUnits = 100;
passedParams.trueThreshold = null;
string path = "cupTrain" + passedParams.NumberOfHiddenUnits + "_lr" + passedParams.learningRate + "_reg" + passedParams.regularizationRate;
//building the architecture
Network n = new Network(new List <Layer>()
{
new Layer(new ActivationIdentity(), true, 10),
new Layer(new ActivationTanh(), true, passedParams.NumberOfHiddenUnits),
// new Layer(new ActivationLeakyRelu(),true,40),
new Layer(new ActivationIdentity(), false, 2),
}, false, AA1_MLP.Enums.WeightsInitialization.Xavier);
passedParams.network = n;
var watch = System.Diagnostics.Stopwatch.StartNew();
List <double[]> learningCurve = trainer.Train(passedParams);
watch.Stop();
var elapsedMs = watch.ElapsedMilliseconds;
Console.WriteLine("elapsed Time:{0} ms", elapsedMs);
File.WriteAllText(path + ".txt", string.Join("\n", learningCurve.Select(s => string.Join(",", s))));
ModelManager.SaveNetowrk(n, path + ".n");
var predictions = ModelManager.GeneratorCUP(FinalTestDS, n);
File.WriteAllText("OMG_LOC-OSM2-TS.txt", string.Join("\n", predictions.Select(s => string.Join(",", s))));
}
private static List <double[]> TrainWithSGD(Network n, DataSet ds, DataSet dt)
{
Gradientdescent br = new Gradientdescent();
//Calling the Train method of the trainer with the desired parameters
//n, ds, learningRate: .3, numberOfEpochs: 200, shuffle: false, debug: n.Debug, nestrov:false, momentum:0.9, resilient: false, resilientUpdateAccelerationRate: 0.3,
//resilientUpdateSlowDownRate: 0.1, regularization: AA1_MLP.Enums.Regularizations.L2, regularizationRate: 0.001, validationSet: dt, batchSize: 7
GradientDescentParams passedParams = new GradientDescentParams();
passedParams.network = n;
passedParams.trainingSet = ds;
passedParams.learningRate = 0.8;
passedParams.numberOfEpochs = 100;
passedParams.shuffle = false;
passedParams.debug = n.Debug;
passedParams.nestrov = false;
passedParams.momentum = 0.7;
passedParams.resilient = false;
passedParams.resilientUpdateAccelerationRate = 0.3;
passedParams.resilientUpdateSlowDownRate = 0.1;
passedParams.regularization = Regularizations.L2;
passedParams.regularizationRate = 0.001;
passedParams.validationSet = dt;
passedParams.batchSize = null;
var learningCurve = br.Train(passedParams);
return(learningCurve);
}
public void ScreenGD(AA1_MLP.Entities.DataSet wholeSet, int k, List <double> momentums, List <double> learningRates, List <double> regularizationRates, List <int> humberOfHiddenNeurons, GradientDescentParams passedParams, int numOfEpochs)
{
//Calling the Train method of the trainer with the desired parameters
//n, ds, learningRate: .3, numberOfEpochs: 200, shuffle: false, debug: n.Debug, nestrov:false, momentum:0.9, resilient: false, resilientUpdateAccelerationRate: 0.3,
//resilientUpdateSlowDownRate: 0.1, regularization: AA1_MLP.Enums.RegularizationRates.L2, regularizationRate: 0.001, validationSet: dt, batchSize: 7
string reportsDirectory = "80SGDKFoldsReportsnonestrov";
if (Directory.Exists(reportsDirectory))
{
Directory.Delete(reportsDirectory, true);
}
Directory.CreateDirectory(reportsDirectory);
/*List<double> momentums = new List<double> { 0, 0.5 };
* List<double> learningRates = new List<double> { 0.005, 0.01 };
* List<double> regularizationRates = new List<double> { 0, 0.001 };
* List<int> humberOfHiddenNeurons = new List<int> { 100, 90, 80, 70, 60, 50, 40, 30, 20, 10 };*/
// GradientDescentParams passedParams = new GradientDescentParams();
IOptimizer trainer = new Gradientdescent();
//AdamParams passedParams = new AdamParams();
//IOptimizer trainer = new Adam();
passedParams.numberOfEpochs = numOfEpochs;
passedParams.batchSize = 10;
for (int idxnh = 0; idxnh < humberOfHiddenNeurons.Count; idxnh++)
{
for (int idxmo = 0; idxmo < momentums.Count; idxmo++)
{
for (int idxLR = 0; idxLR < learningRates.Count; idxLR++)
{
for (int idxReg = 0; idxReg < regularizationRates.Count; idxReg++)
{
int nh = humberOfHiddenNeurons[idxnh];
passedParams.learningRate = learningRates[idxLR];
passedParams.regularization = Regularizations.L2;
passedParams.regularizationRate = regularizationRates[idxReg];
passedParams.momentum = momentums[idxmo];
passedParams.NumberOfHiddenUnits = nh;
RunKFoldWithSetOfParams(wholeSet, k, passedParams, trainer, reportsDirectory);
}
}
}
}
}
public void Screen(DataSet wholeSet, int k = 0)
{
//wholeSet.Inputs = wholeSet.Inputs.NormalizeColumns(2) ;
// wholeSet.Labels = wholeSet.Labels/10;
//standardiing data
//x=(x-mean)/std
/* for (int idxdataFold = 0; idxdataFold < wholeSet.Inputs.ColumnCount; idxdataFold++)
* {
* double mean = wholeSet.Inputs.Column(idxdataFold).Average();
* double std = Math.Sqrt((wholeSet.Inputs.Column(idxdataFold) - mean).PointwisePower(2).Sum() / wholeSet.Inputs.Column(idxdataFold).Count);
* wholeSet.Inputs.SetColumn(idxdataFold, (wholeSet.Inputs.Column(idxdataFold) - mean) / std);
*
*
* }*/
/* for (int idxdataFold = 0; idxdataFold < wholeSet.Inputs.ColumnCount; idxdataFold++)
* {
* double min = wholeSet.Inputs.Column(idxdataFold).Min();
* double max = wholeSet.Inputs.Column(idxdataFold).Max();
* double max_min = max - min;
* wholeSet.Inputs.SetColumn(idxdataFold, (wholeSet.Inputs.Column(idxdataFold) - min) / max_min);
*
* }*/
//the training set split
int trainSplit = (int)(0.6 * wholeSet.Inputs.RowCount);
DataSet TrainDataset = new DataSet(
inputs: wholeSet.Inputs.SubMatrix(0, trainSplit, 0, wholeSet.Inputs.ColumnCount),
labels: wholeSet.Labels.SubMatrix(0, trainSplit, 0, wholeSet.Labels.ColumnCount));
//the validation set
DataSet ValidationSplit = new DataSet(
inputs: wholeSet.Inputs.SubMatrix(trainSplit, (wholeSet.Inputs.RowCount - trainSplit) / 2, 0, wholeSet.Inputs.ColumnCount),
labels: wholeSet.Labels.SubMatrix(trainSplit, (wholeSet.Inputs.RowCount - trainSplit) / 2, 0, wholeSet.Labels.ColumnCount));
//the hold out set for reporting the MEE of the model on the data
DataSet TestDatasetSplit = new DataSet(
inputs: wholeSet.Inputs.SubMatrix(trainSplit + (wholeSet.Inputs.RowCount - trainSplit) / 2, (int)Math.Ceiling((double)(wholeSet.Inputs.RowCount - trainSplit) / 2), 0, wholeSet.Inputs.ColumnCount),
labels: wholeSet.Labels.SubMatrix(trainSplit + (wholeSet.Inputs.RowCount - trainSplit) / 2, (int)Math.Ceiling((double)(wholeSet.Inputs.RowCount - trainSplit) / 2), 0, wholeSet.Labels.ColumnCount));
Gradientdescent bp = new Gradientdescent();
//will hold a number of possible values for the hidden units to try
List <int> PossibleHiddenUnits = new List <int>();
for (int numberOfUnits = 10; numberOfUnits <= 70; numberOfUnits += 10)
{
PossibleHiddenUnits.Add(numberOfUnits);
}
//holds different values for the Regularization to try
List <double> RegularizationRates = new List <double>()
{
0.001, 0.0001
};
//holds different values for the momentum to try for training
List <double> Momentums = new List <double>()
{
0.5
};
//holds different values for the learning rate to try for training
List <double> learningRate = new List <double>()
{
0.001, 0.001, 0.0001
};
//these directories will hold the experiments results
Directory.CreateDirectory("learningCurves");
Directory.CreateDirectory("scatters");
Directory.CreateDirectory("models");
// a simple resume mechanism, in case the search was interrupted
if (!File.Exists("passed.txt"))
{
File.WriteAllText("passed.txt", "0,0,0,0\n");
}
var init = File.ReadLines("passed.txt").Last().Split(',');
int vs = int.Parse(init[0]), bs = int.Parse(init[1]), ms = int.Parse(init[2]), us = int.Parse(init[3]);
// File.Delete("passed.txt");
for (int v = vs; v < PossibleHiddenUnits.Count; v++)
{
var hidn = PossibleHiddenUnits[v];
for (int b = bs; b < RegularizationRates.Count; b++)
{
var reg = RegularizationRates[b];
for (int m = ms; m < Momentums.Count; m++)
{
var mo = Momentums[m];
for (int u = us; u < learningRate.Count; u++)
{
var lr = learningRate[u];
string pre = string.Format("hidn{0}_reg{1}_mo{2}_lr{3}", hidn, reg, mo, lr);
//building the architecture
Network n = new Network(new List <Layer>()
{
new Layer(new ActivationIdentity(), true, 10),
new Layer(new ActivationTanh(), true, hidn),
// new Layer(new ActivationLeakyRelu(),true,40),
new Layer(new ActivationIdentity(), false, 2),
}, false, AA1_MLP.Enums.WeightsInitialization.Xavier);
/*try
* {*/
/* Gradientdescent br = new Gradientdescent();
*
* //Calling the Train method of the trainer with the desired parameters
* //n, ds, learningRate: .3, numberOfEpochs: 200, shuffle: false, debug: n.Debug, nestrov:false, momentum:0.9, resilient: false, resilientUpdateAccelerationRate: 0.3,
* //resilientUpdateSlowDownRate: 0.1, regularization: AA1_MLP.Enums.RegularizationRates.L2, regularizationRate: 0.001, validationSet: dt, batchSize: 7
* GradientDescentParams passedParams = new GradientDescentParams();
* passedParams.network = n;
* passedParams.trainingSet = TrainDataset;
* passedParams.learningRate = lr;
* passedParams.numberOfEpochs = 10000;
* passedParams.shuffle = false;
* passedParams.debug = n.Debug;
* passedParams.nestrov = true;
* passedParams.momentum = mo;
* passedParams.resilient = false;
* passedParams.resilientUpdateAccelerationRate = 0.3;
* passedParams.resilientUpdateSlowDownRate = 0.1;
* passedParams.regularization = Regularizations.L2;
* passedParams.regularizationRate = reg;
* passedParams.validationSet = TestDatasetSplit;
* passedParams.batchSize = 50;
* passedParams.MEE = true;
*
*
*
* var learningCurve = br.Train(passedParams);
*/
Adam br = new Adam();
//Calling the Train method of the trainer with the desired parameters
//n, ds, learningRate: .3, numberOfEpochs: 200, shuffle: false, debug: n.Debug, nestrov:false, momentum:0.9, resilient: false, resilientUpdateAccelerationRate: 0.3,
//resilientUpdateSlowDownRate: 0.1, regularization: AA1_MLP.Enums.RegularizationRates.L2, regularizationRate: 0.001, validationSet: dt, batchSize: 7
AdamParams passedParams = new AdamParams();
passedParams.network = n;
passedParams.trainingSet = TrainDataset;
passedParams.learningRate = lr;
passedParams.numberOfEpochs = 10000;
passedParams.shuffle = true;
passedParams.debug = n.Debug;
passedParams.regularization = Regularizations.L2;
passedParams.regularizationRate = reg;
passedParams.validationSet = TestDatasetSplit;
passedParams.batchSize = 50;
var learningCurve = br.Train(passedParams);
////the training loop
// var learningCurve = bp.Train(n,
// TrainDataset,
// lr,
// 100000,
// true,
// regularizationRate: reg,
// regularization: AA1_MLP.Enums.RegularizationRates.L2,
// momentum: mo,
// validationSet: TestDatasetSplit,
// MEE: true
// /* resilient: true, resilientUpdateAccelerationRate: 10, resilientUpdateSlowDownRate: 1,
// reduceLearningRate: true,
// learningRateReduction: 0.8,
// numberOfReductions: 3,
// learningRateReductionAfterEpochs: 7500*/
// );
//writing the learning curve data to desk (ugly for memory, but simple)
File.WriteAllText("learningCurves/" + pre + "learningCurve.txt", string.Join("\n", learningCurve.Select(s => string.Join(",", s))));
//saving the trained model
AA1_MLP.Utilities.ModelManager.SaveNetowrk(n, "models/" + pre + "_model.AA1");
// var n = AA1_MLP.Utilities.ModelManager.LoadNetwork("model.AA1");
double MEE = 0, MSE = 0;
var log = ModelManager.TesterCUPRegression(TestDatasetSplit, n, out MEE, out MSE);
//reporting the scatter plot of the output against the actual predictions on the held out dataset split
File.WriteAllText("scatters/" + pre + "scatter.txt", string.Join("\n", log.Select(s => string.Join(",", s))));
File.AppendAllText("MEEs.txt", pre + ":" + MEE + "\n");
Console.WriteLine(MEE);
/* }
* catch (Exception e)
* {
* Console.WriteLine(pre + " Failed!");
* File.AppendAllText("fails.txt", pre + "\n");
*
* Console.WriteLine(e.Message);
* }*/
us = (u + 1) % learningRate.Count == 0 ? 0 : us;
File.AppendAllText("passed.txt", string.Format("{0},{1},{2},{3}\n", (((u + 1) % learningRate.Count == 0) && ((m + 1) % Momentums.Count == 0) && ((b + 1) % RegularizationRates.Count == 0)) ? v + 1 : v, ((((u + 1) % learningRate.Count == 0) && (m + 1) % Momentums.Count == 0) ? b + 1 : b) % RegularizationRates.Count, ((u + 1) % learningRate.Count == 0 ? m + 1 : m) % Momentums.Count, (u + 1) % learningRate.Count));
}
ms = (m + 1) % Momentums.Count == 0 ? 0 : ms;
}
bs = (b + 1) % RegularizationRates.Count == 0 ? 0 : bs;
}
}
}
