static void Main(string[] args) { //Loading and parsing cup dataset /* CupDataManager dm = new CupDataManager(); * DataSet wholeSet = dm.LoadData(Properties.Settings.Default.TrainingSetLocation, 10, 2, permute: true, seed: 1); * List<double> momentums = new List<double> { 0, 0.5 }; * List<double> learningRates = new List<double> { 0.005, 0.01 }; * List<double> regularizationRates = new List<double> { 0, 0.001 }; * List<int> humberOfHiddenNeurons = new List<int> { 80 }; * //screening SGD+Momentum experiments * GradientDescentParams passedParams = new GradientDescentParams(); * passedParams.nestrov = false; * passedParams.resilient = false; * passedParams.resilientUpdateAccelerationRate = 0.3; * passedParams.resilientUpdateSlowDownRate = 0.1; * new KFoldValidation().ScreenGD(wholeSet, 5, momentums, learningRates, regularizationRates, humberOfHiddenNeurons, passedParams,5000);*/ //screening Adam //new KFoldValidation().ScreenAdam(wholeSet, 5, learningRates, regularizationRates, humberOfHiddenNeurons, 5000); //ReportHowCloseWeightsAcquiredFromDifferentSeedsAre(); AA1_MLP.DataManagers.CupDataManager dm = new AA1_MLP.DataManagers.CupDataManager(); DataSet trainDS = dm.LoadData(@"C:\Users\Ronin\Documents\monks\Monks\UsedFiles\TrainValSplits\60percenttrain.txt", 10, 2, standardize: true); DataSet testDS = dm.LoadData(@"C:\Users\Ronin\Documents\monks\Monks\UsedFiles\TrainValSplits\60percenttest.txt", 10, 2, standardize: true); Console.WriteLine("Training Adamax"); AdamParams adampassedParams = new AdamParams(); IOptimizer adamtrainer = new Adam(); //adampassedParams.numberOfEpochs = 10000; //adampassedParams.batchSize = 10; //adampassedParams.trainingSet = trainDS; //adampassedParams.validationSet = testDS; //adampassedParams.learningRate = 0.001; //adampassedParams.regularization = Regularizations.L2; //adampassedParams.regularizationRate = 0.001; //adampassedParams.NumberOfHiddenUnits = 100; //adampassedParams.parallelize = false; //LastTrain(testDS, adampassedParams, adamtrainer, "10kt100adam", 1); adamtrainer = new Adamax(); adampassedParams.numberOfEpochs = 100000; adampassedParams.batchSize = 10; adampassedParams.trainingSet = trainDS; adampassedParams.validationSet = testDS; adampassedParams.learningRate = 0.001; adampassedParams.regularization = Regularizations.L2; adampassedParams.regularizationRate = 0.001; adampassedParams.NumberOfHiddenUnits = 100; //adampassedParams.PrintLoss = true; adampassedParams.parallelize = false; LastTrain(testDS, adampassedParams, adamtrainer, "100kadamax", 1); /*Console.WriteLine("Training Adam"); * AdamParams adampassedParams = new AdamParams(); * IOptimizer adamtrainer = new Adamax(); * * adampassedParams.numberOfEpochs = 100; * adampassedParams.batchSize = 10; * adampassedParams.trainingSet = trainDS; * adampassedParams.validationSet = testDS; * adampassedParams.learningRate = 0.001; * adampassedParams.regularization = Regularizations.L2; * adampassedParams.regularizationRate = 0.001; * adampassedParams.NumberOfHiddenUnits = 100; * adampassedParams.t = 1000000000; * * adampassedParams.parallelize = false; * LastTrain(testDS, adampassedParams, adamtrainer, "100adam", 1);*/ /* Console.WriteLine("training SGD"); * GradientDescentParams passedParams = new GradientDescentParams(); * Gradientdescent trainer = new Gradientdescent(); * passedParams.numberOfEpochs = 100; * passedParams.batchSize = 10; * passedParams.trainingSet = trainDS; * passedParams.validationSet = testDS; * passedParams.learningRate = 0.001; * passedParams.regularization = Regularizations.L2; * passedParams.regularizationRate = 0.001; * passedParams.nestrov = true; * passedParams.resilient = false; * passedParams.resilientUpdateAccelerationRate = 2; * passedParams.resilientUpdateSlowDownRate = 0.5; * passedParams.momentum = 0.5; * passedParams.NumberOfHiddenUnits = 100; * passedParams.parallelize = true; * LastTrain(testDS, passedParams, trainer, "5kepochsprofiling_seq", 1); * * Console.WriteLine();*/ /* * List<int> seeds = new List<int>() { 1,15,40,4,73,2}; * * foreach (var seed in seeds) * { * Console.WriteLine("Seed:{0}",seed); * * /*AdamParams passedParams = new AdamParams(); * IOptimizer trainer = new Adam(); * Console.WriteLine("training SGD"); * GradientDescentParams passedParams = new GradientDescentParams(); * Gradientdescent trainer = new Gradientdescent(); * passedParams.numberOfEpochs = 20000; * passedParams.batchSize = 10; * passedParams.trainingSet = trainDS; * passedParams.validationSet = testDS; * passedParams.learningRate = 0.001; * passedParams.regularization = Regularizations.L2; * passedParams.regularizationRate = 0.001; * passedParams.nestrov = true; * passedParams.resilient = false; * passedParams.resilientUpdateAccelerationRate = 2; * passedParams.resilientUpdateSlowDownRate = 0.5; * * passedParams.momentum = 0.5; * passedParams.NumberOfHiddenUnits = 100; * * LastTrain(testDS, passedParams, trainer, "20kseed_"+seed+"_", seed); * }*/ /* Console.WriteLine("Training Adam"); * AdamParams adampassedParams = new AdamParams(); * IOptimizer adamtrainer = new Adam(); * * adampassedParams.numberOfEpochs = 30000; * adampassedParams.batchSize = 50; * adampassedParams.trainingSet = trainDS; * adampassedParams.validationSet = testDS; * adampassedParams.learningRate = 0.001; * adampassedParams.regularization = Regularizations.L2; * adampassedParams.regularizationRate = 0.001; * adampassedParams.NumberOfHiddenUnits = 100; * * LastTrain(testDS, adampassedParams, adamtrainer, "BS50_30kepochs_100_final_adam_hdn", 1); */ //Loading and parsing cup dataset // CupDataManager dm = new CupDataManager(); //Loading the test dataset //DataSet TestSet = dm.LoadData(Properties.Settings.Default.TestSetLocation, 10, reportOsutput: false); //Loading the trained model //var n = AA1_MLP.Utilities.ModelManager.LoadNetwork("Final_hidn18_reg0.01_mo0.5_lr9E-06_model.AA1"); //double MEE = 0; //applying the model on the test data //var predictions = ModelManager.GeneratorCUP(TestSet, n); //writing the results // File.WriteAllText("OMG_LOC-OSM2-TS.txt", string.Join("\n", predictions.Select(s => string.Join(",", s)))); }
public void InitNetwork(ECostType costType, CostSettings costSettings, EOptimizerType optimizerType, OptimizerSettings optimizerSettings) { Utility.Dims InShape; Utility.Dims OutShape; Utility.Dims WShape; for (int i = 1; i < Layers.Count; i++) { Data.Data["a" + i.ToString()] = new Matrix(Layers[i].NCount, 1); InShape = new Utility.Dims(Layers[i].NCount, 1); Data.Data["b" + i.ToString()] = Matrix.RandomMatrix(Layers[i].NCount, 1, 1, EDistrubution.Gaussian); OutShape = new Utility.Dims(Layers[i].NCount, 1); Data.Data["W" + i.ToString()] = Matrix.RandomMatrix(Layers[i - 1].NCount, Layers[i].NCount, 1, EDistrubution.Gaussian); WShape = new Utility.Dims(Layers[i - 1].NCount, Layers[i].NCount); Layers[i].SetSettings(new LayerSettings(InShape, OutShape, WShape)); } Data.Data["a0"] = new Matrix(Layers[0].NCount, 1); InShape = new Utility.Dims(Layers[0].NCount, 1); Data.Data["b0"] = new Matrix(Layers[0].NCount, 1); OutShape = new Utility.Dims(Layers[0].NCount, 1); Data.Data["W0"] = new Matrix(Layers[0].NCount * Layers[1].NCount, Layers[1].NCount); WShape = new Utility.Dims(Layers[0].NCount * Layers[1].NCount, Layers[1].NCount); Layers[0].SetSettings(new LayerSettings(InShape, OutShape, WShape)); switch (costType) { case ECostType.Invalid: throw new ArgumentException("Invalid Cost Function Selected!"); case ECostType.CrossEntropyCost: CostFunction = new CrossEntropyCost((CrossEntropyCostSettings)costSettings); break; case ECostType.ExponentionalCost: CostFunction = new ExponentionalCost((ExponentionalCostSettings)costSettings); break; case ECostType.GeneralizedKullbackLeiblerDivergence: CostFunction = new GeneralizedKullbackLeiblerDivergence((GeneralizedKullbackLeiblerDivergenceSettings)costSettings); break; case ECostType.HellingerDistance: CostFunction = new HellingerDistance((HellingerDistanceSettings)costSettings); break; case ECostType.ItakuraSaitoDistance: CostFunction = new ItakuraSaitoDistance((ItakuraSaitoDistanceSettings)costSettings); break; case ECostType.KullbackLeiblerDivergence: CostFunction = new KullbackLeiblerDivergence((KullbackLeiblerDivergenceSettings)costSettings); break; case ECostType.QuadraticCost: CostFunction = new QuadraticCost((QuadraticCostSettings)costSettings); break; default: throw new ArgumentException("Invalid Cost Function Selected!"); } switch (optimizerType) { case EOptimizerType.Invalid: throw new ArgumentException("Invalid Optimizer Function Selected!"); case EOptimizerType.AdaDelta: OptimizerFunction = new AdaDelta((AdaDeltaSettings)optimizerSettings); break; case EOptimizerType.AdaGrad: OptimizerFunction = new AdaGrad((AdaGradSettings)optimizerSettings); break; case EOptimizerType.Adam: OptimizerFunction = new Adam((AdamSettings)optimizerSettings); break; case EOptimizerType.Adamax: OptimizerFunction = new Adamax((AdamaxSettings)optimizerSettings); break; case EOptimizerType.GradientDescent: OptimizerFunction = new GradientDescent((GradientDescentSettings)optimizerSettings); break; case EOptimizerType.Momentum: OptimizerFunction = new Momentum((MomentumSettings)optimizerSettings); break; case EOptimizerType.Nadam: OptimizerFunction = new Nadam((NadamSettings)optimizerSettings); break; case EOptimizerType.NesterovMomentum: OptimizerFunction = new NesterovMomentum((NesterovMomentumSettings)optimizerSettings); break; case EOptimizerType.RMSProp: OptimizerFunction = new RMSProp((RMSPropSettings)optimizerSettings); break; default: throw new ArgumentException("Invalid Optimizer Function Selected!"); } }