private static void Train() { Logger.LogFile = "RNNSharpConsole.log"; if (File.Exists(strTagFile) == false) { Logger.WriteLine(Logger.Level.err, "FAILED: The tag mapping file {0} isn't existed.", strTagFile); UsageTrain(); return; } //Load tag id and its name from file TagSet tagSet = new TagSet(strTagFile); //Create configuration instance and set parameters ModelSetting RNNConfig = new ModelSetting(); RNNConfig.ModelFile = strModelFile; RNNConfig.NumHidden = layersize; RNNConfig.IsCRFTraining = (iCRF == 1) ? true : false; RNNConfig.ModelDirection = iDir; RNNConfig.ModelType = modelType; RNNConfig.MaxIteration = maxIter; RNNConfig.SaveStep = savestep; RNNConfig.LearningRate = alpha; RNNConfig.Dropout = dropout; RNNConfig.Bptt = bptt; //Dump RNN setting on console RNNConfig.DumpSetting(); if (File.Exists(strFeatureConfigFile) == false) { Logger.WriteLine(Logger.Level.err, "FAILED: The feature configuration file {0} doesn't exist.", strFeatureConfigFile); UsageTrain(); return; } //Create feature extractors and load word embedding data from file Featurizer featurizer = new Featurizer(strFeatureConfigFile, tagSet); featurizer.ShowFeatureSize(); if (featurizer.IsRunTimeFeatureUsed() == true && iDir == 1) { Logger.WriteLine(Logger.Level.err, "FAILED: Run time feature is not available for bi-directional RNN model."); UsageTrain(); return; } if (File.Exists(strTrainFile) == false) { Logger.WriteLine(Logger.Level.err, "FAILED: The training corpus doesn't exist."); UsageTrain(); return; } if (File.Exists(strValidFile) == false) { Logger.WriteLine(Logger.Level.err, "FAILED: The validation corpus doesn't exist."); UsageTrain(); return; } //Create RNN encoder and save necessary parameters RNNEncoder encoder = new RNNEncoder(RNNConfig); //LoadFeatureConfig training corpus and extract feature set encoder.TrainingSet = new DataSet(tagSet.GetSize()); LoadDataset(strTrainFile, featurizer, encoder.TrainingSet); //LoadFeatureConfig validated corpus and extract feature set encoder.ValidationSet = new DataSet(tagSet.GetSize()); LoadDataset(strValidFile, featurizer, encoder.ValidationSet); if (iCRF == 1) { Logger.WriteLine(Logger.Level.info, "Initialize output tag bigram transition probability..."); //Build tag bigram transition matrix encoder.TrainingSet.BuildLabelBigramTransition(); } //Start to train the model encoder.Train(); }
private static void Train() { if (File.Exists(strTagFile) == false) { Console.WriteLine("FAILED: The tag mapping file {0} isn't existed.", strTagFile); UsageTrain(); return; } //Load tag id and its name from file TagSet tagSet = new TagSet(strTagFile); //Create configuration instance and set parameters ModelSetting RNNConfig = new ModelSetting(); RNNConfig.SetModelFile(strModelFile); RNNConfig.SetNumHidden(layersize); RNNConfig.SetCRFTraining((iCRF == 1) ? true : false); RNNConfig.SetDir(iDir); RNNConfig.SetModelType(modelType); RNNConfig.SetMaxIteration(maxIter); RNNConfig.SetSaveStep(savestep); RNNConfig.SetLearningRate(alpha); RNNConfig.SetRegularization(beta); RNNConfig.SetBptt(bptt); //Dump RNN setting on console RNNConfig.DumpSetting(); if (File.Exists(strFeatureConfigFile) == false) { Console.WriteLine("FAILED: The feature configuration file {0} isn't existed.", strFeatureConfigFile); UsageTrain(); return; } //Create feature extractors and load word embedding data from file Featurizer featurizer = new Featurizer(strFeatureConfigFile, tagSet); featurizer.ShowFeatureSize(); if (File.Exists(strTrainFile) == false) { Console.WriteLine("FAILED: The training corpus {0} isn't existed.", strTrainFile); UsageTrain(); return; } //LoadFeatureConfig training corpus and extract feature set DataSet dataSetTrain = new DataSet(tagSet.GetSize()); LoadDataset(strTrainFile, featurizer, dataSetTrain); if (File.Exists(strValidFile) == false) { Console.WriteLine("FAILED: The validated corpus {0} isn't existed.", strValidFile); UsageTrain(); return; } //LoadFeatureConfig validated corpus and extract feature set DataSet dataSetValidation = new DataSet(tagSet.GetSize()); LoadDataset(strValidFile, featurizer, dataSetValidation); //Create RNN encoder and save necessary parameters RNNEncoder encoder = new RNNEncoder(RNNConfig); encoder.SetTrainingSet(dataSetTrain); encoder.SetValidationSet(dataSetValidation); if (iCRF == 1) { Console.WriteLine("Initialize output tag bigram transition probability..."); //Build tag bigram transition matrix dataSetTrain.BuildLabelBigramTransition(); encoder.SetLabelBigramTransition(dataSetTrain.GetLabelBigramTransition()); } //Start to train the model encoder.Train(); }
private static void Test() { if (String.IsNullOrEmpty(strTagFile) == true) { Logger.WriteLine(Logger.Level.err, "FAILED: The tag mapping file {0} isn't specified.", strTagFile); UsageTest(); return; } //Load tag name TagSet tagSet = new TagSet(strTagFile); if (String.IsNullOrEmpty(strModelFile) == true) { Logger.WriteLine(Logger.Level.err, "FAILED: The model file {0} isn't specified.", strModelFile); UsageTest(); return; } if (String.IsNullOrEmpty(strFeatureConfigFile) == true) { Logger.WriteLine(Logger.Level.err, "FAILED: The feature configuration file {0} isn't specified.", strFeatureConfigFile); UsageTest(); return; } if (strOutputFile.Length == 0) { Logger.WriteLine(Logger.Level.err, "FAILED: The output file name should not be empty."); UsageTest(); return; } //Create feature extractors and load word embedding data from file Featurizer featurizer = new Featurizer(strFeatureConfigFile, tagSet); featurizer.ShowFeatureSize(); //Create instance for decoder RNNSharp.RNNDecoder decoder = new RNNSharp.RNNDecoder(strModelFile, featurizer); if (File.Exists(strTestFile) == false) { Logger.WriteLine(Logger.Level.err, "FAILED: The test corpus {0} isn't existed.", strTestFile); UsageTest(); return; } StreamReader sr = new StreamReader(strTestFile); StreamWriter sw = new StreamWriter(strOutputFile); while (true) { Sentence sent = new Sentence(ReadRecord(sr)); if (sent.TokensList.Count <= 2) { //No more record, it only contains <s> and </s> break; } if (nBest == 1) { int[] output = decoder.Process(sent); //Output decoded result //Append the decoded result into the end of feature set of each token StringBuilder sb = new StringBuilder(); for (int i = 0; i < sent.TokensList.Count; i++) { string tokens = String.Join("\t", sent.TokensList[i]); sb.Append(tokens); sb.Append("\t"); sb.Append(tagSet.GetTagName(output[i])); sb.AppendLine(); } sw.WriteLine(sb.ToString()); } else { int[][] output = decoder.ProcessNBest(sent, nBest); StringBuilder sb = new StringBuilder(); for (int i = 0; i < nBest; i++) { for (int j = 0; j < sent.TokensList.Count; j++) { string tokens = String.Join("\t", sent.TokensList[i]); sb.Append(tokens); sb.Append("\t"); sb.Append(tagSet.GetTagName(output[i][j])); sb.AppendLine(); } sb.AppendLine(); } sw.WriteLine(sb.ToString()); } } sr.Close(); sw.Close(); }
private static void Test() { if (File.Exists(strTagFile) == false) { Console.WriteLine("FAILED: The tag mapping file {0} isn't existed.", strTagFile); UsageTest(); return; } //Load tag id and its name from file TagSet tagSet = new TagSet(strTagFile); if (File.Exists(strModelFile) == false) { Console.WriteLine("FAILED: The model file {0} isn't existed.", strModelFile); UsageTest(); return; } if (File.Exists(strFeatureConfigFile) == false) { Console.WriteLine("FAILED: The feature configuration file {0} isn't existed.", strFeatureConfigFile); UsageTest(); return; } if (strOutputFile.Length == 0) { Console.WriteLine("FAILED: The output file name should not be empty."); UsageTest(); return; } //Create feature extractors and load word embedding data from file Featurizer featurizer = new Featurizer(strFeatureConfigFile, tagSet); featurizer.ShowFeatureSize(); //Create an instance for the model // Model model = new Model(strModelFile); //Create instance for decoder RNNSharp.RNNDecoder decoder = new RNNSharp.RNNDecoder(strModelFile, featurizer); if (File.Exists(strTestFile) == false) { Console.WriteLine("FAILED: The test corpus {0} isn't existed.", strTestFile); UsageTest(); return; } StreamReader sr = new StreamReader(strTestFile); StreamWriter sw = new StreamWriter(strOutputFile); while (true) { List<string> tokenList = ReadRecord(sr); if (tokenList.Count == 0) { //No more record break; } Sentence sent = new Sentence(); sent.SetFeatures(tokenList); if (nBest == 1) { int[] output = decoder.Process(sent); //Output decoded result //Append the decoded result into the end of feature set of each token StringBuilder sb = new StringBuilder(); for (int i = 0; i < tokenList.Count; i++) { sb.Append(tokenList[i]); sb.Append("\t"); sb.Append(tagSet.GetTagName(output[i])); sb.AppendLine(); } sw.WriteLine(sb.ToString()); } else { int[][] output = decoder.ProcessNBest(sent, nBest); if (output == null) { Console.WriteLine("FAILED: decode failed. Dump current sentence..."); sent.DumpFeatures(); return; } StringBuilder sb = new StringBuilder(); for (int i = 0; i < nBest; i++) { for (int j = 0; j < tokenList.Count; j++) { sb.Append(tokenList[j]); sb.Append("\t"); sb.Append(tagSet.GetTagName(output[i][j])); sb.AppendLine(); } sb.AppendLine(); } sw.WriteLine(sb.ToString()); } } sr.Close(); sw.Close(); }