//The format of configuration file public void LoadFeatureConfigFromFile(string configFilePath) { //Load configuration file config = new ConfigUtils(); config.LoadFile(configFilePath); //Get current directory from configuration file currentDirectory = config.GetValueOptional(CURRENT_DIRECTORY); if (string.IsNullOrEmpty(currentDirectory)) { currentDirectory = Environment.CurrentDirectory; } Logger.WriteLine($"Current directory : {currentDirectory}"); //Get model file path ModelFilePath = GetFilePath(currentDirectory, config.GetValueRequired(MODEL_FILEPATH)); Logger.WriteLine($"Main model is located at {ModelFilePath}"); featureContext = new Dictionary <string, List <int> >(); SetHiddenLayers(); SetOutputLayers(); SetPretrainedModel(); SetTFeatures(); var isCRFTraining = config.GetValueOptional(CRF_LAYER); IsCRFTraining = false; if (string.IsNullOrEmpty(isCRFTraining) == false) { IsCRFTraining = bool.Parse(isCRFTraining); } //Load model type ModelType = config.GetValueRequired(MODEL_TYPE) .Equals(MODELTYPE.SeqLabel.ToString(), StringComparison.InvariantCultureIgnoreCase) ? MODELTYPE.SeqLabel : MODELTYPE.Seq2Seq; Logger.WriteLine($"Model type: {ModelType}"); ModelDirection = config.GetValueRequired(MODEL_DIRECTION) .Equals(MODELDIRECTION.Forward.ToString(), StringComparison.InvariantCultureIgnoreCase) ? MODELDIRECTION.Forward : MODELDIRECTION.BiDirectional; Logger.WriteLine($"Model direction: {ModelDirection}"); //Load auto-encoder model for sequence-to-sequence. This model is used to encode source sequence if (ModelType == MODELTYPE.Seq2Seq) { var seqAutoEncoderConfigFilePath = GetFilePath(currentDirectory, config.GetValueRequired(SEQ2SEQ_AUTOENCODER_CONFIG)); Logger.WriteLine( $"Loading auto encoder model for sequnce-to-sequence task. Config file = '{seqAutoEncoderConfigFilePath}'"); Seq2SeqAutoEncoder = InitializeAutoEncoder(seqAutoEncoderConfigFilePath); } //Check if settings are validated CheckSettings(); }
public static void CheckModelFileType(string filename, out MODELDIRECTION modelDir) { using (StreamReader sr = new StreamReader(filename)) { BinaryReader br = new BinaryReader(sr.BaseStream); int modelType = br.ReadInt32(); modelDir = (MODELDIRECTION)br.ReadInt32(); } Logger.WriteLine("Get model direction: {0}", modelDir); }
public static void CheckModelFileType(string filename, out MODELDIRECTION modelDir, out MODELTYPE modelType) { using (StreamReader sr = new StreamReader(filename)) { BinaryReader br = new BinaryReader(sr.BaseStream); LAYERTYPE layerType = (LAYERTYPE)br.ReadInt32(); modelDir = (MODELDIRECTION)br.ReadInt32(); modelType = (MODELTYPE)br.ReadInt32(); Logger.WriteLine("Model information:"); Logger.WriteLine("File name : '{0}'", filename); Logger.WriteLine("Direction: '{0}'", modelDir); Logger.WriteLine("Model type: '{0}'", modelType); Logger.WriteLine("Layer type: '{0}'", layerType); } }
public RNNDecoder(string strModelFileName, Featurizer featurizer) { MODELDIRECTION modelDir = MODELDIRECTION.FORWARD; RNNHelper.CheckModelFileType(strModelFileName, out modelDir); if (modelDir == MODELDIRECTION.BI_DIRECTIONAL) { Logger.WriteLine("Model Structure: Bi-directional RNN"); m_Rnn = new BiRNN(); } else { Logger.WriteLine("Model Structure: Simple RNN"); m_Rnn = new ForwardRNN(); } m_Rnn.LoadModel(strModelFileName); Logger.WriteLine("CRF Model: {0}", m_Rnn.IsCRFTraining); m_Featurizer = featurizer; }
public RNNDecoder(string strModelFileName) { MODELDIRECTION modelDir = MODELDIRECTION.FORWARD; MODELTYPE modelType; RNNHelper.CheckModelFileType(strModelFileName, out modelDir, out modelType); if (modelDir == MODELDIRECTION.BI_DIRECTIONAL) { Logger.WriteLine("Model Structure: Bi-directional RNN"); rnn = new BiRNN <Sequence>(); } else { Logger.WriteLine("Model Structure: Simple RNN"); rnn = new ForwardRNN <Sequence>(); } ModelType = modelType; rnn.LoadModel(strModelFileName); Logger.WriteLine("CRF Model: {0}", rnn.IsCRFTraining); }
public RNNDecoder(string strModelFileName, Featurizer featurizer) { MODELTYPE modelType = MODELTYPE.SIMPLE; MODELDIRECTION modelDir = MODELDIRECTION.FORWARD; RNN.CheckModelFileType(strModelFileName, out modelType, out modelDir); if (modelDir == MODELDIRECTION.BI_DIRECTIONAL) { Logger.WriteLine("Model Structure: Bi-directional RNN"); if (modelType == MODELTYPE.SIMPLE) { m_Rnn = new BiRNN(new SimpleRNN(), new SimpleRNN()); } else { m_Rnn = new BiRNN(new LSTMRNN(), new LSTMRNN()); } } else { if (modelType == MODELTYPE.SIMPLE) { Logger.WriteLine("Model Structure: Simple RNN"); m_Rnn = new SimpleRNN(); } else { Logger.WriteLine("Model Structure: LSTM-RNN"); m_Rnn = new LSTMRNN(); } } m_Rnn.LoadModel(strModelFileName); Logger.WriteLine("CRF Model: {0}", m_Rnn.IsCRFTraining); m_Featurizer = featurizer; }