コード例 #1
1
ファイル: BiRNN.cs プロジェクト: zxz/RNNSharp
        public BiRNN(int modeltype)
        {
            if (modeltype == 0)
            {
                SimpleRNN s_forwardRNN = new SimpleRNN();
                SimpleRNN s_backwardRNN = new SimpleRNN();

                s_forwardRNN.setBPTT(4 + 1);
                s_forwardRNN.setBPTTBlock(30);

                s_backwardRNN.setBPTT(4 + 1);
                s_backwardRNN.setBPTTBlock(30);

                forwardRNN = s_forwardRNN;
                backwardRNN = s_backwardRNN;
            }
            else
            {
                forwardRNN = new LSTMRNN();
                backwardRNN = new LSTMRNN();
            }

            m_modeldirection = MODELDIRECTION.BI_DIRECTIONAL;
        }
コード例 #2
0
ファイル: BiRNN.cs プロジェクト: zhangweiabc/RNNSharp
        public BiRNN(RNN s_forwardRNN, RNN s_backwardRNN)
        {
            forwardRNN = s_forwardRNN;
            backwardRNN = s_backwardRNN;

            ModelType = forwardRNN.ModelType;
            ModelDirection = MODELDIRECTION.BI_DIRECTIONAL;
        }
コード例 #3
0
ファイル: BiRNN.cs プロジェクト: My-Khan/RNNSharp
        public BiRNN(RNN s_forwardRNN, RNN s_backwardRNN)
        {
            forwardRNN  = s_forwardRNN;
            backwardRNN = s_backwardRNN;

            ModelType      = forwardRNN.ModelType;
            ModelDirection = MODELDIRECTION.BI_DIRECTIONAL;
        }
コード例 #4
0
ファイル: RNNDecoder.cs プロジェクト: vahpetr/RNNSharp
        private RNN <Sequence> GetRNNInstance()
        {
            RNN <Sequence> r = null;

            while (qRNNs.TryDequeue(out r) == false)
            {
                Thread.Yield();
            }

            return(r);
        }
コード例 #5
0
ファイル: RNNDecoder.cs プロジェクト: zli69/RNNSharp
        private RNN <Sequence> GetRNNInstance()
        {
            RNN <Sequence> r = null;

            if (qRNNs.TryDequeue(out r) == false)
            {
                r = rnn.Clone();
            }

            return(r);
        }
コード例 #6
0
ファイル: RNNDecoder.cs プロジェクト: alrehamy/RNNSharp
        public RNNDecoder(Config config)
        {
            Config = config;
            if (Config.ModelDirection == MODELDIRECTION.BiDirectional)
            {
                Logger.WriteLine("Model Structure: Bi-directional RNN");
                rnn = new BiRNN <Sequence>();
            }
            else
            {
                Logger.WriteLine("Model Structure: Simple RNN");
                rnn = new ForwardRNN <Sequence>();
            }

            rnn.LoadModel(config.ModelFilePath);
            Logger.WriteLine("CRF Model: {0}", rnn.IsCRFTraining);
        }
コード例 #7
0
ファイル: RNNDecoder.cs プロジェクト: vahpetr/RNNSharp
        public RNNDecoder(Config config)
        {
            Config = config;
            RNN <Sequence> rnn = RNN <Sequence> .CreateRNN(Config.NetworkType);

            rnn.LoadModel(config.ModelFilePath);
            rnn.MaxSeqLength = config.MaxSequenceLength;

            Logger.WriteLine("CRF Model: {0}", rnn.IsCRFTraining);
            Logger.WriteLine($"Max Sequence Length: {rnn.MaxSeqLength}");
            Logger.WriteLine($"Processor Count: {Environment.ProcessorCount}");

            qRNNs = new ConcurrentQueue <RNN <Sequence> >();
            for (var i = 0; i < Environment.ProcessorCount; i++)
            {
                qRNNs.Enqueue(rnn.Clone());
            }
        }
コード例 #8
0
        public RNNDecoder(string strModelFileName, Featurizer featurizer)
        {
            MODELDIRECTION modelDir = MODELDIRECTION.FORWARD;

            RNNHelper.CheckModelFileType(strModelFileName, out modelDir);
            if (modelDir == MODELDIRECTION.BI_DIRECTIONAL)
            {
                Logger.WriteLine("Model Structure: Bi-directional RNN");
                m_Rnn = new BiRNN();
            }
            else
            {
                Logger.WriteLine("Model Structure: Simple RNN");
                m_Rnn = new ForwardRNN();
            }

            m_Rnn.LoadModel(strModelFileName);
            Logger.WriteLine("CRF Model: {0}", m_Rnn.IsCRFTraining);
            m_Featurizer = featurizer;
        }
コード例 #9
0
ファイル: RNNDecoder.cs プロジェクト: dmit25/RNNSharp
        public RNNDecoder(string strModelFileName, Featurizer featurizer)
        {
            MODELDIRECTION modelDir = MODELDIRECTION.FORWARD;

            RNNHelper.CheckModelFileType(strModelFileName, out modelDir);
            if (modelDir == MODELDIRECTION.BI_DIRECTIONAL)
            {
                Logger.WriteLine("Model Structure: Bi-directional RNN");
                m_Rnn = new BiRNN();
            }
            else
            {
                Logger.WriteLine("Model Structure: Simple RNN");
                m_Rnn = new ForwardRNN();
            }

            m_Rnn.LoadModel(strModelFileName);
            Logger.WriteLine("CRF Model: {0}", m_Rnn.IsCRFTraining);
            m_Featurizer = featurizer;
        }
コード例 #10
0
ファイル: RNNDecoder.cs プロジェクト: shaoxuan92/RNNSharp
        public RNNDecoder(string strModelFileName)
        {
            MODELDIRECTION modelDir = MODELDIRECTION.FORWARD;
            MODELTYPE      modelType;

            RNNHelper.CheckModelFileType(strModelFileName, out modelDir, out modelType);
            if (modelDir == MODELDIRECTION.BI_DIRECTIONAL)
            {
                Logger.WriteLine("Model Structure: Bi-directional RNN");
                rnn = new BiRNN <Sequence>();
            }
            else
            {
                Logger.WriteLine("Model Structure: Simple RNN");
                rnn = new ForwardRNN <Sequence>();
            }
            ModelType = modelType;

            rnn.LoadModel(strModelFileName);
            Logger.WriteLine("CRF Model: {0}", rnn.IsCRFTraining);
        }
コード例 #11
0
ファイル: RNNDecoder.cs プロジェクト: zxz/RNNSharp
        public RNNDecoder(string strModelFileName, Featurizer featurizer)
        {
            MODELTYPE model_type = RNN.CheckModelFileType(strModelFileName);

            if (model_type == MODELTYPE.SIMPLE)
            {
                Console.WriteLine("Model Structure: Simple RNN");
                SimpleRNN sRNN = new SimpleRNN();
                m_Rnn = sRNN;
            }
            else
            {
                Console.WriteLine("Model Structure: LSTM-RNN");
                LSTMRNN lstmRNN = new LSTMRNN();
                m_Rnn = lstmRNN;
            }

            m_Rnn.loadNetBin(strModelFileName);
            Console.WriteLine("CRF Model: {0}", m_Rnn.IsCRFModel());
            m_Featurizer = featurizer;
        }
コード例 #12
0
ファイル: RNNDecoder.cs プロジェクト: zhangweiabc/RNNSharp
        public RNNDecoder(string strModelFileName, Featurizer featurizer)
        {
            MODELTYPE modelType = MODELTYPE.SIMPLE;
            MODELDIRECTION modelDir = MODELDIRECTION.FORWARD;

            RNN.CheckModelFileType(strModelFileName, out modelType, out modelDir);

            if (modelDir == MODELDIRECTION.BI_DIRECTIONAL)
            {
                Logger.WriteLine(Logger.Level.info, "Model Structure: Bi-directional RNN");
                if (modelType == MODELTYPE.SIMPLE)
                {
                    m_Rnn = new BiRNN(new SimpleRNN(), new SimpleRNN());
                }
                else
                {
                    m_Rnn = new BiRNN(new LSTMRNN(), new LSTMRNN());
                }
            }
            else
            {
                if (modelType == MODELTYPE.SIMPLE)
                {
                    Logger.WriteLine(Logger.Level.info, "Model Structure: Simple RNN");
                    m_Rnn = new SimpleRNN();
                }
                else
                {
                    Logger.WriteLine(Logger.Level.info, "Model Structure: LSTM-RNN");
                    m_Rnn = new LSTMRNN();
                }
            }

            m_Rnn.loadNetBin(strModelFileName);
            Logger.WriteLine(Logger.Level.info, "CRF Model: {0}", m_Rnn.IsCRFTraining);
            m_Featurizer = featurizer;
        }
コード例 #13
0
        public RNNDecoder(string strModelFileName, Featurizer featurizer)
        {
            MODELTYPE      modelType = MODELTYPE.SIMPLE;
            MODELDIRECTION modelDir  = MODELDIRECTION.FORWARD;

            RNN.CheckModelFileType(strModelFileName, out modelType, out modelDir);

            if (modelDir == MODELDIRECTION.BI_DIRECTIONAL)
            {
                Logger.WriteLine("Model Structure: Bi-directional RNN");
                if (modelType == MODELTYPE.SIMPLE)
                {
                    m_Rnn = new BiRNN(new SimpleRNN(), new SimpleRNN());
                }
                else
                {
                    m_Rnn = new BiRNN(new LSTMRNN(), new LSTMRNN());
                }
            }
            else
            {
                if (modelType == MODELTYPE.SIMPLE)
                {
                    Logger.WriteLine("Model Structure: Simple RNN");
                    m_Rnn = new SimpleRNN();
                }
                else
                {
                    Logger.WriteLine("Model Structure: LSTM-RNN");
                    m_Rnn = new LSTMRNN();
                }
            }

            m_Rnn.LoadModel(strModelFileName);
            Logger.WriteLine("CRF Model: {0}", m_Rnn.IsCRFTraining);
            m_Featurizer = featurizer;
        }
コード例 #14
0
ファイル: RNNDecoder.cs プロジェクト: vahpetr/RNNSharp
 private void FreeRNNInstance(RNN <Sequence> r)
 {
     qRNNs.Enqueue(r);
 }
コード例 #15
0
ファイル: RNNEncoder.cs プロジェクト: vahpetr/RNNSharp
        public void Train()
        {
            //Create neural net work
            Logger.WriteLine("Create a new network according settings in configuration file.");
            Logger.WriteLine($"Processor Count = {Environment.ProcessorCount}");

            RNN <T> rnn = RNN <T> .CreateRNN(networkType);

            if (ModelSettings.IncrementalTrain)
            {
                Logger.WriteLine($"Loading previous trained model from {modelFilePath}.");
                rnn.LoadModel(modelFilePath, true);
            }
            else
            {
                Logger.WriteLine("Create a new network.");
                rnn.CreateNetwork(hiddenLayersConfig, outputLayerConfig, TrainingSet, featurizer);
                //Create tag-bigram transition probability matrix only for sequence RNN mode
                if (IsCRFTraining)
                {
                    Logger.WriteLine("Initialize bigram transition for CRF output layer.");
                    rnn.InitializeCRFWeights(TrainingSet.CRFLabelBigramTransition);
                }
            }

            rnn.MaxSeqLength  = maxSequenceLength;
            rnn.bVQ           = ModelSettings.VQ != 0 ? true : false;
            rnn.IsCRFTraining = IsCRFTraining;

            int             N    = Environment.ProcessorCount * 2;
            List <RNN <T> > rnns = new List <RNN <T> >();

            rnns.Add(rnn);

            for (int i = 1; i < N; i++)
            {
                rnns.Add(rnn.Clone());
            }

            //Initialize RNNHelper
            RNNHelper.LearningRate          = ModelSettings.LearningRate;
            RNNHelper.vecNormalLearningRate = new Vector <float>(RNNHelper.LearningRate);

            RNNHelper.GradientCutoff = ModelSettings.GradientCutoff;
            RNNHelper.vecMaxGrad     = new Vector <float>(RNNHelper.GradientCutoff);
            RNNHelper.vecMinGrad     = new Vector <float>(-RNNHelper.GradientCutoff);
            RNNHelper.IsConstAlpha   = ModelSettings.IsConstAlpha;
            RNNHelper.MiniBatchSize  = ModelSettings.MiniBatchSize;

            Logger.WriteLine("");

            Logger.WriteLine("Iterative training begins ...");
            var             bestTrainTknErrCnt = long.MaxValue;
            var             bestValidTknErrCnt = long.MaxValue;
            var             lastAlpha          = RNNHelper.LearningRate;
            var             iter           = 0;
            ParallelOptions parallelOption = new ParallelOptions();

            parallelOption.MaxDegreeOfParallelism = -1;
            while (true)
            {
                if (ModelSettings.MaxIteration > 0 && iter > ModelSettings.MaxIteration)
                {
                    Logger.WriteLine("We have trained this model {0} iteration, exit.");
                    break;
                }

                var start = DateTime.Now;
                Logger.WriteLine($"Start to training {iter} iteration. learning rate = {RNNHelper.LearningRate}");

                //Clean all RNN instances' status for training
                foreach (var r in rnns)
                {
                    r.CleanStatusForTraining();
                }
                Process(rnns, TrainingSet, RunningMode.Training);

                var duration = DateTime.Now.Subtract(start);

                Logger.WriteLine($"End {iter} iteration. Time duration = {duration}");
                Logger.WriteLine("");

                if (tknErrCnt >= bestTrainTknErrCnt && lastAlpha != RNNHelper.LearningRate)
                {
                    //Although we reduce alpha value, we still cannot get better result.
                    Logger.WriteLine(
                        $"Current token error count({(double)tknErrCnt / (double)processedWordCnt * 100.0}%) is larger than the previous one({(double)bestTrainTknErrCnt / (double)processedWordCnt * 100.0}%) on training set. End training early.");
                    Logger.WriteLine("Current alpha: {0}, the previous alpha: {1}", RNNHelper.LearningRate, lastAlpha);
                    break;
                }
                lastAlpha = RNNHelper.LearningRate;

                int trainTknErrCnt = tknErrCnt;
                //Validate the model by validated corpus
                if (ValidationSet != null)
                {
                    Logger.WriteLine("Verify model on validated corpus.");
                    Process(rnns, ValidationSet, RunningMode.Validate);
                    Logger.WriteLine("End model verification.");
                    Logger.WriteLine("");

                    if (tknErrCnt < bestValidTknErrCnt)
                    {
                        //We got better result on validated corpus, save this model
                        Logger.WriteLine($"Saving better model into file {modelFilePath}, since we got a better result on validation set.");
                        Logger.WriteLine($"Error token percent: {(double)tknErrCnt / (double)processedWordCnt * 100.0}%, Error sequence percent: {(double)sentErrCnt / (double)processedSequence * 100.0}%");

                        rnn.SaveModel(modelFilePath);
                        bestValidTknErrCnt = tknErrCnt;
                    }
                }
                else if (trainTknErrCnt < bestTrainTknErrCnt)
                {
                    //We don't have validate corpus, but we get a better result on training corpus
                    //We got better result on validated corpus, save this model
                    Logger.WriteLine($"Saving better model into file {modelFilePath}, although validation set doesn't exist, we have better result on training set.");
                    Logger.WriteLine($"Error token percent: {(double)trainTknErrCnt / (double)processedWordCnt * 100.0}%, Error sequence percent: {(double)sentErrCnt / (double)processedSequence * 100.0}%");

                    rnn.SaveModel(modelFilePath);
                }

                Logger.WriteLine("");

                if (trainTknErrCnt >= bestTrainTknErrCnt)
                {
                    //We don't have better result on training set, so reduce learning rate
                    RNNHelper.LearningRate          = RNNHelper.LearningRate / 2.0f;
                    RNNHelper.vecNormalLearningRate = new Vector <float>(RNNHelper.LearningRate);
                }
                else
                {
                    bestTrainTknErrCnt = trainTknErrCnt;
                }

                iter++;
            }
        }
コード例 #16
0
ファイル: RNNEncoder.cs プロジェクト: deTrident/RNNSharp
        public void Process(RNN <T> rnn, DataSet <T> trainingSet, RunningMode runningMode, int totalSequenceNum)
        {
            //Shffle training corpus
            trainingSet.Shuffle();

            for (var i = 0; i < trainingSet.SequenceList.Count; i++)
            {
                var pSequence = trainingSet.SequenceList[i];

                int wordCnt = 0;
                if (pSequence is Sequence)
                {
                    wordCnt = (pSequence as Sequence).States.Length;
                }
                else
                {
                    SequencePair sp = pSequence as SequencePair;
                    if (sp.srcSentence.TokensList.Count > rnn.MaxSeqLength)
                    {
                        continue;
                    }

                    wordCnt = sp.tgtSequence.States.Length;
                }

                if (wordCnt > rnn.MaxSeqLength)
                {
                    continue;
                }

                Interlocked.Add(ref processedWordCnt, wordCnt);

                int[] predicted;
                if (IsCRFTraining)
                {
                    predicted = rnn.ProcessSequenceCRF(pSequence as Sequence, runningMode);
                }
                else
                {
                    Matrix <float> m;
                    predicted = rnn.ProcessSequence(pSequence, runningMode, false, out m);
                }

                int newTknErrCnt;
                if (pSequence is Sequence)
                {
                    newTknErrCnt = GetErrorTokenNum(pSequence as Sequence, predicted);
                }
                else
                {
                    newTknErrCnt = GetErrorTokenNum((pSequence as SequencePair).tgtSequence, predicted);
                }

                Interlocked.Add(ref tknErrCnt, newTknErrCnt);
                if (newTknErrCnt > 0)
                {
                    Interlocked.Increment(ref sentErrCnt);
                }

                Interlocked.Increment(ref processedSequence);

                if (processedSequence % 1000 == 0)
                {
                    Logger.WriteLine("Progress = {0} ", processedSequence / 1000 + "K/" + totalSequenceNum / 1000.0 + "K");
                    Logger.WriteLine(" Error token ratio = {0}%", (double)tknErrCnt / (double)processedWordCnt * 100.0);
                    Logger.WriteLine(" Error sentence ratio = {0}%", (double)sentErrCnt / (double)processedSequence * 100.0);
                }

                if (ModelSettings.SaveStep > 0 && processedSequence % ModelSettings.SaveStep == 0)
                {
                    //After processed every m_SaveStep sentences, save current model into a temporary file
                    Logger.WriteLine("Saving temporary model into file...");
                    rnn.SaveModel("model.tmp");
                }
            }
        }