Пример #1
0
 public static List<string> Viterbi(
     List<string> str,
     Trigram q,
     Bigram e,
     Unigram freqList,
     HashSet<string> tags)
 {
     return Viterbi(str,q,e,freqList,tags,false);
 }
Пример #2
0
        public static List<string> Viterbi(
           List<string> str,
           Trigram q,
           Bigram e,
           Unigram freqList,
           HashSet<string> tags,
           bool splitRare)
        {
            List<string> y = new List<string>();
            Dictionary<string, double> current = new Dictionary<string, double>();
            Dictionary<string, double> previous = new Dictionary<string, double>();
            string key;
            double prob = 0;
            double max = 0;

            if (freqList.Contains(str[0]))
            {
                y.Add("STOP");

                foreach (string v in tags)
                {
                    prob = q.Qml(v, "*", "*") * e.Qml(str[0], v);
                    current.Add("*:" + v, prob);

                    if (prob >= max)
                    {
                        max = prob;
                        y.RemoveAt(y.Count - 1);
                        y.Add(v);
                    }
                }
            }
            else
            {
                string v = (splitRare) ? ProperRare(str[0]) : "_RARE_";
                y.Add(v);

                prob = Math.Max(q.Qml("_RARE_", "*", "*"), 0);
                current.Add("*:" + "_RARE_", prob);

            }

            previous = current;
            current = new Dictionary<string, double>();
            max = 0;
            if (freqList.Contains(str[1]))
            {
                y.Add("STOP");

                foreach (string u in tags)
                {
                    foreach (string v in tags)
                    {
                        try
                        {
                            key = "*:" + u;
                            prob = previous[key] * q.Qml(v, "*", u) * e.Qml(str[1], v);
                            current.Add(string.Format("{0}:{1}", u, v), prob);
                            if (prob >= max)
                            {
                                max = prob;
                                y.RemoveAt(y.Count - 1);
                                y.Add(v);
                            }
                        }
                        catch (Exception)
                        {
                            continue;

                        }
                    }
                }
            }
            else
            {
                string v = (splitRare) ? ProperRare(str[1]) : "_RARE_";
                y.Add(v);

                foreach (string u in tags)
                {
                    try
                    {
                        key = "*:" + u;
                        prob = previous[key] * q.Qml(v, "*", u) * e.Qml(str[1], v);
                        current.Add(string.Format("{0}:{1}", u, v), prob);
                        if (prob >= max)
                        {
                            max = prob;
                            y.RemoveAt(y.Count - 1);
                            y.Add(v);
                        }
                    }
                    catch (Exception)
                    {
                        continue;

                    }
                }
            }

            for (int i = 2; i < str.Count; i++)
            {
                previous = current;
                current = new Dictionary<string, double>();
                max = 0;

                if (freqList.Contains(str[i]))
                {
                    y.Add("STOP");

                    foreach (string v in tags)
                    {
                        foreach (string u in tags)
                        {
                            foreach (string w in tags)
                            {
                                key = w + ":" + u;

                                try
                                {
                                    prob = previous[key] * q.Qml(v, w, u) * e.Qml(str[i], v);
                                }
                                catch (Exception)
                                {

                                    prob = 0;
                                }

                                try
                                {
                                    if (prob > 0)
                                        current.Add(string.Format("{0}:{1}", u, v), prob);

                                }
                                catch (Exception)
                                {
                                    if (prob > current[string.Format("{0}:{1}", u, v)])
                                    {
                                        current.Remove(string.Format("{0}:{1}", u, v));
                                        current.Add(string.Format("{0}:{1}", u, v), prob);
                                    }
                                }

                                if (prob >= max)
                                {
                                    max = prob;
                                    y.RemoveAt(y.Count - 1);
                                    y.Add(v);
                                }

                            }
                        }
                    }
                }
                else
                {
                    string v = (splitRare) ? ProperRare(str[i]) : "_RARE_";
                    y.Add(v);

                    foreach (string u in tags)
                    {
                        foreach (string w in tags)
                        {
                            key = w + ":" + u;

                            try
                            {
                                prob = previous[key] * q.Qml(v, w, u);
                            }
                            catch (Exception)
                            {
                                prob = 0;
                            }

                            try
                            {
                                if (prob > 0)
                                    current.Add(string.Format("{0}:{1}", u, v), prob);

                            }
                            catch (Exception)
                            {
                                if (prob > current[string.Format("{0}:{1}", u, v)])
                                {
                                    current.Remove(string.Format("{0}:{1}", u, v));
                                    current.Add(string.Format("{0}:{1}", u, v), prob);
                                }
                            }

                            if (prob >= max)
                            {
                                max = prob;
                                y.RemoveAt(y.Count - 1);
                                y.Add(v);
                            }

                        }
                    }

                }
            }

            return y;
        }
Пример #3
0
Файл: HMM.cs Проект: kicasta/NLP
        public static void Train()
        {
            if (!(File.Exists(trigramPath) && File.Exists(bigramPath) && File.Exists(tagsPath)))
            {
                if (File.Exists("tmp.train"))
                    ParseTrainingData("tmp.train");
                else
                    ParseTrainingData(ReplaceTrainingFile(trainingPath, BuildFreqList(trainingPath), true));

                SerializeModel(trigramPath, transition);
                SerializeModel(bigramPath, emission);
                SerializeModel(unigramPath, freqList);
                SerializeTags(tagsPath);
            }
            else
            {
                transition = (Trigram)DeserializeModel(trigramPath);
                emission = (Bigram)DeserializeModel(bigramPath);
                freqList = (Unigram)DeserializeModel(unigramPath);
                tags = DeserializeTags(tagsPath);
            }
        }