Пример #1
0
 public ComplexSeg(Dictionary dic)
     : base(dic)
 {
     otherRules.Add(new LargestAvgLenRule());
     otherRules.Add(new SmallestVarianceRule());
     otherRules.Add(new LargestSumDegreeFreedomRule());
 }
Пример #2
0
 /// <summary>
 /// 销毁,释放资源
 /// </summary>
 void destroy()
 {
     dicPath = null;
     dict.Clear();
     dict = null;
     unit.Clear();
     unit = null;
 }
Пример #3
0
 Dictionary<char,object> loadUnit(FileInfo path)
 {
     string[] lines = null;
     if (path.Exists)
     {
         lines = File.ReadAllLines(path.FullName);
         addLastTime(path);
     }
     else
     {
         lines = getLinesFromResources("units.dic");
     }
     Dictionary<char, object> unit = new Dictionary<char, object>();
     long s = now;
     int lineNum = 0;
    
         foreach (string line in lines)
         {
             if (line.Length != 1) continue;
             if (!unit.ContainsKey(line[0]))
                 unit.Add(line[0], typeof(Dictionary));
             ++lineNum;
         }
     
     Log.Info("unit loaded time={0}ms,line={1},on file={2}", now - s, lineNum, path);
     return unit;
 }
Пример #4
0
 void loadWord(string[] buffers, Dictionary<char, CharNode> dic)
 {
     long s = now;
     int lineNum = WordsLoading(buffers, dic);
     Log.Info("words loaded time={0}ms,line={1},on file=words.dic", now - s, lineNum);
 }
 public MMSegAnalyzer(Dictionary p_dic)
     : base()
 {
     dic = p_dic;
 }
Пример #6
0
 static int WordsLoading(string[] buffers, Dictionary<char, CharNode> dic)
 {
     CharNode cn = null;
     int count = 0;
     foreach (string line in buffers)
     {
         if(line.Length < 2) continue;
         cn = null;
         if (dic.ContainsKey(line[0]))
             cn = dic[line[0]];
         if (cn == null)
         {
             cn = new CharNode();
             dic.Add(line[0], cn);
         }
         ++count;
         cn.AddWordTail(tail(line));
     }
     return count;
 }
 public MaxWordAnalyzer(Dictionary dic) : base(dic) { }
Пример #8
0
 void init(FileInfo path)
 {
     dicPath = path;
     wordsLastTime = new Dictionary<string, long>();
     reload();
 }
Пример #9
0
 public TreeNode(char key)
 {
     this.key = key;
     subNodes = new Dictionary<char, TreeNode>();
 }
Пример #10
0
 public SimpleSeg(Dictionary dic) : base(dic) { }
Пример #11
0
 public MaxWordSeg(Dictionary dic)
     : base(dic)
 { }
 public SimpleAnalyzer(Dictionary dic) : base(dic) { }
Пример #13
0
 public static Dictionary getInstance(FileInfo path)
 {
     Dictionary dic = null;
     lock (m_syncOjbect)
     {
         if (dics.ContainsKey(path))
         {
             dic = dics[path];
         }
         else
         {
             dic = new Dictionary(path);
             dics.Add(path, dic);
         }
     }
     return dic;
 }
Пример #14
0
 public static Dictionary getInstance(FileInfo path)
 {
     Dictionary dic = null;
     string key = path.ToString();
     lock (m_syncOjbect)
     {
         if (dics.ContainsKey(key))
         {
             dic = dics[key];
         }
         else
         {
             dic = new Dictionary(path);
             dics.Add(key, dic);
         }
     }
     return dic;
 }
 public MMSegAnalyzer()
 {
     dic = Dictionary.getInstance();
 }
Пример #16
0
        /// <summary>
        /// 全部加载词库,没有成功加载会回滚
        /// 注意:重新加载时,务必有两倍的词库树结构的内存,默认词库是 50M/个 左右。否则抛出 OOM。
        /// </summary>
        /// <returns></returns>
        public bool reload()
        {
            Dictionary<string, long> oldWordsLastTime = new Dictionary<string, long>(wordsLastTime);
            Dictionary<char, CharNode> oldDict = dict;
            Dictionary<char, object> oldUnit = unit;

            try
            {
                wordsLastTime.Clear();
                dict = loadDic(dicPath);
                unit = loadUnit(dicPath);
                lastLoadTime = now;
            }
            catch (Exception ex)
            {
                //rollback
                foreach (KeyValuePair<string, long> key in oldWordsLastTime)
                {
                    wordsLastTime.Add(key.Key, key.Value);
                }
                dict = oldDict;
                unit = oldUnit;
                Log.Info("reload dic error! dic={0},and rollbacked.{1}", dicPath, ex);
                return false;
            }

            return true;
        }
 /// <summary>
 /// 构造函数
 /// </summary>
 /// <param name="path">词库目录</param>
 public MMSegAnalyzer(string path)
 {
     dic = Dictionary.getInstance(path);
 }
Пример #18
0
        /// <summary>
        /// 文件总行数
        /// </summary>
        /// <param name="file"></param>
        /// <returns></returns>
        static int load(string[] buffers, Dictionary<char, CharNode> dic)
        {
            if (buffers == null) return 0;
            int n = 0;
            string[] w = null;
            CharNode cn = null;
            foreach (string line in buffers)
            {
                if (string.IsNullOrEmpty(line)) continue;
                if (line.StartsWith("#")) continue;
                cn = new CharNode();
                w = line.Split(' ');
                if (w.Length == 2)
                {
                    try
                    {
                        //字频计算出自由度
                        cn.Freq = (int)(Math.Log(Int32.Parse(w[1])) * 100);
                    }
                    catch
                    {
                    }
                }

                if (!dic.ContainsKey(w[0][0]))
                {

                    dic.Add(w[0][0], cn);
                }
                else
                    dic[w[0][0]] = cn;
                ++n;
            }
            return n;
        }
 public MMSegAnalyzer(FileInfo path)
 {
     dic = Dictionary.getInstance(path);
 }
Пример #20
0
        Dictionary<char, CharNode> loadDic(FileInfo wordsPath)
        {
            string path = wordsPath.FullName;
            string[] data = null;
            if(!wordsPath.Exists)
                wordsPath = new FileInfo(string.Format("{0}\\data\\chars.dic",wordsPath.FullName));
            if (wordsPath.Exists)
            {
                data = File.ReadAllLines(wordsPath.FullName);
                addLastTime(wordsPath);
            }
            else
            {
                data = getLinesFromResources("chars.dic");
            }
            Dictionary<char,CharNode> dic = new Dictionary<char,CharNode>();
            int lineNum = 0;
            long s = now;
            long ss = s;
            lineNum = load(data, dic);
            Log.Info("chars loaded time={0}ms,line={1},on file={2}", now - s, lineNum, wordsPath);

            //try load words.dic
            string[] wordsDicIn = getLinesFromResources("words.dic");
            if (wordsDicIn != null)
            {
                loadWord(wordsDicIn, dic);
            }

            string[] words = listWordsFiles();
            if (words != null)
            {  
                foreach (string wordsFile in words)
                {
                    if (!File.Exists(wordsFile)) continue;
                    loadWord(File.ReadAllLines(wordsFile), dic);
                    addLastTime(new FileInfo(wordsFile));
                }
            }

            Log.Info("load all dic user time={0}ms", now - ss);

            return dic;
        }
 public ComplexAnalyzer(Dictionary dic) : base(dic) { }