private async Task <WordDictionaryFile> LoadFromTextFile(String fileName) { InitConvertDictionary(); WordDictionaryFile dictFile = new WordDictionaryFile(); try { dictFile = new WordDictionaryFile(); dictFile.Dicts = new List <WordAttribute>(); using (Stream s = await GlobalAccess.DirectoryService.OpenFile(fileName, FileModeEnum.Open)) { using (StreamReader sr = new StreamReader(s, Encoding.UTF8)) { while (!sr.EndOfStream) { string line = sr.ReadLine(); string[] strs = line.Split(new char[] { '|' }); if (strs.Length == 3) { string word = strs[0].Trim(); POS pos = (POS)Enum.Parse(typeof(POS), strs[1]); double frequency = double.Parse(strs[2]); WordAttribute dict = new WordAttribute(word, pos, frequency); dictFile.Dicts.Add(dict); } } } } return(dictFile); } catch (Exception) { throw; } }
/// <summary> /// /// </summary> /// <param name="fileName"></param> /// <param name="textFile"></param> /// <param name="version">输出字典的版本号</param> public async Task Load(String fileName, bool textFile) { _WordDict = new Dictionary <string, WordAttribute>(); _FirstCharDict = new Dictionary <char, WordAttribute>(); _DoubleCharDict = new Dictionary <uint, WordAttribute>(); _TripleCharDict = new Dictionary <long, byte[]>(); List <WordAttribute> waList = null; WordDictionaryFile wdf = null; if (!textFile) { wdf = await LoadFromBinFile(fileName); waList = wdf.Dicts; } else { wdf = await LoadFromTextFile(fileName); waList = wdf.Dicts; } foreach (WordAttribute wa in waList) { string key = wa.Word.ToLower(); if (key.Length == 1) { if (!_FirstCharDict.ContainsKey(key[0])) { _FirstCharDict.Add(key[0], wa); continue; } } if (key.Length == 2) { uint doubleChar = ((uint)key[0] * 65536) + key[1]; if (!_DoubleCharDict.ContainsKey(doubleChar)) { _DoubleCharDict.Add(doubleChar, wa); continue; } } if (!_WordDict.ContainsKey(key)) { _WordDict.Add(key, wa); long tripleChar = ((long)key[0]) * 0x100000000 + (uint)(key[1] * 65536) + key[2]; byte[] wordLenArray; if (!_TripleCharDict.TryGetValue(tripleChar, out wordLenArray)) { wordLenArray = new byte[4]; wordLenArray[0] = (byte)key.Length; _TripleCharDict.Add(tripleChar, wordLenArray); } else { bool find = false; int i; for (i = 0; i < wordLenArray.Length; i++) { byte len = wordLenArray[i]; if (len == key.Length) { find = true; break; } if (len == 0) { wordLenArray[i] = (byte)key.Length; find = true; break; } } if (!find) { byte[] temp = new byte[wordLenArray.Length * 2]; wordLenArray.CopyTo(temp, 0); wordLenArray = temp; wordLenArray[i] = (byte)key.Length; _TripleCharDict[tripleChar] = wordLenArray; } } } } }