private WordDictionaryFile LoadFromTextFile(String fileName) { WordDictionaryFile dictFile = new WordDictionaryFile(); dictFile.Dicts = new List<IDataNode>(); using (StreamReader sr = new StreamReader(fileName, Encoding.UTF8)) { while (!sr.EndOfStream) { string line = sr.ReadLine(); string[] strs = line.Split(new char[] { '|' }); if (strs.Length == 3) { string word = strs[0].Trim(); POSType pos = (POSType)int.Parse(strs[1].Substring(2, strs[1].Length - 2), System.Globalization.NumberStyles.HexNumber); double frequency = double.Parse(strs[2]); WordAttribute dict = new WordAttribute(word, pos, frequency); dictFile.Dicts.Add(dict); } } } return dictFile; }
private WordDictionaryFile LoadFromBinFile(String fileName, out string verNumStr) { WordDictionaryFile dictFile = new WordDictionaryFile(); dictFile.Dicts = new List<IDataNode>(); FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read); byte[] version = new byte[32]; fs.Read(version, 0, version.Length); String ver = Encoding.UTF8.GetString(version, 0, version.Length); int zeroPosition = ver.IndexOf('\0'); if (zeroPosition >= 0) { ver = ver.Substring(0, zeroPosition); } var matches = verRegex.Matches(ver); if (matches.Count > 0) verNumStr = matches[0].Value; else verNumStr = null; while (fs.Position < fs.Length) { byte[] buf = new byte[sizeof(int)]; fs.Read(buf, 0, buf.Length); int length = BitConverter.ToInt32(buf, 0); buf = new byte[length]; fs.Read(buf, 0, buf.Length); string word = Encoding.UTF8.GetString(buf, 0, length - sizeof(int) - sizeof(double)); POSType pos = (POSType)BitConverter.ToInt32(buf, length - sizeof(int) - sizeof(double)); double frequency = BitConverter.ToDouble(buf, length - sizeof(double)); WordAttribute dict = new WordAttribute(word, pos, frequency); string.Intern(dict.Word); dictFile.Dicts.Add(dict); } fs.Close(); return dictFile; }