private void LoadModel() { var stopWatch = new Stopwatch(); stopWatch.Start(); _prevStatus = new Dictionary <char, char[]>() { { 'B', new [] { 'E', 'S' } }, { 'M', new [] { 'M', 'B' } }, { 'S', new [] { 'S', 'E' } }, { 'E', new [] { 'B', 'M' } } }; _startProbs = new Dictionary <char, double>() { { 'B', -0.26268660809250016 }, { 'E', -3.14e+100 }, { 'M', -3.14e+100 }, { 'S', -1.4652633398537678 } }; var transJson = ManifestResourceReader.ReadAllText(ConfigManager.ProbTransFile, Encoding.UTF8); // File.ReadAllText(Path.GetFullPath(ConfigManager.ProbTransFile)); _transProbs = ZCompileNLP.Json.JsonUtil.ToObject <IDictionary <char, IDictionary <char, double> > >(transJson); var emitJson = ManifestResourceReader.ReadAllText(ConfigManager.ProbEmitFile, Encoding.UTF8); //File.ReadAllText(Path.GetFullPath(ConfigManager.ProbEmitFile)); _emitProbs = ZCompileNLP.Json.JsonUtil.ToObject <IDictionary <char, IDictionary <char, double> > >(emitJson); stopWatch.Stop(); Debug.WriteLine("model loading finished, time elapsed {0} ms.", stopWatch.ElapsedMilliseconds); }
private static void LoadWordTagTab() { if (!ConfigManager.LoadMainDictFile) { return; } try { _wordTagTab = new Dictionary <string, string>(); var lines = ManifestResourceReader.ReadText(ConfigManager.MainDictFile, Encoding.UTF8);// File.ReadAllLines(ConfigManager.MainDictFile, Encoding.UTF8); foreach (var line in lines) { var tokens = line.Split(' '); if (tokens.Length < 2) { Debug.Fail(string.Format("Invalid line: {0}", line)); continue; } var word = tokens[0]; var tag = tokens[2]; _wordTagTab[word] = tag; } } catch (IOException e) { Debug.Fail(string.Format("Word tag table load failure, reason: {0}", e.Message)); } catch (FormatException fe) { Debug.Fail(fe.Message); } }
private void LoadDict() { if (!ConfigManager.LoadMainDictFile) { return; } try { var stopWatch = new Stopwatch(); stopWatch.Start(); Stream stream = ManifestResourceReader.GetStream(ConfigManager.MainDictFile); using (var sr = new StreamReader(stream, Encoding.UTF8)) //MainDict, Encoding.UTF8)) { string line = null; while ((line = sr.ReadLine()) != null) { if (string.IsNullOrWhiteSpace(line)) { continue; } var tokens = line.Split(' '); if (tokens.Length < 2) { Debug.Fail(string.Format("Invalid line: {0}", line)); continue; } var word = tokens[0]; var freq = int.Parse(tokens[1]); Trie[word] = freq; Total += freq; foreach (var ch in Enumerable.Range(0, word.Length)) { var wfrag = word.Sub(0, ch + 1); if (!Trie.ContainsKey(wfrag)) { Trie[wfrag] = 0; } } } } stopWatch.Stop(); Debug.WriteLine("main dict load finished, time elapsed {0} ms", stopWatch.ElapsedMilliseconds); } catch (IOException e) { Debug.Fail(string.Format("{0} load failure, reason: {1}", ConfigManager.MainDictFile, e.Message)); } catch (FormatException fe) { Debug.Fail(fe.Message); } }
private static void LoadModel() { var startJson = ManifestResourceReader.ReadAllText(ConfigManager.PosProbStartFile, Encoding.UTF8); //File.ReadAllText(Path.GetFullPath(ConfigManager.PosProbStartFile)); _startProbs = ZCompileNLP.Json.JsonUtil.ToObject <IDictionary <string, double> >(startJson); // JsonConvert.DeserializeObject<IDictionary<string, double>>(startJson); var transJson = ManifestResourceReader.ReadAllText(ConfigManager.PosProbTransFile, Encoding.UTF8); //File.ReadAllText(Path.GetFullPath(ConfigManager.PosProbTransFile)); _transProbs = ZCompileNLP.Json.JsonUtil.ToObject <IDictionary <string, IDictionary <string, double> > >(transJson); var emitJson = ManifestResourceReader.ReadAllText(ConfigManager.PosProbEmitFile, Encoding.UTF8); //File.ReadAllText(Path.GetFullPath(ConfigManager.PosProbEmitFile)); _emitProbs = ZCompileNLP.Json.JsonUtil.ToObject <IDictionary <string, IDictionary <char, double> > >(emitJson); var tabJson = ManifestResourceReader.ReadAllText(ConfigManager.CharStateTabFile, Encoding.UTF8); //File.ReadAllText(Path.GetFullPath(ConfigManager.CharStateTabFile)); _stateTab = ZCompileNLP.Json.JsonUtil.ToObject <IDictionary <char, List <string> > >(tabJson); }