private void LoadModel()
{
var stopWatch = new Stopwatch();
stopWatch.Start();
_prevStatus = new Dictionary <char, char[]>()
{
{ 'B', new [] { 'E', 'S' } },
{ 'M', new [] { 'M', 'B' } },
{ 'S', new [] { 'S', 'E' } },
{ 'E', new [] { 'B', 'M' } }
};
_startProbs = new Dictionary <char, double>()
{
{ 'B', -0.26268660809250016 },
{ 'E', -3.14e+100 },
{ 'M', -3.14e+100 },
{ 'S', -1.4652633398537678 }
};
var transJson = ManifestResourceReader.ReadAllText(ConfigManager.ProbTransFile, Encoding.UTF8);
// File.ReadAllText(Path.GetFullPath(ConfigManager.ProbTransFile));
_transProbs = ZCompileNLP.Json.JsonUtil.ToObject <IDictionary <char, IDictionary <char, double> > >(transJson);
var emitJson = ManifestResourceReader.ReadAllText(ConfigManager.ProbEmitFile, Encoding.UTF8);
//File.ReadAllText(Path.GetFullPath(ConfigManager.ProbEmitFile));
_emitProbs = ZCompileNLP.Json.JsonUtil.ToObject <IDictionary <char, IDictionary <char, double> > >(emitJson);
stopWatch.Stop();
Debug.WriteLine("model loading finished, time elapsed {0} ms.", stopWatch.ElapsedMilliseconds);
}
private static void LoadWordTagTab()
{
if (!ConfigManager.LoadMainDictFile)
{
return;
}
try
{
_wordTagTab = new Dictionary <string, string>();
var lines = ManifestResourceReader.ReadText(ConfigManager.MainDictFile, Encoding.UTF8);// File.ReadAllLines(ConfigManager.MainDictFile, Encoding.UTF8);
foreach (var line in lines)
{
var tokens = line.Split(' ');
if (tokens.Length < 2)
{
Debug.Fail(string.Format("Invalid line: {0}", line));
continue;
}
var word = tokens[0];
var tag = tokens[2];
_wordTagTab[word] = tag;
}
}
catch (IOException e)
{
Debug.Fail(string.Format("Word tag table load failure, reason: {0}", e.Message));
}
catch (FormatException fe)
{
Debug.Fail(fe.Message);
}
}
private void LoadDict()
{
if (!ConfigManager.LoadMainDictFile)
{
return;
}
try
{
var stopWatch = new Stopwatch();
stopWatch.Start();
Stream stream = ManifestResourceReader.GetStream(ConfigManager.MainDictFile);
using (var sr = new StreamReader(stream, Encoding.UTF8)) //MainDict, Encoding.UTF8))
{
string line = null;
while ((line = sr.ReadLine()) != null)
{
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
var tokens = line.Split(' ');
if (tokens.Length < 2)
{
Debug.Fail(string.Format("Invalid line: {0}", line));
continue;
}
var word = tokens[0];
var freq = int.Parse(tokens[1]);
Trie[word] = freq;
Total += freq;
foreach (var ch in Enumerable.Range(0, word.Length))
{
var wfrag = word.Sub(0, ch + 1);
if (!Trie.ContainsKey(wfrag))
{
Trie[wfrag] = 0;
}
}
}
}
stopWatch.Stop();
Debug.WriteLine("main dict load finished, time elapsed {0} ms", stopWatch.ElapsedMilliseconds);
}
catch (IOException e)
{
Debug.Fail(string.Format("{0} load failure, reason: {1}", ConfigManager.MainDictFile, e.Message));
}
catch (FormatException fe)
{
Debug.Fail(fe.Message);
}
}
private static void LoadModel()
{
var startJson = ManifestResourceReader.ReadAllText(ConfigManager.PosProbStartFile, Encoding.UTF8);
//File.ReadAllText(Path.GetFullPath(ConfigManager.PosProbStartFile));
_startProbs = ZCompileNLP.Json.JsonUtil.ToObject <IDictionary <string, double> >(startJson);
// JsonConvert.DeserializeObject<IDictionary<string, double>>(startJson);
var transJson = ManifestResourceReader.ReadAllText(ConfigManager.PosProbTransFile, Encoding.UTF8);
//File.ReadAllText(Path.GetFullPath(ConfigManager.PosProbTransFile));
_transProbs = ZCompileNLP.Json.JsonUtil.ToObject <IDictionary <string, IDictionary <string, double> > >(transJson);
var emitJson = ManifestResourceReader.ReadAllText(ConfigManager.PosProbEmitFile, Encoding.UTF8);
//File.ReadAllText(Path.GetFullPath(ConfigManager.PosProbEmitFile));
_emitProbs = ZCompileNLP.Json.JsonUtil.ToObject <IDictionary <string, IDictionary <char, double> > >(emitJson);
var tabJson = ManifestResourceReader.ReadAllText(ConfigManager.CharStateTabFile, Encoding.UTF8);
//File.ReadAllText(Path.GetFullPath(ConfigManager.CharStateTabFile));
_stateTab = ZCompileNLP.Json.JsonUtil.ToObject <IDictionary <char, List <string> > >(tabJson);
}
