/// <summary>
/// 加载主词典及扩展词典
/// </summary>
private void LoadMainDict()
{
var files = Directory.GetFiles(cfg.MainDictionary, "*.dict");
//读取量词词典文件
if (!files.Contains(cfg.MainDictFile))
{
throw new InvalidOperationException("未发现主词库词典!!!");
}
//建立一个主词典实例
mainDict = new DictSegment((char)0);
foreach (var file in files)
{
var theWord = File.ReadAllLines(file, Encoding.UTF8);
foreach (var word in theWord)
{
if (IsValidWord(word))
{
var vals = word.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
WordNode node = new WordNode()
{
Frequency = vals.Length > 1 ? double.Parse(vals[1]) : 1 //默认词频1
};
//词
mainDict.FillSegment(vals[0].ToLower().ToCharArray(), node);
}
}
}
}
public void TestRoundTrip_FileNames()
{
var a1 = new WordNode("DB", PartOfSpeechTag.Preamble);
var a2 = new WordNode("Get", PartOfSpeechTag.Verb);
var t1 = new WordNode("Hydro", PartOfSpeechTag.NounModifier);
var t2 = new WordNode("Fixed", PartOfSpeechTag.NounModifier);
var t3 = new WordNode("Schedule", PartOfSpeechTag.Noun);
var f1 = @"C:\foo\bar.cpp";
var f2 = @"C:\foo\baz\xyzzy.h";
var f3 = "test.cpp";
var sdr = new SwumDataRecord();
sdr.ParsedAction = new PhraseNode(new[] { a1, a2 }, Location.None, false);
sdr.Action = sdr.ParsedAction.ToPlainString();
sdr.ParsedTheme = new PhraseNode(new[] { t1, t2, t3 }, Location.None, false);
sdr.Theme = sdr.ParsedTheme.ToPlainString();
sdr.FileNames.Add(f1);
sdr.FileNames.Add(f2);
sdr.FileNames.Add(f3);
var actual = SwumDataRecord.Parse(sdr.ToString());
Assert.IsTrue(SwumDataRecordsAreEqual(sdr, actual));
}
public bool SearchNode(string word, WordNode node, int level)
{
if (node == null)
{
return(false);
}
if (level == word.Length)
{
return(node.isWordFinished);
}
char c = word[level];
if (c != '.')
{
return(node.children[c - 'a'] != null && SearchNode(word, node.children[c - 'a'], level + 1));
}
else
{
for (int i = 0; i < 26; i++)
{
if (node.children[i] != null && SearchNode(word, node.children[i], level + 1))
{
return(true);
}
}
}
return(false);
}
public void TestConstructor()
{
word2 = new WordNode("Testing");
string test = word2.Word;
Assert.AreEqual(test, "Testing");
}
public int Compare(WordNode x, WordNode y)
{
if (x == null && y == null)
{
return(0);
}
if (x == null)
{
return(-1);
}
if (y == null)
{
return(1);
}
if (x.count > y.count)
{
return(1);
}
else if (x.count < y.count)
{
return(-1);
}
return(0);
}
public void TestParse()
{
var actual = WordNode.Parse("cowboy(Noun)");
var expected = new WordNode("cowboy", PartOfSpeechTag.Noun);
Assert.IsTrue(WordNodesAreEqual(actual, expected));
}
public static bool WordNodesAreEqual(WordNode wn1, WordNode wn2)
{
return(wn1.Text == wn2.Text &&
wn1.Tag == wn2.Tag &&
wn1.Confidence == wn2.Confidence &&
wn1.Location == wn2.Location);
}
public void addChild(String cTag, String cWord, WordNode node)
{
if (childs != null)
{
WordNode[] temps = new WordNode[childs.Length];
for (int i = 0; i < childs.Length; i++)
{
temps[i] = childs[i];
}
numberOfChild++;
childs = new WordNode[childs.Length + 1];
for (int i = 0; i < childs.Length - 1; i++)
{
childs[i] = temps[i];
}
childs[childs.Length - 1] = new WordNode(cWord, cTag);
if (node.childs != null)
{
childs[childs.Length - 1].childs = node.childs;
}
}
else
{
childs = new WordNode[1];
childs[0] = new WordNode(cWord, cTag);
numberOfChild++;
}
}
public WordNode Add(char c)
{
if (!wordNode.ContainsKey(c))
{
wordNode[c] = new WordNode();
}
return(wordNode[c]);
}
public void TestConstructor_AllParams()
{
WordNode wn = new WordNode("watermelon", PartOfSpeechTag.Noun, 5.7);
Assert.AreEqual("watermelon", wn.Text);
Assert.AreEqual(PartOfSpeechTag.Noun, wn.Tag);
Assert.AreEqual(5.7, wn.Confidence);
Assert.AreEqual(Location.None, wn.Location);
}
public void TestGetParse()
{
WordNode wn = new WordNode("cowboy", PartOfSpeechTag.Noun);
var parse = wn.GetParse();
Assert.AreEqual(1, parse.Size());
Assert.AreEqual(wn, parse[0]);
}
public void TestConstructor_Default()
{
WordNode wn = new WordNode();
Assert.AreEqual(string.Empty, wn.Text);
Assert.AreEqual(PartOfSpeechTag.Unknown, wn.Tag);
Assert.AreEqual(0.0, wn.Confidence);
Assert.AreEqual(Location.None, wn.Location);
}
/// <summary>
/// Visit Word node in DFS manner.
/// </summary>
/// <param name="node">Word node that will be visited.</param>
public void Visit(WordNode node)
{
foreach (var item in node.Descendants)
{
item.Accept(this);
}
node.Accept(_visitor);
}
public void TestAllCaps()
{
WordNode wn = new WordNode("HoWdy");
Assert.IsFalse(wn.IsAllCaps());
wn = new WordNode("HOWDY");
Assert.IsTrue(wn.IsAllCaps());
}
public void TestConstructor_AllParams()
{
WordNode wn = new WordNode("watermelon", PartOfSpeechTag.Noun, 5.7);
Assert.AreEqual("watermelon", wn.Text);
Assert.AreEqual(PartOfSpeechTag.Noun, wn.Tag);
Assert.AreEqual(5.7, wn.Confidence);
Assert.AreEqual(Location.None, wn.Location);
}
public void TestConstructor_Default()
{
WordNode wn = new WordNode();
Assert.AreEqual(string.Empty, wn.Text);
Assert.AreEqual(PartOfSpeechTag.Unknown, wn.Tag);
Assert.AreEqual(0.0, wn.Confidence);
Assert.AreEqual(Location.None, wn.Location);
}
public void TestAllCaps()
{
WordNode wn = new WordNode("HoWdy");
Assert.IsFalse(wn.IsAllCaps());
wn = new WordNode("HOWDY");
Assert.IsTrue(wn.IsAllCaps());
}
/**
* /// Constructs a LexTreeWordState
*
* /// @param wordNode the word node
* /// @param lastNode the previous word node
* /// @param wordSequence the sequence of words triphone context
*
*
* /// @param logProbability the probability of this word occurring
*/
public LexTreeEndWordState(WordNode wordNode, HMMNode lastNode,
WordSequence wordSequence, float smearTerm, float smearProb,
float logProbability, LexTreeLinguist parent)
: base(wordNode, lastNode, wordSequence, smearTerm, smearProb,
logProbability, parent)
{
//Trace.WriteLine(string.Format("LexTreeEndWordState Created with values wordNode: {0}, lastNode: {1}, wordSequence: {2}, smearTerm: {3}, smearProb: {4}, logProbability: {5}",
// wordNode,lastNode,wordSequence,smearTerm,smearProb, logProbability));
}
public void TestGetNewWord_Default()
{
WordNode wn = new WordNode("watermelon", PartOfSpeechTag.Noun);
object obj = wn.GetNewWord();
Assert.IsInstanceOf(wn.GetType(), obj);
Assert.IsFalse(wn.Equals(obj));
WordNode wn2 = new WordNode();
Assert.IsTrue(WordNodesAreEqual(wn2, obj as WordNode));
}
public void TestGetNewWord_Parameters()
{
WordNode wn = new WordNode("watermelon", PartOfSpeechTag.Noun);
object obj = wn.GetNewWord("beef", PartOfSpeechTag.Verb);
Assert.IsInstanceOf(wn.GetType(), obj);
WordNode wn2 = new WordNode("beef", PartOfSpeechTag.Verb);
Assert.IsTrue(WordNodesAreEqual(wn2, obj as WordNode));
}
public void Parse_PlainTextSpacedWord_Successfully(string text, string expectedValue)
{
var tokens = tokenizer.Tokenize(text);
var expected = new WordNode(WordType.SpacedWord, expectedValue);
Parser.ParseSpacedWord(tokens)
.Should()
.BeEquivalentTo(expected);
}
public static TextNode ParseEmphasisText(Deque <Token> tokens)
{
var underscore = tokens.PopFirst();
if (tokens.Count == 0)
{
return(ParseText(new Deque <Token> {
new Token(TokenType.Text, underscore.Value)
}));
}
if (tokens.PeekFirst().Type == TokenType.Space)
{
tokens.Insert(0, new Token(TokenType.Text, underscore.Value));
return(ParseText(tokens));
}
var emphasisText = new TextNode(TextType.Emphasis);
while (true)
{
if (tokens.Count == 0)
{
emphasisText.Words.Insert(0, new WordNode(WordType.SimpleWord, underscore.Value));
return(new TextNode(TextType.Text, emphasisText.Words));
}
var currentToken = tokens.PeekFirst();
switch (currentToken.Type)
{
case TokenType.Underscore:
tokens.PopFirst();
if (emphasisText.Words.Last().Type == WordType.Space)
{
emphasisText.Words.Insert(0, new WordNode(WordType.SimpleWord, underscore.Value));
return(new TextNode(TextType.Text, emphasisText.Words));
}
return(emphasisText);
case TokenType.DoubleUnderscore:
tokens.PopFirst();
var doubleUnderscore = new WordNode(WordType.SimpleWord, underscore.Value + underscore.Value);
emphasisText.Add(doubleUnderscore);
break;
default:
var innerText = ParseText(tokens);
var innerWords = innerText.Words;
emphasisText.AddRange(innerWords);
break;
}
}
}
public void Parse_Space_Successfully()
{
var text = " ";
var tokens = tokenizer.Tokenize(text);
var expected = new WordNode(WordType.Space, " ");
Parser.ParseSpacedWord(tokens)
.Should()
.BeEquivalentTo(expected);
}
public void TestToString() {
var wn1 = new WordNode("Eat", PartOfSpeechTag.Verb);
var wn2 = new WordNode("More", PartOfSpeechTag.NounModifier);
var wn3 = new WordNode("Chicken", PartOfSpeechTag.Noun);
var words = new WordNode[] {wn1, wn2, wn3};
PhraseNode pn = new PhraseNode(words, Location.Name, true);
string expected = string.Format("{0} {1} {2}", wn1, wn2, wn3);
Assert.AreEqual(expected, pn.ToString());
}
public void TestRoundTrip()
{
var wn1 = new WordNode("Get", PartOfSpeechTag.Verb);
var wn2 = new WordNode("Fixed", PartOfSpeechTag.NounModifier);
var wn3 = new WordNode("Hydro", PartOfSpeechTag.NounModifier);
var wn4 = new WordNode("Schedule", PartOfSpeechTag.Noun);
var pn = new PhraseNode(new[] { wn1, wn2, wn3, wn4 }, Location.Name, false);
Assert.IsTrue(PhraseNodesAreEqual(pn, PhraseNode.Parse(pn.ToString())));
}
public void TestConstructor_EnumerableWordNodesEmpty() {
var words = new WordNode[] {};
PhraseNode pn = new PhraseNode(words, Location.Name, true);
Assert.AreEqual(words.Length, pn.Size());
for(int i = 0; i < pn.Size(); i++) {
Assert.AreEqual(words[i], pn[i]);
}
Assert.AreEqual(Location.Name, pn.Location);
}
public void TestGetNewWord_Parameters()
{
WordNode wn = new WordNode("watermelon", PartOfSpeechTag.Noun);
object obj = wn.GetNewWord("beef", PartOfSpeechTag.Verb);
Assert.IsInstanceOf(wn.GetType(), obj);
WordNode wn2 = new WordNode("beef", PartOfSpeechTag.Verb);
Assert.IsTrue(WordNodesAreEqual(wn2, obj as WordNode));
}
public void TestGetNewWord_Default()
{
WordNode wn = new WordNode("watermelon", PartOfSpeechTag.Noun);
object obj = wn.GetNewWord();
Assert.IsInstanceOf(wn.GetType(), obj);
Assert.IsFalse(wn.Equals(obj));
WordNode wn2 = new WordNode();
Assert.IsTrue(WordNodesAreEqual(wn2, obj as WordNode));
}
public void TestToString()
{
var wn1 = new WordNode("Eat", PartOfSpeechTag.Verb);
var wn2 = new WordNode("More", PartOfSpeechTag.NounModifier);
var wn3 = new WordNode("Chicken", PartOfSpeechTag.Noun);
var words = new WordNode[] { wn1, wn2, wn3 };
PhraseNode pn = new PhraseNode(words, Location.Name, true);
string expected = string.Format("{0} {1} {2}", wn1, wn2, wn3);
Assert.AreEqual(expected, pn.ToString());
}
public WordNode AddChildWorkNode(char word, bool isBreakUpNode = false)
{
var newWordNode = new WordNode();
newWordNode.word = word;
newWordNode.parent = this;
newWordNode.isBreakUpNode = isBreakUpNode;
children.Add(newWordNode);
return(newWordNode);
}
public void TestConstructor_EnumerableWordNodesEmpty()
{
var words = new WordNode[] {};
PhraseNode pn = new PhraseNode(words, Location.Name, true);
Assert.AreEqual(words.Length, pn.Size());
for (int i = 0; i < pn.Size(); i++)
{
Assert.AreEqual(words[i], pn[i]);
}
Assert.AreEqual(Location.Name, pn.Location);
}
public void TestParse()
{
var wn1 = new WordNode("Eat", PartOfSpeechTag.Verb);
var wn2 = new WordNode("More", PartOfSpeechTag.NounModifier);
var wn3 = new WordNode("Chicken", PartOfSpeechTag.Noun);
var words = new WordNode[] { wn1, wn2, wn3 };
var expected = new PhraseNode(words, Location.Name, false);
var actual = PhraseNode.Parse("Eat(Verb) More(NounModifier) Chicken(Noun)");
Assert.IsTrue(PhraseNodesAreEqual(actual, expected));
}
public void TestGetParse()
{
var wn1 = new WordNode("Eat", PartOfSpeechTag.Verb);
var wn2 = new WordNode("More", PartOfSpeechTag.NounModifier);
var wn3 = new WordNode("Chicken", PartOfSpeechTag.Noun);
var words = new WordNode[] { wn1, wn2, wn3 };
PhraseNode pn = new PhraseNode(words, Location.Name, true);
var parse = pn.GetParse();
Assert.AreEqual(pn, parse);
}
public object VisitWordNode(WordNode node, Context parameter)
{
parameter.IsAvailable = true;
var word = node.Word;
if (node.IsReserved)
{
parameter.AppendWhitespaceIfNecessary();
}
parameter.AppendRawSql(word);
parameter.AppendFormattedSql(word);
return(null);
}
public void copy(WordNode node)
{
word = node.word;
tag = node.tag;
if (node.childs != null)
{
childs = node.childs;
}
else
{
childs = null;
}
}
public void TestConstructor_EnumerableWordNodes() {
var wn1 = new WordNode("Eat", PartOfSpeechTag.Verb);
var wn2 = new WordNode("More", PartOfSpeechTag.NounModifier);
var wn3 = new WordNode("Chicken", PartOfSpeechTag.Noun);
var words = new WordNode[] {wn1, wn2, wn3};
PhraseNode pn = new PhraseNode(words, Location.Name, true);
Assert.AreEqual(words.Length, pn.Size());
for(int i = 0; i < pn.Size(); i++) {
Assert.AreEqual(words[i], pn[i]);
}
Assert.AreEqual(Location.Name, pn.Location);
}
public void TestGetPhrase()
{
var wn1 = new WordNode("Eat", PartOfSpeechTag.Verb);
var wn2 = new WordNode("More", PartOfSpeechTag.NounModifier);
var wn3 = new WordNode("Chicken", PartOfSpeechTag.Noun);
var words = new WordNode[] { wn1, wn2, wn3 };
PhraseNode pn = new PhraseNode(words, Location.Name, true);
var phrase = pn.GetPhrase();
Assert.AreEqual(words.Length, phrase.Count);
for (int i = 0; i < phrase.Count; i++)
{
Assert.AreEqual(pn[i], words[i]);
}
}
public void TestConstructor_EnumerableWordNodes()
{
var wn1 = new WordNode("Eat", PartOfSpeechTag.Verb);
var wn2 = new WordNode("More", PartOfSpeechTag.NounModifier);
var wn3 = new WordNode("Chicken", PartOfSpeechTag.Noun);
var words = new WordNode[] { wn1, wn2, wn3 };
PhraseNode pn = new PhraseNode(words, Location.Name, true);
Assert.AreEqual(words.Length, pn.Size());
for (int i = 0; i < pn.Size(); i++)
{
Assert.AreEqual(words[i], pn[i]);
}
Assert.AreEqual(Location.Name, pn.Location);
}
public void TestToString()
{
WordNode wn = new WordNode("Test", PartOfSpeechTag.Noun);
Assert.AreEqual("Test(Noun)", wn.ToString());
}
public void TestParse() {
var actual = WordNode.Parse("cowboy(Noun)");
var expected = new WordNode("cowboy", PartOfSpeechTag.Noun);
Assert.IsTrue(WordNodesAreEqual(actual, expected));
}
public void TestGetParse() {
WordNode wn = new WordNode("cowboy", PartOfSpeechTag.Noun);
var parse = wn.GetParse();
Assert.AreEqual(1, parse.Size());
Assert.AreEqual(wn, parse[0]);
}
public void TestParse() {
var wn1 = new WordNode("Eat", PartOfSpeechTag.Verb);
var wn2 = new WordNode("More", PartOfSpeechTag.NounModifier);
var wn3 = new WordNode("Chicken", PartOfSpeechTag.Noun);
var words = new WordNode[] { wn1, wn2, wn3 };
var expected = new PhraseNode(words, Location.Name, false);
var actual = PhraseNode.Parse("Eat(Verb) More(NounModifier) Chicken(Noun)");
Assert.IsTrue(PhraseNodesAreEqual(actual, expected));
}
public void TestGetParse() {
var wn1 = new WordNode("Eat", PartOfSpeechTag.Verb);
var wn2 = new WordNode("More", PartOfSpeechTag.NounModifier);
var wn3 = new WordNode("Chicken", PartOfSpeechTag.Noun);
var words = new WordNode[] {wn1, wn2, wn3};
PhraseNode pn = new PhraseNode(words, Location.Name, true);
var parse = pn.GetParse();
Assert.AreEqual(pn, parse);
}
public void TestRoundTrip() {
var wn1 = new WordNode("Get", PartOfSpeechTag.Verb);
var wn2 = new WordNode("Fixed", PartOfSpeechTag.NounModifier);
var wn3 = new WordNode("Hydro", PartOfSpeechTag.NounModifier);
var wn4 = new WordNode("Schedule", PartOfSpeechTag.Noun);
var pn = new PhraseNode(new[] { wn1, wn2, wn3, wn4 }, Location.Name, false);
Assert.IsTrue(PhraseNodesAreEqual(pn, PhraseNode.Parse(pn.ToString())));
}
public WordNode()
{
ChildNodes = new WordNode[26];
}
public void TestGetPhrase() {
var wn1 = new WordNode("Eat", PartOfSpeechTag.Verb);
var wn2 = new WordNode("More", PartOfSpeechTag.NounModifier);
var wn3 = new WordNode("Chicken", PartOfSpeechTag.Noun);
var words = new WordNode[] {wn1, wn2, wn3};
PhraseNode pn = new PhraseNode(words, Location.Name, true);
var phrase = pn.GetPhrase();
Assert.AreEqual(words.Length, phrase.Count);
for(int i = 0; i < phrase.Count; i++) {
Assert.AreEqual(pn[i], words[i]);
}
}
public static bool WordNodesAreEqual(WordNode wn1, WordNode wn2)
{
return wn1.Text == wn2.Text
&& wn1.Tag == wn2.Tag
&& wn1.Confidence == wn2.Confidence
&& wn1.Location == wn2.Location;
}
public void TestRoundTrip() {
var wn = new WordNode("mangoes", PartOfSpeechTag.NounPlural);
Assert.IsTrue(WordNodesAreEqual(wn, WordNode.Parse(wn.ToString())));
}
void UpdateEndpoint(WordNode node) {
Log.WriteLine("Updating endpoint");
if (Endpoint == null) {
Endpoint = new WordNode(node);
Endpoint.PathsLeadingHere = node.PathsLeadingHere;
}
else {
Endpoint.Parents.Add(node);
Endpoint.PathsLeadingHere += node.PathsLeadingHere;
}
}
void DecodeBFS() {
Memorized[0] = new WordNode(null);
for (int j = 0; j < Memorized.Length; j++) {
if (Memorized[j] != null) {
WordNode node = Memorized[j];
Log.WriteLine("Examining character {0}, path count: {1}", j, node.PathsLeadingHere);
int i = 0;
MorseNode morse = MorseRoot;
while (i + j < MorseInput.Length) {
int k = i + j;
morse = morse.Next(MorseInput[k]);
if (morse != null) {
if (morse.WordsEndingHere.Count > 0) {
Log.WriteLine("Word found at position {0}", k);
if (k == MorseInput.Length - 1) {
Log.WriteLine("Updating endpoint");
if (Endpoint == null) {
Endpoint = new WordNode(node);
Endpoint.PathsLeadingHere = 0;
}
Endpoint.PathsLeadingHere += node.PathsLeadingHere * morse.WordsEndingHere.Count;
}
else {
//Update the node after this one so it becomes a start point
if (Memorized[k + 1] == null) {
Memorized[k + 1] = new WordNode(node);
Memorized[k + 1].PathsLeadingHere = 0;
}
Log.WriteLine("In-node word found at {0}", k + 1);
Memorized[k + 1].PathsLeadingHere += node.PathsLeadingHere * morse.WordsEndingHere.Count;
}
}
i++;
}
else {
Log.WriteLine("Dead end");
break;
}
}
}
}
}
public long Solve() {
foreach (Word w in Words) {
Encode(w, MorseRoot);
}
WordNode wordRoot = new WordNode(null);
wordRoot.PathsLeadingHere = 1;
Log.TimeStamp();
DecodeBFS();
Log.TimeStamp();
if (Endpoint == null) {
Log.WriteLine("No endpoint exists.");
return 0;
}
else {
Log.WriteLine("Endpoint exists.");
return Endpoint.PathsLeadingHere;
}
}
public WordNode(WordNode parent) {
if (parent != null) {
Parents.Add(parent);
parent.Children.Add(this);
}
}
