public void Expectation(TrieNode p, Stack <char> stack_list, StreamWriter outfile)
{
if (p != null)
{
stack_list.Push(p.letter);
List <Data> data_list = new List <Data>();
if ((p.endOfWord) && (p.NumOfWords > 1) && (p.frequency >= 5)) //unigrams need not be considered: see paper for reason and we want bi/trigram to occur atleast 10 times to be considered
{
TrieNode new_p;
string temp = string.Join("", stack_list.ToArray().Reverse());
temp = temp.Trim();
string[] words = temp.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
string word;
///**************************************************** Finding the intersecting LISTS ***************/
for (int i = 0; i < words.Length; i++)
{
word = " " + words[i];//root has " " to begin with
new_p = this.FindWord(word);
if (new_p == null)
{
continue;// we coudl have removed that word during pruning
}
//throw new Exception("You are trying to get expectation value for a words whose unigram is not present");
if (i == 0)
{
data_list = new_p.Query_list;
}
else if (data_list != null)
{
data_list = Data.intersect(data_list, new_p.Query_list); // this gives the number of queies which contain all the 3 words
}
else
{
throw new Exception("Expectation calculation:Unigram is stored without a query id");
}
}
p.Set_frequency = data_list.Count;
/* *************************************************** Finding the Hoefding's ************** */
p.Set_expectation = 0;
for (int i = 0; i < data_list.Count; i++)
{
int number = this.factorial(data_list[i].Q_length, p.NumOfWords);
float num_inverse = 0;
if (number < 10000000 && number != 0)
{
num_inverse = 1 / (( float )number);
p.Set_expectation += num_inverse;
}
}
/*
* Hoefding's inequality: F-E > sqrt(-K*log(s)/2)
* let hoefding'score -log(s)=((F-E)^2)*2/k;
*/
if (p.frequency > p.Set_expectation)
{
p.Hoefding_score = (((p.frequency - p.Set_expectation) * (p.frequency - p.Set_expectation)) * 2) / 2 * p.Set_frequency;
outfile.WriteLine(p.Hoefding_score + "\t" + temp);
}
}
string test = String.Join("", stack_list.ToArray().Reverse());
Expectation(p.daughter, stack_list, outfile);
stack_list.Pop();
Expectation(p.sibling, stack_list, outfile);
}
return;
}
public void insert(string word, Data new_data, int num_words) //num_words tells whether it is trigram or bigram. Remember each word in a bigram must be separated from other by " "
{
TrieNode p = this.root;
int i = 0;
while (i < word.Length - 1)
{
if (p == null)
{
break;
}
char test = word[i];
char test2 = word[i + 1];
char pLetter = p.letter;
if (p.letter == word[i])
{
if (p.daughter != null)
{
p = p.daughter;
}
else
{
p.daughter = new TrieNode(word[i + 1]);
p = p.daughter;
this.TotalNodes++;
}
//go to next letter in the word
++i;
}
else
{
if (p.sibling != null)
{
p = p.sibling;
}
else
{
p.sibling = new TrieNode(word[i]);
p = p.sibling;
this.TotalNodes++;
}
}
}
//handling the last character as a special case
while (i < word.Length)
{
if (p == null)
{
break;
}
if (p.letter == word[i])
{
++i;
}
else if (p.letter != word[i] && p.sibling != null)
{
p = p.sibling;
continue;
}
else if (p.letter != word[i]) // and we have no sibling
{
p.sibling = new TrieNode(word[i]);
this.TotalNodes++;
p = p.sibling;
}
p.endOfWord = true;
p.frequency++;
p.NumOfWords = num_words;
if (num_words == 1)
{
p.Query_list.Add(new_data);
}
}
}
public Trie()
{
root = new TrieNode();
TotalNodes = 1;
root.frequency = 100;
}