public IDataTree setUpDataTree(IBaseTree baseTree)
{
IDataTree data = DataTreeBuilder.CreateDocumentMappedTree(baseTree);
DataTreeBuilder.AddToDataTree(data, toMap);
return(data);
}
public void DocumentMappingTest2()
{
IBaseTree tree = setUpTree();
IDataTree dataTree = DataTreeBuilder.CreateDocumentMappedTree(tree, toMap);
//make sure the root has the correct number of children
Assert.AreEqual(dataTree.Root.Children.Count, 2);
//make sure each of the children has the correct weight
Assert.AreEqual(dataTree.Root.Children.Where(c => c.EndPoint.Keyword == "tree").FirstOrDefault().Weight, 3);
Assert.AreEqual(dataTree.Root.Children.Where(c => c.EndPoint.Keyword == "others").FirstOrDefault().Weight, 2);
//make sure a branch on the content tree which is not existent in the mapped tree does not exist
Assert.AreEqual(dataTree.Root.Children.Where(c => c.EndPoint.Keyword == "nonexistent").FirstOrDefault(), null);
//make sure each child's connections have their own correct weight
Assert.AreEqual(dataTree.Root.Children.Where(c => c.EndPoint.Keyword == "tree").FirstOrDefault().EndPoint.Children.Where(x => x.EndPoint.Keyword == "content").FirstOrDefault().Weight, 1);
Assert.AreEqual(dataTree.Root.Children.Where(c => c.EndPoint.Keyword == "tree").FirstOrDefault().EndPoint.Children.Where(x => x.EndPoint.Keyword == "mapped").FirstOrDefault().Weight, 1);
Assert.AreEqual(dataTree.Root.Children.Where(c => c.EndPoint.Keyword == "others").FirstOrDefault().EndPoint.Children.Where(x => x.EndPoint.Keyword == "series").FirstOrDefault().Weight, 1);
Assert.AreEqual(dataTree.Root.Children.Where(c => c.EndPoint.Keyword == "others").FirstOrDefault().EndPoint.Children.Where(x => x.EndPoint.Keyword == "going").FirstOrDefault().Weight, 1);
//make sure children connections do not cross
Assert.AreEqual(dataTree.Root.Children.Where(c => c.EndPoint.Keyword == "others").FirstOrDefault().EndPoint.Children.Where(x => x.EndPoint.Keyword == "content").FirstOrDefault(), null);
//make sure leaf children do not have any connections
Assert.AreEqual(dataTree.Root.Children.Where(c => c.EndPoint.Keyword == "tree").FirstOrDefault().EndPoint.Children.Where(x => x.EndPoint.Keyword == "content").FirstOrDefault().EndPoint.Children.Count, 0);
}
public static void MakeTrees(string contentTreeName)
{
IIO io = new FileIO();
IEnumerable<string> file = io.ReadSourceIterable(testpath + "TIME.ALL");
ITextExtractor it = new BeginMarkerExtraction(file, "*TEXT");
ITreeIO tio = new TreeIO();
IBaseTree tree = tio.LoadBaseTree(testpath + contentTreeName);
int count = 1;
while (it.HasNextContent()) {
string content = it.FindNextContent();
//Console.WriteLine("-----");
string name = "" + count;
//Console.WriteLine(name);
content = Helpers.ConsumeName(content);
//Console.WriteLine(content);
IDataTree datatree = DataTreeBuilder.CreateDocumentMappedTree(tree);
//Console.WriteLine(tree);
//Console.WriteLine(datatree.GetBaseTree());
DataTreeBuilder.AddToDataTree(datatree, content);
datatree.Name = name;
tio.SaveDataTree(datatree, testpath + @"\datatrees\" + name + ".dtree");
//Console.WriteLine(datatree.MappedWords);
count++;
}
}
public static void BuildContentTreeGoodMatches()
{
ITreeIO tio = new TreeIO();
WordSuggestor ws = tio.LoadObject(testpath + "WordSuggestions") as WordSuggestor;
IBaseTree tree = ws.BuildTreeGoodMatches();
tio.SaveBaseTree(tree, testpath + "AutoTree2.tree");
}
public void DocumentMapNonexistentContentEnumberable()
{
IBaseTree tree = setUpTree();
IDataTree dataTree = DataTreeBuilder.CreateDocumentMappedTree(tree);
DataTreeBuilder.AddAllToDataTree(dataTree, null);
}
private void New_Click(object sender, RoutedEventArgs e)
{
baseTree = new DuplicatesAllowedBaseTree();
baseTree.AddWord((string)null, "Root");
tree = new ViewableTree(baseTree);
nodeList.ItemsSource = tree.Root;
}
public MainWindow()
{
InitializeComponent();
baseTree = new DuplicatesAllowedBaseTree();
baseTree.AddWord((string)null, "Root");
tree = new ViewableTree(baseTree);
nodeList.ItemsSource = tree.Root;
}
public static IDataTree CreateStemmedDocumentMapTree(IBaseTree baseTree)
{
if (baseTree == null)
{
throw new ArgumentNullException();
}
IDataTree toReturn = new StemmedDocumentMap(new DocumentMappedTree(baseTree));
return(toReturn);
}
public static IDataTree CreateDocumentMappedTree(IBaseTree baseTree, string content)
{
if (baseTree == null || content == null)
{
throw new ArgumentNullException();
}
IDataTree toReturn = CreateDocumentMappedTree(baseTree);
AddToDataTree(toReturn, content);
return(toReturn);
}
public void DocumentMapChangeBaseTree()
{
IBaseTree tree = setUpTree();
IDataTree dataTree = DataTreeBuilder.CreateDocumentMappedTree(tree);
DataTreeBuilder.AddAllToDataTree(dataTree, toMapEnumerable);
dataTree.SetBaseTree(null);
Assert.AreEqual(dataTree.GetBaseTree(), null);
}
public void LoadInvalidFileTest2()
{
ITreeIO io = new TreeIO();
IBaseTree basetree = setUpBaseTree();
IDataTree datatree = setUpDataTree(basetree);
io.SaveDataTree(datatree, location);
Assert.IsTrue(File.Exists(location));
IBaseTree loadedDatatree = io.LoadBaseTree(location);
}
public void SaveBaseTreeTest()
{
ITreeIO io = new TreeIO();
IBaseTree basetree = setUpBaseTree();
io.SaveBaseTree(basetree, location);
Assert.IsTrue(File.Exists(location));
IBaseTree loadedBasetree = io.LoadBaseTree(location);
Assert.AreNotSame(basetree, loadedBasetree);
Assert.AreEqual(basetree.Root.KeyWord, loadedBasetree.Root.KeyWord);
}
public void SetBaseTree(IBaseTree tree)
{
baseTree = tree;
this.tree.SetBaseTree(tree);
var enumer = baseTree.GetEnumerator();
List <Node> nodes = new List <Node>();
while (enumer.MoveNext())
{
nodes.Add((Node)enumer.Current);
}
for (int i = 0; i < nodes.Count; i++)
{
Node s = nodes[i];
baseTree.Rename(s, StringFunctions.StemmedWord(s.KeyWord));
}
}
private void loadContentTreeButton_Click(object sender, RoutedEventArgs e)
{
OpenFileDialog ofd = new OpenFileDialog();
ofd.FileName = "Tree";
ofd.DefaultExt = ".tree";
ofd.Filter = "Tree Files (.tree)|*.tree";
Nullable <bool> result = ofd.ShowDialog();
if (result == true)
{
string filename = ofd.FileName;
TreeIO io = new TreeIO();
baseTree = io.LoadBaseTree(filename);
contentTreeLabel.Content = ofd.FileName;
}
}
public void SaveDataTreeTest()
{
ITreeIO io = new TreeIO();
IBaseTree basetree = setUpBaseTree();
IDataTree datatree = setUpDataTree(basetree);
io.SaveDataTree(datatree, location);
Assert.IsTrue(File.Exists(location));
IDataTree loadedDatatree = io.LoadDataTree(location);
Assert.AreNotSame(basetree, loadedDatatree);
Assert.AreEqual(basetree.Root.KeyWord, loadedDatatree.Root.Keyword);
Assert.IsNull(loadedDatatree.GetBaseTree());
Assert.IsNotNull(datatree.GetBaseTree());
}
public ViewableTree(IBaseTree basetree)
{
List <ViewableTreeNode> temp = new List <ViewableTreeNode>();
List <Node> orphans = new List <Node>();
Dictionary <Node, ViewableTreeNode> nodes = new Dictionary <Node, ViewableTreeNode>();
tree = basetree;
Root = new ObservableCollection <ViewableTreeNode>();
ViewableTreeNode rootNode = new ViewableTreeNode(tree.Root);
temp.Add(rootNode);
nodes.Add(tree.Root, rootNode);
Root.Add(rootNode);
foreach (Node n in tree)
{
if (n != tree.Root)
{
ViewableTreeNode node = new ViewableTreeNode(n);
temp.Add(node);
nodes.Add(n, node);
var result = temp.Where(t => n.Parent != null && t.Keyword == n.Parent.KeyWord).FirstOrDefault();
if (result != null)
{
result.Children.Add(node);
}
else
{
orphans.Add(n);
}
}
}
foreach (Node n in orphans)
{
var result = temp.Where(t => n.Parent != null && t.Keyword == n.Parent.KeyWord).FirstOrDefault();
if (result != null)
{
result.Children.Add(nodes[n]);
}
}
}
public void CompareTrees()
{
IBaseTree tree = setUpTree();
IDataTree originalDataTree = DataTreeBuilder.CreateDocumentMappedTree(tree);
DataTreeBuilder.AddToDataTree(originalDataTree, toMap);
IDataTree probablyMatches = DataTreeBuilder.CreateDocumentMappedTree(tree);
DataTreeBuilder.AddToDataTree(probablyMatches, shouldProbablyMatch);
IDataTree notAMatch = DataTreeBuilder.CreateDocumentMappedTree(tree);
DataTreeBuilder.AddToDataTree(notAMatch, shouldProbablyNotMatch);
Assert.IsTrue(originalDataTree.CompareTo(originalDataTree));
Assert.IsTrue(originalDataTree.CompareTo(probablyMatches));
Assert.IsFalse(originalDataTree.CompareTo(notAMatch));
}
private void Load_Click(object sender, RoutedEventArgs e)
{
OpenFileDialog ofd = new OpenFileDialog();
ofd.FileName = "Tree";
ofd.DefaultExt = ".tree";
ofd.Filter = "Tree Files (.tree)|*.tree";
Nullable <bool> result = ofd.ShowDialog();
if (result == true)
{
string filename = ofd.FileName;
TreeIO io = new TreeIO();
baseTree = io.LoadBaseTree(filename);
tree = new ViewableTree(baseTree);
nodeList.ItemsSource = tree.Root;
}
}
internal DocumentMappedTree(IBaseTree tree)
{
BaseTree = tree;
Root = new DataNode(tree.Root.KeyWord);
}
public void SetBaseTree(IBaseTree tree)
{
this.BaseTree = tree;
}
public void DocumentMapNonexistentContent2()
{
IBaseTree tree = setUpTree();
IDataTree dataTree = DataTreeBuilder.CreateDocumentMappedTree(tree, null);
}
public static void Compare(string contentTreeName)
{
Console.SetBufferSize(100, 20000);
IIO io = new FileIO();
IEnumerable<string> file = io.ReadSourceIterable(testpath + "TIME.QUE");
IEnumerable<string> expectedResults = io.ReadSourceIterable(testpath + "TIME.REL");
var resultsEnum = expectedResults.GetEnumerator();
ITextExtractor it = new BeginMarkerExtraction(file, "*FIND");
ITreeIO tio = new TreeIO();
IBaseTree tree = tio.LoadBaseTree(testpath + contentTreeName);
double totalRecall = 0;
double totalPrecision = 0;
double bestRecall = -1;
double worstRecall = 2;
double bestPrecision = -1;
double worstPrecision = 2;
double totalGoodPrecision = 0;
double count = 0;
while (it.HasNextContent()) {
string query = it.FindNextContent();
Console.WriteLine("---------------------------------");
string queryName = Helpers.GetNameWhenFirst(query);
Console.WriteLine("Query: " + queryName);
query = Helpers.ConsumeName(query);
Console.WriteLine(query);
IDataTree queryTree = DataTreeBuilder.CreateDocumentMappedTree(tree);
DataTreeBuilder.AddToDataTree(queryTree, query);
queryTree.PrintDataTree();
Console.WriteLine("Expected Results: ");
while(string.IsNullOrEmpty(resultsEnum.Current))
resultsEnum.MoveNext();
string expected = Helpers.ConsumeName(resultsEnum.Current);
Console.WriteLine(expected);
resultsEnum.MoveNext();
expected = expected.Trim();
string[] expectedArray = expected.Split(' ');
double relevant = 0;
double totalRetrieved = 0;
Console.WriteLine("Actual Results: ");
foreach(String s in Directory.EnumerateFiles(testpath + @"\datatrees")) {
IDataTree docTree = tio.LoadDataTree(s);
if (queryTree.CompareTo(docTree)) {
Console.WriteLine(" Found: " + docTree.Name);
//docTree.PrintDataTree();
totalRetrieved++;
if (expectedArray.Contains(docTree.Name)) {
relevant++;
}
} else if (expectedArray.Contains(docTree.Name)) {
Console.WriteLine(" Expected: " + docTree.Name);
//docTree.PrintDataTree();
}
}
Console.WriteLine();
Console.WriteLine("Precision: " + relevant + "/" + totalRetrieved );
Console.WriteLine("Recall: " + relevant + "/" + (expectedArray.Length));
Console.WriteLine();
//Console.ReadLine();
count++;
double recall = relevant / expectedArray.Length;
double precision = 0;
if (totalRetrieved > 0) {
precision = relevant / totalRetrieved;
}
totalPrecision += precision;
totalRecall += recall;
if (precision > bestPrecision) {
bestPrecision = precision;
}
if (precision < worstPrecision) {
worstPrecision = precision;
}
if(recall > bestRecall) {
bestRecall = recall;
}
if(recall < worstRecall) {
worstRecall = recall;
}
if (recall > .5) {
totalGoodPrecision += precision;
}
}
Console.WriteLine("-------------------");
Console.WriteLine("Average Precision: " + totalPrecision / count);
Console.WriteLine("Average Recall: " + totalRecall / count);
Console.WriteLine("Worst Precision: " + worstPrecision);
Console.WriteLine("Worst Recall: " + worstRecall);
Console.WriteLine("Best Precision: " + bestPrecision);
Console.WriteLine("Best Recall: " + bestRecall);
//Console.WriteLine("Average Good Recall Precision: " + totalGoodPrecision / count);
}
public void SaveBaseTree(IBaseTree tree, string location)
{
SaveObject(tree, location);
}
public static void getDocsForQuery(string contentTreeName)
{
Console.SetBufferSize(100, 2000);
IIO io = new FileIO();
IEnumerable<string> file = io.ReadSourceIterable(testpath + "TIME.QUE");
IEnumerable<string> expectedResults = io.ReadSourceIterable(testpath + "TIME.REL");
var resultsEnum = expectedResults.GetEnumerator();
ITextExtractor it = new BeginMarkerExtraction(file, "*FIND");
ITreeIO tio = new TreeIO();
IBaseTree tree = tio.LoadBaseTree(testpath + contentTreeName);
string query = it.FindNextContent();
Console.WriteLine("---------------------------------");
string queryName = Helpers.GetNameWhenFirst(query);
Console.WriteLine("Query: " + queryName);
query = Helpers.ConsumeName(query);
Console.WriteLine(query);
IDataTree queryTree = DataTreeBuilder.CreateStemmedDocumentMapTree(tree);
DataTreeBuilder.AddToDataTreeBoyerMoore(queryTree, query);
queryTree.PrintDataTree();
Console.WriteLine("Expected Results: ");
while (string.IsNullOrEmpty(resultsEnum.Current))
resultsEnum.MoveNext();
string expected = Helpers.ConsumeName(resultsEnum.Current);
Console.WriteLine(expected);
resultsEnum.MoveNext();
expected = expected.Trim();
string[] expectedArray = expected.Split(' ');
double relevant = 0;
double totalRetrieved = 0;
Console.WriteLine("Actual Results: ");
List<string> retrieved = new List<string>();
foreach (String s in Directory.EnumerateFiles(testpath + @"\datatrees")) {
IDataTree docTree = tio.LoadDataTree(s);
if (queryTree.CompareTo(docTree)) {
Console.Write(" " + docTree.Name);
retrieved.Add(docTree.Name);
totalRetrieved++;
if (expectedArray.Contains(docTree.Name)) {
relevant++;
}
Console.WriteLine("Found---");
Console.WriteLine(docTree.Name);
docTree.PrintDataTree();
Console.WriteLine("---");
}
if (expectedArray.Contains(docTree.Name)) {
Console.WriteLine("Expected---");
Console.WriteLine(docTree.Name);
docTree.PrintDataTree();
Console.WriteLine("---");
}
}
Console.WriteLine();
Console.WriteLine("Precision: " + relevant + "/" + totalRetrieved);
Console.WriteLine("Recall: " + relevant + "/" + (expectedArray.Length));
Console.WriteLine();
Console.WriteLine("---------------------------------");
Thread.Sleep(10000);
IEnumerable<string> fileAll = io.ReadSourceIterable(testpath + "TIME.ALL");
ITextExtractor itAll = new BeginMarkerExtraction(fileAll, "*TEXT");
//int count = 1;
//while (itAll.HasNextContent()) {
// string content = itAll.FindNextContent();
// string name = "" + count;
// if (retrieved.Contains(name) || expectedArray.ToList().Contains(name)) {
// Console.WriteLine("------------------------------------------------------------");
// Console.WriteLine("------------------------------------------------------------");
// Console.WriteLine(name);
// content = Helpers.ConsumeName(content);
// Console.WriteLine(content);
// Console.WriteLine("------------------------------------------------------------");
// Console.WriteLine("------------------------------------------------------------");
// }
// count++;
//}
}