public LuceneApp()
{
luceneIndexDirectory = null;
analyzer = null;
writer = null;
newSimilarity = new NewSimilarity();
parserFields = new string[] { DOC_TITLE, DOC_AUTHOR, DOC_BIB, DOC_BODY };
fieldWeights = new Dictionary <string, float>();
foreach (string field in parserFields)
{
fieldWeights.Add(field, 1);
}
// Init WordNet
// Src: https://developer.syn.co.in/tutorial/wordnet/tutorial.html
var directory = "../../../wordnetdic";
wordNetEngine = new WordNetEngine();
// data sources
wordNetEngine.AddDataSource(new StreamReader(Path.Combine(directory, "data.adj")), PartOfSpeech.Adjective);
wordNetEngine.AddDataSource(new StreamReader(Path.Combine(directory, "data.adv")), PartOfSpeech.Adverb);
wordNetEngine.AddDataSource(new StreamReader(Path.Combine(directory, "data.noun")), PartOfSpeech.Noun);
wordNetEngine.AddDataSource(new StreamReader(Path.Combine(directory, "data.verb")), PartOfSpeech.Verb);
// indexes
wordNetEngine.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adj")), PartOfSpeech.Adjective);
wordNetEngine.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adv")), PartOfSpeech.Adverb);
wordNetEngine.AddIndexSource(new StreamReader(Path.Combine(directory, "index.noun")), PartOfSpeech.Noun);
wordNetEngine.AddIndexSource(new StreamReader(Path.Combine(directory, "index.verb")), PartOfSpeech.Verb);
Console.WriteLine("Loading database...");
wordNetEngine.Load();
Console.WriteLine("Load completed.");
}
public static WordNetEngine wordNet; // Set WordNet object
public MainSearchBox()
{
InitializeComponent();
var directory = System.IO.Directory.GetCurrentDirectory() + "\\wordnet"; // Set WordNet directory
wordNet = new WordNetEngine(); // Initiate WordNet object
// Load WordNet data and index
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adj")), PartOfSpeech.Adjective);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adv")), PartOfSpeech.Adverb);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.noun")), PartOfSpeech.Noun);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.verb")), PartOfSpeech.Verb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adj")), PartOfSpeech.Adjective);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adv")), PartOfSpeech.Adverb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.noun")), PartOfSpeech.Noun);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.verb")), PartOfSpeech.Verb);
wordNet.Load();
// Create the column headers for the list view
resultListView.Columns.Add("Rank", 90); // 90 is the width for the column
resultListView.Columns.Add("DocID", 100);
resultListView.Columns.Add("Title", 180);
resultListView.Columns.Add("Author", 150);
resultListView.Columns.Add("Bibliography", 150);
resultListView.Columns.Add("Abstract", 400);
}
//Implementation of advanced system
//WordNet
public static List <string> WordNet(string query)
{
//Can read more files in database but dont know how to use those file yet
//var FileList = Directory.GetFiles(@"C:\Users\USER\Documents\GitHub\IFN647_Advanced\testConvertOrPresentJson\bin\Debug\dict");
//var wordNet = new WordNetEngine();
//foreach (var file in FileList)
//{
// wordNet.AddDataSource(new StreamReader(file));
// // Do some work
//}
var directory = Directory.GetCurrentDirectory();
var wordNet = new WordNetEngine();
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "./dict/", "data.adj")), PartOfSpeech.Adjective);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "./dict/", "data.adv")), PartOfSpeech.Adverb);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "./dict/", "data.noun")), PartOfSpeech.Noun);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "./dict/", "data.verb")), PartOfSpeech.Verb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "./dict/", "index.adj")), PartOfSpeech.Adjective);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "./dict/", "index.adv")), PartOfSpeech.Adverb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "./dict/", "index.noun")), PartOfSpeech.Noun);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "./dict/", "index.verb")), PartOfSpeech.Verb);
Console.WriteLine("Loading database...");
wordNet.Load();
Console.WriteLine("Load completed.");
string word = $"{query}";
var synSetList = new List <Syn.WordNet.SynSet>();
if (word != null)
{
synSetList = wordNet.GetSynSets(word);
if (synSetList.Count == 0)
{
Console.WriteLine($"No SynSet found for '{word}'");
}
}
var synonyms = new List <string>();
foreach (var synSet in synSetList)
{
var words = string.Join(", ", synSet.Words);
synonyms.Add(words);
Console.WriteLine($"\nWords: {words}");
Console.WriteLine($"POS: {synSet.PartOfSpeech}");
Console.WriteLine($"Gloss: {synSet.Gloss}");
}
return(synonyms);
}
public void load()
{
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adj")), PartOfSpeech.Adjective);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adv")), PartOfSpeech.Adverb);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.noun")), PartOfSpeech.Noun);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.verb")), PartOfSpeech.Verb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adj")), PartOfSpeech.Adjective);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adv")), PartOfSpeech.Adverb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.noun")), PartOfSpeech.Noun);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.verb")), PartOfSpeech.Verb);
Console.WriteLine("Loading data files...");
wordNet.Load();
Console.WriteLine("Files loaded successfully.");
}
public ActionResult Login([FromBody] LinguisticRequest linguisticRequest)
{
var directory = Directory.GetCurrentDirectory();
directory = Path.Combine(directory, "Artifacts");
var wordNet = new WordNetEngine();
var words = new List <String>();
try
{
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adj")), PartOfSpeech.Adjective);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adv")), PartOfSpeech.Adverb);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.noun")), PartOfSpeech.Noun);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.verb")), PartOfSpeech.Verb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adj")), PartOfSpeech.Adjective);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adv")), PartOfSpeech.Adverb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.noun")), PartOfSpeech.Noun);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.verb")), PartOfSpeech.Verb);
wordNet.Load();
var synSetList = wordNet.GetSynSets(linguisticRequest.Word);
if (synSetList.Count == 0)
{
return(BadRequest(new { message = "No Words Found." }));
}
foreach (var synSet in synSetList)
{
foreach (string word in synSet.Words)
{
words.Add(word);
}
}
return(Ok(words));
}
catch (Exception ex)
{
return(BadRequest(new { message = ex.Message }));
}
}
public WordNetWrapper()
{
var directory = "./resources/wordnet";
wordNet = new WordNetEngine();
// add all resources
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adj")), PartOfSpeech.Adjective);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adv")), PartOfSpeech.Adverb);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.noun")), PartOfSpeech.Noun);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.verb")), PartOfSpeech.Verb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adj")), PartOfSpeech.Adjective);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adv")), PartOfSpeech.Adverb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.noun")), PartOfSpeech.Noun);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.verb")), PartOfSpeech.Verb);
wordNet.Load();
}
public wordnetComponent()
: base("wordnet", "wordnet",
"Description",
"wordnet", "wordnet")
{
// load wordnet database
var directory = Directory.GetCurrentDirectory();
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adj")), PartOfSpeech.Adjective);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adv")), PartOfSpeech.Adverb);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.noun")), PartOfSpeech.Noun);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.verb")), PartOfSpeech.Verb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adj")), PartOfSpeech.Adjective);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adv")), PartOfSpeech.Adverb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.noun")), PartOfSpeech.Noun);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.verb")), PartOfSpeech.Verb);
//Console.WriteLine("Loading database...");
wordNet.Load();
//Console.WriteLine("Load completed.");
}
private void LoadWordNet()
{
string directory = WORDNETDB_PATH;//Directory.GetCurrentDirectory();
wordNet = new WordNetEngine();
Console.WriteLine("Loading database...");
wordNet.LoadFromDirectory(directory);
Console.WriteLine("Load completed.");
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adj")), PartOfSpeech.Adjective);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adv")), PartOfSpeech.Adverb);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.noun")), PartOfSpeech.Noun);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.verb")), PartOfSpeech.Verb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adj")), PartOfSpeech.Adjective);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adv")), PartOfSpeech.Adverb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.noun")), PartOfSpeech.Noun);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.verb")), PartOfSpeech.Verb);
Console.WriteLine("Loading database...");
wordNet.Load();
Console.WriteLine("Load completed.");
}
private void LoadSource(string type, PartOfSpeech partOfSpeech)
{
string dataSource = wordNetDir + $"data.{type}";
string indexSource = wordNetDir + $"index.{type}";
wordNet.AddDataSource(
new StreamReader(
new FileStream(dataSource, FileMode.Open, FileAccess.Read)),
partOfSpeech);
wordNet.AddIndexSource(
new StreamReader(
new FileStream(indexSource, FileMode.Open, FileAccess.Read)),
partOfSpeech);
}
//This method will initite the search process.
private void searchQuery()
{
if (!LuceneSearch.isIndexAvailale())
{
MessageBox.Show("Please create/load an index first", "Index Error");
return;
}
if (String.IsNullOrEmpty(QueryBox.Text))
{
MessageBox.Show("Please enter a query to search.", "Retry");
return;
}
resultsTable.Rows.Clear();
resultsTable.Refresh();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
if (queryExpansionCheckBox.Checked == false && multiTermCheckBox.Checked == false)
{
//Lucene search without query expansion
LuceneSearch.CreateSearcher();
LuceneSearch.CreateParser(LuceneSearch.FIELDS_FN[2]);
Query parsedQuery = LuceneSearch.ParseQuery(QueryBox.Text);
queryTextBox.Text = parsedQuery.ToString();
TopDocs resultDocs = LuceneSearch.SearchText(parsedQuery);
DisplaySearch(resultDocs);
}
else if (queryExpansionCheckBox.Checked == true)
{
//Lucene search with query expansion
var directory = Directory.GetCurrentDirectory() + "/wordnet";
var wordNet = new WordNetEngine();
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adj")), PartOfSpeech.Adjective);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adv")), PartOfSpeech.Adverb);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.noun")), PartOfSpeech.Noun);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.verb")), PartOfSpeech.Verb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adj")), PartOfSpeech.Adjective);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adv")), PartOfSpeech.Adverb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.noun")), PartOfSpeech.Noun);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.verb")), PartOfSpeech.Verb);
wordNet.Load();
string baseQuery = QueryBox.Text.ToLower();
List <string> tokenizedBaseQuery = LuceneSearch.TokenizeString(baseQuery);
string synonyms = "";
string finalQuery = "";
foreach (string token in tokenizedBaseQuery)
{
finalQuery += token + "^5 ";
List <SynSet> synSetList = wordNet.GetSynSets(token);
if (synSetList.Count != 0)
{
foreach (SynSet synSet in synSetList)
{
foreach (string word in synSet.Words)
{
if (!word.Equals(token))
{
synonyms += word + " ";
}
}
}
}
}
finalQuery += synonyms;
LuceneSearch.CreateSearcher();
LuceneSearch.CreateParser(LuceneSearch.FIELDS_FN[2]);
Query parsedQuery = LuceneSearch.ParseQuery(finalQuery);
queryTextBox.Text = parsedQuery.ToString();
TopDocs resultDocs = LuceneSearch.SearchText(parsedQuery);
DisplaySearch(resultDocs);
}
else if (multiTermCheckBox.Checked == true)
{
LuceneSearch.CreateSearcher();
LuceneSearch.CreateParser(LuceneSearch.FIELDS_FN[2]);
Query parsedQuery = LuceneSearch.ParseQuery("\"" + QueryBox.Text + "\"");
queryTextBox.Text = parsedQuery.ToString();
TopDocs resultDocs = LuceneSearch.SearchText(parsedQuery);
DisplaySearch(resultDocs);
}
stopwatch.Stop();
TimeSpan elapsed = stopwatch.Elapsed;
string elapsedTime = elapsed.ToString(@"hh\:mm\:ss\.fff");
SearchTimeLabel.Text = "Search time elapsed: " + elapsedTime;
}
public static List <Dictionary <string, string> > Search_Click(string querytext, bool asIsCheckBox, bool QECheckbox, bool advCheckBox, bool SCCheckbox, string cond, out string finalQueryTxt)
{
List <List <string> > tempList = new List <List <string> >(); // Create a list of lists for receiving output from SearchText method
List <Dictionary <string, string> > resultListDict = new List <Dictionary <string, string> >(); // Create a list of dictionaries for outputting to GUI
myLuceneApp.CreateSearcher(); // Create searcher
if (SCCheckbox)
{
querytext = SpellCheck.Query_SpellCheck(querytext);
}
if (QECheckbox)
{
if (!QECheck)
{
var directory = System.IO.Directory.GetCurrentDirectory() + "\\wordnet"; // Set WordNet directory
// Load WordNet data and index
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adj")), PartOfSpeech.Adjective);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.adv")), PartOfSpeech.Adverb);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.noun")), PartOfSpeech.Noun);
wordNet.AddDataSource(new StreamReader(Path.Combine(directory, "data.verb")), PartOfSpeech.Verb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adj")), PartOfSpeech.Adjective);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.adv")), PartOfSpeech.Adverb);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.noun")), PartOfSpeech.Noun);
wordNet.AddIndexSource(new StreamReader(Path.Combine(directory, "index.verb")), PartOfSpeech.Verb);
Console.WriteLine("Loading WordNet database...");
wordNet.Load();
Console.WriteLine("Load completed.");
QECheck = true;
}
if (advCheckBox)
{
int indexT = querytext.IndexOf("Title:"); // Get again the index just in case it has been changed
int indexA = querytext.IndexOf("Author:"); // Get again the index just in case it has been changed
if (indexT != -1 && indexA != -1)
{
string title_query = querytext.Substring(indexT + 6, ((indexA - 1 - (indexT + 6)) > 0) ? (indexA - 1 - (indexT + 6)) : 0); // Get title string
string author_query = querytext.Substring(indexA + 7, ((querytext.Length - (indexA + 7)) > 0) ? (querytext.Length - (indexA + 7)) : 0); // Get author string
string querytitle = myLuceneApp.PreProcess(myStemmer, title_query); // Pre-process query texts if checked
string queryauthor = myLuceneApp.PreProcess(myStemmer, author_query); // Pre-process query texts if checked
querytext = "Title:" + querytitle + " Author:" + queryauthor;
}//When both the title and author are queried
if (indexT == -1)
{
string author_query = querytext.Substring(indexA + 7, ((querytext.Length - (indexA + 7)) > 0) ? (querytext.Length - (indexA + 7)) : 0); // Get author string
string queryauthor = myLuceneApp.PreProcess(myStemmer, author_query); // Pre-process query texts if checked
querytext = "Author:" + queryauthor;
}//When only the Author is queried
if (indexA == -1)
{
string title_query = querytext.Substring(indexT + 6, ((querytext.Length - (indexT + 6)) > 0) ? (querytext.Length - (indexT + 6)) : 0); // Get title string
string querytitle = myLuceneApp.PreProcess(myStemmer, title_query); // Pre-process query texts if checked
querytext = "Title:" + querytitle;
}//When only the title is queried
}
else
{
querytext = querytext.Replace("Title:", "").Replace("Author:", "");
querytext = myLuceneApp.PreProcess(myStemmer, querytext); // Pre-process query texts if checked
}
}
tempList = myLuceneApp.SearchText(querytext, asIsCheckBox, QECheckbox, advCheckBox, cond, out finalQueryTxt); // Get search result list of lists
myLuceneApp.CleanUpSearcher(); // Clean searcher
int rank = 0;
foreach (List <string> result in tempList) // Go through each resulting document
{
rank++;
//Retrieving values indexed as TEXT
string text = result[0]; // Get whole text from input list
int indexI = text.IndexOf(".I ") + 3; // Get ID starting index
int indexT = text.IndexOf(".T\n"); // Get title starting index
int indexA = text.IndexOf(".A\n"); // Get author starting index
int indexB = text.IndexOf(".B\n"); // Get bibliography starting index
int indexW = text.IndexOf(".W\n"); // Get words starting index
string id = text.Substring(indexI, indexT - 1 - indexI); // Get ID string
string title = text.Substring(indexT + 3, ((indexA - 1 - (indexT + 3)) > 0) ? (indexA - 1 - (indexT + 3)) : 0); // Get title string
string author = text.Substring(indexA + 3, ((indexB - 1 - (indexA + 3)) > 0) ? (indexB - 1 - (indexA + 3)) : 0); // Get author string
string biblio = text.Substring(indexB + 3, ((indexW - 1 - (indexB + 3)) > 0) ? (indexW - 1 - (indexB + 3)) : 0); // Get bibliography string
string abst = text.Substring(indexW + 3, ((text.Length - 1 - (indexW + 3)) > 0) ? (text.Length - 1 - (indexW + 3)) : 0); // Get abstract string
//Handling issues in the abstract to display it
abst = abst.Replace("\n", " "); // Replace abstract LF
Regex rx = new Regex("^.*?[.?!]", RegexOptions.Compiled | RegexOptions.IgnoreCase); // Set the RE to match first sentence of abstract
MatchCollection abst_first = rx.Matches(abst); // Get RE match for first sentence of abstract
string abst_fir = ""; // Create first sentence variable
if (abst_first.Count != 0) // Check if there is no RE match (usually abstract is empty)
{
abst_fir = abst_first[0].Value;
}
// Add everything into the created list of dictionaries
resultListDict.Add(new Dictionary <string, string> {
{ "rank", rank.ToString() }, { "id", id }, { "title", title }, { "author", author },
{ "biblio", biblio }, { "abstract", abst }, { "abstract_first", abst_fir }, { "score", result[1] }
});
}
return(resultListDict);
}
