public static termExploreModel getSynonymsWithSerbianWordNet(termExploreModel model, ILogBuilder response)
{
tokenGraph result = new tokenGraph(model.lemma.inputForm);
languageManagerWordnet.manager.queryWithGraph(result, response, WordnetSource.serbian, WordnetQueryType.getSymsetCodesByWord);
model.wordnetSecondarySymsets.AddRange(result.getAllLeafs().getNames());
if (response != null)
{
response.consoleAltColorToggle();
string rst = result.ToStringTreeview();
response.Append(rst);
response.consoleAltColorToggle();
}
languageManagerWordnet.manager.queryWithGraph(result, response, WordnetSource.serbian, WordnetQueryType.getWordsBySymsetCode);
model.synonyms.AddRange(result.getAllLeafs().getNames());
if (response != null)
{
response.consoleAltColorToggle();
string rst = result.ToStringTreeview();
response.Append(rst);
response.consoleAltColorToggle();
}
model.graph = result;
return(model);
}
/// <summary>
/// Logs a multiline description of the gramCaseSet
/// </summary>
/// <param name="log">The log.</param>
public void ToString(ILogBuilder log, string itemName = "Instance", bool expanded = false)
{
//StringBuilder sb = new StringBuilder();
log.AppendLine(itemName + ": " + inputForm);
log.consoleAltColorToggle();
gramSet.ToString(log, expanded);
log.consoleAltColorToggle();
}
protected override void stageExecute(ILogBuilder response)
{
foreach (string word in state.entryList)
{
// <------------------------------------------------------------------------------------------------ Exploration
termExploreModel output = null;
if (state.verbose)
{
output = termExploreProcedures.exploreWithUnitex(word, response);
}
else
{
output = termExploreProcedures.exploreWithUnitex(word, null);
}
if (output.wasExploreFailed)
{
if (state.debug)
{
response.consoleAltColorToggle();
response.AppendLine("--- running debug search for [" + word + "]");
var exp = languageManagerUnitex.manager.operatorDelaf.Search(word, false, 25);
exp.ToString(response, true);
string debugLines = exp.ToString();
string debugPath = semanticLexiconManager.manager.constructor.projectFolderStructure[lexiconConstructorProjectFolder.logs].pathFor(word + "_failDebug.txt");
debugLines.saveStringToFile(debugPath, getWritableFileMode.overwrite);
response.consoleAltColorToggle();
}
state.shadowBuffer.Add(word);
state.failedBuffer.Add(word);
}
else
{
if (state.saveModel)
{
semanticLexiconManager.manager.constructor.saveTermModel(output);
}
state.shadowBuffer.Add(word);
state.shadowBuffer.AddRangeUnique(output.GetShadow());
try
{
semanticLexiconManager.manager.constructor.addTermModelToLexicon(output);
response.AppendLine("Lexicon update: Lemma [" + output.lemma.inputForm + "][" + output.instances.Count() + "]");
state.processedBuffer.Add(output.lemma.inputForm);
}
catch (Exception ex)
{
state.failedBuffer.Add(word);
response.AppendLine("Lexicon term update failed for [" + word + "][" + output.lemmaForm + "]");
output.ToString(response, true);
}
}
}
}
public void ToString(ILogBuilder loger, bool expanded = false)
{
foreach (termExploreModel md in this)
{
md.ToString(loger, expanded);
loger.consoleAltColorToggle();
}
}
public static termExploreModel getSynonymsByCorpus(termExploreModel model, ILogBuilder response)
{
tokenGraph result = new tokenGraph(model.lemma.inputForm);
var lines = semanticLexiconManager.manager.settings.sourceFiles.getOperater(lexiconSourceTypeEnum.corpus).Search(model.lemma.inputForm);
result.Add(lines.getLineContentList(), tokenGraphNodeType.word_srb);
model.synonyms.AddRange(result.getAllLeafs().getNames());
if (response != null)
{
response.consoleAltColorToggle();
string rst = result.ToStringTreeview();
response.Append(rst);
response.consoleAltColorToggle();
}
model.graph = result;
return(model);
}
/// <summary>
/// Explores definition on an unknown term
/// </summary>
/// <param name="term">The term.</param>
/// <param name="loger">The loger.</param>
/// <param name="shortExplore">if set to <c>true</c> [short explore].</param>
/// <param name="debug">if set to <c>true</c> [debug].</param>
/// <returns></returns>
public List <termExploreModel> explore(string term, ILogBuilder loger, bool shortExplore = true, bool debug = true, termExploreModel exploreModel = null)
{
term = term.Trim();
List <termExploreModel> output = new List <termExploreModel>();
if (modelRegistry.ContainsKey(term))
{
return(modelRegistry[term]);
}
if (missing.Contains(term))
{
return(GetModels(term));
}
if (term.isNumber())
{
termExploreModel tmp = makeTempModel(term, pos_type.NUMnumerical);
tmp.flags = termExploreItemEnumFlag.datapoint;
if (loger != null)
{
loger.AppendLine("Term [" + term + "] detected to be number.");
}
AddModel(tmp);
return(GetModels(term));
}
// <----- drugi test
exploreModel = termExploreProcedures.exploreWithHunspell(new termExploreItem(term), loger);
List <string> suggests = new List <string>();
exploreModel.instances.ForEach(x => suggests.Add(x.inputForm));
//languageManagerDBNamedEntities.manager.exploreEntities(exploreModel.rootWord, exploreModel);
suggests.Add(exploreModel.rootWord);
// s
apertiumDictionaryResult result = languageManagerApertium.manager.query(suggests, apertiumDictQueryScope.exact, apertiumDictNeedleSide.serbian);
if (result.Any())
{
List <termExploreItem> gramCheck = new List <termExploreItem>();
gramFlags gr = null;
if (result.termVsGramFlags.ContainsKey(exploreModel.inputForm))
{
exploreModel.gramSet.Add(new gramFlags(result.termVsGramFlags[exploreModel.inputForm]));
if (exploreModel.lemma == null)
{
exploreModel.lemma = exploreModel.instances[exploreModel.inputForm];
}
gramCheck.Add(exploreModel);
if (debug)
{
if (loger != null)
{
loger.AppendLine("Apertium discovered model [" + exploreModel.inputForm + "]");
}
}
}
else
{
//if (loger != null) loger.AppendLine("Apertium failed to discover [" + exploreModel.inputForm + "]");
}
foreach (termExploreItem item in exploreModel.instances)
{
if (result.termVsGramFlags.ContainsKey(item.inputForm))
{
item.gramSet.Add(new gramFlags(result.termVsGramFlags[item.inputForm]));
gramCheck.Add(exploreModel);
exploreModel.lemmaForm = item.inputForm;
if (exploreModel.lemma == null)
{
exploreModel.lemma = item;
}
if (debug)
{
if (loger != null)
{
loger.AppendLine("Apertium discovered model [" + item.inputForm + "]");
}
}
}
else
{
//if (loger != null) loger.AppendLine("Apertium failed to discover [" + item.inputForm + "]");
}
}
exploreModel.translations.AddRange(result.GetEnglish());
gramCheck.RemoveAll(x => posConverter.posTypeVsPattern[x.gramSet.getPosType()].Count() == 0);
int disc = 0;
foreach (var gram in gramCheck)
{
if (discoverGram(gram, loger, debug))
{
disc++;
}
}
if (loger != null)
{
loger.AppendLine("Gram [" + term + "] autodiscovered for [" + disc + "] / [" + gramCheck.Count() + "]");
}
if (debug)
{
if (loger != null)
{
exploreModel.ToString(loger, true, false);
manager.constructor.saveTermModel(exploreModel, "Apertium_");
}
}
AddModel(exploreModel);
exploreModel.flags = termExploreItemEnumFlag.aper;
if (shortExplore)
{
return(GetModels(term));
}
}
else
{
if (loger != null)
{
loger.AppendLine("Apertium failed to discover any information on [" + term + "]");
}
}
if (loger != null)
{
loger.consoleAltColorToggle();
}
// <------------------ APERTIUM ^^
foreach (string s in suggests)
{
languageManagerDBNamedEntities.manager.exploreEntities(s, exploreModel);
}
if (exploreModel.flags == termExploreItemEnumFlag.namedEntity)
{
AddModel(exploreModel);
if (debug)
{
if (loger != null)
{
exploreModel.ToString(loger, true, false);
manager.constructor.saveTermModel(exploreModel, "NamedEntity_");
loger.AppendLine("Named entities discovered model [" + exploreModel.inputForm + "]:" + exploreModel.gramSet.ToString());
}
}
if (shortExplore)
{
return(GetModels(term));
}
}
else
{
if (loger != null)
{
if (debug)
{
loger.AppendLine("Named entities found nothing for [" + exploreModel.inputForm + "]:" + exploreModel.gramSet.ToString());
}
}
}
if (loger != null)
{
loger.consoleAltColorToggle();
}
// <------------------ NAMED ENTITY ^^
// <----------------- Wordnet
wordnetSymsetResults resSrWordnet = languageManagerWordnet.manager.query_srb(suggests, loger);
bool found = false;
if (resSrWordnet.Any())
{
foreach (termExploreItem item in exploreModel.instances)
{
if (resSrWordnet.GetByKey(item.inputForm).Any())
{
exploreModel.lemma = item;
exploreModel.lemmaForm = item.inputForm;
exploreModel.translations.AddRange(resSrWordnet.GetValues());
exploreModel.synonyms.AddRange(resSrWordnet.GetKeys());
exploreModel.flags = termExploreItemEnumFlag.srWNet;
found = true;
item.gramSet.Add(new gramFlags(new Enum[] { resSrWordnet.models[item.inputForm].gramSet.getPosType() }));
}
}
foreach (termExploreItem item in exploreModel.instances)
{
discoverGram(item, loger, debug);
}
}
if (found)
{
if (loger != null)
{
loger.AppendLine("SerbianWordNet discovered model [" + term + "]:" + exploreModel.gramSet.ToString());
}
if (debug)
{
if (loger != null)
{
exploreModel.ToString(loger, true, false);
manager.constructor.saveTermModel(exploreModel, "SrWordNet_");;
}
}
AddModel(exploreModel);
exploreModel.flags = termExploreItemEnumFlag.srWNet;
if (shortExplore)
{
return(GetModels(term));
}
}
else
{
if (loger != null)
{
if (debug)
{
loger.AppendLine("Serbian wordnet found nothing for [" + term + "]");
}
}
}
// <------------------ SERBIAN WORD NET ^^
bool failed = discoverGram(exploreModel, loger, debug);
exploreModel.instances.ForEach(x => discoverGram(x, loger, debug));
int d = 0;
List <termExploreItem> lastCheck = new List <termExploreItem>();
foreach (var gram in lastCheck)
{
if (discoverGram(gram, loger, debug))
{
d++;
}
}
if (debug)
{
if (loger != null)
{
loger.AppendLine("The last check [" + term + "] autodiscovered for [" + d + "] / [" + lastCheck.Count() + "]");
}
}
if (d == 0)
{
failed = true;
}
if (loger != null)
{
loger.consoleAltColorToggle();
}
// <------------------ LAST CHECK ^^
if (!failed)
{
exploreModel.flags = termExploreItemEnumFlag.termExplorer;
AddModel(exploreModel);
return(GetModels(term));
}
else
{
if (debug)
{
if (loger != null)
{
loger.AppendLine("Exploration failed for [" + term + "] -- creating temporary term model");
}
}
output.Add(makeTempModel(term, pos_type.TEMP));
missing.Add(term);
return(output);
}
}
/// <summary>
/// Builds a term model out from Word input
/// </summary>
/// <param name="word">The word.</param>
/// <param name="response">The response.</param>
/// <returns></returns>
public static termExploreModel exploreWithUnitex(string word, ILogBuilder response, bool wordIsLemma = false)
{
termExploreModel output = new termExploreModel();
output.modelSource = termExploreModelSource.fromToken;
output.inputForm = word;
string lemma = word;
var tls = semanticLexiconManager.manager.resolve(word);
if (tls != null)
{
if (Enumerable.Count(tls) > 0)
{
if (response != null)
{
response.AppendLine("#1 Lemma already defined in the triplestore [" + word + "] ");
}
output = semanticLexiconManager.manager.constructor.getTermModel(Enumerable.First(tls));
return(output);
}
}
if (!wordIsLemma)
{
if (response != null)
{
response.AppendLine("#1 Finding Lemma for [" + word + "] ");
}
string query = string.Format(posConverter.REGEX_UNITEX_InstanceToLemmaFormat, word);
fileTextSearchResult reslt = languageManagerUnitex.manager.operatorDelaf.Search(query, true, 1, RegexOptions.IgnoreCase);
if (response != null)
{
reslt.ToString(response, true);
}
Regex instanceToLemmaReg = new Regex(query);
if (reslt.Count() > 0)
{
var lnp = reslt.First();
Match mch = instanceToLemmaReg.Match(lnp.Value);
lemma = mch.Groups[1].Value;
}
}
else
{
if (response != null)
{
response.AppendLine("#1 The word is trusted to be a lemma [" + word + "] - skipping search");
}
}
// <------------------------------------------------------------------- preparing chache ---------------
var cache = languageManagerUnitex.manager.operatorDelaf.Search(lemma, false, 300);
if (response != null)
{
response.AppendLine("Cached definitions [" + cache.Count() + "] ");
}
// <------------------------------------------------------------ 2. finding lemma definition
output.lemmaForm = lemma;
output.lemma = new termExploreItem(lemma);
if (response != null)
{
response.AppendLine("#2 Finding Lemma definition [" + lemma + "] ");
}
string lemmaQuery = string.Format(posConverter.REGEX_UNITEX_DeclarationForLemma, lemma);
Regex lemmaQueryRegex = new Regex(lemmaQuery);
fileTextSearchResult lemmaResult = languageManagerUnitex.manager.operatorDelaf.Search(cache, lemmaQuery, true, 5, RegexOptions.IgnoreCase);
if (response != null)
{
lemmaResult.ToString(response, true);
}
if (lemmaResult.Count() == 0)
{
if (response != null)
{
response.consoleAltColorToggle();
response.AppendLine("Failed to find lemma definition for [" + word + "]. Aborting exploration.");
response.consoleAltColorToggle();
}
output.wasExploreFailed = true;
return(output);
}
foreach (var lr_lnp in lemmaResult)
{
Match lmch = lemmaQueryRegex.Match(lr_lnp.Value);
if (lmch.Success)
{
output.lemma.gramSet.Add(lmch.Groups[1].Value);
}
}
if (response != null)
{
output.lemma.ToString(response);
}
// <------------------------------------------------------------ 3. getting all instances for the lemma
if (response != null)
{
response.AppendLine("#3 Extracting all instances for the Lemma [" + lemma + "] ");
}
string instanceQuery = string.Format(posConverter.REGEX_UNITEX_LemmaToInstanceFormat, lemma);
string instanceUnitexQuery = "," + lemma + ".";
Regex instanceQueryRegex = new Regex(instanceQuery);
fileTextSearchResult instanceResult = languageManagerUnitex.manager.operatorDelaf.Search(cache, instanceUnitexQuery, false, 100, RegexOptions.IgnoreCase);
if (response != null)
{
instanceResult.ToString(response, true);
}
foreach (var lr_lnp in instanceResult)
{
Match lmch = instanceQueryRegex.Match(lr_lnp.Value);
output.instances.Add(lmch.Groups[1].Value, lmch.Groups[2].Value);
}
// <------------------------------------------------------------ 4. Resulting term model
if (response != null)
{
response.AppendLine("#4 Resulting term model [" + lemma + "] ");
output.ToString(response);
}
return(output);
}
/// <summary>
/// Method: word -- translation --- synset ---- other synsets --- collecting all words --- translation --- word
/// </summary>
/// <param name="model">The model.</param>
/// <param name="response">The response.</param>
/// <param name="disableCodePrefixFilter">if set to <c>true</c> [disable code prefix filter].</param>
/// <param name="disableCodeBranchFilter">if set to <c>true</c> [disable code branch filter].</param>
/// <returns></returns>
public static termExploreModel getSynonymsWithWordnetViaApertium(termExploreModel model, ILogBuilder response, bool disableCodePrefixFilter = false, bool disableCodeBranchFilter = false)
{
tokenGraph result = model.graph;
result = languageManagerApertium.manager.queryForGraph(model.lemma.inputForm, apertiumDictQueryScope.exact);
model.translations.AddRange(result.getAllLeafs().getNames());
if (response != null)
{
response.consoleAltColorToggle();
string rst = result.ToStringTreeview();
response.Append(rst);
response.consoleAltColorToggle();
}
languageManagerWordnet.manager.queryWithGraph(result, response, WordnetSource.english, WordnetQueryType.getSymsetCodesByWord);
if (response != null)
{
response.consoleAltColorToggle();
string st = result.ToStringTreeview();
response.Append(st);
response.consoleAltColorToggle();
}
model.wordnetSecondarySymsets.AddRange(result.getAllLeafs().getDeepest().getNames());
if (!disableCodePrefixFilter)
{
string codeStart = model.lemma.gramSet.getPosType().GetWordNetCodeStart().ToString();
Regex codeCriteria = new Regex("^" + codeStart + "");
var badCodes = result.getAllLeafs(codeCriteria, true);
if (response != null)
{
response.AppendHorizontalLine();
response.AppendLine("Reducing to proper codes [" + codeStart + "]->filtered-out[" + badCodes.Count() + "]");
}
badCodes.removeFromParent();
if (response != null)
{
response.consoleAltColorToggle();
string rst = result.ToStringTreeview();
response.Append(rst);
response.consoleAltColorToggle();
}
}
model.wordnetPrimarySymsets.AddRange(result.getAllLeafs().getDeepest().getNames());
languageManagerWordnet.manager.queryWithGraph(result, response, WordnetSource.english, WordnetQueryType.getWordsBySymsetCode);
model.translationRelated.AddRange(result.getAllLeafs().getDeepest().getNames());
if (response != null)
{
response.AppendHorizontalLine();
response.AppendLine("Getting English words by symsetcodes via WordNet");
response.consoleAltColorToggle();
string rst = result.ToStringTreeview();
response.Append(rst);
response.consoleAltColorToggle();
}
languageManagerApertium.manager.queryByGraphNode(result, apertiumDictQueryScope.exact, apertiumDictNeedleSide.translated);
model.wordnetSynonyms.AddRange(result.getAllLeafs().getDeepest().getNames());
if (response != null)
{
response.AppendHorizontalLine();
response.AppendLine("Translating back to Serbian via Apertium");
response.consoleAltColorToggle();
string rst = result.ToStringTreeview();
response.Append(rst);
response.consoleAltColorToggle();
}
if (!disableCodeBranchFilter) // <------ removes the symset nodes that contain none of first-level translation words
{
var codeLevel = result.getAllChildren().getOnLevel(3);
List <IObjectWithPathAndChildren> toTakeOut = new List <IObjectWithPathAndChildren>();
foreach (var clb in codeLevel)
{
foreach (var clb_c in clb)
{
bool takeOut = true;
foreach (var clb_cc in clb_c)
{
if (clb_cc.name == model.lemma.inputForm)
{
takeOut = false;
break;
}
}
if (takeOut)
{
if (response != null)
{
response.AppendLine("-- take out: " + clb.path);
}
toTakeOut.Add(clb);
break;
}
}
}
toTakeOut.removeFromParent();
int wps = Enumerable.Count(model.wordnetSecondarySymsets);
int tr = Enumerable.Count(model.translationRelated);
int ws = Enumerable.Count(model.wordnetSynonyms);
if (response != null)
{
response.AppendLine("----- Branch-node filter ----");
response.AppendLine("Symsets: " + wps);
response.AppendLine("Translations: " + tr);
response.AppendLine("Terms: " + ws);
response.consoleAltColorToggle();
string rst = result.ToStringTreeview();
response.Append(rst);
response.consoleAltColorToggle();
}
model.wordnetPrimarySymsets = result.getAllChildren().getOnLevel(3).getNames(true);
model.translations = result.getAllChildren().getOnLevel(4).getNames(true);
model.synonyms = result.getAllChildren().getOnLevel(5).getNames(true);
wps = wps - Enumerable.Count(model.wordnetPrimarySymsets);
tr = tr - Enumerable.Count(model.translations);
ws = ws - Enumerable.Count(model.synonyms);
if (response != null)
{
//response.AppendLine("----- Branch-node filter ----");
response.AppendLine("Reduction of Symsets: " + wps);
response.AppendLine("Reduction of Translations: " + tr);
response.AppendLine("Reduction of Terms: " + ws);
response.consoleAltColorToggle();
string rst = result.ToStringTreeview();
response.Append(rst);
response.consoleAltColorToggle();
}
}
/*
* String rgex_pat = "^([\\w]*\\\\[\\w]*\\\\[\\w]*\\\\[\\w]*\\\\{0}$)";
*
* Regex rgex = new Regex(String.Format(rgex_pat, model.lemma.inputForm));
*
* var onlyWithLemma = result.getAllLeafs().getFilterOut(rgex);
*/
//languageManagerApertium.manager.queryByGraphNode(result, apertiumDictQueryScope.exact, apertiumDictNeedleSide.english);
model.graph = result;
return(model);
}
public static termExploreModel explore(string word, ILogBuilder response, termExploreMode mode, bool verbose = false)
{
termExploreModel model = new termExploreModel(word);
termExploreModelSet outset = semanticLexiconManager.manager.constructor.loadTermModels(word, true);
if (response != null)
{
response.consoleAltColorToggle();
response.AppendHorizontalLine();
response.AppendLine("Exploring term[" + model.inputForm + "] with [" + mode.ToString() + "]");
response.consoleAltColorToggle();
}
if (Enumerable.Any(outset))
{
model = Enumerable.First(outset);
if (response != null)
{
response.AppendLine("term[" + model.inputForm + "]->lemma[" + model.lemma.inputForm + "]");
}
}
else
{
model.lemmaForm = "";
if (response != null)
{
response.AppendLine("term[" + word + "]->missingLemma[]");
}
}
var output = response;
if (!verbose)
{
response = null;
}
switch (mode)
{
case termExploreMode.apertium_direct:
model = getSynonymsWithApertium(model, response);
break;
case termExploreMode.apertium_wordnet_eng:
model = getSynonymsWithWordnetViaApertium(model, response);
break;
case termExploreMode.apertium_wordnet_srb:
model = getSynonymsWithSerbianWordNetAndApertium(model, response);
break;
case termExploreMode.corpus:
model = getSynonymsByCorpus(model, response);
break;
case termExploreMode.hunspell_srb:
model = getSynonymsWithHunspell(model, response);
break;
case termExploreMode.none:
break;
case termExploreMode.wordnet_srb:
model = getSynonymsWithSerbianWordNet(model, response);
break;
case termExploreMode.unitex:
model = exploreWithUnitex(word, response);
break;
}
model.PostProcess();
if (output != null)
{
model.ToString(output, verbose, false);
}
return(model);
}
