public AnnotatedSentence.AnnotatedSentence Convert(ParseTreeDrawable parseTree, List <TreeEnsembleModel> models)
{
if (parseTree != null)
{
var annotatedSentence = new AnnotatedSentence.AnnotatedSentence();
var nodeDrawableCollector = new NodeDrawableCollector((ParseNodeDrawable)parseTree.GetRoot(), new IsLeafNode());
var leafList = nodeDrawableCollector.Collect();
var wordNodePairList = new List <WordNodePair>();
for (var i = 0; i < leafList.Count; i++)
{
var parseNode = leafList[i];
var wordNodePair = new WordNodePair(parseNode, i + 1);
wordNodePair.UpdateNode();
if (wordNodePair.GetNode().GetParent() != null && wordNodePair.GetNode().GetParent().NumberOfChildren() == 1)
{
wordNodePair.UpdateNode();
Console.WriteLine("check this");
return(null);
}
annotatedSentence.AddWord(wordNodePair.GetWord());
wordNodePairList.Add(wordNodePair);
}
ConstructDependenciesFromTree(wordNodePairList, models);
return(annotatedSentence);
}
return(null);
}
public void AutoSemantic(AnnotatedSentence.AnnotatedSentence sentence)
{
if (AutoLabelSingleSemantics(sentence))
{
sentence.Save();
}
}
protected override bool AutoLabelSingleSemantics(AnnotatedSentence.AnnotatedSentence sentence)
{
var random = new Random(1);
var done = false;
for (var i = 0; i < sentence.WordCount(); i++)
{
var synSets = GetCandidateSynSets(_turkishWordNet, _fsm, sentence, i);
var maxIntersection = -1;
for (var j = 0; j < synSets.Count; j++)
{
var synSet = synSets[j];
var intersectionCount = Intersection(synSet, sentence);
if (intersectionCount > maxIntersection)
{
maxIntersection = intersectionCount;
}
}
var maxSynSets = new List <SynSet>();
for (var j = 0; j < synSets.Count; j++)
{
var synSet = synSets[j];
if (Intersection(synSet, sentence) == maxIntersection)
{
maxSynSets.Add(synSet);
}
}
if (maxSynSets.Count > 0)
{
done = true;
((AnnotatedWord)sentence.GetWord(i)).SetSemantic(maxSynSets[random.Next(maxSynSets.Count)].GetId());
}
}
return(done);
}
private int Intersection(SynSet synSet, AnnotatedSentence.AnnotatedSentence sentence)
{
string[] words1;
if (synSet.GetExample() != null)
{
words1 = (synSet.GetLongDefinition() + " " + synSet.GetExample()).Split(" ");
}
else
{
words1 = synSet.GetLongDefinition().Split(" ");
}
var words2 = sentence.ToWords().Split(" ");
var count = 0;
foreach (var word1 in words1)
{
foreach (var word2 in words2)
{
if (word1.ToLower(new CultureInfo("tr")).Equals(word2.ToLower(new CultureInfo("tr"))))
{
count++;
}
}
}
return(count);
}
public PolarityType AutoSentiment(AnnotatedSentence.AnnotatedSentence sentence)
{
var polarityValue = 0.0;
for (var i = 0; i < sentence.WordCount(); i++)
{
var word = (AnnotatedWord)sentence.GetWord(i);
try {
if (word != null && word.GetSemantic() != null)
{
var sentiSynSet = _sentiNet.GetSentiSynSet(word.GetSemantic());
var value = System.Math.Max(sentiSynSet.GetNegativeScore(), sentiSynSet.GetPositiveScore());
switch (GetPolarity(sentiSynSet.GetPolarity(), sentence, i))
{
case PolarityType.POSITIVE:
polarityValue += value;
break;
case PolarityType.NEGATIVE:
polarityValue -= value;
break;
}
}
}
catch (Exception e) {
if (e is ArgumentNullException || e is KeyNotFoundException)
{
}
}
}
return(FindPolarityType(polarityValue));
}
protected override PolarityType GetPolarity(PolarityType polarityType, AnnotatedSentence.AnnotatedSentence sentence, int index)
{
if (((AnnotatedWord)sentence.GetWord(index)).GetParse().ContainsTag(MorphologicalTag.NEGATIVE))
{
if (polarityType.Equals(PolarityType.POSITIVE))
{
polarityType = PolarityType.NEGATIVE;
}
else if (polarityType.Equals(PolarityType.NEGATIVE))
{
polarityType = PolarityType.POSITIVE;
}
}
if (index + 1 < sentence.WordCount())
{
var nextWord = (AnnotatedWord)sentence.GetWord(index + 1);
if (nextWord.GetParse().GetWord().GetName().ToLower().Equals("değil"))
{
if (polarityType.Equals(PolarityType.POSITIVE))
{
return(PolarityType.NEGATIVE);
}
else if (polarityType.Equals(PolarityType.NEGATIVE))
{
return(PolarityType.POSITIVE);
}
}
}
return(polarityType);
}
/**
* <summary> The method checks the LOCATION gazetteer, and if the word exists in the gazetteer, it assigns the LOCATION tag.</summary>
* <param name="sentence">The sentence for which LOCATION named entities checked.</param>
*/
protected override void AutoDetectLocation(AnnotatedSentence.AnnotatedSentence sentence)
{
for (var i = 0; i < sentence.WordCount(); i++)
{
var word = (AnnotatedWord)sentence.GetWord(i);
if (word.GetParse() != null)
{
word.CheckGazetteer(locationGazetteer);
}
}
}
protected override bool AutoLabelSingleSemantics(AnnotatedSentence.AnnotatedSentence sentence)
{
var random = new Random(1);
for (var i = 0; i < sentence.WordCount(); i++)
{
var synSets = GetCandidateSynSets(_turkishWordNet, _fsm, sentence, i);
if (synSets.Count > 0)
{
((AnnotatedWord)sentence.GetWord(i)).SetSemantic(synSets[random.Next(synSets.Count)].GetId());
}
}
return(true);
}
/**
* <summary> The method disambiguates words with multiple possible root words in its morphological parses. If the word
* is already morphologically disambiguated, the method does not disambiguate that word. The method first check
* for multiple root words by using rootWords method. If there are multiple root words, the method select the most
* occurring root word (if its occurence wrt other root words occurence is above some threshold) for that word
* using the bestRootWord method. If root word is selected, then the case for single root word is called.</summary>
* <param name="sentence">The sentence to be disambiguated automatically.</param>
*/
protected override void AutoDisambiguateMultipleRootWords(AnnotatedSentence.AnnotatedSentence sentence)
{
FsmParseList[] fsmParses = morphologicalAnalyzer.RobustMorphologicalAnalysis(sentence);
List <FsmParse> correctParses = longestRootFirstDisambiguation.Disambiguate(fsmParses);
for (int i = 0; i < sentence.WordCount(); i++)
{
AnnotatedWord word = (AnnotatedWord)sentence.GetWord(i);
if (word.GetParse() == null)
{
SetParseAutomatically(correctParses[i], word);
}
}
}
/**
* <summary> Given the sentence for which the predicate(s) were determined before, this method automatically assigns
* semantic role labels to some/all words in the sentence. The method first finds the first predicate, then assuming
* that the shallow parse tags were preassigned, assigns ÖZNE tagged words ARG0; NESNE tagged words ARG1. If the
* verb is in passive form, ÖZNE tagged words are assigned as ARG1.</summary>
* <param name="sentence">The sentence for which semantic roles will be determined automatically.</param>
* <returns>If the method assigned at least one word a semantic role label, the method returns true; false otherwise.</returns>
*/
public override bool AutoArgument(AnnotatedSentence.AnnotatedSentence sentence)
{
var modified = false;
string predicateId = null;
for (var i = 0; i < sentence.WordCount(); i++)
{
var word = (AnnotatedWord)sentence.GetWord(i);
if (word.GetArgument() != null && word.GetArgument().GetArgumentType().Equals("PREDICATE"))
{
predicateId = word.GetArgument().GetId();
break;
}
}
if (predicateId != null)
{
for (var i = 0; i < sentence.WordCount(); i++)
{
var word = (AnnotatedWord)sentence.GetWord(i);
if (word.GetArgument() == null)
{
if (word.GetShallowParse() != null && word.GetShallowParse().Equals("ÖZNE"))
{
if (word.GetParse() != null && word.GetParse().ContainsTag(MorphologicalTag.PASSIVE))
{
word.SetArgument("ARG1$" + predicateId);
}
else
{
word.SetArgument("ARG0$" + predicateId);
}
modified = true;
}
else
{
if (word.GetShallowParse() != null && word.GetShallowParse().Equals("NESNE"))
{
word.SetArgument("ARG1$" + predicateId);
modified = true;
}
}
}
}
}
return(modified);
}
protected override bool AutoLabelSingleSemantics(AnnotatedSentence.AnnotatedSentence sentence)
{
for (var i = 0; i < sentence.WordCount(); i++)
{
var synSets = GetCandidateSynSets(_turkishWordNet, _fsm, sentence, i);
if (synSets.Count > 0)
{
var best = MostFrequent(synSets, ((AnnotatedWord)sentence.GetWord(i)).GetParse().GetWord().GetName());
if (best != null)
{
((AnnotatedWord)sentence.GetWord(i)).SetSemantic(best.GetId());
}
}
}
return(true);
}
public AnnotatedSentence.AnnotatedSentence GenerateAnnotatedSentence(string language)
{
var sentence = new AnnotatedSentence.AnnotatedSentence("");
var nodeDrawableCollector =
new NodeDrawableCollector((ParseNodeDrawable)root, new IsEnglishLeafNode());
var leafList = nodeDrawableCollector.Collect();
foreach (var parseNode in leafList)
{
var newWord = new AnnotatedWord("{" + language + "=" + parseNode.GetData().GetName() + "}{posTag="
+ parseNode.GetParent().GetData().GetName() + "}");
sentence.AddWord(newWord);
}
return(sentence);
}
/**
* <summary> The method assigns the words "corp.", "inc.", and "co" ORGANIZATION tag. The method also checks the
* ORGANIZATION gazetteer, and if the word exists in the gazetteer, it assigns ORGANIZATION tag.</summary>
* <param name="sentence">The sentence for which ORGANIZATION named entities checked.</param>
*/
protected override void AutoDetectOrganization(AnnotatedSentence.AnnotatedSentence sentence)
{
for (var i = 0; i < sentence.WordCount(); i++)
{
var word = (AnnotatedWord)sentence.GetWord(i);
if (word.GetParse() != null)
{
if (Word.IsOrganization(word.GetName()))
{
word.SetNamedEntityType("ORGANIZATION");
}
word.CheckGazetteer(organizationGazetteer);
}
}
}
/**
* <summary> The method assigns the words "bay" and "bayan" PERSON tag. The method also checks the PERSON gazetteer, and if
* the word exists in the gazetteer, it assigns PERSON tag.</summary>
* <param name="sentence">The sentence for which PERSON named entities checked.</param>
*/
protected override void AutoDetectPerson(AnnotatedSentence.AnnotatedSentence sentence)
{
for (var i = 0; i < sentence.WordCount(); i++)
{
var word = (AnnotatedWord)sentence.GetWord(i);
if (word.GetParse() != null)
{
if (Word.IsHonorific(word.GetName()))
{
word.SetNamedEntityType("PERSON");
}
word.CheckGazetteer(personGazetteer);
}
}
}
private List <WordNodePair> ConstructWordPairList(AnnotatedSentence.AnnotatedSentence sentence, String fileName)
{
var wordNodePairs = new List <WordNodePair>();
for (var i = 0; i < sentence.WordCount(); i++)
{
var annotatedWord1 = (AnnotatedWord)sentence.GetWord(i);
if (annotatedWord1.GetParse() == null)
{
throw new MorphologicalAnalysisNotExistsException(fileName);
}
if (annotatedWord1.GetUniversalDependency() == null)
{
throw new UniversalDependencyNotExistsException(fileName);
}
var toWord1 = annotatedWord1.GetUniversalDependency().To() - 1;
wordNodePairs.Add(new WordNodePair(annotatedWord1, i));
for (var j = 0; j < sentence.WordCount(); j++)
{
if (i == j)
{
continue;
}
var annotatedWord2 = (AnnotatedWord)sentence.GetWord(j);
if (annotatedWord2.GetUniversalDependency() == null)
{
throw new UniversalDependencyNotExistsException(fileName);
}
var toWord2 = annotatedWord2.GetUniversalDependency().To() - 1;
if (i > j)
{
if (toWord2 > i && toWord1 > toWord2)
{
throw new NonProjectiveDependencyException(fileName);
}
}
else
{
if (toWord1 > j && toWord1 < toWord2)
{
throw new NonProjectiveDependencyException(fileName);
}
}
}
}
return(wordNodePairs);
}
public AnnotatedSentence.AnnotatedSentence GenerateAnnotatedSentence()
{
var sentence = new AnnotatedSentence.AnnotatedSentence("");
var nodeDrawableCollector =
new NodeDrawableCollector((ParseNodeDrawable)root, new IsTurkishLeafNode());
var leafList = nodeDrawableCollector.Collect();
foreach (var parseNode in leafList)
{
var layers = parseNode.GetLayerInfo();
for (var i = 0; i < layers.GetNumberOfWords(); i++)
{
sentence.AddWord(layers.ToAnnotatedWord(i));
}
}
return(sentence);
}
/**
* <summary> Creates a morphological disambiguation corpus from the corpus.</summary>
*
* <returns>Created disambiguation corpus.</returns>
*/
public DisambiguationCorpus Generate()
{
var corpus = new DisambiguationCorpus();
for (var i = 0; i < _annotatedCorpus.SentenceCount(); i++)
{
var sentence = _annotatedCorpus.GetSentence(i);
var disambiguationSentence = new AnnotatedSentence.AnnotatedSentence("");
for (var j = 0; j < sentence.WordCount(); j++)
{
disambiguationSentence.AddWord(new DisambiguatedWord(sentence.GetWord(j).GetName(),
((AnnotatedWord)sentence.GetWord(j)).GetParse()));
}
corpus.AddSentence(disambiguationSentence);
}
return(corpus);
}
public ParseTree.ParseTree Convert(AnnotatedSentence.AnnotatedSentence annotatedSentence, List <TreeEnsembleModel> models)
{
try {
var wordNodePairs = ConstructWordPairList(annotatedSentence, annotatedSentence.GetFileName());
var dependencyMap = SetDependencyMap(wordNodePairs);
if (wordNodePairs.Count > 1)
{
return(ConstructTreeFromWords(wordNodePairs, dependencyMap, models));
}
else
{
var parent = new ParseNodeDrawable(new Symbol("S"));
parent.AddChild(wordNodePairs[0].GetNode());
return(new ParseTree.ParseTree(parent));
}
} catch (Exception e) {
if (e is UniversalDependencyNotExistsException || e is NonProjectiveDependencyException || e is MorphologicalAnalysisNotExistsException)
{
Console.WriteLine(e.ToString());
}
}
return(null);
}
/**
* <summary> Creates a morphological disambiguation corpus from the treeBank. Calls generateAnnotatedSentence for each parse
* tree in the treebank.</summary>
*
* <returns>Created disambiguation corpus.</returns>
*/
public DisambiguationCorpus Generate()
{
var corpus = new DisambiguationCorpus();
for (var i = 0; i < _treeBank.Size(); i++)
{
var parseTree = _treeBank.Get(i);
if (parseTree.LayerAll(ViewLayerType.INFLECTIONAL_GROUP))
{
var sentence = parseTree.GenerateAnnotatedSentence();
var disambiguationSentence = new AnnotatedSentence.AnnotatedSentence("");
for (var j = 0; j < sentence.WordCount(); j++)
{
disambiguationSentence.AddWord(new DisambiguatedWord(sentence.GetWord(j).GetName(),
((AnnotatedWord)sentence.GetWord(j)).GetParse()));
}
corpus.AddSentence(sentence);
}
}
return(corpus);
}
/**
* <summary> The method checks for the TIME entities using regular expressions. After that, if the expression is a TIME
* expression, it also assigns the previous texts, which are numbers, TIME tag.</summary>
* <param name="sentence">The sentence for which TIME named entities checked.</param>
*/
protected override void AutoDetectTime(AnnotatedSentence.AnnotatedSentence sentence)
{
for (var i = 0; i < sentence.WordCount(); i++)
{
var word = (AnnotatedWord)sentence.GetWord(i);
var wordLowercase = word.GetName().ToLower(new CultureInfo("tr"));
if (word.GetParse() != null)
{
if (Word.IsTime(wordLowercase))
{
word.SetNamedEntityType("TIME");
if (i > 0)
{
AnnotatedWord previous = (AnnotatedWord)sentence.GetWord(i - 1);
if (previous.GetParse().ContainsTag(MorphologicalTag.CARDINAL))
{
previous.SetNamedEntityType("TIME");
}
}
}
}
}
}
/**
* <summary> The method checks for the MONEY entities using regular expressions. After that, if the expression is a MONEY
* expression, it also assigns the previous text, which may included numbers or some monetarial texts, MONEY tag.</summary>
* <param name="sentence">The sentence for which MONEY named entities checked.</param>
*/
protected override void AutoDetectMoney(AnnotatedSentence.AnnotatedSentence sentence)
{
for (var i = 0; i < sentence.WordCount(); i++)
{
var word = (AnnotatedWord)sentence.GetWord(i);
var wordLowercase = word.GetName().ToLower(new CultureInfo("tr"));
if (word.GetParse() != null)
{
if (Word.IsMoney(wordLowercase))
{
word.SetNamedEntityType("MONEY");
var j = i - 1;
while (j >= 0)
{
AnnotatedWord previous = (AnnotatedWord)sentence.GetWord(j);
if (previous.GetParse() != null && (previous.GetName().Equals("amerikan") ||
previous.GetParse()
.ContainsTag(MorphologicalTag.REAL) ||
previous.GetParse()
.ContainsTag(MorphologicalTag.CARDINAL) ||
previous.GetParse()
.ContainsTag(MorphologicalTag.NUMBER)))
{
previous.SetNamedEntityType("MONEY");
}
else
{
break;
}
j--;
}
}
}
}
}
protected abstract PolarityType GetPolarity(PolarityType polarityType, AnnotatedSentence.AnnotatedSentence sentence, int index);
/**
* <summary> The method checks
* 1. the previous two words and the current word; the previous, current and next word, current and the next
* two words for a three word multiword expression that occurs in the Turkish wordnet.
* 2. the previous word and current word; current word and the next word for a two word multiword expression that
* occurs in the Turkish wordnet.
* 3. the current word
* if it has only one sense. If there is only one sense for that multiword expression or word; it sets that sense.</summary>
* <param name="sentence">The sentence for which word sense disambiguation will be determined automatically.</param>
*/
protected override bool AutoLabelSingleSemantics(AnnotatedSentence.AnnotatedSentence sentence)
{
AnnotatedWord twoPrevious = null, previous = null;
AnnotatedWord twoNext = null, next = null;
for (var i = 0; i < sentence.WordCount(); i++)
{
var current = (AnnotatedWord)sentence.GetWord(i);
if (i > 1)
{
twoPrevious = (AnnotatedWord)sentence.GetWord(i - 2);
}
if (i > 0)
{
previous = (AnnotatedWord)sentence.GetWord(i - 1);
}
if (i != sentence.WordCount() - 1)
{
next = (AnnotatedWord)sentence.GetWord(i + 1);
}
if (i < sentence.WordCount() - 2)
{
twoNext = (AnnotatedWord)sentence.GetWord(i + 2);
}
if (current.GetSemantic() == null && current.GetParse() != null)
{
if (previous != null && twoPrevious != null && twoPrevious.GetParse() != null &&
previous.GetParse() != null)
{
var idioms = _turkishWordNet.ConstructIdiomSynSets(twoPrevious.GetParse(),
previous.GetParse(), current.GetParse(), twoPrevious.GetMetamorphicParse(),
previous.GetMetamorphicParse(), current.GetMetamorphicParse(), _fsm);
if (idioms.Count == 1)
{
current.SetSemantic(idioms[0].GetId());
continue;
}
}
if (previous != null && previous.GetParse() != null && next != null && next.GetParse() != null)
{
var idioms = _turkishWordNet.ConstructIdiomSynSets(previous.GetParse(),
current.GetParse(), next.GetParse(), previous.GetMetamorphicParse(),
current.GetMetamorphicParse(), next.GetMetamorphicParse(), _fsm);
if (idioms.Count == 1)
{
current.SetSemantic(idioms[0].GetId());
continue;
}
}
if (next != null && next.GetParse() != null && twoNext != null && twoNext.GetParse() != null)
{
var idioms = _turkishWordNet.ConstructIdiomSynSets(current.GetParse(),
next.GetParse(), twoNext.GetParse(), current.GetMetamorphicParse(),
next.GetMetamorphicParse(), twoNext.GetMetamorphicParse(), _fsm);
if (idioms.Count == 1)
{
current.SetSemantic(idioms[0].GetId());
continue;
}
}
if (previous != null && previous.GetParse() != null)
{
var idioms = _turkishWordNet.ConstructIdiomSynSets(previous.GetParse(),
current.GetParse(), previous.GetMetamorphicParse(), current.GetMetamorphicParse(), _fsm);
if (idioms.Count == 1)
{
current.SetSemantic(idioms[0].GetId());
continue;
}
}
if (current.GetSemantic() == null && next != null && next.GetParse() != null)
{
var idioms = _turkishWordNet.ConstructIdiomSynSets(current.GetParse(),
next.GetParse(), current.GetMetamorphicParse(), next.GetMetamorphicParse(), _fsm);
if (idioms.Count == 1)
{
current.SetSemantic(idioms[0].GetId());
continue;
}
}
var meanings = _turkishWordNet.ConstructSynSets(current.GetParse().GetWord().GetName(),
current.GetParse(), current.GetMetamorphicParse(), _fsm);
if (current.GetSemantic() == null && meanings.Count == 1)
{
current.SetSemantic(meanings[0].GetId());
}
}
}
return(true);
}
/** * <summary> The method should set the senses of all words, for which there is only one possible sense.</summary> * <param name="sentence">The sentence for which word sense disambiguation will be determined automatically.</param> */ protected abstract bool AutoLabelSingleSemantics(AnnotatedSentence.AnnotatedSentence sentence);
protected List <SynSet> GetCandidateSynSets(WordNet.WordNet wordNet, FsmMorphologicalAnalyzer fsm,
AnnotatedSentence.AnnotatedSentence sentence, int index)
{
AnnotatedWord twoPrevious = null, previous = null, current, twoNext = null, next = null;
var synSets = new List <SynSet>();
current = (AnnotatedWord)sentence.GetWord(index);
if (index > 1)
{
twoPrevious = (AnnotatedWord)sentence.GetWord(index - 2);
}
if (index > 0)
{
previous = (AnnotatedWord)sentence.GetWord(index - 1);
}
if (index != sentence.WordCount() - 1)
{
next = (AnnotatedWord)sentence.GetWord(index + 1);
}
if (index < sentence.WordCount() - 2)
{
twoNext = (AnnotatedWord)sentence.GetWord(index + 2);
}
synSets = wordNet.ConstructSynSets(current.GetParse().GetWord().GetName(),
current.GetParse(), current.GetMetamorphicParse(), fsm);
if (twoPrevious?.GetParse() != null && previous?.GetParse() != null)
{
synSets.AddRange(wordNet.ConstructIdiomSynSets(twoPrevious.GetParse(), previous.GetParse(),
current.GetParse(),
twoPrevious.GetMetamorphicParse(), previous.GetMetamorphicParse(), current.GetMetamorphicParse(),
fsm));
}
if (previous?.GetParse() != null && next?.GetParse() != null)
{
synSets.AddRange(wordNet.ConstructIdiomSynSets(previous.GetParse(), current.GetParse(), next.GetParse(),
previous.GetMetamorphicParse(), current.GetMetamorphicParse(), next.GetMetamorphicParse(), fsm));
}
if (next?.GetParse() != null && twoNext?.GetParse() != null)
{
synSets.AddRange(wordNet.ConstructIdiomSynSets(current.GetParse(), next.GetParse(), twoNext.GetParse(),
current.GetMetamorphicParse(), next.GetMetamorphicParse(), twoNext.GetMetamorphicParse(), fsm));
}
if (previous?.GetParse() != null)
{
synSets.AddRange(wordNet.ConstructIdiomSynSets(previous.GetParse(), current.GetParse(),
previous.GetMetamorphicParse(), current.GetMetamorphicParse(), fsm));
}
if (next?.GetParse() != null)
{
synSets.AddRange(wordNet.ConstructIdiomSynSets(current.GetParse(), next.GetParse(),
current.GetMetamorphicParse(), next.GetMetamorphicParse(), fsm));
}
return(synSets);
}
