static void _addNormals(KeywordReferent kref, List <Pullenti.Semantic.Utils.DerivateGroup> grs, string norm)
{
if (grs == null || grs.Count == 0)
{
return;
}
for (int k = 0; k < grs.Count; k++)
{
bool ch = false;
for (int i = 0; i < (grs.Count - 1); i++)
{
if (_calcRank(grs[i]) < _calcRank(grs[i + 1]))
{
Pullenti.Semantic.Utils.DerivateGroup gr = grs[i];
grs[i] = grs[i + 1];
grs[i + 1] = gr;
ch = true;
}
}
if (!ch)
{
break;
}
}
for (int i = 0; (i < 3) && (i < grs.Count); i++)
{
if (!grs[i].IsDummy && grs[i].Words.Count > 0)
{
if (grs[i].Words[0].Spelling != norm)
{
kref.AddSlot(KeywordReferent.ATTR_NORMAL, grs[i].Words[0].Spelling, false, 0);
}
}
}
}
int _getChildWords(KeywordReferent root, int lev)
{
if (lev > 5)
{
return(0);
}
int res = 0;
foreach (Pullenti.Ner.Slot s in Slots)
{
if (s.TypeName == ATTR_REF && (s.Value is KeywordReferent))
{
if (s.Value == root)
{
return(0);
}
res += (s.Value as KeywordReferent)._getChildWords(root, lev + 1);
}
}
if (res == 0)
{
res = 1;
}
return(res);
}
internal void Union(KeywordReferent kw1, KeywordReferent kw2, string word2)
{
Typ = kw1.Typ;
List <string> tmp = new List <string>();
StringBuilder tmp2 = new StringBuilder();
foreach (string v in kw1.GetStringValues(ATTR_VALUE))
{
this.AddSlot(ATTR_VALUE, string.Format("{0} {1}", v, word2), false, 0);
}
List <string> norms1 = kw1.GetStringValues(ATTR_NORMAL);
if (norms1.Count == 0 && kw1.ChildWords == 1)
{
norms1 = kw1.GetStringValues(ATTR_VALUE);
}
List <string> norms2 = kw2.GetStringValues(ATTR_NORMAL);
if (norms2.Count == 0 && kw2.ChildWords == 1)
{
norms2 = kw2.GetStringValues(ATTR_VALUE);
}
foreach (string n1 in norms1)
{
foreach (string n2 in norms2)
{
tmp.Clear();
tmp.AddRange(n1.Split(' '));
foreach (string n in n2.Split(' '))
{
if (!tmp.Contains(n))
{
tmp.Add(n);
}
}
tmp.Sort();
tmp2.Length = 0;
for (int i = 0; i < tmp.Count; i++)
{
if (i > 0)
{
tmp2.Append(' ');
}
tmp2.Append(tmp[i]);
}
this.AddSlot(ATTR_NORMAL, tmp2.ToString(), false, 0);
}
}
this.AddSlot(ATTR_REF, kw1, false, 0);
this.AddSlot(ATTR_REF, kw2, false, 0);
}
public override bool CanBeEquals(Pullenti.Ner.Referent obj, Pullenti.Ner.Core.ReferentsEqualType typ = Pullenti.Ner.Core.ReferentsEqualType.WithinOneText)
{
KeywordReferent kw = obj as KeywordReferent;
if (kw == null)
{
return(false);
}
KeywordType ki = Typ;
if (ki != kw.Typ)
{
return(false);
}
if (ki == KeywordType.Referent)
{
Pullenti.Ner.Referent re = this.GetSlotValue(ATTR_REF) as Pullenti.Ner.Referent;
if (re == null)
{
return(false);
}
Pullenti.Ner.Referent re2 = kw.GetSlotValue(ATTR_REF) as Pullenti.Ner.Referent;
if (re2 == null)
{
return(false);
}
if (re.CanBeEquals(re2, typ))
{
return(true);
}
}
foreach (Pullenti.Ner.Slot s in Slots)
{
if (s.TypeName == ATTR_NORMAL || s.TypeName == ATTR_VALUE)
{
if (kw.FindSlot(ATTR_NORMAL, s.Value, true) != null)
{
return(true);
}
if (kw.FindSlot(ATTR_VALUE, s.Value, true) != null)
{
return(true);
}
}
}
return(false);
}
static void _setRank(KeywordReferent kr, int cur, int max)
{
double rank = (double)1;
KeywordType ty = kr.Typ;
if (ty == KeywordType.Predicate)
{
rank = 1;
}
else if (ty == KeywordType.Object)
{
string v = kr.GetStringValue(KeywordReferent.ATTR_VALUE) ?? kr.GetStringValue(KeywordReferent.ATTR_NORMAL);
if (v != null)
{
for (int i = 0; i < v.Length; i++)
{
if (v[i] == ' ' || v[i] == '-')
{
rank++;
}
}
}
}
else if (ty == KeywordType.Referent)
{
rank = 3;
Pullenti.Ner.Referent r = kr.GetSlotValue(KeywordReferent.ATTR_REF) as Pullenti.Ner.Referent;
if (r != null)
{
if (r.TypeName == "PERSON")
{
rank = 4;
}
}
}
if (max > 0)
{
rank *= ((1 - (((0.5 * cur) / max))));
}
kr.Rank += rank;
}
Pullenti.Ner.Token _addReferents(Pullenti.Ner.Core.AnalyzerData ad, Pullenti.Ner.Token t, int cur, int max)
{
if (!(t is Pullenti.Ner.ReferentToken))
{
return(t);
}
Pullenti.Ner.Referent r = t.GetReferent();
if (r == null)
{
return(t);
}
if (r is Pullenti.Ner.Denomination.DenominationReferent)
{
Pullenti.Ner.Denomination.DenominationReferent dr = r as Pullenti.Ner.Denomination.DenominationReferent;
KeywordReferent kref0 = new KeywordReferent()
{
Typ = KeywordType.Referent
};
foreach (Pullenti.Ner.Slot s in dr.Slots)
{
if (s.TypeName == Pullenti.Ner.Denomination.DenominationReferent.ATTR_VALUE)
{
kref0.AddSlot(KeywordReferent.ATTR_NORMAL, s.Value, false, 0);
}
}
kref0.AddSlot(KeywordReferent.ATTR_REF, dr, false, 0);
Pullenti.Ner.ReferentToken rt0 = new Pullenti.Ner.ReferentToken(ad.RegisterReferent(kref0), t, t);
t.Kit.EmbedToken(rt0);
return(rt0);
}
if ((r is Pullenti.Ner.Phone.PhoneReferent) || (r is Pullenti.Ner.Uri.UriReferent) || (r is Pullenti.Ner.Bank.BankDataReferent))
{
return(t);
}
if (r is Pullenti.Ner.Money.MoneyReferent)
{
Pullenti.Ner.Money.MoneyReferent mr = r as Pullenti.Ner.Money.MoneyReferent;
KeywordReferent kref0 = new KeywordReferent()
{
Typ = KeywordType.Object
};
kref0.AddSlot(KeywordReferent.ATTR_NORMAL, mr.Currency, false, 0);
Pullenti.Ner.ReferentToken rt0 = new Pullenti.Ner.ReferentToken(ad.RegisterReferent(kref0), t, t);
t.Kit.EmbedToken(rt0);
return(rt0);
}
if (r.TypeName == "DATE" || r.TypeName == "DATERANGE" || r.TypeName == "BOOKLINKREF")
{
return(t);
}
for (Pullenti.Ner.Token tt = (t as Pullenti.Ner.MetaToken).BeginToken; tt != null && tt.EndChar <= t.EndChar; tt = tt.Next)
{
if (tt is Pullenti.Ner.ReferentToken)
{
this._addReferents(ad, tt, cur, max);
}
}
KeywordReferent kref = new KeywordReferent()
{
Typ = KeywordType.Referent
};
string norm = null;
if (r.TypeName == "GEO")
{
norm = r.GetStringValue("ALPHA2");
}
if (norm == null)
{
norm = r.ToString(true, null, 0);
}
if (norm != null)
{
kref.AddSlot(KeywordReferent.ATTR_NORMAL, norm.ToUpper(), false, 0);
}
kref.AddSlot(KeywordReferent.ATTR_REF, t.GetReferent(), false, 0);
_setRank(kref, cur, max);
Pullenti.Ner.ReferentToken rt1 = new Pullenti.Ner.ReferentToken(ad.RegisterReferent(kref), t, t);
t.Kit.EmbedToken(rt1);
return(rt1);
}
// Основная функция выделения телефонов
public override void Process(Pullenti.Ner.Core.AnalysisKit kit)
{
Pullenti.Ner.Core.AnalyzerData ad = kit.GetAnalyzerData(this);
bool hasDenoms = false;
foreach (Pullenti.Ner.Analyzer a in kit.Processor.Analyzers)
{
if ((a is Pullenti.Ner.Denomination.DenominationAnalyzer) && !a.IgnoreThisAnalyzer)
{
hasDenoms = true;
}
}
if (!hasDenoms)
{
Pullenti.Ner.Denomination.DenominationAnalyzer a = new Pullenti.Ner.Denomination.DenominationAnalyzer();
a.Process(kit);
}
List <KeywordReferent> li = new List <KeywordReferent>();
StringBuilder tmp = new StringBuilder();
List <string> tmp2 = new List <string>();
int max = 0;
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
max++;
}
int cur = 0;
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next, cur++)
{
Pullenti.Ner.Referent r = t.GetReferent();
if (r != null)
{
t = this._addReferents(ad, t, cur, max);
continue;
}
if (!(t is Pullenti.Ner.TextToken))
{
continue;
}
if (!t.Chars.IsLetter || (t.LengthChar < 3))
{
continue;
}
string term = (t as Pullenti.Ner.TextToken).Term;
if (term == "ЕСТЬ")
{
if ((t.Previous is Pullenti.Ner.TextToken) && t.Previous.Morph.Class.IsVerb)
{
}
else
{
continue;
}
}
Pullenti.Ner.Core.NounPhraseToken npt = null;
npt = Pullenti.Ner.Core.NounPhraseHelper.TryParse(t, Pullenti.Ner.Core.NounPhraseParseAttr.AdjectiveCanBeLast | Pullenti.Ner.Core.NounPhraseParseAttr.ParsePreposition, 0, null);
if (npt == null)
{
Pullenti.Morph.MorphClass mc = t.GetMorphClassInDictionary();
if (mc.IsVerb && !mc.IsPreposition)
{
if ((t as Pullenti.Ner.TextToken).IsVerbBe)
{
continue;
}
if (t.IsValue("МОЧЬ", null) || t.IsValue("WOULD", null))
{
continue;
}
KeywordReferent kref = new KeywordReferent()
{
Typ = KeywordType.Predicate
};
string norm = t.GetNormalCaseText(Pullenti.Morph.MorphClass.Verb, Pullenti.Morph.MorphNumber.Singular, Pullenti.Morph.MorphGender.Undefined, false);
if (norm == null)
{
norm = (t as Pullenti.Ner.TextToken).Lemma;
}
if (norm.EndsWith("ЬСЯ"))
{
norm = norm.Substring(0, norm.Length - 2);
}
kref.AddSlot(KeywordReferent.ATTR_VALUE, norm, false, 0);
List <Pullenti.Semantic.Utils.DerivateGroup> drv = Pullenti.Semantic.Utils.DerivateService.FindDerivates(norm, true, t.Morph.Language);
_addNormals(kref, drv, norm);
kref = ad.RegisterReferent(kref) as KeywordReferent;
_setRank(kref, cur, max);
Pullenti.Ner.ReferentToken rt1 = new Pullenti.Ner.ReferentToken(ad.RegisterReferent(kref), t, t)
{
Morph = t.Morph
};
kit.EmbedToken(rt1);
t = rt1;
continue;
}
continue;
}
if (npt.InternalNoun != null)
{
continue;
}
if (npt.EndToken.IsValue("ЦЕЛОМ", null) || npt.EndToken.IsValue("ЧАСТНОСТИ", null))
{
if (npt.Preposition != null)
{
t = npt.EndToken;
continue;
}
}
if (npt.EndToken.IsValue("СТОРОНЫ", null) && npt.Preposition != null && npt.Preposition.Normal == "С")
{
t = npt.EndToken;
continue;
}
if (npt.BeginToken == npt.EndToken)
{
Pullenti.Morph.MorphClass mc = t.GetMorphClassInDictionary();
if (mc.IsPreposition)
{
continue;
}
else if (mc.IsAdverb)
{
if (t.IsValue("ПОТОМ", null))
{
continue;
}
}
}
else
{
}
li.Clear();
Pullenti.Ner.Token t0 = t;
for (Pullenti.Ner.Token tt = t; tt != null && tt.EndChar <= npt.EndChar; tt = tt.Next)
{
if (!(tt is Pullenti.Ner.TextToken))
{
continue;
}
if (tt.IsValue("NATURAL", null))
{
}
if ((tt.LengthChar < 3) || !tt.Chars.IsLetter)
{
continue;
}
Pullenti.Morph.MorphClass mc = tt.GetMorphClassInDictionary();
if ((mc.IsPreposition || mc.IsPronoun || mc.IsPersonalPronoun) || mc.IsConjunction)
{
if (tt.IsValue("ОТНОШЕНИЕ", null))
{
}
else
{
continue;
}
}
if (mc.IsMisc)
{
if (Pullenti.Ner.Core.MiscHelper.IsEngArticle(tt))
{
continue;
}
}
KeywordReferent kref = new KeywordReferent()
{
Typ = KeywordType.Object
};
string norm = (tt as Pullenti.Ner.TextToken).Lemma;
kref.AddSlot(KeywordReferent.ATTR_VALUE, norm, false, 0);
if (norm != "ЕСТЬ")
{
List <Pullenti.Semantic.Utils.DerivateGroup> drv = Pullenti.Semantic.Utils.DerivateService.FindDerivates(norm, true, tt.Morph.Language);
_addNormals(kref, drv, norm);
}
kref = ad.RegisterReferent(kref) as KeywordReferent;
_setRank(kref, cur, max);
Pullenti.Ner.ReferentToken rt1 = new Pullenti.Ner.ReferentToken(kref, tt, tt)
{
Morph = tt.Morph
};
kit.EmbedToken(rt1);
if (tt == t && li.Count == 0)
{
t0 = rt1;
}
t = rt1;
li.Add(kref);
}
if (li.Count > 1)
{
KeywordReferent kref = new KeywordReferent()
{
Typ = KeywordType.Object
};
tmp.Length = 0;
tmp2.Clear();
bool hasNorm = false;
foreach (KeywordReferent kw in li)
{
string s = kw.GetStringValue(KeywordReferent.ATTR_VALUE);
if (tmp.Length > 0)
{
tmp.Append(' ');
}
tmp.Append(s);
string n = kw.GetStringValue(KeywordReferent.ATTR_NORMAL);
if (n != null)
{
hasNorm = true;
tmp2.Add(n);
}
else
{
tmp2.Add(s);
}
kref.AddSlot(KeywordReferent.ATTR_REF, kw, false, 0);
}
string val = npt.GetNormalCaseText(null, Pullenti.Morph.MorphNumber.Singular, Pullenti.Morph.MorphGender.Undefined, false);
kref.AddSlot(KeywordReferent.ATTR_VALUE, val, false, 0);
tmp.Length = 0;
tmp2.Sort();
foreach (string s in tmp2)
{
if (tmp.Length > 0)
{
tmp.Append(' ');
}
tmp.Append(s);
}
string norm = tmp.ToString();
if (norm != val)
{
kref.AddSlot(KeywordReferent.ATTR_NORMAL, norm, false, 0);
}
kref = ad.RegisterReferent(kref) as KeywordReferent;
_setRank(kref, cur, max);
Pullenti.Ner.ReferentToken rt1 = new Pullenti.Ner.ReferentToken(kref, t0, t)
{
Morph = npt.Morph
};
kit.EmbedToken(rt1);
t = rt1;
}
}
cur = 0;
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next, cur++)
{
KeywordReferent kw = t.GetReferent() as KeywordReferent;
if (kw == null || kw.Typ != KeywordType.Object)
{
continue;
}
if (t.Next == null || kw.ChildWords > 2)
{
continue;
}
Pullenti.Ner.Token t1 = t.Next;
if (t1.IsValue("OF", null) && (t1.WhitespacesAfterCount < 3) && t1.Next != null)
{
t1 = t1.Next;
if ((t1 is Pullenti.Ner.TextToken) && Pullenti.Ner.Core.MiscHelper.IsEngArticle(t1) && t1.Next != null)
{
t1 = t1.Next;
}
}
else if (!t1.Morph.Case.IsGenitive || t.WhitespacesAfterCount > 1)
{
continue;
}
KeywordReferent kw2 = t1.GetReferent() as KeywordReferent;
if (kw2 == null)
{
continue;
}
if (kw == kw2)
{
continue;
}
if (kw2.Typ != KeywordType.Object || (kw.ChildWords + kw2.ChildWords) > 3)
{
continue;
}
KeywordReferent kwUn = new KeywordReferent();
kwUn.Union(kw, kw2, Pullenti.Ner.Core.MiscHelper.GetTextValue(t1, t1, Pullenti.Ner.Core.GetTextAttr.No));
kwUn = ad.RegisterReferent(kwUn) as KeywordReferent;
_setRank(kwUn, cur, max);
Pullenti.Ner.ReferentToken rt1 = new Pullenti.Ner.ReferentToken(kwUn, t, t1)
{
Morph = t.Morph
};
kit.EmbedToken(rt1);
t = rt1;
}
if (SortKeywordsByRank)
{
List <Pullenti.Ner.Referent> all = new List <Pullenti.Ner.Referent>(ad.Referents);
all.Sort(new CompByRank());
ad.Referents = all;
}
if (AnnotationMaxSentences > 0)
{
KeywordReferent ano = Pullenti.Ner.Keyword.Internal.AutoannoSentToken.CreateAnnotation(kit, AnnotationMaxSentences);
if (ano != null)
{
ad.RegisterReferent(ano);
}
}
}