public override void Process(Pullenti.Ner.Core.AnalysisKit kit)
{
Pullenti.Ner.Core.AnalyzerData ad = kit.GetAnalyzerData(this);
int delta = 100000;
int parts = (((kit.Sofa.Text.Length + delta) - 1)) / delta;
if (parts == 0)
{
parts = 1;
}
int cur = 0;
int nextPos = 0;
List <GoodReferent> goods = new List <GoodReferent>();
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
if (!t.IsNewlineBefore)
{
continue;
}
if (t.BeginChar > nextPos)
{
nextPos += delta;
cur++;
if (!this.OnProgress(cur, parts, kit))
{
break;
}
}
if (!t.Chars.IsLetter && t.Next != null)
{
t = t.Next;
}
List <Pullenti.Ner.ReferentToken> rts = Pullenti.Ner.Goods.Internal.GoodAttrToken.TryParseList(t);
if (rts == null || rts.Count == 0)
{
continue;
}
GoodReferent good = new GoodReferent();
foreach (Pullenti.Ner.ReferentToken rt in rts)
{
rt.Referent = ad.RegisterReferent(rt.Referent);
if (good.FindSlot(GoodReferent.ATTR_ATTR, rt.Referent, true) == null)
{
good.AddSlot(GoodReferent.ATTR_ATTR, rt.Referent, false, 0);
}
kit.EmbedToken(rt);
}
goods.Add(good);
Pullenti.Ner.ReferentToken rt0 = new Pullenti.Ner.ReferentToken(good, rts[0], rts[rts.Count - 1]);
kit.EmbedToken(rt0);
t = rt0;
}
foreach (GoodReferent g in goods)
{
ad.Referents.Add(g);
}
}
public override void Process(Pullenti.Ner.Core.AnalysisKit kit)
{
Pullenti.Ner.Core.AnalyzerData ad = kit.GetAnalyzerData(this);
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
Pullenti.Ner.ReferentToken rt = null;
if (t.Chars.IsLetter)
{
Pullenti.Ner.Core.TerminToken tok = m_Ontology.TryParse(t, Pullenti.Ner.Core.TerminParseAttr.No);
if (tok != null)
{
Pullenti.Ner.Token tt = tok.EndToken.Next;
if (tt != null && tt.IsChar(':'))
{
tt = tt.Next;
}
rt = this.TryAttach(tt, true);
if (rt != null)
{
rt.BeginToken = t;
}
}
}
if (rt == null && (((t is Pullenti.Ner.ReferentToken) || t.IsNewlineBefore)))
{
rt = this.TryAttach(t, false);
}
if (rt != null)
{
rt.Referent = ad.RegisterReferent(rt.Referent);
kit.EmbedToken(rt);
t = rt;
}
}
}
public override void Process(Pullenti.Ner.Core.AnalysisKit kit)
{
Pullenti.Ner.Core.AnalyzerData ad = kit.GetAnalyzerData(this);
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
Pullenti.Ner.ReferentToken mon = TryParse(t);
if (mon != null)
{
mon.Referent = ad.RegisterReferent(mon.Referent);
kit.EmbedToken(mon);
t = mon;
continue;
}
}
}
public static void Process(Pullenti.Ner.Core.AnalysisKit kit, Pullenti.Ner.Core.AnalyzerData ad)
{
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
if (!Pullenti.Ner.Core.MiscHelper.CanBeStartOfSentence(t))
{
continue;
}
Pullenti.Ner.ReferentToken rt = TryParseThesis(t);
if (rt == null)
{
continue;
}
rt.Referent = ad.RegisterReferent(rt.Referent);
kit.EmbedToken(rt);
t = rt;
}
}
public override void Process(Pullenti.Ner.Core.AnalysisKit kit)
{
Pullenti.Ner.Core.AnalyzerDataWithOntology ad = kit.GetAnalyzerData(this) as Pullenti.Ner.Core.AnalyzerDataWithOntology;
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
List <Pullenti.Ner.Named.Internal.NamedItemToken> li = Pullenti.Ner.Named.Internal.NamedItemToken.TryParseList(t, ad.LocalOntology);
if (li == null || li.Count == 0)
{
continue;
}
Pullenti.Ner.ReferentToken rt = _tryAttach(li);
if (rt != null)
{
rt.Referent = ad.RegisterReferent(rt.Referent);
kit.EmbedToken(rt);
t = rt;
continue;
}
}
}
public override void Process(Pullenti.Ner.Core.AnalysisKit kit)
{
Pullenti.Ner.Core.AnalyzerData ad = kit.GetAnalyzerData(this);
int delta = 100000;
int parts = (((kit.Sofa.Text.Length + delta) - 1)) / delta;
if (parts == 0)
{
parts = 1;
}
int cur = 0;
int nextPos = 0;
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
if (t.BeginChar > nextPos)
{
nextPos += delta;
cur++;
if (!this.OnProgress(cur, parts, kit))
{
break;
}
}
Pullenti.Ner.Goods.Internal.GoodAttrToken at = Pullenti.Ner.Goods.Internal.GoodAttrToken.TryParse(t, null, true, true);
if (at == null)
{
continue;
}
GoodAttributeReferent attr = at._createAttr();
if (attr == null)
{
t = at.EndToken;
continue;
}
Pullenti.Ner.ReferentToken rt = new Pullenti.Ner.ReferentToken(attr, at.BeginToken, at.EndToken);
rt.Referent = ad.RegisterReferent(attr);
kit.EmbedToken(rt);
t = rt;
}
}
// Основная функция выделения объектов
public override void Process(Pullenti.Ner.Core.AnalysisKit kit)
{
Pullenti.Ner.Core.AnalyzerDataWithOntology ad = kit.GetAnalyzerData(this) as Pullenti.Ner.Core.AnalyzerDataWithOntology;
for (int k = 0; k < 2; k++)
{
bool detectNewDenoms = false;
DateTime dt = DateTime.Now;
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
if (t.IsWhitespaceBefore)
{
}
else if (t.Previous != null && ((t.Previous.IsCharOf(",") || Pullenti.Ner.Core.BracketHelper.CanBeStartOfSequence(t.Previous, false, false))))
{
}
else
{
continue;
}
Pullenti.Ner.ReferentToken rt0 = this.TryAttachSpec(t);
if (rt0 != null)
{
rt0.Referent = ad.RegisterReferent(rt0.Referent);
kit.EmbedToken(rt0);
t = rt0;
continue;
}
if (!t.Chars.IsLetter)
{
continue;
}
if (!this.CanBeStartOfDenom(t))
{
continue;
}
if (((DateTime.Now - dt)).TotalMinutes > 1)
{
break;
}
List <Pullenti.Ner.Core.IntOntologyToken> ot = null;
ot = ad.LocalOntology.TryAttach(t, null, false);
if (ot != null && (ot[0].Item.Referent is DenominationReferent))
{
if (this.CheckAttach(ot[0].BeginToken, ot[0].EndToken))
{
DenominationReferent cl = ot[0].Item.Referent.Clone() as DenominationReferent;
cl.Occurrence.Clear();
Pullenti.Ner.ReferentToken rt = new Pullenti.Ner.ReferentToken(cl, ot[0].BeginToken, ot[0].EndToken);
kit.EmbedToken(rt);
t = rt;
continue;
}
}
if (k > 0)
{
continue;
}
if (t != null && t.Kit.Ontology != null)
{
if ((((ot = t.Kit.Ontology.AttachToken(DenominationReferent.OBJ_TYPENAME, t)))) != null)
{
if (this.CheckAttach(ot[0].BeginToken, ot[0].EndToken))
{
DenominationReferent dr = new DenominationReferent();
dr.MergeSlots(ot[0].Item.Referent, true);
Pullenti.Ner.ReferentToken rt = new Pullenti.Ner.ReferentToken(ad.RegisterReferent(dr), ot[0].BeginToken, ot[0].EndToken);
kit.EmbedToken(rt);
t = rt;
continue;
}
}
}
rt0 = this.TryAttach(t, false);
if (rt0 != null)
{
rt0.Referent = ad.RegisterReferent(rt0.Referent);
kit.EmbedToken(rt0);
detectNewDenoms = true;
t = rt0;
if (ad.LocalOntology.Items.Count > 1000)
{
break;
}
}
}
if (!detectNewDenoms)
{
break;
}
}
}
public override void Process(Pullenti.Ner.Core.AnalysisKit kit)
{
PhoneAnalizerData ad = kit.GetAnalyzerData(this) as PhoneAnalizerData;
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
List<Pullenti.Ner.Phone.Internal.PhoneItemToken> pli = Pullenti.Ner.Phone.Internal.PhoneItemToken.TryAttachAll(t, 15);
if (pli == null || pli.Count == 0)
continue;
PhoneReferent prevPhone = null;
int kkk = 0;
for (Pullenti.Ner.Token tt = t.Previous; tt != null; tt = tt.Previous)
{
if (tt.GetReferent() is PhoneReferent)
{
prevPhone = tt.GetReferent() as PhoneReferent;
break;
}
else if (tt is Pullenti.Ner.ReferentToken)
{
}
else if (tt.IsChar(')'))
{
Pullenti.Ner.Token ttt = tt.Previous;
int cou = 0;
for (; ttt != null; ttt = ttt.Previous)
{
if (ttt.IsChar('('))
break;
else if ((++cou) > 100)
break;
}
if (ttt == null || !ttt.IsChar('('))
break;
tt = ttt;
}
else if (!tt.IsCharOf(",;/\\") && !tt.IsAnd)
{
if ((++kkk) > 5)
break;
if (tt.IsNewlineBefore || tt.IsNewlineAfter)
break;
}
}
int j = 0;
bool isPhoneBefore = false;
bool isPref = false;
PhoneKind ki = PhoneKind.Undefined;
while (j < pli.Count)
{
if (pli[j].ItemType == Pullenti.Ner.Phone.Internal.PhoneItemToken.PhoneItemType.Prefix)
{
if (ki == PhoneKind.Undefined)
ki = pli[j].Kind;
isPref = true;
isPhoneBefore = true;
j++;
if ((j < pli.Count) && pli[j].ItemType == Pullenti.Ner.Phone.Internal.PhoneItemToken.PhoneItemType.Delim)
j++;
}
else if (((j + 1) < pli.Count) && pli[j + 1].ItemType == Pullenti.Ner.Phone.Internal.PhoneItemToken.PhoneItemType.Prefix && j == 0)
{
if (ki == PhoneKind.Undefined)
ki = pli[0].Kind;
isPref = true;
pli.RemoveAt(0);
}
else
break;
}
if (prevPhone != null)
isPhoneBefore = true;
if (pli.Count == 1 && pli[0].ItemType == Pullenti.Ner.Phone.Internal.PhoneItemToken.PhoneItemType.Number)
{
Pullenti.Ner.Token tt = t.Previous;
if ((tt is Pullenti.Ner.TextToken) && !tt.Chars.IsLetter)
tt = tt.Previous;
if (tt is Pullenti.Ner.TextToken)
{
if (Pullenti.Ner.Uri.UriAnalyzer.m_Schemes.TryParse(tt, Pullenti.Ner.Core.TerminParseAttr.No) != null)
continue;
}
}
List<Pullenti.Ner.ReferentToken> rts = this.TryAttach(pli, j, isPhoneBefore, prevPhone);
if (rts == null)
{
for (j = 1; j < pli.Count; j++)
{
if (pli[j].ItemType == Pullenti.Ner.Phone.Internal.PhoneItemToken.PhoneItemType.Prefix)
{
pli.RemoveRange(0, j);
rts = this.TryAttach(pli, 1, true, prevPhone);
break;
}
}
}
if (rts == null)
t = pli[pli.Count - 1].EndToken;
else
{
if ((ki == PhoneKind.Undefined && prevPhone != null && !isPref) && prevPhone.Kind != PhoneKind.Mobile && kkk == 0)
ki = prevPhone.Kind;
foreach (Pullenti.Ner.ReferentToken rt in rts)
{
PhoneReferent ph = rt.Referent as PhoneReferent;
if (ki != PhoneKind.Undefined)
ph.Kind = ki;
else
{
if (rt == rts[0] && (rt.WhitespacesBeforeCount < 3))
{
Pullenti.Ner.Token tt1 = rt.BeginToken.Previous;
if (tt1 != null && tt1.IsTableControlChar)
tt1 = tt1.Previous;
if ((tt1 is Pullenti.Ner.TextToken) && ((tt1.IsNewlineBefore || ((tt1.Previous != null && tt1.Previous.IsTableControlChar)))))
{
string term = (tt1 as Pullenti.Ner.TextToken).Term;
if (term == "T" || term == "Т")
rt.BeginToken = tt1;
else if (term == "Ф" || term == "F")
{
ph.Kind = (ki = PhoneKind.Fax);
rt.BeginToken = tt1;
}
else if (term == "M" || term == "М")
{
ph.Kind = (ki = PhoneKind.Mobile);
rt.BeginToken = tt1;
}
}
}
ph.Correct();
}
rt.Referent = ad.RegisterReferent(rt.Referent);
kit.EmbedToken(rt);
t = rt;
}
}
}
}
public override void Process(Pullenti.Ner.Core.AnalysisKit kit)
{
Pullenti.Ner.Core.AnalyzerData ad = kit.GetAnalyzerData(this);
Pullenti.Ner.Core.TerminCollection models = new Pullenti.Ner.Core.TerminCollection();
Dictionary <string, List <Pullenti.Ner.Referent> > objsByModel = new Dictionary <string, List <Pullenti.Ner.Referent> >();
Pullenti.Ner.Core.TerminCollection objByNames = new Pullenti.Ner.Core.TerminCollection();
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
List <Pullenti.Ner.Weapon.Internal.WeaponItemToken> its = Pullenti.Ner.Weapon.Internal.WeaponItemToken.TryParseList(t, 10);
if (its == null)
{
continue;
}
List <Pullenti.Ner.ReferentToken> rts = this.TryAttach(its, false);
if (rts != null)
{
foreach (Pullenti.Ner.ReferentToken rt in rts)
{
rt.Referent = ad.RegisterReferent(rt.Referent);
kit.EmbedToken(rt);
t = rt;
foreach (Pullenti.Ner.Slot s in rt.Referent.Slots)
{
if (s.TypeName == WeaponReferent.ATTR_MODEL)
{
string mod = s.Value.ToString();
for (int k = 0; k < 2; k++)
{
if (!char.IsDigit(mod[0]))
{
List <Pullenti.Ner.Referent> li;
if (!objsByModel.TryGetValue(mod, out li))
{
objsByModel.Add(mod, (li = new List <Pullenti.Ner.Referent>()));
}
if (!li.Contains(rt.Referent))
{
li.Add(rt.Referent);
}
models.AddString(mod, li, null, false);
}
if (k > 0)
{
break;
}
string brand = rt.Referent.GetStringValue(WeaponReferent.ATTR_BRAND);
if (brand == null)
{
break;
}
mod = string.Format("{0} {1}", brand, mod);
}
}
else if (s.TypeName == WeaponReferent.ATTR_NAME)
{
objByNames.Add(new Pullenti.Ner.Core.Termin(s.Value.ToString())
{
Tag = rt.Referent
});
}
}
}
}
}
if (objsByModel.Count == 0 && objByNames.Termins.Count == 0)
{
return;
}
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
Pullenti.Ner.Core.BracketSequenceToken br = Pullenti.Ner.Core.BracketHelper.TryParse(t, Pullenti.Ner.Core.BracketParseAttr.No, 10);
if (br != null)
{
Pullenti.Ner.Core.TerminToken toks = objByNames.TryParse(t.Next, Pullenti.Ner.Core.TerminParseAttr.No);
if (toks != null && toks.EndToken.Next == br.EndToken)
{
Pullenti.Ner.ReferentToken rt0 = new Pullenti.Ner.ReferentToken(toks.Termin.Tag as Pullenti.Ner.Referent, br.BeginToken, br.EndToken);
kit.EmbedToken(rt0);
t = rt0;
continue;
}
}
if (!(t is Pullenti.Ner.TextToken))
{
continue;
}
if (!t.Chars.IsLetter)
{
continue;
}
Pullenti.Ner.Core.TerminToken tok = models.TryParse(t, Pullenti.Ner.Core.TerminParseAttr.No);
if (tok == null)
{
if (!t.Chars.IsAllLower)
{
tok = objByNames.TryParse(t, Pullenti.Ner.Core.TerminParseAttr.No);
}
if (tok == null)
{
continue;
}
}
if (!tok.IsWhitespaceAfter)
{
if (tok.EndToken.Next == null || !tok.EndToken.Next.IsCharOf(",.)"))
{
if (!Pullenti.Ner.Core.BracketHelper.IsBracket(tok.EndToken.Next, false))
{
continue;
}
}
}
Pullenti.Ner.Referent tr = null;
List <Pullenti.Ner.Referent> li = tok.Termin.Tag as List <Pullenti.Ner.Referent>;
if (li != null && li.Count == 1)
{
tr = li[0];
}
else
{
tr = tok.Termin.Tag as Pullenti.Ner.Referent;
}
if (tr != null)
{
Pullenti.Ner.Weapon.Internal.WeaponItemToken tit = Pullenti.Ner.Weapon.Internal.WeaponItemToken.TryParse(tok.BeginToken.Previous, null, false, true);
if (tit != null && tit.Typ == Pullenti.Ner.Weapon.Internal.WeaponItemToken.Typs.Brand)
{
tr.AddSlot(WeaponReferent.ATTR_BRAND, tit.Value, false, 0);
tok.BeginToken = tit.BeginToken;
}
Pullenti.Ner.ReferentToken rt0 = new Pullenti.Ner.ReferentToken(tr, tok.BeginToken, tok.EndToken);
kit.EmbedToken(rt0);
t = rt0;
continue;
}
}
}
// Основная функция выделения телефонов
public override void Process(Pullenti.Ner.Core.AnalysisKit kit)
{
Pullenti.Ner.Core.AnalyzerData ad = kit.GetAnalyzerData(this);
bool hasDenoms = false;
foreach (Pullenti.Ner.Analyzer a in kit.Processor.Analyzers)
{
if ((a is Pullenti.Ner.Denomination.DenominationAnalyzer) && !a.IgnoreThisAnalyzer)
{
hasDenoms = true;
}
}
if (!hasDenoms)
{
Pullenti.Ner.Denomination.DenominationAnalyzer a = new Pullenti.Ner.Denomination.DenominationAnalyzer();
a.Process(kit);
}
List <KeywordReferent> li = new List <KeywordReferent>();
StringBuilder tmp = new StringBuilder();
List <string> tmp2 = new List <string>();
int max = 0;
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
max++;
}
int cur = 0;
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next, cur++)
{
Pullenti.Ner.Referent r = t.GetReferent();
if (r != null)
{
t = this._addReferents(ad, t, cur, max);
continue;
}
if (!(t is Pullenti.Ner.TextToken))
{
continue;
}
if (!t.Chars.IsLetter || (t.LengthChar < 3))
{
continue;
}
string term = (t as Pullenti.Ner.TextToken).Term;
if (term == "ЕСТЬ")
{
if ((t.Previous is Pullenti.Ner.TextToken) && t.Previous.Morph.Class.IsVerb)
{
}
else
{
continue;
}
}
Pullenti.Ner.Core.NounPhraseToken npt = null;
npt = Pullenti.Ner.Core.NounPhraseHelper.TryParse(t, Pullenti.Ner.Core.NounPhraseParseAttr.AdjectiveCanBeLast | Pullenti.Ner.Core.NounPhraseParseAttr.ParsePreposition, 0, null);
if (npt == null)
{
Pullenti.Morph.MorphClass mc = t.GetMorphClassInDictionary();
if (mc.IsVerb && !mc.IsPreposition)
{
if ((t as Pullenti.Ner.TextToken).IsVerbBe)
{
continue;
}
if (t.IsValue("МОЧЬ", null) || t.IsValue("WOULD", null))
{
continue;
}
KeywordReferent kref = new KeywordReferent()
{
Typ = KeywordType.Predicate
};
string norm = t.GetNormalCaseText(Pullenti.Morph.MorphClass.Verb, Pullenti.Morph.MorphNumber.Singular, Pullenti.Morph.MorphGender.Undefined, false);
if (norm == null)
{
norm = (t as Pullenti.Ner.TextToken).Lemma;
}
if (norm.EndsWith("ЬСЯ"))
{
norm = norm.Substring(0, norm.Length - 2);
}
kref.AddSlot(KeywordReferent.ATTR_VALUE, norm, false, 0);
List <Pullenti.Semantic.Utils.DerivateGroup> drv = Pullenti.Semantic.Utils.DerivateService.FindDerivates(norm, true, t.Morph.Language);
_addNormals(kref, drv, norm);
kref = ad.RegisterReferent(kref) as KeywordReferent;
_setRank(kref, cur, max);
Pullenti.Ner.ReferentToken rt1 = new Pullenti.Ner.ReferentToken(ad.RegisterReferent(kref), t, t)
{
Morph = t.Morph
};
kit.EmbedToken(rt1);
t = rt1;
continue;
}
continue;
}
if (npt.InternalNoun != null)
{
continue;
}
if (npt.EndToken.IsValue("ЦЕЛОМ", null) || npt.EndToken.IsValue("ЧАСТНОСТИ", null))
{
if (npt.Preposition != null)
{
t = npt.EndToken;
continue;
}
}
if (npt.EndToken.IsValue("СТОРОНЫ", null) && npt.Preposition != null && npt.Preposition.Normal == "С")
{
t = npt.EndToken;
continue;
}
if (npt.BeginToken == npt.EndToken)
{
Pullenti.Morph.MorphClass mc = t.GetMorphClassInDictionary();
if (mc.IsPreposition)
{
continue;
}
else if (mc.IsAdverb)
{
if (t.IsValue("ПОТОМ", null))
{
continue;
}
}
}
else
{
}
li.Clear();
Pullenti.Ner.Token t0 = t;
for (Pullenti.Ner.Token tt = t; tt != null && tt.EndChar <= npt.EndChar; tt = tt.Next)
{
if (!(tt is Pullenti.Ner.TextToken))
{
continue;
}
if (tt.IsValue("NATURAL", null))
{
}
if ((tt.LengthChar < 3) || !tt.Chars.IsLetter)
{
continue;
}
Pullenti.Morph.MorphClass mc = tt.GetMorphClassInDictionary();
if ((mc.IsPreposition || mc.IsPronoun || mc.IsPersonalPronoun) || mc.IsConjunction)
{
if (tt.IsValue("ОТНОШЕНИЕ", null))
{
}
else
{
continue;
}
}
if (mc.IsMisc)
{
if (Pullenti.Ner.Core.MiscHelper.IsEngArticle(tt))
{
continue;
}
}
KeywordReferent kref = new KeywordReferent()
{
Typ = KeywordType.Object
};
string norm = (tt as Pullenti.Ner.TextToken).Lemma;
kref.AddSlot(KeywordReferent.ATTR_VALUE, norm, false, 0);
if (norm != "ЕСТЬ")
{
List <Pullenti.Semantic.Utils.DerivateGroup> drv = Pullenti.Semantic.Utils.DerivateService.FindDerivates(norm, true, tt.Morph.Language);
_addNormals(kref, drv, norm);
}
kref = ad.RegisterReferent(kref) as KeywordReferent;
_setRank(kref, cur, max);
Pullenti.Ner.ReferentToken rt1 = new Pullenti.Ner.ReferentToken(kref, tt, tt)
{
Morph = tt.Morph
};
kit.EmbedToken(rt1);
if (tt == t && li.Count == 0)
{
t0 = rt1;
}
t = rt1;
li.Add(kref);
}
if (li.Count > 1)
{
KeywordReferent kref = new KeywordReferent()
{
Typ = KeywordType.Object
};
tmp.Length = 0;
tmp2.Clear();
bool hasNorm = false;
foreach (KeywordReferent kw in li)
{
string s = kw.GetStringValue(KeywordReferent.ATTR_VALUE);
if (tmp.Length > 0)
{
tmp.Append(' ');
}
tmp.Append(s);
string n = kw.GetStringValue(KeywordReferent.ATTR_NORMAL);
if (n != null)
{
hasNorm = true;
tmp2.Add(n);
}
else
{
tmp2.Add(s);
}
kref.AddSlot(KeywordReferent.ATTR_REF, kw, false, 0);
}
string val = npt.GetNormalCaseText(null, Pullenti.Morph.MorphNumber.Singular, Pullenti.Morph.MorphGender.Undefined, false);
kref.AddSlot(KeywordReferent.ATTR_VALUE, val, false, 0);
tmp.Length = 0;
tmp2.Sort();
foreach (string s in tmp2)
{
if (tmp.Length > 0)
{
tmp.Append(' ');
}
tmp.Append(s);
}
string norm = tmp.ToString();
if (norm != val)
{
kref.AddSlot(KeywordReferent.ATTR_NORMAL, norm, false, 0);
}
kref = ad.RegisterReferent(kref) as KeywordReferent;
_setRank(kref, cur, max);
Pullenti.Ner.ReferentToken rt1 = new Pullenti.Ner.ReferentToken(kref, t0, t)
{
Morph = npt.Morph
};
kit.EmbedToken(rt1);
t = rt1;
}
}
cur = 0;
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next, cur++)
{
KeywordReferent kw = t.GetReferent() as KeywordReferent;
if (kw == null || kw.Typ != KeywordType.Object)
{
continue;
}
if (t.Next == null || kw.ChildWords > 2)
{
continue;
}
Pullenti.Ner.Token t1 = t.Next;
if (t1.IsValue("OF", null) && (t1.WhitespacesAfterCount < 3) && t1.Next != null)
{
t1 = t1.Next;
if ((t1 is Pullenti.Ner.TextToken) && Pullenti.Ner.Core.MiscHelper.IsEngArticle(t1) && t1.Next != null)
{
t1 = t1.Next;
}
}
else if (!t1.Morph.Case.IsGenitive || t.WhitespacesAfterCount > 1)
{
continue;
}
KeywordReferent kw2 = t1.GetReferent() as KeywordReferent;
if (kw2 == null)
{
continue;
}
if (kw == kw2)
{
continue;
}
if (kw2.Typ != KeywordType.Object || (kw.ChildWords + kw2.ChildWords) > 3)
{
continue;
}
KeywordReferent kwUn = new KeywordReferent();
kwUn.Union(kw, kw2, Pullenti.Ner.Core.MiscHelper.GetTextValue(t1, t1, Pullenti.Ner.Core.GetTextAttr.No));
kwUn = ad.RegisterReferent(kwUn) as KeywordReferent;
_setRank(kwUn, cur, max);
Pullenti.Ner.ReferentToken rt1 = new Pullenti.Ner.ReferentToken(kwUn, t, t1)
{
Morph = t.Morph
};
kit.EmbedToken(rt1);
t = rt1;
}
if (SortKeywordsByRank)
{
List <Pullenti.Ner.Referent> all = new List <Pullenti.Ner.Referent>(ad.Referents);
all.Sort(new CompByRank());
ad.Referents = all;
}
if (AnnotationMaxSentences > 0)
{
KeywordReferent ano = Pullenti.Ner.Keyword.Internal.AutoannoSentToken.CreateAnnotation(kit, AnnotationMaxSentences);
if (ano != null)
{
ad.RegisterReferent(ano);
}
}
}
internal static TitlePageReferent _process(Pullenti.Ner.Token begin, int maxCharPos, Pullenti.Ner.Core.AnalysisKit kit, out Pullenti.Ner.Token endToken)
{
endToken = begin;
TitlePageReferent res = new TitlePageReferent();
Pullenti.Ner.Core.Termin term = null;
List <Pullenti.Ner.Titlepage.Internal.Line> lines = Pullenti.Ner.Titlepage.Internal.Line.Parse(begin, 30, 1500, maxCharPos);
if (lines.Count < 1)
{
return(null);
}
int cou = lines.Count;
int minNewlinesCount = 10;
Dictionary <int, int> linesCountStat = new Dictionary <int, int>();
for (int i = 0; i < lines.Count; i++)
{
if (Pullenti.Ner.Titlepage.Internal.TitleNameToken.CanBeStartOfTextOrContent(lines[i].BeginToken, lines[i].EndToken))
{
cou = i;
break;
}
int j = lines[i].NewlinesBeforeCount;
if (i > 0 && j > 0)
{
if (!linesCountStat.ContainsKey(j))
{
linesCountStat.Add(j, 1);
}
else
{
linesCountStat[j]++;
}
}
}
int max = 0;
foreach (KeyValuePair <int, int> kp in linesCountStat)
{
if (kp.Value > max)
{
max = kp.Value;
minNewlinesCount = kp.Key;
}
}
int endChar = (cou > 0 ? lines[cou - 1].EndChar : 0);
if (maxCharPos > 0 && endChar > maxCharPos)
{
endChar = maxCharPos;
}
List <Pullenti.Ner.Titlepage.Internal.TitleNameToken> names = new List <Pullenti.Ner.Titlepage.Internal.TitleNameToken>();
for (int i = 0; i < cou; i++)
{
if (i == 6)
{
}
for (int j = i; (j < cou) && (j < (i + 5)); j++)
{
if (i == 6 && j == 8)
{
}
if (j > i)
{
if (lines[j - 1].IsPureEn && lines[j].IsPureRu)
{
break;
}
if (lines[j - 1].IsPureRu && lines[j].IsPureEn)
{
break;
}
if (lines[j].NewlinesBeforeCount >= (minNewlinesCount * 2))
{
break;
}
}
Pullenti.Ner.Titlepage.Internal.TitleNameToken ttt = Pullenti.Ner.Titlepage.Internal.TitleNameToken.TryParse(lines[i].BeginToken, lines[j].EndToken, minNewlinesCount);
if (ttt != null)
{
if (lines[i].IsPureEn)
{
ttt.Morph.Language = Pullenti.Morph.MorphLang.EN;
}
else if (lines[i].IsPureRu)
{
ttt.Morph.Language = Pullenti.Morph.MorphLang.RU;
}
names.Add(ttt);
}
}
}
Pullenti.Ner.Titlepage.Internal.TitleNameToken.Sort(names);
Pullenti.Ner.ReferentToken nameRt = null;
if (names.Count > 0)
{
int i0 = 0;
if (names[i0].Morph.Language.IsEn)
{
for (int ii = 1; ii < names.Count; ii++)
{
if (names[ii].Morph.Language.IsRu && names[ii].Rank > 0)
{
i0 = ii;
break;
}
}
}
term = res.AddName(names[i0].BeginNameToken, names[i0].EndNameToken);
if (names[i0].TypeValue != null)
{
res.AddType(names[i0].TypeValue);
}
if (names[i0].Speciality != null)
{
res.Speciality = names[i0].Speciality;
}
Pullenti.Ner.ReferentToken rt = new Pullenti.Ner.ReferentToken(res, names[i0].BeginToken, names[i0].EndToken);
if (kit != null)
{
kit.EmbedToken(rt);
}
else
{
res.AddOccurence(new Pullenti.Ner.TextAnnotation(rt.BeginToken, rt.EndToken));
}
endToken = rt.EndToken;
nameRt = rt;
if (begin.BeginChar == rt.BeginChar)
{
begin = rt;
}
}
if (term != null && kit != null)
{
for (Pullenti.Ner.Token t = kit.FirstToken; t != null; t = t.Next)
{
Pullenti.Ner.Core.TerminToken tok = term.TryParse(t, Pullenti.Ner.Core.TerminParseAttr.No);
if (tok == null)
{
continue;
}
Pullenti.Ner.Token t0 = t;
Pullenti.Ner.Token t1 = tok.EndToken;
if (t1.Next != null && t1.Next.IsChar('.'))
{
t1 = t1.Next;
}
if (Pullenti.Ner.Core.BracketHelper.CanBeStartOfSequence(t0.Previous, false, false) && Pullenti.Ner.Core.BracketHelper.CanBeEndOfSequence(t1.Next, false, null, false))
{
t0 = t0.Previous;
t1 = t1.Next;
}
Pullenti.Ner.ReferentToken rt = new Pullenti.Ner.ReferentToken(res, t0, t1);
kit.EmbedToken(rt);
t = rt;
}
}
Pullenti.Ner.Titlepage.Internal.PersonRelations pr = new Pullenti.Ner.Titlepage.Internal.PersonRelations();
Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types persTyp = Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Undefined;
List <Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types> persTypes = pr.RelTypes;
for (Pullenti.Ner.Token t = begin; t != null; t = t.Next)
{
if (maxCharPos > 0 && t.BeginChar > maxCharPos)
{
break;
}
if (t == nameRt)
{
continue;
}
Pullenti.Ner.Titlepage.Internal.TitleItemToken tpt = Pullenti.Ner.Titlepage.Internal.TitleItemToken.TryAttach(t);
if (tpt != null)
{
persTyp = Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Undefined;
if (tpt.Typ == Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Typ)
{
if (res.Types.Count == 0)
{
res.AddType(tpt.Value);
}
else if (res.Types.Count == 1)
{
string ty = res.Types[0].ToUpper();
if (ty == "РЕФЕРАТ")
{
res.AddType(tpt.Value);
}
else if (ty == "АВТОРЕФЕРАТ")
{
if (tpt.Value == "КАНДИДАТСКАЯ ДИССЕРТАЦИЯ")
{
res.AddSlot(TitlePageReferent.ATTR_TYPE, "автореферат кандидатской диссертации", true, 0);
}
else if (tpt.Value == "ДОКТОРСКАЯ ДИССЕРТАЦИЯ")
{
res.AddSlot(TitlePageReferent.ATTR_TYPE, "автореферат докторской диссертации", true, 0);
}
else if (tpt.Value == "МАГИСТЕРСКАЯ ДИССЕРТАЦИЯ")
{
res.AddSlot(TitlePageReferent.ATTR_TYPE, "автореферат магистерской диссертации", true, 0);
}
else if (tpt.Value == "КАНДИДАТСЬКА ДИСЕРТАЦІЯ")
{
res.AddSlot(TitlePageReferent.ATTR_TYPE, "автореферат кандидатської дисертації", true, 0);
}
else if (tpt.Value == "ДОКТОРСЬКА ДИСЕРТАЦІЯ")
{
res.AddSlot(TitlePageReferent.ATTR_TYPE, "автореферат докторської дисертації", true, 0);
}
else if (tpt.Value == "МАГІСТЕРСЬКА ДИСЕРТАЦІЯ")
{
res.AddSlot(TitlePageReferent.ATTR_TYPE, "автореферат магістерської дисертації", true, 0);
}
else
{
res.AddType(tpt.Value);
}
}
else if (tpt.Value == "РЕФЕРАТ" || tpt.Value == "АВТОРЕФЕРАТ")
{
if (!ty.Contains(tpt.Value))
{
res.AddType(tpt.Value);
}
}
}
}
else if (tpt.Typ == Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Speciality)
{
if (res.Speciality == null)
{
res.Speciality = tpt.Value;
}
}
else if (persTypes.Contains(tpt.Typ))
{
persTyp = tpt.Typ;
}
t = tpt.EndToken;
if (t.EndChar > endToken.EndChar)
{
endToken = t;
}
if (t.Next != null && t.Next.IsCharOf(":-"))
{
t = t.Next;
}
continue;
}
if (t.EndChar > endChar)
{
break;
}
List <Pullenti.Ner.Referent> rli = t.GetReferents();
if (rli == null)
{
continue;
}
if (!t.IsNewlineBefore && (t.Previous is Pullenti.Ner.TextToken))
{
string s = (t.Previous as Pullenti.Ner.TextToken).Term;
if (s == "ИМЕНИ" || s == "ИМ")
{
continue;
}
if (s == "." && t.Previous.Previous != null && t.Previous.Previous.IsValue("ИМ", null))
{
continue;
}
}
foreach (Pullenti.Ner.Referent r in rli)
{
if (r is Pullenti.Ner.Person.PersonReferent)
{
if (r != rli[0])
{
continue;
}
Pullenti.Ner.Person.PersonReferent p = r as Pullenti.Ner.Person.PersonReferent;
if (persTyp != Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Undefined)
{
if (t.Previous != null && t.Previous.IsChar('.'))
{
persTyp = Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Undefined;
}
}
Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types typ = pr.CalcTypFromAttrs(p);
if (typ != Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Undefined)
{
pr.Add(p, typ, 1);
persTyp = typ;
}
else if (persTyp != Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Undefined)
{
pr.Add(p, persTyp, 1);
}
else if (t.Previous != null && t.Previous.IsChar('©'))
{
persTyp = Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Worker;
pr.Add(p, persTyp, 1);
}
else
{
for (Pullenti.Ner.Token tt = t.Next; tt != null; tt = tt.Next)
{
Pullenti.Ner.Referent rr = tt.GetReferent();
if (rr == res)
{
persTyp = Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Worker;
break;
}
if (rr is Pullenti.Ner.Person.PersonReferent)
{
if (pr.CalcTypFromAttrs(r as Pullenti.Ner.Person.PersonReferent) != Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Undefined)
{
break;
}
else
{
continue;
}
}
if (rr != null)
{
break;
}
tpt = Pullenti.Ner.Titlepage.Internal.TitleItemToken.TryAttach(tt);
if (tpt != null)
{
if (tpt.Typ != Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Typ && tpt.Typ != Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.TypAndTheme)
{
break;
}
tt = tpt.EndToken;
if (tt.EndChar > endToken.EndChar)
{
endToken = tt;
}
continue;
}
}
if (persTyp == Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Undefined)
{
for (Pullenti.Ner.Token tt = t.Previous; tt != null; tt = tt.Previous)
{
Pullenti.Ner.Referent rr = tt.GetReferent();
if (rr == res)
{
persTyp = Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Worker;
break;
}
if (rr != null)
{
break;
}
if ((tt.IsValue("СТУДЕНТ", null) || tt.IsValue("СТУДЕНТКА", null) || tt.IsValue("СЛУШАТЕЛЬ", null)) || tt.IsValue("ДИПЛОМНИК", null) || tt.IsValue("ИСПОЛНИТЕЛЬ", null))
{
persTyp = Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Worker;
break;
}
tpt = Pullenti.Ner.Titlepage.Internal.TitleItemToken.TryAttach(tt);
if (tpt != null && tpt.Typ != Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Typ)
{
break;
}
}
}
if (persTyp != Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Undefined)
{
pr.Add(p, persTyp, 1);
}
else
{
pr.Add(p, persTyp, (float)0.5);
}
if (t.EndChar > endToken.EndChar)
{
endToken = t;
}
}
continue;
}
if (r == rli[0])
{
persTyp = Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Undefined;
}
if (r is Pullenti.Ner.Date.DateReferent)
{
if (res.Date == null)
{
res.Date = r as Pullenti.Ner.Date.DateReferent;
if (t.EndChar > endToken.EndChar)
{
endToken = t;
}
}
}
else if (r is Pullenti.Ner.Geo.GeoReferent)
{
if (res.City == null && (r as Pullenti.Ner.Geo.GeoReferent).IsCity)
{
res.City = r as Pullenti.Ner.Geo.GeoReferent;
if (t.EndChar > endToken.EndChar)
{
endToken = t;
}
}
}
if (r is Pullenti.Ner.Org.OrganizationReferent)
{
Pullenti.Ner.Org.OrganizationReferent org = r as Pullenti.Ner.Org.OrganizationReferent;
if (org.Types.Contains("курс") && org.Number != null)
{
int i;
if (int.TryParse(org.Number, out i))
{
if (i > 0 && (i < 8))
{
res.StudentYear = i;
}
}
}
for (; org.Higher != null; org = org.Higher)
{
if (org.Kind != Pullenti.Ner.Org.OrganizationKind.Department)
{
break;
}
}
if (org.Kind != Pullenti.Ner.Org.OrganizationKind.Department)
{
if (res.Org == null)
{
res.Org = org;
}
else if (Pullenti.Ner.Org.OrganizationReferent.CanBeHigher(res.Org, org))
{
res.Org = org;
}
}
if (t.EndChar > endToken.EndChar)
{
endToken = t;
}
}
if ((r is Pullenti.Ner.Uri.UriReferent) || (r is Pullenti.Ner.Geo.GeoReferent))
{
if (t.EndChar > endToken.EndChar)
{
endToken = t;
}
}
}
}
foreach (Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types ty in persTypes)
{
foreach (Pullenti.Ner.Person.PersonReferent p in pr.GetPersons(ty))
{
if (pr.GetAttrNameForType(ty) != null)
{
res.AddSlot(pr.GetAttrNameForType(ty), p, false, 0);
}
}
}
if (res.GetSlotValue(TitlePageReferent.ATTR_AUTHOR) == null)
{
foreach (Pullenti.Ner.Person.PersonReferent p in pr.GetPersons(Pullenti.Ner.Titlepage.Internal.TitleItemToken.Types.Undefined))
{
res.AddSlot(TitlePageReferent.ATTR_AUTHOR, p, false, 0);
break;
}
}
if (res.City == null && res.Org != null)
{
Pullenti.Ner.Slot s = res.Org.FindSlot(Pullenti.Ner.Org.OrganizationReferent.ATTR_GEO, null, true);
if (s != null && (s.Value is Pullenti.Ner.Geo.GeoReferent))
{
if ((s.Value as Pullenti.Ner.Geo.GeoReferent).IsCity)
{
res.City = s.Value as Pullenti.Ner.Geo.GeoReferent;
}
}
}
if (res.Date == null)
{
for (Pullenti.Ner.Token t = begin; t != null && t.EndChar <= endChar; t = t.Next)
{
Pullenti.Ner.Geo.GeoReferent city = t.GetReferent() as Pullenti.Ner.Geo.GeoReferent;
if (city == null)
{
continue;
}
if (t.Next is Pullenti.Ner.TextToken)
{
if (t.Next.IsCharOf(":,") || t.Next.IsHiphen)
{
t = t.Next;
}
}
Pullenti.Ner.ReferentToken rt = t.Kit.ProcessReferent(Pullenti.Ner.Date.DateAnalyzer.ANALYZER_NAME, t.Next);
if (rt != null)
{
rt.SaveToLocalOntology();
res.Date = rt.Referent as Pullenti.Ner.Date.DateReferent;
if (kit != null)
{
kit.EmbedToken(rt);
}
break;
}
}
}
if (res.Slots.Count == 0)
{
return(null);
}
else
{
return(res);
}
}
