public bool Deserialize(Stream stream)
{
int vers = 0;
byte b = (byte)stream.ReadByte();
if (b == 0xAA)
{
b = (byte)stream.ReadByte();
vers = b;
}
else
{
stream.Position--;
}
m_Sofa = new Pullenti.Ner.SourceOfAnalysis(null);
m_Sofa.Deserialize(stream);
BaseLanguage = new Pullenti.Morph.MorphLang()
{
Value = (short)Pullenti.Ner.Core.Internal.SerializerHelper.DeserializeInt(stream)
};
m_Entities = new List <Pullenti.Ner.Referent>();
int cou = Pullenti.Ner.Core.Internal.SerializerHelper.DeserializeInt(stream);
for (int i = 0; i < cou; i++)
{
string typ = Pullenti.Ner.Core.Internal.SerializerHelper.DeserializeString(stream);
Pullenti.Ner.Referent r = Pullenti.Ner.ProcessorService.CreateReferent(typ);
if (r == null)
{
r = new Pullenti.Ner.Referent("UNDEFINED");
}
m_Entities.Add(r);
}
for (int i = 0; i < cou; i++)
{
m_Entities[i].Deserialize(stream, m_Entities, m_Sofa);
}
FirstToken = Pullenti.Ner.Core.Internal.SerializerHelper.DeserializeTokens(stream, this, vers);
this.CreateStatistics();
return(true);
}
public AnalysisKit(Pullenti.Ner.SourceOfAnalysis sofa = null, bool onlyTokenizing = false, Pullenti.Morph.MorphLang lang = null, ProgressChangedEventHandler progress = null)
{
if (sofa == null)
{
return;
}
m_Sofa = sofa;
StartDate = DateTime.Now;
List <Pullenti.Morph.MorphToken> tokens = Pullenti.Morph.MorphologyService.Process(sofa.Text, lang, progress);
Pullenti.Ner.Token t0 = null;
if (tokens != null)
{
for (int ii = 0; ii < tokens.Count; ii++)
{
Pullenti.Morph.MorphToken mt = tokens[ii];
if (mt.BeginChar == 733860)
{
}
Pullenti.Ner.TextToken tt = new Pullenti.Ner.TextToken(mt, this);
if (sofa.CorrectionDict != null)
{
string corw;
if (sofa.CorrectionDict.TryGetValue(mt.Term, out corw))
{
List <Pullenti.Morph.MorphToken> ccc = Pullenti.Morph.MorphologyService.Process(corw, lang, null);
if (ccc != null && ccc.Count == 1)
{
Pullenti.Ner.TextToken tt1 = new Pullenti.Ner.TextToken(ccc[0], this, tt.BeginChar, tt.EndChar)
{
Term0 = tt.Term
};
tt1.Chars = tt.Chars;
tt = tt1;
if (CorrectedTokens == null)
{
CorrectedTokens = new Dictionary <Pullenti.Ner.Token, string>();
}
CorrectedTokens.Add(tt, tt.GetSourceText());
}
}
}
if (t0 == null)
{
FirstToken = tt;
}
else
{
t0.Next = tt;
}
t0 = tt;
}
}
if (sofa.ClearDust)
{
this.ClearDust();
}
if (sofa.DoWordsMergingByMorph)
{
this.CorrectWordsByMerging(lang);
}
if (sofa.DoWordCorrectionByMorph)
{
this.CorrectWordsByMorph(lang);
}
this.MergeLetters();
this.DefineBaseLanguage();
if (sofa.CreateNumberTokens)
{
for (Pullenti.Ner.Token t = FirstToken; t != null; t = t.Next)
{
Pullenti.Ner.NumberToken nt = NumberHelper.TryParseNumber(t);
if (nt == null)
{
continue;
}
this.EmbedToken(nt);
t = nt;
}
}
if (onlyTokenizing)
{
return;
}
for (Pullenti.Ner.Token t = FirstToken; t != null; t = t.Next)
{
if (t.Morph.Class.IsPreposition)
{
continue;
}
Pullenti.Morph.MorphClass mc = t.GetMorphClassInDictionary();
if (mc.IsUndefined && t.Chars.IsCyrillicLetter && t.LengthChar > 4)
{
string tail = sofa.Text.Substring(t.EndChar - 1, 2);
Pullenti.Ner.Token tte = null;
Pullenti.Ner.Token tt = t.Previous;
if (tt != null && ((tt.IsCommaAnd || tt.Morph.Class.IsPreposition || tt.Morph.Class.IsConjunction)))
{
tt = tt.Previous;
}
if ((tt != null && !tt.GetMorphClassInDictionary().IsUndefined&& ((tt.Morph.Class.Value & t.Morph.Class.Value)) != 0) && tt.LengthChar > 4)
{
string tail2 = sofa.Text.Substring(tt.EndChar - 1, 2);
if (tail2 == tail)
{
tte = tt;
}
}
if (tte == null)
{
tt = t.Next;
if (tt != null && ((tt.IsCommaAnd || tt.Morph.Class.IsPreposition || tt.Morph.Class.IsConjunction)))
{
tt = tt.Next;
}
if ((tt != null && !tt.GetMorphClassInDictionary().IsUndefined&& ((tt.Morph.Class.Value & t.Morph.Class.Value)) != 0) && tt.LengthChar > 4)
{
string tail2 = sofa.Text.Substring(tt.EndChar - 1, 2);
if (tail2 == tail)
{
tte = tt;
}
}
}
if (tte != null)
{
t.Morph.RemoveItemsEx(tte.Morph, tte.GetMorphClassInDictionary());
}
}
continue;
}
this.CreateStatistics();
}