public IOperation GetOperation(Encoding encoding, IProcessorState processorState)
{
IToken tokenBytes = StartToken.ToToken(encoding);
IToken endTokenBytes = EndToken.ToToken(encoding);
TokenTrie endTokenMatcher = new TokenTrie();
endTokenMatcher.AddToken(endTokenBytes);
return(new Impl(tokenBytes, endTokenMatcher, this, _id, _initialState));
}
public void AddEntry(byte[] Identifier, EndToken NewEntry)
{
if (this.EndTokens.ContainsKey(Identifier))
{
this.EndTokens[Identifier] = NewEntry;
}
else
{
this.EndTokens.Add(Identifier, NewEntry);
}
}
private void CompositeEndHandler(EndToken token)
{
Type type = typeIds[(int)token.TypeId];
Midwife midwife = parentStack.Pop();
if (midwife.Obj == null)
{
CreateInstance(midwife);
}
midwife.Populate();
SetValue(token.Name, midwife.Obj);
}
/// <summary> /// Formats the specified end token. /// </summary> /// <param name="endToken">The end token.</param> /// <returns>The token in serialized form.</returns> protected abstract T Format(EndToken endToken);
public void AddEntry(byte[] Identifier, EndToken NewEntry)
{
if (this.EndTokens.ContainsKey(Identifier))
this.EndTokens[Identifier] = NewEntry;
else
this.EndTokens.Add(Identifier, NewEntry);
}
public Token GetToken()
{
State = LexicAnalyserState.Initial;
Value = HasNext ? Character.ToString() : "";
Token token = null;
while (HasNext && token == null)
{
switch (State)
{
case LexicAnalyserState.Initial:
HandleInitial();
break;
case LexicAnalyserState.NonTerminal:
token = new NonTerminalToken(Value);
CurrentIndex++;
break;
case LexicAnalyserState.Space:
token = new SpaceToken();
CurrentIndex++;
break;
case LexicAnalyserState.Terminal:
token = new TerminalToken(Value);
CurrentIndex++;
break;
case LexicAnalyserState.Empty:
token = new EmptyToken();
CurrentIndex++;
break;
case LexicAnalyserState.NewLine:
token = new NewLineToken(Value);
CurrentIndex++;
break;
case LexicAnalyserState.Identifier:
token = new IdentifierToken(Value);
CurrentIndex++;
break;
case LexicAnalyserState.Number:
token = new NumberToken(Value);
CurrentIndex++;
break;
case LexicAnalyserState.Var:
token = new VarToken();
CurrentIndex++;
break;
case LexicAnalyserState.Write:
token = new WriteToken();
CurrentIndex++;
break;
case LexicAnalyserState.Read:
token = new ReadToken();
CurrentIndex++;
break;
case LexicAnalyserState.If:
token = new IfToken();
CurrentIndex++;
break;
case LexicAnalyserState.End:
token = new EndToken();
CurrentIndex++;
break;
case LexicAnalyserState.Then:
token = new ThenToken();
CurrentIndex++;
break;
case LexicAnalyserState.Begin:
token = new BeginToken();
CurrentIndex++;
break;
case LexicAnalyserState.While:
token = new WhileToken();
CurrentIndex++;
break;
case LexicAnalyserState.Do:
token = new DoToken();
CurrentIndex++;
break;
case LexicAnalyserState.SemiColon:
token = new SemiColonToken();
CurrentIndex++;
break;
case LexicAnalyserState.OpenParentheses:
token = new OpenParenthesesToken();
CurrentIndex++;
break;
case LexicAnalyserState.CloseParentheses:
token = new CloseParenthesesToken();
CurrentIndex++;
break;
case LexicAnalyserState.Plus:
token = new PlusToken();
CurrentIndex++;
break;
case LexicAnalyserState.Sub:
token = new SubToken();
CurrentIndex++;
break;
case LexicAnalyserState.Great:
token = new GreatToken();
CurrentIndex++;
break;
case LexicAnalyserState.Less:
token = new LessToken();
CurrentIndex++;
break;
case LexicAnalyserState.Equal:
token = new EqualToken();
CurrentIndex++;
break;
case LexicAnalyserState.Repeat:
token = new RepeatToken();
CurrentIndex++;
break;
case LexicAnalyserState.Until:
token = new UntilToken();
CurrentIndex++;
break;
case LexicAnalyserState.Attribution:
token = new AttributionToken();
CurrentIndex++;
break;
case LexicAnalyserState.NotEqual:
token = new NotEqualToken();
CurrentIndex++;
break;
case LexicAnalyserState.GreatOrEqual:
token = new GreatOrEqualToken();
CurrentIndex++;
break;
case LexicAnalyserState.LessOrEqual:
token = new LessOrEqualToken();
CurrentIndex++;
break;
default:
throw new ArgumentOutOfRangeException();
}
}
return(token);
}
/// <summary>
/// Formats the specified end token.
/// </summary>
/// <param name="endToken">The end token.</param>
/// <returns>The token in serialized form.</returns>
protected override string Format(EndToken endToken)
{
isSepReq = true;
return(")");
}
public void Visit(EndToken endToken)
{
}
/// <summary>
/// Formats the specified end token.
/// </summary>
/// <param name="endToken">The end token.</param>
/// <returns>The token in serialized form.</returns>
protected override string Format(EndToken endToken)
{
return(CreateNodeEnd(XmlStringConstants.COMPOSITE));
}
public override void Cancel()
{
EndToken.Cancel(true);
}
bool CalcRankAndValue(int minNewlinesCount)
{
Rank = 0;
if (BeginToken.Chars.IsAllLower)
{
Rank -= 30;
}
int words = 0;
int upWords = 0;
int notwords = 0;
int lineNumber = 0;
Pullenti.Ner.Token tstart = BeginToken;
Pullenti.Ner.Token tend = EndToken;
for (Pullenti.Ner.Token t = BeginToken; t != EndToken.Next && t != null && t.EndChar <= EndToken.EndChar; t = t.Next)
{
if (t.IsNewlineBefore)
{
}
TitleItemToken tit = TitleItemToken.TryAttach(t);
if (tit != null)
{
if (tit.Typ == TitleItemToken.Types.Theme || tit.Typ == TitleItemToken.Types.TypAndTheme)
{
if (t != BeginToken)
{
if (lineNumber > 0)
{
return(false);
}
words = (upWords = (notwords = 0));
tstart = tit.EndToken.Next;
}
t = tit.EndToken;
if (t.Next == null)
{
return(false);
}
if (t.Next.Chars.IsLetter && t.Next.Chars.IsAllLower)
{
Rank += 20;
}
else
{
Rank += 100;
}
tstart = t.Next;
if (tit.Typ == TitleItemToken.Types.TypAndTheme)
{
TypeValue = tit.Value;
}
continue;
}
if (tit.Typ == TitleItemToken.Types.Typ)
{
if (t == BeginToken)
{
if (tit.EndToken.IsNewlineAfter)
{
TypeValue = tit.Value;
Rank += 5;
tstart = tit.EndToken.Next;
}
}
t = tit.EndToken;
words++;
if (tit.BeginToken != tit.EndToken)
{
words++;
}
if (tit.Chars.IsAllUpper)
{
upWords++;
}
continue;
}
if (tit.Typ == TitleItemToken.Types.Dust || tit.Typ == TitleItemToken.Types.Speciality)
{
if (t == BeginToken)
{
return(false);
}
Rank -= 20;
if (tit.Typ == TitleItemToken.Types.Speciality)
{
Speciality = tit.Value;
}
t = tit.EndToken;
continue;
}
if (tit.Typ == TitleItemToken.Types.Consultant || tit.Typ == TitleItemToken.Types.Boss || tit.Typ == TitleItemToken.Types.Editor)
{
t = tit.EndToken;
if (t.Next != null && ((t.Next.IsCharOf(":") || t.Next.IsHiphen || t.WhitespacesAfterCount > 4)))
{
Rank -= 10;
}
else
{
Rank -= 2;
}
continue;
}
return(false);
}
Pullenti.Ner.Booklink.Internal.BookLinkToken blt = Pullenti.Ner.Booklink.Internal.BookLinkToken.TryParse(t, 0);
if (blt != null)
{
if (blt.Typ == Pullenti.Ner.Booklink.Internal.BookLinkTyp.Misc || blt.Typ == Pullenti.Ner.Booklink.Internal.BookLinkTyp.N || blt.Typ == Pullenti.Ner.Booklink.Internal.BookLinkTyp.Pages)
{
Rank -= 10;
}
else if (blt.Typ == Pullenti.Ner.Booklink.Internal.BookLinkTyp.N || blt.Typ == Pullenti.Ner.Booklink.Internal.BookLinkTyp.PageRange)
{
Rank -= 20;
}
}
if (t == BeginToken && Pullenti.Ner.Booklink.Internal.BookLinkToken.TryParseAuthor(t, Pullenti.Ner.Person.Internal.FioTemplateType.Undefined) != null)
{
Rank -= 20;
}
if (t.IsNewlineBefore && t != BeginToken)
{
lineNumber++;
if (lineNumber > 4)
{
return(false);
}
if (t.Chars.IsAllLower)
{
Rank += 10;
}
else if (t.Previous.IsChar('.'))
{
Rank -= 10;
}
else if (t.Previous.IsCharOf(",-"))
{
Rank += 10;
}
else
{
Pullenti.Ner.Core.NounPhraseToken npt = Pullenti.Ner.Core.NounPhraseHelper.TryParse(t.Previous, Pullenti.Ner.Core.NounPhraseParseAttr.No, 0, null);
if (npt != null && npt.EndChar >= t.EndChar)
{
Rank += 10;
}
}
}
if (t != BeginToken && t.NewlinesBeforeCount > minNewlinesCount)
{
Rank -= (t.NewlinesBeforeCount - minNewlinesCount);
}
Pullenti.Ner.Core.BracketSequenceToken bst = Pullenti.Ner.Core.BracketHelper.TryParse(t, Pullenti.Ner.Core.BracketParseAttr.No, 100);
if (bst != null && bst.IsQuoteType && bst.EndToken.EndChar <= EndToken.EndChar)
{
if (words == 0)
{
tstart = bst.BeginToken;
Rank += 10;
if (bst.EndToken == EndToken)
{
tend = EndToken;
Rank += 10;
}
}
}
List <Pullenti.Ner.Referent> rli = t.GetReferents();
if (rli != null)
{
foreach (Pullenti.Ner.Referent r in rli)
{
if (r is Pullenti.Ner.Org.OrganizationReferent)
{
if (t.IsNewlineBefore)
{
Rank -= 10;
}
else
{
Rank -= 4;
}
continue;
}
if ((r is Pullenti.Ner.Geo.GeoReferent) || (r is Pullenti.Ner.Person.PersonReferent))
{
if (t.IsNewlineBefore)
{
Rank -= 5;
if (t.IsNewlineAfter || t.Next == null)
{
Rank -= 20;
}
else if (t.Next.IsHiphen || (t.Next is Pullenti.Ner.NumberToken) || (t.Next.GetReferent() is Pullenti.Ner.Date.DateReferent))
{
Rank -= 20;
}
else if (t != BeginToken)
{
Rank -= 20;
}
}
continue;
}
if ((r is Pullenti.Ner.Geo.GeoReferent) || (r is Pullenti.Ner.Denomination.DenominationReferent))
{
continue;
}
if ((r is Pullenti.Ner.Uri.UriReferent) || (r is Pullenti.Ner.Phone.PhoneReferent))
{
return(false);
}
if (t.IsNewlineBefore)
{
Rank -= 4;
}
else
{
Rank -= 2;
}
if (t == BeginToken && (EndToken.GetReferent() is Pullenti.Ner.Person.PersonReferent))
{
Rank -= 10;
}
}
words++;
if (t.Chars.IsAllUpper)
{
upWords++;
}
if (t == BeginToken)
{
if (t.IsNewlineAfter)
{
Rank -= 10;
}
else if (t.Next != null && t.Next.IsChar('.') && t.Next.IsNewlineAfter)
{
Rank -= 10;
}
}
continue;
}
if (t is Pullenti.Ner.NumberToken)
{
if ((t as Pullenti.Ner.NumberToken).Typ == Pullenti.Ner.NumberSpellingType.Words)
{
words++;
if (t.Chars.IsAllUpper)
{
upWords++;
}
}
else
{
notwords++;
}
continue;
}
Pullenti.Ner.Person.Internal.PersonAttrToken pat = Pullenti.Ner.Person.Internal.PersonAttrToken.TryAttach(t, null, Pullenti.Ner.Person.Internal.PersonAttrToken.PersonAttrAttachAttrs.No);
if (pat != null)
{
if (t.IsNewlineBefore)
{
if (!pat.Morph.Case.IsUndefined && !pat.Morph.Case.IsNominative)
{
}
else if (pat.Chars.IsAllUpper)
{
}
else
{
Rank -= 20;
}
}
else if (t.Chars.IsAllLower)
{
Rank--;
}
for (; t != null; t = t.Next)
{
words++;
if (t.Chars.IsAllUpper)
{
upWords++;
}
if (t == pat.EndToken)
{
break;
}
}
continue;
}
Pullenti.Ner.Org.Internal.OrgItemTypeToken oitt = Pullenti.Ner.Org.Internal.OrgItemTypeToken.TryAttach(t, true, null);
if (oitt != null)
{
if (oitt.Morph.Number != Pullenti.Morph.MorphNumber.Plural && !oitt.IsDoubtRootWord)
{
if (!oitt.Morph.Case.IsUndefined && !oitt.Morph.Case.IsNominative)
{
words++;
if (t.Chars.IsAllUpper)
{
upWords++;
}
}
else
{
Rank -= 4;
if (t == BeginToken)
{
Rank -= 5;
}
}
}
else
{
words += 1;
if (t.Chars.IsAllUpper)
{
upWords++;
}
}
t = oitt.EndToken;
continue;
}
Pullenti.Ner.TextToken tt = t as Pullenti.Ner.TextToken;
if (tt != null)
{
if (tt.IsChar('©'))
{
Rank -= 10;
}
if (tt.IsChar('_'))
{
Rank--;
}
if (tt.Chars.IsLetter)
{
if (tt.LengthChar > 2)
{
words++;
if (t.Chars.IsAllUpper)
{
upWords++;
}
}
}
else if (!tt.IsChar(','))
{
notwords++;
}
if (tt.IsPureVerb)
{
{
Rank -= 30;
words--;
}
break;
}
if (tt == EndToken)
{
if (tt.Morph.Class.IsPreposition || tt.Morph.Class.IsConjunction)
{
Rank -= 10;
}
else if (tt.IsChar('.'))
{
Rank += 5;
}
}
else if (tt.IsCharOf("._"))
{
Rank -= 5;
}
}
}
Rank += words;
Rank -= notwords;
if ((words < 1) && (Rank < 50))
{
return(false);
}
if (tstart == null || tend == null)
{
return(false);
}
if (tstart.EndChar > tend.EndChar)
{
return(false);
}
TitleItemToken tit1 = TitleItemToken.TryAttach(EndToken.Next);
if (tit1 != null && ((tit1.Typ == TitleItemToken.Types.Typ || tit1.Typ == TitleItemToken.Types.Speciality)))
{
if (tit1.EndToken.IsNewlineAfter)
{
Rank += 15;
}
else
{
Rank += 10;
}
if (tit1.Typ == TitleItemToken.Types.Speciality)
{
Speciality = tit1.Value;
}
}
if (upWords > 4 && upWords > ((int)((0.8 * words))))
{
if (tstart.Previous != null && (tstart.Previous.GetReferent() is Pullenti.Ner.Person.PersonReferent))
{
Rank += (5 + upWords);
}
}
BeginNameToken = tstart;
EndNameToken = tend;
return(true);
}
/// <summary>
/// Formats the specified end token.
/// </summary>
/// <param name="endToken">The end token.</param>
/// <returns>The token in serialized form.</returns>
protected override string Format(EndToken endToken)
{
return(CreateNodeEnd(typeCache[endToken.TypeId]));
}
