public override bool Parse(T[] dataToTokenize, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser) { //for all the suffixes this parses to, finds the nearest one (nongreedy) int closestIdx = -1; T[] suffix = null; closestIdx = dataToTokenize.FindNearestIndexOf(this.Suffixes, out suffix, currentPosition); //if we can't find a suffix, we kack if (closestIdx == -1) { newParser = null; newToken = null; newPosition = -1; return(false); } newPosition = closestIdx; //get string between old and new positions var tokenText = dataToTokenize.GetSegment(currentPosition, newPosition - currentPosition); //returns a suffixed natural token newToken = NaturalToken <T> .New(tokenText).HasSuffix(suffix, this.IsInclusive); //we don't know what parser to use next newParser = null; return(true); }
public bool Parse(string text, int currentPosition, object state, IToken currentToken, out int newPosition, out IToken newToken, out IForwardMovingTokenizer newParser) { //if we can't find a , or ) then we kack var idx = text.IndexOf(",", currentPosition); if (idx == -1) { idx = text.IndexOf(")", currentPosition); if (idx == -1) { newPosition = -1; newToken = null; newParser = null; return(false); } var substring = text.Substring(currentPosition, idx - currentPosition); newPosition = idx + 1; newToken = NaturalToken.New(substring).HasDPCTokenType(DPCTokenType.Item); newParser = new ToDotParser(); return(true); } else { var substring = text.Substring(currentPosition, idx - currentPosition); newPosition = idx + 1; newToken = NaturalToken.New(substring).HasDPCTokenType(DPCTokenType.Item); newParser = new ToCommaOrEndParenthesisParser(); return(true); } }
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser) { //Debug.WriteLine("Parsing @ {0} of {1}", currentPosition, string.Join("", source)); //get the new tokenizer var tokenizer = GetTokenizer(source, currentPosition, state, currentToken); int newPositionOUT = 0; IToken <T> newTokenOUT = null; IForwardMovingTokenizer <T> newParserOUT = null; bool rv = tokenizer != null; if (rv) { //Debug.WriteLine("located tokenizer delegated to"); IForwardMovingTokenizer <T> alg = tokenizer.As <IForwardMovingTokenizer <T> >().GetOuterDecorator() as IForwardMovingTokenizer <T>; var cake2 = alg.GetAllDecorations(); rv = alg.Parse(source, currentPosition, state, currentToken, out newPositionOUT, out newTokenOUT, out newParserOUT); } //loop back into router to handle the next token if (OverridesTokenizerRouting) { newParserOUT = this; } //if we classify unrecognized, then we do that here if (!rv && this.TokenizeUnrecognized) { //Debug.WriteLine("classifying unrecognized"); rv = true; newPositionOUT = GetNextRecognizedPosition(source, currentPosition, state, currentToken); //get string between old and new positions var tokenText = source.GetSegment(currentPosition, newPositionOUT - currentPosition); //Debug.WriteLine("unrecognized runs to {0}, producing {1}", newPositionOUT, string.Join("", tokenText)); //returns a suffixed natural token newTokenOUT = NaturalToken <T> .New(tokenText).HasComment("UNRECOGNIZED"); newParserOUT = this; } newParser = newParserOUT; newToken = newTokenOUT; newPosition = newPositionOUT; //if (!rv) // Debug.WriteLine("tokenize fail."); //if (rv) // Debug.WriteLine("tokenize successful. new position = {0}. token = {1}", newPositionOUT, string.Join("", newToken.TokenData)); //Debug.WriteLine("Parsing Complete @ {0} of {1}", currentPosition, string.Join("", source)); return(rv); }
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser) { newPosition = currentPosition + this.TokenValue.Length; //returns a natural token newToken = NaturalToken <T> .New(this.TokenValue); //we don't know what parser to use next newParser = null; return(true); }
public override bool Parse(T[] dataToTokenize, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser) { newPosition = currentPosition + Length; //get string between old and new positions var tokenText = dataToTokenize.GetSegment(currentPosition, newPosition - currentPosition); //returns a suffixed natural token newToken = NaturalToken <T> .New(tokenText); //we don't know what parser to use next newParser = null; return(true); }
public bool Parse(string text, int currentPosition, IToken currentToken, out int newPosition, out IToken newToken, out IForwardMovingTokenParser newParser) { //if we can't find the suffix, we kack var idx = text.IndexOf(this.Suffix, currentPosition); if (idx == -1) { newPosition = -1; newToken = null; newParser = null; return(false); } else { var substring = text.Substring(currentPosition, idx - currentPosition); newPosition = idx + 1; newToken = NaturalToken.New(substring);//build up the token newParser = new ToOpenParenthesisParser(); return(true); } }
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser) { var cursor = ForwardMovingTokenizingCursor <T> .New(source, currentPosition, state, currentToken); var logicRes = this.LengthStrategy.Perform(cursor) as LogicOfTo <ForwardMovingTokenizingCursor <T>, int>; int length = logicRes.Result; newPosition = currentPosition + length; //get string between old and new positions var tokenText = source.GetSegment(currentPosition, newPosition - currentPosition); //returns a suffixed natural token newToken = NaturalToken <T> .New(tokenText); //we don't know what parser to use next newParser = null; return(true); }
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser) { int newPos; var tokens = this.routerParse(source, currentPosition, state, currentToken, out newPos); if (newPos == 0) { newPosition = currentPosition; newToken = null; newParser = null; return(false); } //build the composite var segData = source.GetSegment(currentPosition, newPos); newToken = NaturalToken <T> .New(segData).HasComposite(tokens.ToArray()); newParser = null; newPosition = currentPosition + newPos; return(true); }