public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser) { var rv = this.TokenizingStrategy(ForwardMovingTokenizingCursor <T> .New(source, currentPosition, state, currentToken)); newPosition = rv.CurrentPosition; newToken = rv.CurrentToken; newParser = null; return(true); }
public bool CanHandle(T[] source, int currentPosition, object state, IToken <T> currentToken) { var cursor = ForwardMovingTokenizingCursor <T> .New(source, currentPosition, state, currentToken); //get all IHasHandleConditionTokenizer conditions in the decoration stack //-the idea here is that there is only one instance of ValidatingTokenizerDecoration per decoration stack (it checks during ctor) //-this is the only decoration that will filter //-all the other decorations that affect this decoration (eg. all IHasHandleConditionTokenizers) are pulled from // the decoration stack and applied here var decs = this.GetAllDecorations(); var hasConditions = this.GetAllImplementingDecorations <IHasHandleConditionTokenizer <T> >(); var segmentText = string.Join("", source.GetSegment(currentPosition)); Debug.WriteLine(string.Format("can handle {0}?", segmentText)); Debug.WriteLine(string.Format(" {0}", this.GetDecorationSummary())); var rv = true; if (hasConditions != null) { foreach (var each in hasConditions) { if (each.CanTokenizeCondition == null) { continue; } var canTokenize = each.CanTokenizeCondition.Evaluate(cursor).GetValueOrDefault(); Debug.WriteLine(" layer {0} = {1}", each.GetType().Name, canTokenize.ToString()); if (!canTokenize) { rv = false; break; } } } Debug.WriteLine(" " + rv.ToString()); return(rv); }
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser) { var cursor = ForwardMovingTokenizingCursor <T> .New(source, currentPosition, state, currentToken); var logicRes = this.LengthStrategy.Perform(cursor) as LogicOfTo <ForwardMovingTokenizingCursor <T>, int>; int length = logicRes.Result; newPosition = currentPosition + length; //get string between old and new positions var tokenText = source.GetSegment(currentPosition, newPosition - currentPosition); //returns a suffixed natural token newToken = NaturalToken <T> .New(tokenText); //we don't know what parser to use next newParser = null; return(true); }
/// <summary> /// using the router, and a calculated initial state, performs a tokenize /// </summary> /// <typeparam name="T"></typeparam> /// <param name="rawData"></param> /// <param name="currentPosition"></param> /// <param name="state"></param> /// <param name="currentToken"></param> /// <param name="newPosition"></param> /// <returns></returns> private List <IToken <T> > routerParse <T>(T[] rawData, int currentPosition, object state, IToken <T> currentToken, out int newPosition) { var res = this.StateStrategy.Perform(ForwardMovingTokenizingCursor <T> .New(rawData, currentPosition, state, currentToken)) as LogicOfTo <ForwardMovingTokenizingCursor <T>, object>; object routerInitialState = res.Result; //get the substring from current position var subData = rawData.GetSegment(currentPosition); int newPos; var rv = subData.ForwardMovingTokenize(routerInitialState, this.Router as IForwardMovingTokenizer <T>, out newPos); //calc length to validate int length = 0; rv.WithEach(x => { length += x.TokenData.Length; }); Condition.Requires(length).IsEqualTo(newPos); newPosition = currentPosition + newPos; return(rv); }
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser) { IToken <T> newTokenOUT; int newPosOUT; IForwardMovingTokenizer <T> newTokenizerOUT; //do the initial parse by length var rv = base.Parse(source, currentPosition, state, currentToken, out newPosOUT, out newTokenOUT, out newTokenizerOUT); if (!rv) { newPosition = currentPosition; newToken = null; newParser = null; return(false); } Debug.WriteLine(string.Format("Composite token created. Token={0}", newTokenOUT.DumpToken())); //get the substring to dig into var compositeSeg = newTokenOUT.TokenData; //build state data for recursion var res = this.StateStrategy.Perform(ForwardMovingTokenizingCursor <T> .New(compositeSeg, currentPosition, state, currentToken)) as LogicOfTo <ForwardMovingTokenizingCursor <T>, object>; object routerInitialState = res.Result; int newPos; var childTokens = compositeSeg.ForwardMovingTokenize(routerInitialState, this.Router as IForwardMovingTokenizer <T>, out newPos); newTokenOUT = newTokenOUT.HasComposite(childTokens.ToArray()); newToken = newTokenOUT; newParser = newTokenizerOUT; newPosition = newPosOUT; return(true); }