public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken,
out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
var rv = this.TokenizingStrategy(ForwardMovingTokenizingCursor <T> .New(source, currentPosition, state, currentToken));
newPosition = rv.CurrentPosition;
newToken = rv.CurrentToken;
newParser = null;
return(true);
}
public bool CanHandle(T[] source, int currentPosition, object state, IToken <T> currentToken)
{
var cursor = ForwardMovingTokenizingCursor <T> .New(source, currentPosition, state, currentToken);
//get all IHasHandleConditionTokenizer conditions in the decoration stack
//-the idea here is that there is only one instance of ValidatingTokenizerDecoration per decoration stack (it checks during ctor)
//-this is the only decoration that will filter
//-all the other decorations that affect this decoration (eg. all IHasHandleConditionTokenizers) are pulled from
// the decoration stack and applied here
var decs = this.GetAllDecorations();
var hasConditions = this.GetAllImplementingDecorations <IHasHandleConditionTokenizer <T> >();
var segmentText = string.Join("", source.GetSegment(currentPosition));
Debug.WriteLine(string.Format("can handle {0}?", segmentText));
Debug.WriteLine(string.Format(" {0}", this.GetDecorationSummary()));
var rv = true;
if (hasConditions != null)
{
foreach (var each in hasConditions)
{
if (each.CanTokenizeCondition == null)
{
continue;
}
var canTokenize = each.CanTokenizeCondition.Evaluate(cursor).GetValueOrDefault();
Debug.WriteLine(" layer {0} = {1}", each.GetType().Name, canTokenize.ToString());
if (!canTokenize)
{
rv = false;
break;
}
}
}
Debug.WriteLine(" " + rv.ToString());
return(rv);
}
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken,
out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
var cursor = ForwardMovingTokenizingCursor <T> .New(source, currentPosition, state, currentToken);
var logicRes = this.LengthStrategy.Perform(cursor) as LogicOfTo <ForwardMovingTokenizingCursor <T>, int>;
int length = logicRes.Result;
newPosition = currentPosition + length;
//get string between old and new positions
var tokenText = source.GetSegment(currentPosition, newPosition - currentPosition);
//returns a suffixed natural token
newToken = NaturalToken <T> .New(tokenText);
//we don't know what parser to use next
newParser = null;
return(true);
}
/// <summary>
/// using the router, and a calculated initial state, performs a tokenize
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="rawData"></param>
/// <param name="currentPosition"></param>
/// <param name="state"></param>
/// <param name="currentToken"></param>
/// <param name="newPosition"></param>
/// <returns></returns>
private List <IToken <T> > routerParse <T>(T[] rawData, int currentPosition, object state, IToken <T> currentToken, out int newPosition)
{
var res = this.StateStrategy.Perform(ForwardMovingTokenizingCursor <T> .New(rawData, currentPosition, state, currentToken)) as LogicOfTo <ForwardMovingTokenizingCursor <T>, object>;
object routerInitialState = res.Result;
//get the substring from current position
var subData = rawData.GetSegment(currentPosition);
int newPos;
var rv = subData.ForwardMovingTokenize(routerInitialState, this.Router as IForwardMovingTokenizer <T>, out newPos);
//calc length to validate
int length = 0;
rv.WithEach(x =>
{
length += x.TokenData.Length;
});
Condition.Requires(length).IsEqualTo(newPos);
newPosition = currentPosition + newPos;
return(rv);
}
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
IToken <T> newTokenOUT;
int newPosOUT;
IForwardMovingTokenizer <T> newTokenizerOUT;
//do the initial parse by length
var rv = base.Parse(source, currentPosition, state, currentToken, out newPosOUT, out newTokenOUT, out newTokenizerOUT);
if (!rv)
{
newPosition = currentPosition;
newToken = null;
newParser = null;
return(false);
}
Debug.WriteLine(string.Format("Composite token created. Token={0}", newTokenOUT.DumpToken()));
//get the substring to dig into
var compositeSeg = newTokenOUT.TokenData;
//build state data for recursion
var res = this.StateStrategy.Perform(ForwardMovingTokenizingCursor <T> .New(compositeSeg, currentPosition, state, currentToken)) as LogicOfTo <ForwardMovingTokenizingCursor <T>, object>;
object routerInitialState = res.Result;
int newPos;
var childTokens = compositeSeg.ForwardMovingTokenize(routerInitialState, this.Router as IForwardMovingTokenizer <T>, out newPos);
newTokenOUT = newTokenOUT.HasComposite(childTokens.ToArray());
newToken = newTokenOUT;
newParser = newTokenizerOUT;
newPosition = newPosOUT;
return(true);
}