public override bool Parse(T[] dataToTokenize, int currentPosition, object state, IToken <T> currentToken,
out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
//for all the suffixes this parses to, finds the nearest one (nongreedy)
int closestIdx = -1;
T[] suffix = null;
closestIdx = dataToTokenize.FindNearestIndexOf(this.Suffixes, out suffix, currentPosition);
//if we can't find a suffix, we kack
if (closestIdx == -1)
{
newParser = null;
newToken = null;
newPosition = -1;
return(false);
}
newPosition = closestIdx;
//get string between old and new positions
var tokenText = dataToTokenize.GetSegment(currentPosition, newPosition - currentPosition);
//returns a suffixed natural token
newToken = NaturalToken <T> .New(tokenText).HasSuffix(suffix, this.IsInclusive);
//we don't know what parser to use next
newParser = null;
return(true);
}
public bool Parse(string text, int currentPosition, object state, IToken currentToken, out int newPosition, out IToken newToken, out IForwardMovingTokenizer newParser)
{
//if we can't find a , or ) then we kack
var idx = text.IndexOf(",", currentPosition);
if (idx == -1)
{
idx = text.IndexOf(")", currentPosition);
if (idx == -1)
{
newPosition = -1;
newToken = null;
newParser = null;
return(false);
}
var substring = text.Substring(currentPosition, idx - currentPosition);
newPosition = idx + 1;
newToken = NaturalToken.New(substring).HasDPCTokenType(DPCTokenType.Item);
newParser = new ToDotParser();
return(true);
}
else
{
var substring = text.Substring(currentPosition, idx - currentPosition);
newPosition = idx + 1;
newToken = NaturalToken.New(substring).HasDPCTokenType(DPCTokenType.Item);
newParser = new ToCommaOrEndParenthesisParser();
return(true);
}
}
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
//Debug.WriteLine("Parsing @ {0} of {1}", currentPosition, string.Join("", source));
//get the new tokenizer
var tokenizer = GetTokenizer(source, currentPosition, state, currentToken);
int newPositionOUT = 0;
IToken <T> newTokenOUT = null;
IForwardMovingTokenizer <T> newParserOUT = null;
bool rv = tokenizer != null;
if (rv)
{
//Debug.WriteLine("located tokenizer delegated to");
IForwardMovingTokenizer <T> alg = tokenizer.As <IForwardMovingTokenizer <T> >().GetOuterDecorator() as IForwardMovingTokenizer <T>;
var cake2 = alg.GetAllDecorations();
rv = alg.Parse(source, currentPosition, state, currentToken, out newPositionOUT, out newTokenOUT, out newParserOUT);
}
//loop back into router to handle the next token
if (OverridesTokenizerRouting)
{
newParserOUT = this;
}
//if we classify unrecognized, then we do that here
if (!rv && this.TokenizeUnrecognized)
{
//Debug.WriteLine("classifying unrecognized");
rv = true;
newPositionOUT = GetNextRecognizedPosition(source, currentPosition, state, currentToken);
//get string between old and new positions
var tokenText = source.GetSegment(currentPosition, newPositionOUT - currentPosition);
//Debug.WriteLine("unrecognized runs to {0}, producing {1}", newPositionOUT, string.Join("", tokenText));
//returns a suffixed natural token
newTokenOUT = NaturalToken <T> .New(tokenText).HasComment("UNRECOGNIZED");
newParserOUT = this;
}
newParser = newParserOUT;
newToken = newTokenOUT;
newPosition = newPositionOUT;
//if (!rv)
// Debug.WriteLine("tokenize fail.");
//if (rv)
// Debug.WriteLine("tokenize successful. new position = {0}. token = {1}", newPositionOUT, string.Join("", newToken.TokenData));
//Debug.WriteLine("Parsing Complete @ {0} of {1}", currentPosition, string.Join("", source));
return(rv);
}
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken,
out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
newPosition = currentPosition + this.TokenValue.Length;
//returns a natural token
newToken = NaturalToken <T> .New(this.TokenValue);
//we don't know what parser to use next
newParser = null;
return(true);
}
public override bool Parse(T[] dataToTokenize, int currentPosition, object state, IToken <T> currentToken,
out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
newPosition = currentPosition + Length;
//get string between old and new positions
var tokenText = dataToTokenize.GetSegment(currentPosition, newPosition - currentPosition);
//returns a suffixed natural token
newToken = NaturalToken <T> .New(tokenText);
//we don't know what parser to use next
newParser = null;
return(true);
}
public bool Parse(string text, int currentPosition, IToken currentToken, out int newPosition, out IToken newToken, out IForwardMovingTokenParser newParser)
{
//if we can't find the suffix, we kack
var idx = text.IndexOf(this.Suffix, currentPosition);
if (idx == -1)
{
newPosition = -1;
newToken = null;
newParser = null;
return(false);
}
else
{
var substring = text.Substring(currentPosition, idx - currentPosition);
newPosition = idx + 1;
newToken = NaturalToken.New(substring);//build up the token
newParser = new ToOpenParenthesisParser();
return(true);
}
}
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken,
out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
var cursor = ForwardMovingTokenizingCursor <T> .New(source, currentPosition, state, currentToken);
var logicRes = this.LengthStrategy.Perform(cursor) as LogicOfTo <ForwardMovingTokenizingCursor <T>, int>;
int length = logicRes.Result;
newPosition = currentPosition + length;
//get string between old and new positions
var tokenText = source.GetSegment(currentPosition, newPosition - currentPosition);
//returns a suffixed natural token
newToken = NaturalToken <T> .New(tokenText);
//we don't know what parser to use next
newParser = null;
return(true);
}
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
int newPos;
var tokens = this.routerParse(source, currentPosition, state, currentToken, out newPos);
if (newPos == 0)
{
newPosition = currentPosition;
newToken = null;
newParser = null;
return(false);
}
//build the composite
var segData = source.GetSegment(currentPosition, newPos);
newToken = NaturalToken <T> .New(segData).HasComposite(tokens.ToArray());
newParser = null;
newPosition = currentPosition + newPos;
return(true);
}
