public static CompositeTokenizerDecoration <T> MakeComposite <T>(this IForwardMovingTokenizer <T> decorated,
IRoutingTokenizer <T> router = null,
LogicOfTo <ForwardMovingTokenizingCursor <T>, object> stateStrategy = null)
{
Condition.Requires(decorated).IsNotNull();
return(new CompositeTokenizerDecoration <T>(decorated, router, stateStrategy));
}
public static IForwardMovingTokenizer <T> HasLengthStrategy <T>(this IForwardMovingTokenizer <T> decorated,
LogicOfTo <ForwardMovingTokenizingCursor <T>, int> lengthStrategy)
{
Condition.Requires(decorated).IsNotNull();
return(new HasLengthStrategyTokenizerDecoration <T>(decorated, lengthStrategy).HasValidation());
//NOTE: good practice is to add validation fluently after any decoration that introduces a handling condition
}
public CompositeTokenizerDecoration(IForwardMovingTokenizer <T> decorated,
IRoutingTokenizer <T> router = null,
LogicOfTo <ForwardMovingTokenizingCursor <T>, object> stateStrategy = null)
: base(decorated)
{
if (router == null)
{
this.Router = NaturallyNotImplementedForwardMovingTokenizer <T> .New().HasId("composite router").MakeRouter();
}
else
{
this.Router = router;
}
if (stateStrategy == null)
{
this.StateStrategy = LogicOfTo <ForwardMovingTokenizingCursor <T>, object> .New(cursor =>
{
return(null);
});
}
else
{
this.StateStrategy = stateStrategy;
}
}
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken, out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
//Debug.WriteLine("Parsing @ {0} of {1}", currentPosition, string.Join("", source));
//get the new tokenizer
var tokenizer = GetTokenizer(source, currentPosition, state, currentToken);
int newPositionOUT = 0;
IToken <T> newTokenOUT = null;
IForwardMovingTokenizer <T> newParserOUT = null;
bool rv = tokenizer != null;
if (rv)
{
//Debug.WriteLine("located tokenizer delegated to");
IForwardMovingTokenizer <T> alg = tokenizer.As <IForwardMovingTokenizer <T> >().GetOuterDecorator() as IForwardMovingTokenizer <T>;
var cake2 = alg.GetAllDecorations();
rv = alg.Parse(source, currentPosition, state, currentToken, out newPositionOUT, out newTokenOUT, out newParserOUT);
}
//loop back into router to handle the next token
if (OverridesTokenizerRouting)
{
newParserOUT = this;
}
//if we classify unrecognized, then we do that here
if (!rv && this.TokenizeUnrecognized)
{
//Debug.WriteLine("classifying unrecognized");
rv = true;
newPositionOUT = GetNextRecognizedPosition(source, currentPosition, state, currentToken);
//get string between old and new positions
var tokenText = source.GetSegment(currentPosition, newPositionOUT - currentPosition);
//Debug.WriteLine("unrecognized runs to {0}, producing {1}", newPositionOUT, string.Join("", tokenText));
//returns a suffixed natural token
newTokenOUT = NaturalToken <T> .New(tokenText).HasComment("UNRECOGNIZED");
newParserOUT = this;
}
newParser = newParserOUT;
newToken = newTokenOUT;
newPosition = newPositionOUT;
//if (!rv)
// Debug.WriteLine("tokenize fail.");
//if (rv)
// Debug.WriteLine("tokenize successful. new position = {0}. token = {1}", newPositionOUT, string.Join("", newToken.TokenData));
//Debug.WriteLine("Parsing Complete @ {0} of {1}", currentPosition, string.Join("", source));
return(rv);
}
public override bool Parse(T[] dataToTokenize, int currentPosition, object state, IToken <T> currentToken,
out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
//for all the suffixes this parses to, finds the nearest one (nongreedy)
int closestIdx = -1;
T[] suffix = null;
closestIdx = dataToTokenize.FindNearestIndexOf(this.Suffixes, out suffix, currentPosition);
//if we can't find a suffix, we kack
if (closestIdx == -1)
{
newParser = null;
newToken = null;
newPosition = -1;
return(false);
}
newPosition = closestIdx;
//get string between old and new positions
var tokenText = dataToTokenize.GetSegment(currentPosition, newPosition - currentPosition);
//returns a suffixed natural token
newToken = NaturalToken <T> .New(tokenText).HasSuffix(suffix, this.IsInclusive);
//we don't know what parser to use next
newParser = null;
return(true);
}
public bool Parse(string text, int currentPosition, IToken currentToken, out int newPosition, out IToken newToken, out IForwardMovingTokenizer newParser)
{
//if we can't find a , or ) then we kack
var idx = text.IndexOf(",", currentPosition);
if (idx == -1)
{
idx = text.IndexOf(")", currentPosition);
if (idx == -1)
{
newPosition = -1;
newToken = null;
newParser = null;
return false;
}
var substring = text.Substring(currentPosition, idx - currentPosition);
newPosition = idx + 1;
newToken = NaturalToken.New(substring).HasDPCTokenType(DPCTokenType.Item);
newParser = new ToDotParser();
return true;
}
else
{
var substring = text.Substring(currentPosition, idx - currentPosition);
newPosition = idx + 1;
newToken = NaturalToken.New(substring).HasDPCTokenType(DPCTokenType.Item);
newParser = new ToCommaOrEndParenthesisParser();
return true;
}
}
public static RoutingTokenizerDecoration <T> MakeRouter <T>(this IForwardMovingTokenizer <T> decorated,
bool overridesTokenizerRouting = true,
bool tokenizeUnrecognized = true)
{
Condition.Requires(decorated).IsNotNull();
return(new RoutingTokenizerDecoration <T>(decorated));
}
public HasValueFactoryTokenizerDecoration(IForwardMovingTokenizer <T> decorated,
Func <IToken <T>, object> valueFactory)
: base(decorated)
{
Condition.Requires(valueFactory).IsNotNull();
this.ValueFactory = valueFactory;
}
public HasLengthStrategyTokenizerDecoration(IForwardMovingTokenizer <T> decorated,
LogicOfTo <ForwardMovingTokenizingCursor <T>, int> lengthStrategy)
: base(decorated.KnowsLength())
{
Condition.Requires(lengthStrategy).IsNotNull();
this.LengthStrategy = lengthStrategy;
}
public SuffixDelimitedTokenizerDecoration(IForwardMovingTokenizer <T> decorated, bool isInclusive, params T[][] suffixes)
: base(decorated.KnowsLength())
{
Condition.Requires(suffixes).IsNotEmpty();
this.Suffixes = suffixes;
this.IsInclusive = isInclusive;
}
public static IForwardMovingTokenizer <T> HasImplementation <T>(this IForwardMovingTokenizer <T> decorated,
Func <ForwardMovingTokenizingCursor <T>, ForwardMovingTokenizingCursor <T> > tokenizingStrategy,
IConditionOf <ForwardMovingTokenizingCursor <T> > canHandleCondition = null)
{
Condition.Requires(decorated).IsNotNull();
return(new HasImplementationTokenizerDecoration <T>(decorated, tokenizingStrategy, canHandleCondition));
}
public static IForwardMovingTokenizer <T> HasPredecessorTokenizerIds <T>(this IForwardMovingTokenizer <T> decorated,
params string[] priorTokenizerIds)
{
Condition.Requires(decorated).IsNotNull();
return(new HasPredecessorTokenizerDecoration <T>(decorated, priorTokenizerIds).HasValidation());
//NOTE: good practice is to add validation fluently after any decoration that introduces a handling condition
}
/// <summary>
/// decorates with self direction. if stack already has this decoration, just ands the condition
/// </summary>
/// <param name="decorated"></param>
/// <param name="canHandleStrategy"></param>
/// <returns></returns>
public static IForwardMovingTokenizer <T> HasValidation <T>(this IForwardMovingTokenizer <T> decorated, IConditionOf <ForwardMovingTokenizingCursor <T> > canHandleCondition = null)
{
Condition.Requires(decorated).IsNotNull();
/**NOTE:
* We search for the same decoration in order to prevent a duplication of decoration.
* We MUST ALWAYS return the outermost decoration!!!!!!
* This would naturally happen if we do a simple decoration.
* If we don't return outermost then we can cause layers to disappear.
*/
//if we have a self direction decoration in the stack we return that
var dec = decorated.As <ValidatingTokenizerDecoration <T> >(true);
if (dec != null)
{
if (canHandleCondition != null)
{
dec.CanTokenizeCondition = dec.CanTokenizeCondition.And(canHandleCondition);
}
var outer = dec.GetOuterDecorator() as IForwardMovingTokenizer <T>;
return(outer);
}
return(new ValidatingTokenizerDecoration <T>(decorated, canHandleCondition));
}
/*
* Source string:
* @store.search(#ness.IsThing("x","y"))
* First level Tokens:
* @store
* .search
* (
* #ness
* .IsThing
* (
* "x"
* ,
* "y"
* )
* )
*
* 2nd level UoW Tokens:
* UoW0.Opr = @store
* UoW0.Op = .search
* UoW0.Arg0 = UoW1
*
* UoW1.Opr = #ness
* UoW1.Op = .IsThing
* UoW1.Arg0 = SimpleThing "x"
* UoW1.Arg1 = SimpleThing "y"
*
* The process:
*
* iterating thru the level 1 tokens,
* -is the token a simple thing? currentThing = build SimpleThing
* -is the token a thing? currentThing = build UoW
*
*/
public bool Parse(IToken <char>[] source, int currentPosition, object state, IToken <IToken <char> > currentToken,
out int newPosition, out IToken <IToken <char> > newToken, out IForwardMovingTokenizer <IToken <char> > newParser)
{
//create the UnitOfWork we're going to perform, regardless
UnitOfWork uow = new UnitOfWork();
UnitOfWork rv = uow;
//get context if we have it
UnitOfWork contextUoW = currentToken as UnitOfWork;
//do we have context? this means we're setting args on the context UoW and returning that
//otherwise we return the UoW we
//@store.search(#ness.IsThing("x","y"))
//expect operand
int pos = currentPosition;
var token0 = source[pos];
Condition.Requires(token0.IsOperandToken(), "operand expected");
uow.OperandToken = token0;
//expect operation
pos++;
var token1 = source[pos];
Condition.Requires(token1).IsNotNull();
Condition.Requires(token1.GetTokenizerId()).IsEqualTo(CommandLineLexer.OP);
uow.OperationToken = token1;
//, -> append
//expect ( or . (another op)
pos++;
var token2 = source[pos];
if (token2.GetTokenizerId() == CommandLineLexer.OP)
{
UnitOfWork uow2 = UnitOfWork.New();
uow2.Operand = uow;
uow2.OperationToken = token2;
rv = uow2;
//keep going until it's no longer an op
//keep wrapping rv
}
//if it's a (
// -we recurse.
//if an op, we create a new unit of work
return(true);
}
public HasImplementationTokenizerDecoration(IForwardMovingTokenizer <T> decorated,
Func <ForwardMovingTokenizingCursor <T>, ForwardMovingTokenizingCursor <T> > tokenizingStrategy,
IConditionOf <ForwardMovingTokenizingCursor <T> > canHandleCondition = null)
: base(decorated.HasValidation(canHandleCondition))
{
Condition.Requires(tokenizingStrategy).IsNotNull();
this.TokenizingStrategy = tokenizingStrategy;
}
public HasRoutingTokenizerDecoration(IForwardMovingTokenizer <T> decorated,
IRoutingTokenizer <T> router, bool overrideIfNonNull)
: base(decorated)
{
Condition.Requires(router).IsNotNull();
this.Router = router;
this.OverrideIfNonNull = overrideIfNonNull;
}
public RoutingTokenizerDecoration(IForwardMovingTokenizer <T> decorated,
bool overridesTokenizerRouting = true,
bool tokenizeUnrecognized = true)
: base(decorated)
{
this.TokenizerStore = NaturalInMemoryStore.New().IsOf <TokenizerItem>();
this.OverridesTokenizerRouting = overridesTokenizerRouting;
this.TokenizeUnrecognized = tokenizeUnrecognized;
}
/// <summary>
/// tokenizes a string by a forward only parse. each tokenizer MUST provide the next tokenizer
/// </summary>
/// <param name="rawData"></param>
/// <param name="parser"></param>
/// <returns></returns>
public static List <IToken <T> > ForwardMovingTokenizeToCompletion <T>(this T[] rawData, object state, IForwardMovingTokenizer <T> parser)
{
List <IToken <T> > rv = new List <IToken <T> >();
if (rawData == null)
{
return(rv);
}
if (parser == null)
{
return(rv);
}
int pos = 0;
int maxPos = rawData.Length - 1;
IToken <T> token = null;
IForwardMovingTokenizer <T> currentParser = parser;
IForwardMovingTokenizer <T> lastParser = null;
bool goodParse = true;
int counter = 0;
while (goodParse && currentParser != null && pos > -1 && pos <= maxPos)
{
if (currentParser != null)
{
lastParser = currentParser;
}
counter++;
var priorToken = token;
var startPos = pos;
goodParse = currentParser.Parse(rawData, pos, state, token, out pos, out token, out currentParser);
if (goodParse)
{
if (token != null)
{
token.PriorToken = priorToken;
//decorate token with positional info
token = token.HasStartEnd(startPos, pos);
rv.Add(token);
}
}
}
//if there's been a problem, kack
if (!goodParse)
{
throw new LexingException(string.Format("tokenizing failed at pos {0} with tokenizer {1}", pos, lastParser.GetType()));
}
return(rv);
}
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken,
out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
var rv = this.TokenizingStrategy(ForwardMovingTokenizingCursor <T> .New(source, currentPosition, state, currentToken));
newPosition = rv.CurrentPosition;
newToken = rv.CurrentToken;
newParser = null;
return(true);
}
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken,
out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
newPosition = currentPosition + this.TokenValue.Length;
//returns a natural token
newToken = NaturalToken <T> .New(this.TokenValue);
//we don't know what parser to use next
newParser = null;
return(true);
}
public static CompositeTokenizerDecoration <T> MakeCompositeOf <T>(this IForwardMovingTokenizer <T> decorated, params IForwardMovingTokenizer <T>[] composites)
{
Condition.Requires(decorated).IsNotNull();
var rv = new CompositeTokenizerDecoration <T>(decorated);
composites.WithEach(x =>
{
var tokenizer = x.As <IHasStringIdTokenizer <T> >(false);
if (tokenizer != null)
{
rv.Router.AddTokenizer(tokenizer);
}
});
return(rv);
}
public override bool Parse(T[] dataToTokenize, int currentPosition, object state, IToken <T> currentToken,
out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
newPosition = currentPosition + Length;
//get string between old and new positions
var tokenText = dataToTokenize.GetSegment(currentPosition, newPosition - currentPosition);
//returns a suffixed natural token
newToken = NaturalToken <T> .New(tokenText);
//we don't know what parser to use next
newParser = null;
return(true);
}
public KnowsLengthTokenizerDecoration(IForwardMovingTokenizer <T> decorated)
: base(decorated)
{
//ensure no more than 1 decoration is possible per stack
//checking for this is a bit subtle. at this point in the decoration process
// SetDecorated has been called by the base ctor, which puts ValidatingTokenizerDecoration
// as the topmost decoration. So any AS call will return this, as it walks to topmost first.
//We need to check beneath this layer rather, and do an ASBelow call or we will always kack
var layer = decorated.AsBelow <KnowsLengthTokenizerDecoration <T> >();
if (layer != null)
{
throw new InvalidOperationException("already knows length");
}
}
public static IForwardMovingTokenizer <T> HasPredecessorTokenizers <T>(this IForwardMovingTokenizer <T> decorated,
params IHasStringIdTokenizer <T>[] priorTokenizers)
{
Condition.Requires(decorated).IsNotNull();
List <string> ids = new List <string>();
foreach (var each in priorTokenizers)
{
ids.Add(each.Id);
}
var rv = new HasPredecessorTokenizerDecoration <T>(decorated, ids.ToArray()).HasValidation();
//NOTE: good practice is to add validation fluently after any decoration that introduces a handling condition
//wire up the
return(rv);
}
public bool Parse(string text, int currentPosition, IToken currentToken, out int newPosition, out IToken newToken, out IForwardMovingTokenizer newParser)
{
//if we can't find a . then we kack
var idx = text.IndexOf(".", currentPosition);
if (idx == -1)
{
newPosition = -1;
newToken = null;
newParser = null;
return false;
}
else
{
var substring = text.Substring(currentPosition, idx - currentPosition);
newPosition = idx + 1;
newToken = NaturalToken.New(substring);//build up the token
newParser = new ToOpenParenthesisParser();
return true;
}
}
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken,
out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
var cursor = ForwardMovingTokenizingCursor <T> .New(source, currentPosition, state, currentToken);
var logicRes = this.LengthStrategy.Perform(cursor) as LogicOfTo <ForwardMovingTokenizingCursor <T>, int>;
int length = logicRes.Result;
newPosition = currentPosition + length;
//get string between old and new positions
var tokenText = source.GetSegment(currentPosition, newPosition - currentPosition);
//returns a suffixed natural token
newToken = NaturalToken <T> .New(tokenText);
//we don't know what parser to use next
newParser = null;
return(true);
}
public CompositeTokenizerDecoration(IForwardMovingTokenizer <T> decorated,
IRoutingTokenizer <T> router = null,
LogicOfTo <ForwardMovingTokenizingCursor <T>, int> lengthStrategy = null,
LogicOfTo <ForwardMovingTokenizingCursor <T>, object> stateStrategy = null)
: base(decorated)
{
if (router == null)
{
this.Router = NaturallyNotImplementedForwardMovingTokenizer <T> .New().HasId("composite router").MakeRouter();
}
else
{
this.Router = router;
}
if (lengthStrategy == null)
{
//do a router parse and see how far we go
this.LengthStrategy = LogicOfTo <ForwardMovingTokenizingCursor <T>, int> .New(cursor =>
{
int newPos;
var vals = this.routerParse(cursor.Source, cursor.CurrentPosition, cursor.State, cursor.CurrentToken, out newPos);
return(newPos);
});
}
else
{
this.LengthStrategy = lengthStrategy;
}
if (stateStrategy == null)
{
this.StateStrategy = LogicOfTo <ForwardMovingTokenizingCursor <T>, object> .New(cursor =>
{
return(null);
});
}
else
{
this.StateStrategy = stateStrategy;
}
}
public static IForwardMovingTokenizer <T> HasPairDelimitedLengthStrategy <T>(this IForwardMovingTokenizer <T> decorated,
T[] prefix, T[] suffix)
{
Condition.Requires(decorated).IsNotNull();
Condition.Requires(prefix).IsNotNull().IsNotEmpty();
Condition.Requires(suffix).IsNotNull().IsNotEmpty();
var lengthStrategy = LogicOfTo <ForwardMovingTokenizingCursor <T>, int> .New(x =>
{
var pos = x.Source.GetPositionOfComplement(prefix, suffix, x.CurrentPosition);
var rv = pos - x.CurrentPosition;
if (rv <= 0)
{
return(0);
}
return(rv);
});
return(new HasLengthStrategyTokenizerDecoration <T>(decorated, lengthStrategy).HasValidation());
//NOTE: good practice is to add validation fluently after any decoration that introduces a handling condition
}
public ValidatingTokenizerDecoration(IForwardMovingTokenizer <T> decorated,
IConditionOf <ForwardMovingTokenizingCursor <T> > canHandleCondition = null)
: base(decorated)
{
//ensure no more than 1 selfdirected decoration is possible per stack
//checking for this is a bit subtle. at this point in the decoration process
// SetDecorated has been called by the base ctor, which puts ValidatingTokenizerDecoration
// as the topmost decoration. So any AS call will return this, as it walks to topmost first.
//We need to check beneath this layer rather, and do an ASBelow call or we will always kack
var validator = decorated.AsBelow <ValidatingTokenizerDecoration <T> >();
if (validator != null)
{
throw new InvalidOperationException("already self-directed");
}
if (canHandleCondition == null)
{
////default implementation is to do the base parse
//this.CanTokenizeCondition = StrategizedConditionOf<ForwardMovingTokenizingCursor<T>>.New(cursor =>
//{
// int newPosition;
// IToken<T> newToken;
// IForwardMovingTokenizer<T> newTokenizer;
// return base.Parse(cursor.Source, cursor.CurrentPosition, cursor.State, cursor.CurrentToken,
// out newPosition,
// out newToken,
// out newTokenizer);
//});
}
else
{
this.CanTokenizeCondition = canHandleCondition;
}
}
public override bool Parse(T[] source, int currentPosition, object state, IToken <T> currentToken,
out int newPosition, out IToken <T> newToken, out IForwardMovingTokenizer <T> newParser)
{
IToken <T> newTokenOUT = null;
var rv = this.Decorated.Parse(source, currentPosition, state, currentToken, out newPosition, out newTokenOUT, out newParser);
if (this.IsInclusive)
{
//decorate token with suffix
var suffix = newTokenOUT.TokenData.FindMatchingSuffix(this.Suffixes);
newTokenOUT = newTokenOUT.HasSuffix(suffix, this.IsInclusive);
}
else
{
//find which suffix is first
var suffix = source.FindMatchingPrefix(newPosition, this.Suffixes);
newTokenOUT = newTokenOUT.HasSuffix(suffix, this.IsInclusive);
}
newToken = newTokenOUT;
return(rv);
}
public static SuffixDelimitedTokenizerDecoration <T> New <T>(IForwardMovingTokenizer <T> decorated, bool isInclusive, params T[][] suffixes)
{
return(new SuffixDelimitedTokenizerDecoration <T>(decorated, isInclusive, suffixes));
}
public static IForwardMovingTokenizer <T> HasSuffix <T>(this IForwardMovingTokenizer <T> decorated, bool isInclusive, params T[][] suffixes)
{
Condition.Requires(decorated).IsNotNull();
return(new SuffixDelimitedTokenizerDecoration <T>(decorated, isInclusive, suffixes).HasValidation());
//NOTE: good practice is to add validation fluently after any decoration that introduces a handling condition
}
