public CompositeTokenizerDecoration(IForwardMovingTokenizer <T> decorated,
IRoutingTokenizer <T> router = null,
LogicOfTo <ForwardMovingTokenizingCursor <T>, object> stateStrategy = null)
: base(decorated)
{
if (router == null)
{
this.Router = NaturallyNotImplementedForwardMovingTokenizer <T> .New().HasId("composite router").MakeRouter();
}
else
{
this.Router = router;
}
if (stateStrategy == null)
{
this.StateStrategy = LogicOfTo <ForwardMovingTokenizingCursor <T>, object> .New(cursor =>
{
return(null);
});
}
else
{
this.StateStrategy = stateStrategy;
}
}
public CompositeTokenizerDecoration(IForwardMovingTokenizer <T> decorated,
IRoutingTokenizer <T> router = null,
LogicOfTo <ForwardMovingTokenizingCursor <T>, int> lengthStrategy = null,
LogicOfTo <ForwardMovingTokenizingCursor <T>, object> stateStrategy = null)
: base(decorated)
{
if (router == null)
{
this.Router = NaturallyNotImplementedForwardMovingTokenizer <T> .New().HasId("composite router").MakeRouter();
}
else
{
this.Router = router;
}
if (lengthStrategy == null)
{
//do a router parse and see how far we go
this.LengthStrategy = LogicOfTo <ForwardMovingTokenizingCursor <T>, int> .New(cursor =>
{
int newPos;
var vals = this.routerParse(cursor.Source, cursor.CurrentPosition, cursor.State, cursor.CurrentToken, out newPos);
return(newPos);
});
}
else
{
this.LengthStrategy = lengthStrategy;
}
if (stateStrategy == null)
{
this.StateStrategy = LogicOfTo <ForwardMovingTokenizingCursor <T>, object> .New(cursor =>
{
return(null);
});
}
else
{
this.StateStrategy = stateStrategy;
}
}
public static IForwardMovingTokenizer <char> BuildLexingLogic(CLConfig config)
{
/*
* CommandExamples:
*
* Decorating: @[id1]#HasDateCreated(now)#HasId(id2)#HasName(name1)
* Saving: .Save()
* Deleting: @[id1].Delete()
* Getting Ness Value: @[id1]#HasDateCreated.Date
* Performing Ness Op: @[id1]#HasDateCreated.SetDate(now)#HasBeep.Beep()
* Conditional Ness: #HasDateCreated.IsAfter(now)
*
*
*/
//define common separators/punctuation. these will terminate tokens
var at = "@".ToCharArray();
var dot = ".".ToCharArray();
var comma = ",".ToCharArray();
var openParenthesis = "(".ToCharArray();
var closeParenthesis = ")".ToCharArray();
var hash = "#".ToCharArray();
var openBracket = "[".ToCharArray();
var closeBracket = "]".ToCharArray();
var emptyParenthesis = "()".ToCharArray();
#region Primitive-y Tokenizers - all routing unhydrated
//store token starts with @ and ends with any punctuation
var storeTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New()
.HasSuffixDelimitedLengthStrategy(at, dot, comma, openParenthesis, closeParenthesis, hash, openBracket, closeBracket)
.HasPrefix(at)
.HasSuffix(false, at, dot, comma, openParenthesis, closeParenthesis, hash, openBracket, closeBracket)
.HasValueFactory(token =>
{
string storeName = new string(token.TokenData);
if (string.IsNullOrEmpty(storeName))
{
return(null); //default store strategy
}
else
{
var store = config.StoreOfStores.Get <NamedNaturalInMemoryStore>(storeName);
return(store);
}
})
.HasId("Store");
//id token starts with [ and ends with ]. brackets are non-nesting
var idTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New()
.HasSuffixDelimitedLengthStrategy(closeBracket)
.HasPrefix(openBracket)
.HasSuffix(true, closeBracket)
.HasValueFactory(token =>
{
string id = new string(token.TokenData);
return(id);
}).HasId("Id");
//op token starts with . and ends with ( or .
var opTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New()
.HasSuffixDelimitedLengthStrategy(dot, openParenthesis)
.HasPrefix(dot)
.HasSuffix(false, dot, openParenthesis)
.HasId("Op");
//ness token starts with # and ends with ( or .
var nessTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New()
.HasSuffixDelimitedLengthStrategy(dot, openParenthesis)
.HasPrefix(hash)
.HasSuffix(false, dot, openParenthesis)
.HasValueFactory(token =>
{
string nessName = new string(token.TokenData);
//ness is contextual and depends on who is invoking the ness
//so we have to get the prior token's value
var lastTokenValueFace = token.PriorToken.GetFace <IHasValue>();
var lastTokenValue = lastTokenValueFace.Value;
var rv = config.NessManager.GetNess(lastTokenValue, nessName);
return(rv);
})
.HasId("Ness");
var commaTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New()
.HasConstantValue(comma)
.HasId("Comma");
var emptyParenthesisTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New()
.HasConstantValue(emptyParenthesis)
.HasId("EmptyParenthesis");
#endregion
//now build the compound tokenizers
var mainRouterStack = NaturallyNotImplementedForwardMovingTokenizer <char> .New()
.MakeRouter(true, true)
.HasId("MainRouter");
var mainRouter = mainRouterStack.As <IRoutingTokenizer <char> >(false);
//parenthesis token starts with ( and ends with ), and handles nesting
//is a compound that recurses the whole stack
//uses the mainRouter to recurse, but is not registered with it as it's not a high level token
var parenthesisTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New()
.HasPairDelimitedLengthStrategy(openParenthesis, closeParenthesis)
.HasPrefix(openParenthesis)
.HasSuffix(true, closeParenthesis)
.MakeComposite(mainRouter)
.HasId("Parenthesis");
//Decorating: @[id1]#HasDateCreated(now)#HasId(id2)#HasName(name1)
//var decoratingCmdTokenizer = NaturallyNotImplementedForwardMovingTokenizer<char>.New()
// .MakeCompositeOf(storeTokenizer, idTokenizer, opTokenizer, nessTokenizer, parenthesisTokenizer, commaTokenizer)
// .HasId("Decorating");
//object manipulation
//Saving:@[id1].Save()
//Deleting: @[id1].Delete()
//Getting Ness Value: @[id1]#HasDateCreated.Date
//Performing Ness Op: @[id1]#HasDateCreated.SetDate(now)#HasBeep.Beep()
//Conditional Ness: #HasDateCreated.IsAfter(now)
//var hasIdCmdTokenizer = NaturallyNotImplementedForwardMovingTokenizer<char>.New()
// .MakeCompositeOf(storeTokenizer, idTokenizer, nessTokenizer, parenthesisTokenizer)
// .HasId("HasIdCommand");
// mainRouter.AddTokenizer(hasIdCmdTokenizer);//.AddTokenizer(decoratingCmdTokenizer).
mainRouter.AddTokenizer(storeTokenizer, idTokenizer, opTokenizer, nessTokenizer, commaTokenizer, parenthesisTokenizer);
return(mainRouterStack);
}
public static IForwardMovingTokenizer <char> BuildLexingLogic(CLConfig config)
{
//@store.search(#ness.IsThing("x","y"))
//to parse this type of syntax we use prefix routing - ie. we route via prefix
var router = NaturallyNotImplementedForwardMovingTokenizer <char> .New().MakeRouter();
//parse store name into store token. eg. @store
var storeTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New().HasPrefix("@".ToCharArray()).HasSuffix(".".ToCharArray())
.HasValueFactory(token =>
{
string storeName = new string(token.TokenData);
var store = config.StoreOfStores.Get <NamedNaturalInMemoryStore>(storeName);
return(store);
}).HasId(STORE);
router.AddTokenizer(storeTokenizer);
//parse ness name into ness token. eg. #ness
var nessTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New().HasPrefix("#".ToCharArray()).HasSuffix(".".ToCharArray())
.HasValueFactory(token =>
{
string nessName = new string(token.TokenData);
//ness is contextual and depends on who is invoking the ness
//so we have to get the prior token's value
var lastTokenValueFace = token.PriorToken.GetFace <IHasValue>();
var lastTokenValue = lastTokenValueFace.Value;
var rv = config.NessManager.GetNess(lastTokenValue, nessName);
return(rv);
}).HasId(NESS);
router.AddTokenizer(nessTokenizer);
//parse thing token - isn't a store or a ness or an op since it has no predecessors.
//isn't an arg cos it's not terminated by , or ). eg. "hello"
var thingTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New().HasPredecessorTokenizerIds(null).HasSuffix(".".ToCharArray())
.HasValueFactory(token =>
{
string tokenData = new string(token.TokenData);
return(tokenData);
})
.HasId(THING);
router.AddTokenizer(thingTokenizer);
//parse operation name into op token. eg. .search
var opTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New().HasPrefix(".".ToCharArray()).HasSuffix(".".ToCharArray(), "(".ToCharArray()).HasId(OP);
router.AddTokenizer(opTokenizer);
//open and close brackets. constants
var openParenTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New().HasConstantValue("(".ToCharArray()).HasId(OPENPAREN);
var closeParenTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New().HasConstantValue(")".ToCharArray()).HasId(CLOSEPAREN);
router.AddTokenizer(openParenTokenizer);
router.AddTokenizer(closeParenTokenizer);
//the comma. constant
var commaTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New().HasConstantValue(",".ToCharArray()).HasId(COMMA);
router.AddTokenizer(commaTokenizer);
//args. eg. "x", "y"
//can have no predecessor tokenizer, or ( or ,
var argTokenizer = NaturallyNotImplementedForwardMovingTokenizer <char> .New().HasPredecessorTokenizerIds(null, OPENPAREN, COMMA).HasSuffix(",".ToCharArray(), ")".ToCharArray())
.HasValueFactory(token =>
{
string tokenData = new string(token.TokenData);
return(tokenData);
})
.HasId(ARG);
router.AddTokenizer(argTokenizer);
return(router);
}
