public void AppendCharacter(char character)
{
LsmConstantMatchRule constantRule
= _lastState.MatchRules.GetRuleByConstant(character);
if (constantRule == null)
{
constantRule = new LsmConstantMatchRule(character);
_lastState.MatchRules.Add(constantRule);
if (_lastState == _rootState)
constantRule.Actions.EnsureClearTokenAction();
}
constantRule.Actions.EnsureAcceptCharAction();
constantRule.Actions.EnsureAdvanceAction();
_lastState = constantRule.Actions.GetTransitionDestination();
if (_lastState == null)
{
_lastState = new LsmState();
_document.States.Add(_lastState);
constantRule.Actions.SetTransitionDestination(_lastState);
}
}
public LsmStatePathBuilder(LsmDocument document, LsmState rootState)
{
if (rootState == null)
throw new ArgumentNullException("rootState");
if (document == null)
throw new ArgumentNullException("document");
_document = document;
_rootState = rootState;
_lastState = rootState;
}
private void WriteState(LsmState state)
{
_writer.WriteLine(state.ToString());
foreach (LsmMatchRule matchRule in state.MatchRules)
WriteMatchRule(matchRule);
_writer.WriteLine(" Default");
foreach (LsmAction defaultAction in state.DefaultActions)
_writer.WriteLine(" " + defaultAction.ToString());
_writer.WriteLine(" Post Stream");
foreach (LsmAction postStreamAction in state.PostStreamActions)
_writer.WriteLine(" " + postStreamAction.ToString());
}
public LsmState AppendStringTokenPath(LsmState start, string tokenText, int typeID)
{
LsmState currState = start;
for (int i = 0; i < tokenText.Length; i++)
{
char character = tokenText[i];
LsmState nextState;
LsmConstantMatchRule constantRule
= currState.MatchRules.GetRuleByConstant(character);
if (constantRule == null)
{
constantRule = new LsmConstantMatchRule(character);
nextState = new LsmState();
document.States.Add(nextState);
if (i == 0)
constantRule.Actions.Add(new LsmMarkTokenStartAction());
constantRule.Actions.Add(new LsmAcceptCharAction());
constantRule.Actions.Add(new LsmAdvanceAction());
constantRule.Actions.Add(new LsmStateTransitionAction(nextState));
currState.MatchRules.Add(constantRule);
}
else
nextState = constantRule.Actions.GetTransitionDestination();
if (i == tokenText.Length - 1)
{
nextState.DefaultActions.Add(new LsmAcceptTokenAction(typeID));
nextState.DefaultActions.Add(new LsmClearTokenTextAction());
nextState.DefaultActions.Add(new LsmStateTransitionAction(rootState));
nextState.PostStreamActions.Add(new LsmAcceptTokenAction(typeID));
}
currState.DefaultActions.SetTransitionDestination(rootState);
currState = nextState;
}
return currState;
}
public LsmStateTransitionAction(LsmState newState)
{
_newState = newState;
}
static void Main(string[] args)
{
/*
LsmBuilder builder = new LsmBuilder();
builder.CreateStringTokenPath("H", 1, true);
builder.CreateStringTokenPath("He", 2, true);
builder.CreateStringTokenPath("Heck", 3, true);
builder.CreateStringTokenPath("Hello", 4, true);
builder.CreateStringTokenPath("{", 5, false);
builder.CreateStringTokenPath("}", 6, false);
builder.ApplyIdentifierPaths(-1);
*/
/*
LsmDocument stateDoc = new LsmDocument();
LsmState tokenStartState = new LsmState();
LsmState consumeLetters = new LsmState();
tokenStartState.DefaultActions.Add(new LsmMarkTokenStartAction());
tokenStartState.DefaultActions.Add(new LsmStateTransitionAction(consumeLetters));
tokenStartState.DefaultActions.Add(new LsmClearTokenTextAction());
consumeLetters.PostStreamActions.Add(new LsmAcceptTokenAction(0));
LsmLambdaMatchRule letterRule = new LsmLambdaMatchRule((c) => char.IsLetter(c));
letterRule.Actions.Add(new LsmAdvanceAction());
letterRule.Actions.Add(new LsmAcceptCharAction());
consumeLetters.DefaultActions.Add(new LsmAcceptTokenAction(0));
consumeLetters.DefaultActions.Add(new LsmStateTransitionAction(tokenStartState));
consumeLetters.DefaultActions.Add(new LsmAdvanceAction());
consumeLetters.MatchRules.Add(letterRule);
stateDoc.States.Add(tokenStartState);
stateDoc.States.Add(consumeLetters);
stateDoc.StartState = tokenStartState;
*/
/*
LsmDocument doc = new LsmDocument();
LsmState st = new LsmState();
LsmMatchRule alphaNum = new LsmClassMatchRule(LsmCharacterClass.Alphanumeric);
alphaNum.Actions.Add(new LsmAcceptCharAction());
alphaNum.Actions.Add(new LsmAdvanceAction());
LsmMatchRule thirdChar = new LsmCharIndexMatchRule(2);
thirdChar.Actions.Add(new LsmAcceptTokenAction(10));
thirdChar.Actions.Add(new LsmAdvanceAction());
thirdChar.Actions.Add(new LsmClearTokenTextAction());
st.MatchRules.Add(alphaNum);
st.MatchRules.Add(thirdChar);
doc.States.Add(st);
doc.StartState = st;
*/
LsmDocument doc = new LsmDocument();
LsmState rootState = new LsmState();
LsmStatePathBuilder path = new LsmStatePathBuilder(doc, rootState);
path.AppendText("Cat");
path.CompleteToken(1);
path.MoveToState(rootState);
path.AppendText("Dog");
path.CompleteToken(2);
path.MoveToState(rootState);
path.AppendText("Dog");
path.AppendCharacterClass(LsmCharacterClass.Alphanumeric);
path.AppendCharacterClassLoop(LsmCharacterClass.Alphanumeric);
path.CompleteToken(-1);
path.MoveToState(rootState);
//Note: Does not loop.
path.AppendCharacterClass(LsmCharacterClass.WhiteSpace);
doc.States.Add(rootState);
doc.StartState = rootState;
using (TextWriter writer = File.CreateText(@"C:\Users\Guido\Desktop\LSMSource.txt"))
{
LsmDocumentTextGenerator gen = new LsmDocumentTextGenerator(writer);
gen.WriteDocument(doc);
writer.Flush();
}
LsmTokenizeFunction parseFunc
= doc.BuildLexerFunction();
List<LsmToken> tokens;
while (true)
{
string stuff = Console.ReadLine();
using (StringReader reader = new StringReader(stuff))
parseFunc(reader, tokens = new List<LsmToken>());
foreach (LsmToken token in tokens)
{
Console.WriteLine(token.Text + " : " + token.TypeID + " : " + token.CharIndex);
}
}
}
public void MoveToState(LsmState state)
{
_lastState = state;
}
/*
public int IdentifierTokenTypeID
{
get { return identifierTokenTypeID; }
set { identifierTokenTypeID = value; }
}*/
public void ApplyIdentifierPaths(int tokenTypeID)
{
//LsmState identifierFirst = new LsmState();
LsmState identifierRest = new LsmState();
LsmClassMatchRule alphaNumericRule = new LsmClassMatchRule(LsmCharacterClass.Alphanumeric);
LsmClassMatchRule letterRule = new LsmClassMatchRule(LsmCharacterClass.Letter);
letterRule.Actions.Add(new LsmStateTransitionAction(identifierRest));
letterRule.Actions.Add(new LsmMarkTokenStartAction());
alphaNumericRule.Actions.Add(new LsmAcceptCharAction());
alphaNumericRule.Actions.Add(new LsmAdvanceAction());
alphaNumericRule.Actions.Add(new LsmStateTransitionAction(identifierRest));
rootState.MatchRules.Add(letterRule);
identifierRest.MatchRules.Add(alphaNumericRule);
identifierRest.DefaultActions.Add(new LsmAcceptTokenAction(tokenTypeID));
identifierRest.DefaultActions.Add(new LsmClearTokenTextAction());
identifierRest.DefaultActions.Add(new LsmStateTransitionAction(rootState));
identifierRest.PostStreamActions.Add(new LsmAcceptTokenAction(tokenTypeID));
foreach (LsmState keywordEndState in keywordEndStates)
keywordEndState.MatchRules.Add(alphaNumericRule);
document.States.Add(identifierRest);
}
