public void AddKeyWord(IN token, string keyword)
{
NodeCallback <GenericToken> callback = (FSMMatch <GenericToken> match) =>
{
if (derivedTokens.ContainsKey(GenericToken.Identifier))
{
Dictionary <string, IN> derived = derivedTokens[GenericToken.Identifier];
if (derived.ContainsKey(match.Result.Value))
{
match.Properties[DerivedToken] = derived[match.Result.Value];
}
else if (derived.ContainsKey(defaultIdentifierKey))
{
match.Properties[DerivedToken] = identifierDerivedToken;
}
}
else
{
match.Properties[DerivedToken] = identifierDerivedToken;
}
return(match);
};
AddLexeme(GenericToken.Identifier, token, keyword);
FSMBuilder.GoTo(in_identifier);
var node = FSMBuilder.GetNode(in_identifier);
if (!FSMBuilder.Fsm.HasCallback(node.Id))
{
FSMBuilder.GoTo(in_identifier).
CallBack(callback);
}
}
private static void AddExponentExtension(DoubleExponent token, LexemeAttribute lexem, GenericLexer <DoubleExponent> lexer)
{
if (token == DoubleExponent.DOUBLE_EXP)
{
// callback on end_exponent node
NodeCallback <GenericToken> callback = (FSMMatch <GenericToken> match) =>
{
string[] items = match.Result.Value.Split(new[] { 'e', 'E' });
double radix = 0;
double.TryParse(items[0].Replace(".", ","), out radix);
double exponent = 0;
double.TryParse(items[1], out exponent);
double value = Math.Pow(radix, exponent);
match.Result.SpanValue = value.ToString().AsMemory();
match.Properties[GenericLexer <DoubleExponent> .DerivedToken] = DoubleExponent.DOUBLE_EXP;
return(match);
};
var fsmBuilder = lexer.FSMBuilder;
fsmBuilder.GoTo(GenericLexer <DoubleExponent> .in_double) // start a in_double node
.Transition(new char[] { 'E', 'e' }) // add a transition on '.' with precondition
.Transition(new char[] { '+', '-' })
.Mark("start_exponent_val")
.RangeTransitionTo('0', '9', "start_exponent_val") // first year digit
.Mark("end_exponent")
.End(GenericToken.Extension) // mark as ending node
.CallBack(callback); // set the ending callback
}
}
public void AddKeyWord(IN token, string keyword, BuildResult <ILexer <IN> > result)
{
NodeCallback <GenericToken> callback = match =>
{
IN derivedToken;
if (derivedTokens.TryGetValue(GenericToken.Identifier, out var derived))
{
if (!derived.TryGetValue(match.Result.Value, out derivedToken))
{
derivedToken = identifierDerivedToken;
}
}
else
{
derivedToken = identifierDerivedToken;
}
match.Properties[DerivedToken] = derivedToken;
return(match);
};
AddLexeme(GenericToken.Identifier, result, token, keyword);
var node = FSMBuilder.GetNode(in_identifier);
if (!FSMBuilder.Fsm.HasCallback(node.Id))
{
FSMBuilder.GoTo(in_identifier).CallBack(callback);
}
}
public static void SymbolExtension(LispLexer token, LexemeAttribute lexem, GenericLexer <LispLexer> lexer)
{
if (token == LispLexer.SYMBOL)
{
// callback on end_date node
NodeCallback <GenericToken> callback = (FSMMatch <GenericToken> match) =>
{
// this store the token id the the FSMMatch object to be later returned by GenericLexer.Tokenize
match.Properties[GenericLexer <LispLexer> .DerivedToken] = LispLexer.SYMBOL;
return(match);
};
var fsmBuilder = lexer.FSMBuilder;
var symbolCharExclusions = new char[]
{ '(', ')', '|', '#', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '.', ' ', '\'', '\r', '\n' };
fsmBuilder.GoTo(GenericLexer <LispLexer> .start)
.ExceptTransition(symbolCharExclusions)
.Mark("end_symbol")
.ExceptTransitionTo(symbolCharExclusions, "end_symbol")
.End(GenericToken.Extension)
.CallBack(callback);
var graph = fsmBuilder.Fsm.ToGraphViz();
;
}
}
public FSMLexerBuilder <N> CallBack(NodeCallback <N> callback)
{
if (Fsm.HasState(CurrentState))
{
Fsm.SetCallback(CurrentState, callback);
}
return(this);
}
private static NodeCallback <GenericToken> GetCallbackMulti <IN>(IN token, bool doNotIgnore) where IN : struct
{
NodeCallback <GenericToken> callbackMulti = match =>
{
match.Properties[GenericLexer <IN> .DerivedToken] = token;
match.Result.IsComment = true;
match.Result.Notignored = doNotIgnore;
match.Result.CommentType = CommentType.Multi;
return(match);
};
return(callbackMulti);
}
public void RemoveAgent(Agent agent)
{
if (trvCurPln.InvokeRequired) {
NodeCallback d = new NodeCallback(RemoveAgent);
trvCurPln.Invoke(d, new object[] { agent });
} else {
if (agent is Operator || agent is Controller) {
trvCurPln.Nodes[0].Nodes.Remove(agent);
} else {
trvCurPln.Nodes[1].Nodes[(int)agent.Type].Nodes.Remove(agent);
trvCurPln.Nodes[1].Nodes[(int)agent.Type].Text = agentTypes[(int)agent.Type] + " (" + trvCurPln.Nodes[1].Nodes[(int)agent.Type].Nodes.Count + ")";
}
}
}
public static void AddExtension(Extensions token, LexemeAttribute lexem, GenericLexer <Extensions> lexer)
{
if (token == Extensions.DATE)
{
NodeCallback <GenericToken> callback = match =>
{
match.Properties[GenericLexer <Extensions> .DerivedToken] = Extensions.DATE;
return(match);
};
var fsmBuilder = lexer.FSMBuilder;
fsmBuilder.GoTo(GenericLexer <Extensions> .in_double)
.Transition('.', CheckDate)
.Mark("start_date")
.RepetitionTransition(4, "[0-9]")
.End(GenericToken.Extension)
.CallBack(callback);
}
}
public void AddSugarLexem(IN token, string specialValue)
{
if (char.IsLetter(specialValue[0]))
{
throw new InvalidLexerException($"bad lexem {specialValue} : SugarToken lexeme <{token.ToString()}> can not start with a letter.");
}
NodeCallback <GenericToken> callback = (FSMMatch <GenericToken> match) =>
{
match.Properties[DerivedToken] = token;
return(match);
};
FSMBuilder.GoTo(start);
for (int i = 0; i < specialValue.Length; i++)
{
FSMBuilder.SafeTransition(specialValue[i]);
}
FSMBuilder.End(GenericToken.SugarToken)
.CallBack(callback);
}
public void AddNewAgent(Agent agent)
{
if (trvCurPln.InvokeRequired) {
NodeCallback d = new NodeCallback(AddNewAgent);
trvCurPln.Invoke(d, new object[] { agent });
} else {
// Check if a simulation agent has been added
if ((int)agent.Type < 4) {
trvCurPln.Nodes[1].Nodes[(int)agent.Type].Nodes.Add(agent);
trvCurPln.Nodes[1].Nodes[(int)agent.Type].Text = agentTypes[(int)agent.Type] + " (" + trvCurPln.Nodes[1].Nodes[(int)agent.Type].Nodes.Count + ")";
} else {
int newNode = trvCurPln.Nodes[0].Nodes.Add(agent);
trvCurPln.Nodes[0].Nodes[newNode].Text = agentTypes[(int)agent.Type];
}
int totAgents = 0;
for (int i = 0; i < trvCurPln.Nodes[1].Nodes.Count; i++) {
totAgents += trvCurPln.Nodes[1].Nodes[i].Nodes.Count;
}
trvCurPln.Nodes[1].Text = "Simulation Agents (" + totAgents + ")";
}
}
public void AddSugarLexem(IN token, BuildResult <ILexer <IN> > buildResult, string specialValue, bool isLineEnding = false)
{
if (char.IsLetter(specialValue[0]))
{
buildResult.AddError(new InitializationError(ErrorLevel.FATAL,
$"bad lexem {specialValue} : SugarToken lexeme <{token.ToString()}> can not start with a letter.", ErrorCodes.LEXER_SUGAR_TOKEN_CANNOT_START_WITH_LETTER));
return;
}
NodeCallback <GenericToken> callback = match =>
{
match.Properties[DerivedToken] = token;
return(match);
};
FSMBuilder.GoTo(start);
for (var i = 0; i < specialValue.Length; i++)
{
FSMBuilder.SafeTransition(specialValue[i]);
}
FSMBuilder.End(GenericToken.SugarToken, isLineEnding)
.CallBack(callback);
}
public void AddCharLexem(IN token, BuildResult <ILexer <IN> > result, string charDelimiter, string escapeDelimiterChar = "\\")
{
if (string.IsNullOrEmpty(charDelimiter) || charDelimiter.Length > 1)
{
result.AddError(new InitializationError(ErrorLevel.FATAL,
$"bad lexem {charDelimiter} : CharToken lexeme delimiter char <{token.ToString()}> must be 1 character length.",
ErrorCodes.LEXER_CHAR_ESCAPE_CHAR_MUST_BE_1_CHAR));
}
if (charDelimiter.Length == 1 && char.IsLetterOrDigit(charDelimiter[0]))
{
result.AddError(new InitializationError(ErrorLevel.FATAL,
$"bad lexem {charDelimiter} : CharToken lexeme delimiter char <{token.ToString()}> can not start with a letter or digit.", ErrorCodes.LEXER_CHAR_DELIMITER_CANNOT_BE_LETTER));
}
if (string.IsNullOrEmpty(escapeDelimiterChar) || escapeDelimiterChar.Length > 1)
{
result.AddError(new InitializationError(ErrorLevel.FATAL,
$"bad lexem {escapeDelimiterChar} : CharToken lexeme escape char <{token.ToString()}> must be 1 character length.", ErrorCodes.LEXER_CHAR_ESCAPE_CHAR_MUST_BE_1_CHAR));
}
if (escapeDelimiterChar.Length == 1 && char.IsLetterOrDigit(escapeDelimiterChar[0]))
{
result.AddError(new InitializationError(ErrorLevel.FATAL,
$"bad lexem {escapeDelimiterChar} : CharToken lexeme escape char lexeme <{token.ToString()}> can not start with a letter or digit.", ErrorCodes.LEXER_CHAR_ESCAPE_CHAR_CANNOT_BE_LETTER_OR_DIGIT));
}
CharCounter++;
var charDelimiterChar = charDelimiter[0];
var escapeChar = escapeDelimiterChar[0];
NodeCallback <GenericToken> callback = match =>
{
match.Properties[DerivedToken] = token;
var value = match.Result.SpanValue;
match.Result.SpanValue = value;
return(match);
};
FSMBuilder.GoTo(start);
FSMBuilder.Transition(charDelimiterChar)
.Mark(start_char + "_" + CharCounter)
.ExceptTransition(new[] { charDelimiterChar, escapeChar })
.Mark(in_char + "_" + CharCounter)
.Transition(charDelimiterChar)
.Mark(end_char + "_" + CharCounter)
.End(GenericToken.Char)
.CallBack(callback)
.GoTo(start_char + "_" + CharCounter)
.Transition(escapeChar)
.Mark(escapeChar_char + "_" + CharCounter)
.ExceptTransitionTo(new[] { 'u' }, in_char + "_" + CharCounter)
.CallBack(callback);
FSMBuilder.Fsm.StringDelimiter = charDelimiterChar;
// TODO : unicode transitions
FSMBuilder = FSMBuilder.GoTo(escapeChar_char + "_" + CharCounter)
.Transition('u')
.Mark(unicode_char + "_" + CharCounter)
.RepetitionTransitionTo(in_char + "_" + CharCounter, 4, "[0-9,a-z,A-Z]");
}
public void AddStringLexem(IN token, BuildResult <ILexer <IN> > result, string stringDelimiter,
string escapeDelimiterChar = "\\")
{
if (string.IsNullOrEmpty(stringDelimiter) || stringDelimiter.Length > 1)
{
result.AddError(new LexerInitializationError(ErrorLevel.FATAL, $"bad lexem {stringDelimiter} : StringToken lexeme delimiter char <{token.ToString()}> must be 1 character length.", ErrorCodes.LEXER_STRING_DELIMITER_MUST_BE_1_CHAR));
}
if (stringDelimiter.Length == 1 && char.IsLetterOrDigit(stringDelimiter[0]))
{
result.AddError(new InitializationError(ErrorLevel.FATAL, $"bad lexem {stringDelimiter} : StringToken lexeme delimiter char <{token.ToString()}> can not start with a letter.", ErrorCodes.LEXER_STRING_DELIMITER_CANNOT_BE_LETTER_OR_DIGIT));
}
if (string.IsNullOrEmpty(escapeDelimiterChar) || escapeDelimiterChar.Length > 1)
{
result.AddError(new InitializationError(ErrorLevel.FATAL, $"bad lexem {escapeDelimiterChar} : StringToken lexeme escape char <{token.ToString()}> must be 1 character length.", ErrorCodes.LEXER_STRING_ESCAPE_CHAR_MUST_BE_1_CHAR));
}
if (escapeDelimiterChar.Length == 1 && char.IsLetterOrDigit(escapeDelimiterChar[0]))
{
result.AddError(new InitializationError(ErrorLevel.FATAL, $"bad lexem {escapeDelimiterChar} : StringToken lexeme escape char lexeme <{token.ToString()}> can not start with a letter.", ErrorCodes.LEXER_STRING_ESCAPE_CHAR_CANNOT_BE_LETTER_OR_DIGIT));
}
StringDelimiterChar = (char)0;
var stringDelimiterChar = (char)0;
EscapeStringDelimiterChar = (char)0;
var escapeStringDelimiterChar = (char)0;
if (stringDelimiter.Length == 1)
{
StringCounter++;
StringDelimiterChar = stringDelimiter[0];
stringDelimiterChar = stringDelimiter[0];
EscapeStringDelimiterChar = escapeDelimiterChar[0];
escapeStringDelimiterChar = escapeDelimiterChar[0];
}
NodeCallback <GenericToken> callback = match =>
{
match.Properties[DerivedToken] = token;
var value = match.Result.SpanValue;
match.Result.SpanValue = value;
match.StringDelimiterChar = stringDelimiterChar;
match.IsString = true;
if (stringDelimiterChar != escapeStringDelimiterChar)
{
match.Result.SpanValue = diffCharEscaper(escapeStringDelimiterChar, stringDelimiterChar, match.Result.SpanValue);
}
else
{
match.Result.SpanValue = sameCharEscaper(escapeStringDelimiterChar, stringDelimiterChar, match.Result.SpanValue);
}
return(match);
};
if (stringDelimiterChar != escapeStringDelimiterChar)
{
FSMBuilder.GoTo(start);
FSMBuilder.Transition(stringDelimiterChar)
.Mark(in_string + StringCounter)
.ExceptTransitionTo(new[] { stringDelimiterChar, escapeStringDelimiterChar },
in_string + StringCounter)
.Transition(escapeStringDelimiterChar)
.Mark(escape_string + StringCounter)
.ExceptTransitionTo(new[] { stringDelimiterChar }, in_string + StringCounter)
.GoTo(escape_string + StringCounter)
.TransitionTo(stringDelimiterChar, in_string + StringCounter)
.Transition(stringDelimiterChar)
.End(GenericToken.String)
.Mark(string_end + StringCounter)
.CallBack(callback);
FSMBuilder.Fsm.StringDelimiter = stringDelimiterChar;
}
else
{
var exceptDelimiter = new[] { stringDelimiterChar };
var in_string = "in_string_same";
var escaped = "escaped_same";
var delim = "delim_same";
FSMBuilder.GoTo(start)
.Transition(stringDelimiterChar)
.Mark(in_string + StringCounter)
.ExceptTransitionTo(exceptDelimiter, in_string + StringCounter)
.Transition(stringDelimiterChar)
.Mark(escaped + StringCounter)
.End(GenericToken.String)
.CallBack(callback)
.Transition(stringDelimiterChar)
.Mark(delim + StringCounter)
.ExceptTransitionTo(exceptDelimiter, in_string + StringCounter);
FSMBuilder.GoTo(delim + StringCounter)
.TransitionTo(stringDelimiterChar, escaped + StringCounter)
.ExceptTransitionTo(exceptDelimiter, in_string + StringCounter);
}
}
/// <summary>Invokes a callback on each node</summary>
/// <param name="cb">Callbacke to be invoked</param>
public void ForAllNodes(NodeCallback cb)
{
foreach (Node node in _Nodes)
cb(node);
}
public void AddLexeme(GenericToken generic, IN token)
{
NodeCallback <GenericToken> callback = match =>
{
switch (match.Result.TokenID)
{
case GenericToken.Identifier:
{
if (derivedTokens.ContainsKey(GenericToken.Identifier))
{
var possibleTokens = derivedTokens[GenericToken.Identifier];
if (possibleTokens.ContainsKey(match.Result.Value))
{
match.Properties[DerivedToken] = possibleTokens[match.Result.Value];
}
else
{
match.Properties[DerivedToken] = identifierDerivedToken;
}
}
else
{
match.Properties[DerivedToken] = identifierDerivedToken;
}
break;
}
case GenericToken.Int:
{
match.Properties[DerivedToken] = intDerivedToken;
break;
}
case GenericToken.Double:
{
match.Properties[DerivedToken] = doubleDerivedToken;
break;
}
default:
{
match.Properties[DerivedToken] = token;
break;
}
}
return(match);
};
switch (generic)
{
case GenericToken.Double:
{
doubleDerivedToken = token;
FSMBuilder.GoTo(in_double);
FSMBuilder.CallBack(callback);
break;
}
case GenericToken.Int:
{
intDerivedToken = token;
FSMBuilder.GoTo(in_int);
FSMBuilder.CallBack(callback);
break;
}
case GenericToken.Identifier:
{
identifierDerivedToken = token;
FSMBuilder.GoTo(in_identifier);
FSMBuilder.CallBack(callback);
break;
}
}
}
internal void SetCallback(int nodeId, NodeCallback <N> callback)
{
Callbacks[nodeId] = callback;
}
public void AddStringLexem(IN token, string stringDelimiter, string escapeDelimiterChar = "\\")
{
if (string.IsNullOrEmpty(stringDelimiter) || stringDelimiter.Length > 1)
{
throw new InvalidLexerException($"bad lexem {stringDelimiter} : StringToken lexeme delimiter char <{token.ToString()}> must be 1 character length.");
}
if (char.IsLetterOrDigit(stringDelimiter[0]))
{
throw new InvalidLexerException($"bad lexem {stringDelimiter} : StringToken lexeme delimiter char <{token.ToString()}> can not start with a letter.");
}
if (string.IsNullOrEmpty(escapeDelimiterChar) || escapeDelimiterChar.Length > 1)
{
throw new InvalidLexerException($"bad lexem {escapeDelimiterChar} : StringToken lexeme escape char <{token.ToString()}> must be 1 character length.");
}
if (char.IsLetterOrDigit(escapeDelimiterChar[0]))
{
throw new InvalidLexerException($"bad lexem {escapeDelimiterChar} : StringToken lexeme escape char lexeme <{token.ToString()}> can not start with a letter.");
}
StringCounter++;
StringDelimiterChar = stringDelimiter[0];
EscapeStringDelimiterChar = escapeDelimiterChar[0];
NodeCallback <GenericToken> callback = (FSMMatch <GenericToken> match) =>
{
match.Properties[DerivedToken] = token;
string value = match.Result.Value;
match.Result.Value = value;
return(match);
};
if (StringDelimiterChar != EscapeStringDelimiterChar)
{
FSMBuilder.GoTo(start);
FSMBuilder.Transition(StringDelimiterChar)
.Mark(in_string + StringCounter)
.ExceptTransitionTo(new char[] { StringDelimiterChar, EscapeStringDelimiterChar }, in_string + StringCounter)
.Transition(EscapeStringDelimiterChar)
.Mark(escape_string + StringCounter)
.AnyTransitionTo(' ', in_string + StringCounter)
.Transition(StringDelimiterChar)
.End(GenericToken.String)
.Mark(string_end + StringCounter)
.CallBack(callback);
FSMBuilder.Fsm.StringDelimiter = StringDelimiterChar;
}
else
{
NodeAction collapseDelimiter = (string value) =>
{
if (value.EndsWith("" + StringDelimiterChar + StringDelimiterChar))
{
return(value.Substring(0, value.Length - 2) + StringDelimiterChar);
}
return(value);
};
var exceptDelimiter = new char[] { StringDelimiterChar };
string in_string = "in_string_same";
string escaped = "escaped_same";
string delim = "delim_same";
FSMBuilder.GoTo(start)
.Transition(StringDelimiterChar)
.Mark(in_string + StringCounter)
.ExceptTransitionTo(exceptDelimiter, in_string + StringCounter)
.Transition(StringDelimiterChar)
.Mark(escaped + StringCounter)
.End(GenericToken.String)
.CallBack(callback)
.Transition(StringDelimiterChar)
.Mark(delim + StringCounter)
.Action(collapseDelimiter)
.ExceptTransitionTo(exceptDelimiter, in_string + StringCounter);
FSMBuilder.GoTo(delim + StringCounter)
.TransitionTo(StringDelimiterChar, escaped + StringCounter)
.ExceptTransitionTo(exceptDelimiter, in_string + StringCounter);
}
}
public static void AddExtension(Extensions token, LexemeAttribute lexem, GenericLexer <Extensions> lexer)
{
if (token == Extensions.DATE)
{
NodeCallback <GenericToken> callback = match =>
{
match.Properties[GenericLexer <Extensions> .DerivedToken] = Extensions.DATE;
return(match);
};
var fsmBuilder = lexer.FSMBuilder;
fsmBuilder.GoTo(GenericLexer <Extensions> .in_double)
.Transition('.', CheckDate)
.Mark("start_date")
.RepetitionTransition(4, "[0-9]")
.End(GenericToken.Extension)
.CallBack(callback);
}
else if (token == Extensions.CHAINE)
{
NodeCallback <GenericToken> callback = match =>
{
match.Properties[GenericLexer <Extensions> .DerivedToken] = Extensions.CHAINE;
return(match);
};
var quote = '\'';
NodeAction collapseDelimiter = value =>
{
if (value.EndsWith("" + quote + quote))
{
return(value.Substring(0, value.Length - 2) + quote);
}
return(value);
};
var exceptQuote = new[] { quote };
var in_string = "in_string_same";
var escaped = "escaped_same";
var delim = "delim_same";
var fsmBuilder = lexer.FSMBuilder;
fsmBuilder.GoTo(GenericLexer <Extensions> .start)
.Transition(quote)
.Mark(in_string)
.ExceptTransitionTo(exceptQuote, in_string)
.Transition(quote)
.Mark(escaped)
.End(GenericToken.String)
.CallBack(callback)
.Transition(quote)
.Mark(delim)
.Action(collapseDelimiter)
.ExceptTransitionTo(exceptQuote, in_string);
fsmBuilder.GoTo(delim)
.TransitionTo(quote, escaped)
.ExceptTransitionTo(exceptQuote, in_string);
}
}
private static BuildResult <ILexer <IN> > BuildGenericLexer <IN>(Dictionary <IN, List <LexemeAttribute> > attributes,
BuildExtension <IN> extensionBuilder, BuildResult <ILexer <IN> > result) where IN : struct
{
result = CheckStringAndCharTokens(attributes, result);
var(config, tokens) = GetConfigAndGenericTokens(attributes);
config.ExtensionBuilder = extensionBuilder;
var lexer = new GenericLexer <IN>(config, tokens);
var Extensions = new Dictionary <IN, LexemeAttribute>();
foreach (var pair in attributes)
{
var tokenID = pair.Key;
var lexemes = pair.Value;
foreach (var lexeme in lexemes)
{
try
{
if (lexeme.IsStaticGeneric)
{
lexer.AddLexeme(lexeme.GenericToken, tokenID);
}
if (lexeme.IsKeyWord)
{
foreach (var param in lexeme.GenericTokenParameters)
{
lexer.AddKeyWord(tokenID, param);
}
}
if (lexeme.IsSugar)
{
foreach (var param in lexeme.GenericTokenParameters)
{
lexer.AddSugarLexem(tokenID, param, lexeme.IsLineEnding);
}
}
if (lexeme.IsString)
{
var(delimiter, escape) = GetDelimiters(lexeme, "\"", "\\");
lexer.AddStringLexem(tokenID, delimiter, escape);
}
if (lexeme.IsChar)
{
var(delimiter, escape) = GetDelimiters(lexeme, "'", "\\");
lexer.AddCharLexem(tokenID, delimiter, escape);
}
if (lexeme.IsExtension)
{
Extensions[tokenID] = lexeme;
}
}
catch (Exception e)
{
result.AddError(new InitializationError(ErrorLevel.FATAL, e.Message));
}
}
}
AddExtensions(Extensions, extensionBuilder, lexer);
var comments = GetCommentsAttribute(result);
if (!result.IsError)
{
foreach (var comment in comments)
{
NodeCallback <GenericToken> callbackSingle = match =>
{
match.Properties[GenericLexer <IN> .DerivedToken] = comment.Key;
match.Result.IsComment = true;
match.Result.CommentType = CommentType.Single;
return(match);
};
NodeCallback <GenericToken> callbackMulti = match =>
{
match.Properties[GenericLexer <IN> .DerivedToken] = comment.Key;
match.Result.IsComment = true;
match.Result.CommentType = CommentType.Multi;
return(match);
};
foreach (var commentAttr in comment.Value)
{
var fsmBuilder = lexer.FSMBuilder;
var hasSingleLine = !string.IsNullOrWhiteSpace(commentAttr.SingleLineCommentStart);
if (hasSingleLine)
{
lexer.SingleLineComment = commentAttr.SingleLineCommentStart;
fsmBuilder.GoTo(GenericLexer <IN> .start);
fsmBuilder.ConstantTransition(commentAttr.SingleLineCommentStart);
fsmBuilder.Mark(GenericLexer <IN> .single_line_comment_start);
fsmBuilder.End(GenericToken.Comment);
fsmBuilder.CallBack(callbackSingle);
}
var hasMultiLine = !string.IsNullOrWhiteSpace(commentAttr.MultiLineCommentStart);
if (hasMultiLine)
{
lexer.MultiLineCommentStart = commentAttr.MultiLineCommentStart;
lexer.MultiLineCommentEnd = commentAttr.MultiLineCommentEnd;
fsmBuilder.GoTo(GenericLexer <IN> .start);
fsmBuilder.ConstantTransition(commentAttr.MultiLineCommentStart);
fsmBuilder.Mark(GenericLexer <IN> .multi_line_comment_start);
fsmBuilder.End(GenericToken.Comment);
fsmBuilder.CallBack(callbackMulti);
}
}
}
}
result.Result = lexer;
return(result);
}
public void AddCharLexem(IN token, string charDelimiter, string escapeDelimiterChar = "\\")
{
if (string.IsNullOrEmpty(charDelimiter) || charDelimiter.Length > 1)
{
throw new InvalidLexerException(
$"bad lexem {charDelimiter} : CharToken lexeme delimiter char <{token.ToString()}> must be 1 character length.");
}
if (char.IsLetterOrDigit(charDelimiter[0]))
{
throw new InvalidLexerException(
$"bad lexem {charDelimiter} : CharToken lexeme delimiter char <{token.ToString()}> can not start with a letter.");
}
if (string.IsNullOrEmpty(escapeDelimiterChar) || escapeDelimiterChar.Length > 1)
{
throw new InvalidLexerException(
$"bad lexem {escapeDelimiterChar} : CharToken lexeme escape char <{token.ToString()}> must be 1 character length.");
}
if (char.IsLetterOrDigit(escapeDelimiterChar[0]))
{
throw new InvalidLexerException(
$"bad lexem {escapeDelimiterChar} : CharToken lexeme escape char lexeme <{token.ToString()}> can not start with a letter.");
}
CharCounter++;
var charDelimiterChar = charDelimiter[0];
var escapeChar = escapeDelimiterChar[0];
NodeCallback <GenericToken> callback = match =>
{
match.Properties[DerivedToken] = token;
var value = match.Result.SpanValue;
match.Result.SpanValue = value;
return(match);
};
FSMBuilder.GoTo(start);
FSMBuilder.Transition(charDelimiterChar)
.Mark(start_char + "_" + CharCounter)
.ExceptTransition(new[] { charDelimiterChar, escapeChar })
.Mark(in_char + "_" + CharCounter)
.Transition(charDelimiterChar)
.Mark(end_char + "_" + CharCounter)
.End(GenericToken.Char)
.CallBack(callback)
.GoTo(start_char + "_" + CharCounter)
.Transition(escapeChar)
.Mark(escapeChar_char + "_" + CharCounter)
.ExceptTransitionTo(new[] { 'u' }, in_char + "_" + CharCounter)
.CallBack(callback);
FSMBuilder.Fsm.StringDelimiter = charDelimiterChar;
// TODO : unicode transitions
FSMBuilder = FSMBuilder.GoTo(escapeChar_char + "_" + CharCounter)
.Transition('u')
.Mark(unicode_char + "_" + CharCounter)
.RepetitionTransitionTo(in_char + "_" + CharCounter, 4, "[0-9,a-z,A-Z]");
}
public void AddStringLexem(IN token, string stringDelimiter, string escapeDelimiterChar = "\\")
{
if (string.IsNullOrEmpty(stringDelimiter) || stringDelimiter.Length > 1)
{
throw new InvalidLexerException(
$"bad lexem {stringDelimiter} : StringToken lexeme delimiter char <{token.ToString()}> must be 1 character length.");
}
if (char.IsLetterOrDigit(stringDelimiter[0]))
{
throw new InvalidLexerException(
$"bad lexem {stringDelimiter} : StringToken lexeme delimiter char <{token.ToString()}> can not start with a letter.");
}
if (string.IsNullOrEmpty(escapeDelimiterChar) || escapeDelimiterChar.Length > 1)
{
throw new InvalidLexerException(
$"bad lexem {escapeDelimiterChar} : StringToken lexeme escape char <{token.ToString()}> must be 1 character length.");
}
if (char.IsLetterOrDigit(escapeDelimiterChar[0]))
{
throw new InvalidLexerException(
$"bad lexem {escapeDelimiterChar} : StringToken lexeme escape char lexeme <{token.ToString()}> can not start with a letter.");
}
StringCounter++;
StringDelimiterChar = stringDelimiter[0];
var stringDelimiterChar = stringDelimiter[0];
EscapeStringDelimiterChar = escapeDelimiterChar[0];
var escapeStringDelimiterChar = escapeDelimiterChar[0];
NodeCallback <GenericToken> callback = match =>
{
match.Properties[DerivedToken] = token;
var value = match.Result.SpanValue;
match.Result.SpanValue = value;
if (stringDelimiterChar != escapeStringDelimiterChar)
{
match.Result.SpanValue = diffCharEscaper(escapeStringDelimiterChar, stringDelimiterChar, match.Result.SpanValue);
}
else
{
match.Result.SpanValue = sameCharEscaper(escapeStringDelimiterChar, stringDelimiterChar, match.Result.SpanValue);
}
return(match);
};
if (stringDelimiterChar != escapeStringDelimiterChar)
{
FSMBuilder.GoTo(start);
FSMBuilder.Transition(stringDelimiterChar)
.Mark(in_string + StringCounter)
.ExceptTransitionTo(new[] { stringDelimiterChar, escapeStringDelimiterChar },
in_string + StringCounter)
.Transition(escapeStringDelimiterChar)
.Mark(escape_string + StringCounter)
.ExceptTransitionTo(new[] { stringDelimiterChar }, in_string + StringCounter)
.GoTo(escape_string + StringCounter)
.TransitionTo(stringDelimiterChar, in_string + StringCounter)
.Transition(stringDelimiterChar)
.End(GenericToken.String)
.Mark(string_end + StringCounter)
.CallBack(callback);
FSMBuilder.Fsm.StringDelimiter = stringDelimiterChar;
}
else
{
var exceptDelimiter = new[] { stringDelimiterChar };
var in_string = "in_string_same";
var escaped = "escaped_same";
var delim = "delim_same";
FSMBuilder.GoTo(start)
.Transition(stringDelimiterChar)
.Mark(in_string + StringCounter)
.ExceptTransitionTo(exceptDelimiter, in_string + StringCounter)
.Transition(stringDelimiterChar)
.Mark(escaped + StringCounter)
.End(GenericToken.String)
.CallBack(callback)
.Transition(stringDelimiterChar)
.Mark(delim + StringCounter)
.ExceptTransitionTo(exceptDelimiter, in_string + StringCounter);
FSMBuilder.GoTo(delim + StringCounter)
.TransitionTo(stringDelimiterChar, escaped + StringCounter)
.ExceptTransitionTo(exceptDelimiter, in_string + StringCounter);
}
}
public TreeIterator(System.Windows.Forms.TreeView tree, NodeCallback cb)
{
m_tree = tree;
m_callback = cb;
}
public TreeIterator(System.Windows.Forms.TreeView tree, NodeCallback cb)
{
m_tree = tree;
m_callback = cb;
}
public void AddLexeme(GenericToken generic, IN token)
{
if (generic == GenericToken.Identifier)
{
identifierDerivedToken = token;
var derived = new Dictionary <string, IN>();
if (derivedTokens.ContainsKey(generic))
{
derived = derivedTokens[generic];
}
derived[defaultIdentifierKey] = token;
//return;
}
if (generic == GenericToken.Double)
{
doubleDerivedToken = token;
}
if (generic == GenericToken.Int)
{
intDerivedToken = token;
}
NodeCallback <GenericToken> callback = (FSMMatch <GenericToken> match) =>
{
switch (match.Result.TokenID)
{
case GenericToken.Identifier:
{
if (derivedTokens.ContainsKey(GenericToken.Identifier))
{
var possibleTokens = derivedTokens[GenericToken.Identifier];
if (possibleTokens.ContainsKey(match.Result.Value))
{
match.Properties[DerivedToken] = possibleTokens[match.Result.Value];
}
else
{
match.Properties[DerivedToken] = identifierDerivedToken;
}
}
else
{
match.Properties[DerivedToken] = identifierDerivedToken;
}
;
break;
}
case GenericToken.Int:
{
match.Properties[DerivedToken] = intDerivedToken;
break;
}
case GenericToken.Double:
{
match.Properties[DerivedToken] = doubleDerivedToken;
break;
}
default:
{
match.Properties[DerivedToken] = token;
break;
}
}
return(match);
};
switch (generic)
{
case GenericToken.Double:
{
FSMBuilder.GoTo(in_double);
FSMBuilder.CallBack(callback);
break;
}
case GenericToken.Int:
{
FSMBuilder.GoTo(in_int);
FSMBuilder.CallBack(callback);
break;
}
case GenericToken.Identifier:
{
FSMBuilder.GoTo(in_identifier);
FSMBuilder.CallBack(callback);
break;
}
}
}
public TreeIterator(TreeView tree, NodeCallback cb)
{
m_tree = tree;
m_callback = cb;
}
