public GenericLexer(Config config, GenericToken[] staticTokens)
{
derivedTokens = new Dictionary <GenericToken, Dictionary <string, IN> >();
ExtensionBuilder = config.ExtensionBuilder;
KeyWordComparer = config.KeyWordComparer;
InitializeStaticLexer(config, staticTokens);
}
public GenericLexer(IdentifierType idType = IdentifierType.Alpha, BuildExtension <IN> extensionBuilder = null,
params GenericToken[] staticTokens)
{
InitializeStaticLexer(idType, staticTokens);
derivedTokens = new Dictionary <GenericToken, Dictionary <string, IN> >();
ExtensionBuilder = extensionBuilder;
}
public GenericLexer(IdentifierType idType = IdentifierType.Alpha,
BuildExtension <IN> extensionBuilder = null,
params GenericToken[] staticTokens)
: this(new Config {
IdType = idType, ExtensionBuilder = extensionBuilder
}, staticTokens)
{
}
public override BuildResult <Parser <IN, OUT> > BuildParser(object parserInstance, ParserType parserType,
string rootRule, BuildExtension <IN> extensionBuilder = null)
{
var ruleparser = new RuleParser <IN>();
var builder = new ParserBuilder <EbnfTokenGeneric, GrammarNode <IN> >();
var grammarParser = builder.BuildParser(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule").Result;
var result = new BuildResult <Parser <IN, OUT> >();
ParserConfiguration <IN, OUT> configuration = null;
try
{
configuration = ExtractEbnfParserConfiguration(parserInstance.GetType(), grammarParser);
configuration.StartingRule = rootRule;
}
catch (Exception e)
{
result.AddError(new ParserInitializationError(ErrorLevel.ERROR, e.Message, ErrorCodes.PARSER_UNKNOWN_ERROR));
return(result);
}
var syntaxParser = BuildSyntaxParser(configuration, parserType, rootRule);
SyntaxTreeVisitor <IN, OUT> visitor = null;
if (parserType == ParserType.LL_RECURSIVE_DESCENT)
{
new SyntaxTreeVisitor <IN, OUT>(configuration, parserInstance);
}
else if (parserType == ParserType.EBNF_LL_RECURSIVE_DESCENT)
{
visitor = new EBNFSyntaxTreeVisitor <IN, OUT>(configuration, parserInstance);
}
var parser = new Parser <IN, OUT>(syntaxParser, visitor);
parser.Configuration = configuration;
var lexerResult = BuildLexer(extensionBuilder);
if (lexerResult.IsError)
{
foreach (var lexerResultError in lexerResult.Errors)
{
result.AddError(lexerResultError);
}
return(result);
}
else
{
parser.Lexer = lexerResult.Result;
}
parser.Instance = parserInstance;
result.Result = parser;
return(result);
}
public static BuildResult <ILexer <IN> > BuildLexer <IN>(BuildResult <ILexer <IN> > result,
BuildExtension <IN> extensionBuilder = null) where IN : struct
{
var attributes = GetLexemes(result);
result = Build(attributes, result, extensionBuilder);
return(result);
}
private static void AddExtensions <IN>(Dictionary <IN, LexemeAttribute> extensions,
BuildExtension <IN> extensionBuilder, GenericLexer <IN> lexer) where IN : struct
{
if (extensionBuilder != null)
{
foreach (var attr in extensions)
{
extensionBuilder(attr.Key, attr.Value, lexer);
}
}
}
/// <summary>
/// Builds a parser (lexer, syntax parser and syntax tree visitor) according to a parser definition instance
/// </summary>
/// <typeparam name="IN"></typeparam>
/// <param name="parserInstance">
/// a parser definition instance , containing
/// [Reduction] methods for grammar rules
/// <param name="parserType">
/// a ParserType enum value stating the analyser type (LR, LL ...) for now only LL recurive
/// descent parser available
/// </param>
/// <param name="rootRule">the name of the root non terminal of the grammar</param>
/// <returns></returns>
public virtual BuildResult <Parser <IN, OUT> > BuildParser(object parserInstance, ParserType parserType,
string rootRule, BuildExtension <IN> extensionBuilder = null)
{
Parser <IN, OUT> parser = null;
var result = new BuildResult <Parser <IN, OUT> >();
if (parserType == ParserType.LL_RECURSIVE_DESCENT)
{
var configuration = ExtractParserConfiguration(parserInstance.GetType());
configuration.StartingRule = rootRule;
var syntaxParser = BuildSyntaxParser(configuration, parserType, rootRule);
var visitor = new SyntaxTreeVisitor <IN, OUT>(configuration, parserInstance);
parser = new Parser <IN, OUT>(syntaxParser, visitor);
var lexerResult = BuildLexer(extensionBuilder);
parser.Lexer = lexerResult.Result;
if (lexerResult.IsError)
{
result.Errors.AddRange(lexerResult.Errors);
return(result);
}
parser.Instance = parserInstance;
parser.Configuration = configuration;
result.Result = parser;
}
else if (parserType == ParserType.EBNF_LL_RECURSIVE_DESCENT)
{
var builder = new EBNFParserBuilder <IN, OUT>();
result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, rootRule,
extensionBuilder);
}
parser = result.Result;
if (!result.IsError)
{
var expressionResult = parser.BuildExpressionParser(result, rootRule);
if (expressionResult.IsError)
{
result.AddErrors(expressionResult.Errors);
}
result.Result.Configuration = expressionResult.Result;
result = CheckParser(result);
if (result.IsError)
{
result.Result = null;
}
}
else
{
result.Result = null;
}
return(result);
}
public static BuildResult <ILexer <IN> > BuildLexer <IN>(BuildResult <ILexer <IN> > result,
BuildExtension <IN> extensionBuilder = null) where IN : struct
{
var type = typeof(IN);
var typeInfo = type.GetTypeInfo();
ILexer <IN> lexer = new Lexer <IN>();
var attributes = GetLexemes(result);
result = Build(attributes, result, extensionBuilder);
return(result);
}
private static BuildResult <ILexer <IN> > Build <IN>(Dictionary <IN, List <LexemeAttribute> > attributes,
BuildResult <ILexer <IN> > result, BuildExtension <IN> extensionBuilder = null) where IN : struct
{
var hasRegexLexem = IsRegexLexer(attributes);
var hasGenericLexem = IsGenericLexer(attributes);
if (hasGenericLexem && hasRegexLexem)
{
result.AddError(new LexerInitializationError(ErrorLevel.WARN,
"cannot mix Regex lexemes and Generic lexemes in same lexer"));
result.IsError = true;
}
else
{
if (hasRegexLexem)
{
result = BuildRegexLexer(attributes, result);
}
else if (hasGenericLexem)
{
result = BuildGenericLexer(attributes, extensionBuilder, result);
}
}
return(result);
}
/// <summary>
/// Builds a parser (lexer, syntax parser and syntax tree visitor) according to a parser definition instance
/// </summary>
/// <typeparam name="IN"></typeparam>
/// <param name="parserInstance">
/// a parser definition instance , containing
/// [Reduction] methods for grammar rules
/// <param name="parserType">
/// a ParserType enum value stating the analyser type (LR, LL ...) for now only LL recurive
/// descent parser available
/// </param>
/// <param name="rootRule">the name of the root non terminal of the grammar</param>
/// <returns></returns>
public virtual BuildResult <Parser <IN, OUT> > BuildParser(object parserInstance, ParserType parserType,
string rootRule, BuildExtension <IN> extensionBuilder = null)
{
Parser <IN, OUT> parser = null;
var result = new BuildResult <Parser <IN, OUT> >();
if (parserType == ParserType.LL_RECURSIVE_DESCENT)
{
var configuration = ExtractParserConfiguration(parserInstance.GetType());
var(foundRecursion, recursions) = LeftRecursionChecker <IN, OUT> .CheckLeftRecursion(configuration);
if (foundRecursion)
{
var recs = string.Join("\n", recursions.Select(x => string.Join(" > ", x)));
result.AddError(new ParserInitializationError(ErrorLevel.FATAL, $"left recursion detected : {recs}", ErrorCodes.PARSER_LEFT_RECURSIVE));
return(result);
}
configuration.StartingRule = rootRule;
var syntaxParser = BuildSyntaxParser(configuration, parserType, rootRule);
var visitor = new SyntaxTreeVisitor <IN, OUT>(configuration, parserInstance);
parser = new Parser <IN, OUT>(syntaxParser, visitor);
var lexerResult = BuildLexer(extensionBuilder);
parser.Lexer = lexerResult.Result;
if (lexerResult.IsError)
{
result.Errors.AddRange(lexerResult.Errors);
return(result);
}
parser.Instance = parserInstance;
parser.Configuration = configuration;
result.Result = parser;
}
else if (parserType == ParserType.EBNF_LL_RECURSIVE_DESCENT)
{
var builder = new EBNFParserBuilder <IN, OUT>();
result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, rootRule,
extensionBuilder);
}
parser = result.Result;
if (!result.IsError)
{
var expressionResult = parser.BuildExpressionParser(result, rootRule);
if (expressionResult.IsError)
{
result.AddErrors(expressionResult.Errors);
}
result.Result.Configuration = expressionResult.Result;
result = CheckParser(result);
if (result.IsError)
{
result.Result = null;
}
}
else
{
result.Result = null;
}
return(result);
}
protected virtual BuildResult <ILexer <IN> > BuildLexer(BuildExtension <IN> extensionBuilder = null)
{
var lexer = LexerBuilder.BuildLexer(new BuildResult <ILexer <IN> >(), extensionBuilder);
return(lexer);
}
public static BuildResult <ILexer <IN> > BuildLexer <IN>(BuildExtension <IN> extensionBuilder = null) where IN : struct
{
return(BuildLexer(new BuildResult <ILexer <IN> >(), extensionBuilder));
}
private static BuildResult <ILexer <IN> > BuildGenericLexer <IN>(Dictionary <IN, List <LexemeAttribute> > attributes,
BuildExtension <IN> extensionBuilder, BuildResult <ILexer <IN> > result) where IN : struct
{
result = CheckStringAndCharTokens(attributes, result);
var(config, tokens) = GetConfigAndGenericTokens(attributes);
config.ExtensionBuilder = extensionBuilder;
var lexer = new GenericLexer <IN>(config, tokens);
var Extensions = new Dictionary <IN, LexemeAttribute>();
foreach (var pair in attributes)
{
var tokenID = pair.Key;
var lexemes = pair.Value;
foreach (var lexeme in lexemes)
{
try
{
if (lexeme.IsStaticGeneric)
{
lexer.AddLexeme(lexeme.GenericToken, tokenID);
}
if (lexeme.IsKeyWord)
{
foreach (var param in lexeme.GenericTokenParameters)
{
lexer.AddKeyWord(tokenID, param);
}
}
if (lexeme.IsSugar)
{
foreach (var param in lexeme.GenericTokenParameters)
{
lexer.AddSugarLexem(tokenID, param, lexeme.IsLineEnding);
}
}
if (lexeme.IsString)
{
var(delimiter, escape) = GetDelimiters(lexeme, "\"", "\\");
lexer.AddStringLexem(tokenID, delimiter, escape);
}
if (lexeme.IsChar)
{
var(delimiter, escape) = GetDelimiters(lexeme, "'", "\\");
lexer.AddCharLexem(tokenID, delimiter, escape);
}
if (lexeme.IsExtension)
{
Extensions[tokenID] = lexeme;
}
}
catch (Exception e)
{
result.AddError(new InitializationError(ErrorLevel.FATAL, e.Message));
}
}
}
AddExtensions(Extensions, extensionBuilder, lexer);
var comments = GetCommentsAttribute(result);
if (!result.IsError)
{
foreach (var comment in comments)
{
NodeCallback <GenericToken> callbackSingle = match =>
{
match.Properties[GenericLexer <IN> .DerivedToken] = comment.Key;
match.Result.IsComment = true;
match.Result.CommentType = CommentType.Single;
return(match);
};
NodeCallback <GenericToken> callbackMulti = match =>
{
match.Properties[GenericLexer <IN> .DerivedToken] = comment.Key;
match.Result.IsComment = true;
match.Result.CommentType = CommentType.Multi;
return(match);
};
foreach (var commentAttr in comment.Value)
{
var fsmBuilder = lexer.FSMBuilder;
var hasSingleLine = !string.IsNullOrWhiteSpace(commentAttr.SingleLineCommentStart);
if (hasSingleLine)
{
lexer.SingleLineComment = commentAttr.SingleLineCommentStart;
fsmBuilder.GoTo(GenericLexer <IN> .start);
fsmBuilder.ConstantTransition(commentAttr.SingleLineCommentStart);
fsmBuilder.Mark(GenericLexer <IN> .single_line_comment_start);
fsmBuilder.End(GenericToken.Comment);
fsmBuilder.CallBack(callbackSingle);
}
var hasMultiLine = !string.IsNullOrWhiteSpace(commentAttr.MultiLineCommentStart);
if (hasMultiLine)
{
lexer.MultiLineCommentStart = commentAttr.MultiLineCommentStart;
lexer.MultiLineCommentEnd = commentAttr.MultiLineCommentEnd;
fsmBuilder.GoTo(GenericLexer <IN> .start);
fsmBuilder.ConstantTransition(commentAttr.MultiLineCommentStart);
fsmBuilder.Mark(GenericLexer <IN> .multi_line_comment_start);
fsmBuilder.End(GenericToken.Comment);
fsmBuilder.CallBack(callbackMulti);
}
}
}
}
result.Result = lexer;
return(result);
}
private static BuildResult <ILexer <IN> > Build <IN>(Dictionary <IN, List <LexemeAttribute> > attributes,
BuildResult <ILexer <IN> > result, BuildExtension <IN> extensionBuilder = null) where IN : struct
{
var hasRegexLexemes = IsRegexLexer(attributes);
var hasGenericLexemes = IsGenericLexer(attributes);
if (hasGenericLexemes && hasRegexLexemes)
{
result.AddError(new LexerInitializationError(ErrorLevel.ERROR,
"cannot mix Regex lexemes and Generic lexemes in same lexer", ErrorCodes.LEXER_CANNOT_MIX_GENERIC_AND_REGEX));
}
else
{
if (hasRegexLexemes)
{
result = BuildRegexLexer(attributes, result);
}
else if (hasGenericLexemes)
{
result = BuildGenericLexer(attributes, extensionBuilder, result);
}
}
return(result);
}
