/// <summary>Given Rules paired with LNodes produced by <see cref="StageOneParser"/>,
/// this method translates each LNode into a <see cref="Pred"/> and updates
/// <see cref="Rule.Pred"/> to point to the new Pred.</summary>
public void Parse(IEnumerable <Pair <Rule, LNode> > rules)
{
_rules = LLParserGenerator.AddRulesToDict(rules.Select(p => p.A));
foreach (var pair in rules)
{
Debug.Assert(pair.A.Pred == null);
pair.A.Pred = NodeToPred(pair.B);
if (pair.A.HasRecognizerVersion) // oops, need to keep recognizer in sync with main rule
{
pair.A.MakeRecognizerVersion().Pred = pair.A.Pred;
}
}
Remove_any_in_Labels();
}
private static void ApplyOptions(LNode node, LLParserGenerator lllpg, IMessageSink sink)
{
for (int i = 0; i < node.Attrs.Count; i++)
{
var attr = node.Attrs[i];
switch (attr.Name.Name)
{
case "FullLLk":
ReadOption <bool>(sink, attr, v => lllpg.FullLLk = v, true);
break;
case "Verbosity":
ReadOption <int>(sink, attr, v => lllpg.Verbosity = v, null);
break;
case "NoDefaultArm":
ReadOption <bool>(sink, attr, v => lllpg.NoDefaultArm = v, true);
break;
case "DefaultK":
case "k":
case "K":
case "LL":
ReadOption <int>(sink, attr, v => lllpg.DefaultK = v, null);
break;
case "AddComments":
ReadOption <bool>(sink, attr, v => lllpg.AddComments = v, true);
break;
default:
sink.Write(Severity.Error, attr,
"Unrecognized attribute. LLLPG supports the following options: " +
"FullLLk(bool), Verbosity(0..3), NoDefaultArm(bool), and DefaultK(1..9)");
break;
}
}
}
/// <summary>Given Rules paired with LNodes produced by <see cref="StageOneParser"/>,
/// this method translates each LNode into a <see cref="Pred"/> and updates
/// <see cref="Rule.Pred"/> to point to the new Pred.</summary>
public void Parse(IEnumerable <Pair <Rule, LNode> > rules)
{
_rules = LLParserGenerator.AddRulesToDict(rules.Select(p => p.A));
foreach (var pair in rules)
{
Debug.Assert(pair.A.Pred == null);
_parsingRecognizerVersion = _ruleChangesInRecognizer = false;
pair.A.Pred = NodeToPred(pair.B);
if (_ruleChangesInRecognizer)
{
_parsingRecognizerVersion = true;
pair.A.GetOrMakeRecognizerVersion().Pred = NodeToPred(pair.B);
}
else if (pair.A.HasRecognizerVersion)
{
pair.A.Recognizer.Pred = pair.A.Pred;
}
}
Remove_any_in_Labels();
}
public GenerateCodeVisitor(LLParserGenerator llpg)
{
LLPG = llpg;
F = new LNodeFactory(llpg._sourceFile);
_classBody = llpg._classBody;
}
public bool HasDefaultErrorBranch(LLParserGenerator llpg)
{
return(ErrorBranch == null ? llpg.NoDefaultArm : ErrorBranch == DefaultErrorBranch.Value);
}
public bool HasErrorBranch(LLParserGenerator llpg)
{
return(ErrorBranch != null || (DefaultArm == null && llpg.NoDefaultArm));
}
int _ruleDepth = 0; // Used to avoid stack overflow, which crashes VS in SFG
public ApplyPrematchVisitor(LLParserGenerator llpg)
{
LLPG = llpg;
}
public PrematchAnalysisVisitor(LLParserGenerator llpg)
{
LLPG = llpg;
Anything = LLPG.CodeGenHelper.EmptySet.Inverted();
_apply = new ApplyPrematchVisitor(llpg);
}
public PredictionAnalysisVisitor(LLParserGenerator llpg)
{
LLPG = llpg;
}
public static LNode run_LLLPG(LNode node, IMessageSink sink)
{
IPGCodeGenHelper helper;
LNode body;
bool hasBraces = true;
if (node.ArgCount != 2 ||
(helper = node.Args[0].Value as IPGCodeGenHelper) == null ||
!(hasBraces = (body = node.Args[1]).Calls(S.Braces)))
{
string msg = Localize.From("Expected run_LLLPG(helper_object, {...}).");
if (hasBraces)
{
msg = " " + Localize.From("An auxiliary macro is required to supply the helper object.");
}
sink.Write(Severity.Note, node, msg);
return(null);
}
helper = helper ?? new GeneralCodeGenHelper();
var rules = new List <Pair <Rule, LNode> >();
var stmts = new List <LNode>();
// Let helper preprocess the code if it wants to
foreach (var stmt in body.Args)
{
var stmt2 = helper.VisitInput(stmt, sink) ?? stmt;
if (stmt2.Calls(S.Splice))
{
stmts.AddRange(stmt2.Args);
}
else
{
stmts.Add(stmt2);
}
}
// Find rule definitions, create Rule objects
for (int i = 0; i < stmts.Count; i++)
{
LNode stmt = stmts[i];
bool isToken;
if ((isToken = stmt.Calls(_hash_token, 4)) || stmt.Calls(_hash_rule, 4))
{
LNode basis = stmt.WithTarget(S.Fn);
LNode methodBody = stmt.Args.Last;
// basis has the form #fn(ReturnType, Name, #(Args))
var rule = MakeRuleObject(isToken, ref basis, sink);
if (rule != null)
{
var prev = rules.FirstOrDefault(pair => pair.A.Name == rule.Name);
if (prev.A != null)
{
sink.Write(Severity.Error, rule.Basis, "The rule name «{0}» was used before at {1}", rule.Name, prev.A.Basis.Range.Start);
}
else
{
rules.Add(Pair.Create(rule, methodBody));
stmts[i] = null; // remove processed rules from the list
}
}
}
else
{
if (stmt.Calls(_rule) || stmt.Calls(_token))
{
sink.Write(Severity.Error, stmt, "A rule should have the form rule(Name(Args)::ReturnType, @[...])");
}
}
}
if (rules.Count == 0)
{
sink.Write(Severity.Warning, node, "No grammar rules were found in LLLPG block");
}
// Parse the rule definitions (now that we know the names of all the
// rules, we can decide if an Id refers to a rule; if not, it's assumed
// to refer to a terminal).
new StageTwoParser(helper, sink).Parse(rules);
// Process the grammar & generate code
var lllpg = new LLParserGenerator(helper, sink);
ApplyOptions(node, lllpg, sink); // Read attributes such as [DefaultK(3)]
foreach (var pair in rules)
{
lllpg.AddRule(pair.A);
}
// TODO: change lllpg so we can interleave generated code with other
// user code, to preserve the order of the original code.
var results = lllpg.Run(node.Source);
return(F.Call(S.Splice, stmts.Where(p => p != null).Concat(results.Args)));
}
public LNode GenerateLexerCode()
{
_pg = new LLParserGenerator(new IntStreamCodeGenHelper(), MessageSink.Console);
// Whitespace & comments
var Newline = Rule("Newline", ((C('\r') + Opt(C('\n'))) | '\n')
+ Stmt("_lineStartAt = InputPosition")
+ Stmt("_lineNumber++")
+ Stmt("_value = WhitespaceTag.Value"), Token);
var DotIndent = Rule("DotIndent", And("_startPosition == _lineStartAt")
+ Stmt("_type = TT.Spaces")
+ Plus(C('.') + Plus(C('\t') | ' '))
+ Stmt("_indentLevel = MeasureIndent(_indent = Source.Substring(_startPosition, InputPosition - _startPosition))")
+ Stmt("_value = WhitespaceTag.Value"), Private);
var Spaces = Rule("Spaces", Plus(C(' ') | '\t')
+ Stmt("if (_lineStartAt == _startPosition) "
+ "_indentLevel = MeasureIndent(_indent = Source.Substring(_startPosition, InputPosition - _startPosition))")
+ Stmt("_value = WhitespaceTag.Value"), Token);
var SLComment = Rule("SLComment", Seq("//") + Star(Set("[^\r\n]")) + Stmt("_value = WhitespaceTag.Value"), Token);
var MLCommentRef = new RuleRef(null, null);
var MLComment = Rule("MLComment",
Seq("/*") +
Star(MLCommentRef / AnyCh, false) +
Seq("*/") +
Stmt("_value = WhitespaceTag.Value"), Token, 3);
MLCommentRef.Rule = MLComment;
_pg.AddRules(Newline, DotIndent, Spaces, SLComment, MLComment);
// Strings
var SQString = Rule("SQString", Stmt("_parseNeeded = false") +
C('\'')
+ ((C('\\') + Set("[^\r\n]") + Stmt("_parseNeeded = true"))
/ Set("[^'\r\n\\\\]")
/ (Seq("") + Expr("_parseNeeded = true")))
+ Star(Set("[^' \t\n\r]") + Stmt("_parseNeeded = true"))
+ (C('\'') / (Seq("") + Stmt("_parseNeeded = true")))
+ Call("ParseSQStringValue"), Token);
var TQString = Rule("TQString", Stmt("_parseNeeded = true")
+ (Stmt("_style = NodeStyle.Alternate") +
Seq(@"""""""") + Star(Seq(@"\\""") / AnyCh, false) + Seq(@"""""""")
| Stmt("_style = NodeStyle.Alternate | NodeStyle.Alternate2") +
Seq(@"'''") + Star(Seq(@"\\'") / AnyCh, false) + Seq(@"'''"))
+ Stmt("ParseStringValue(true)"), Token, 4);
var DQString = Rule("DQString", Stmt("_parseNeeded = false") +
(C('"') + Star(C('\\') + AnyCh + Stmt("_parseNeeded = true") | Set("[^\"\\\\\r\n]")) + '"'
| (Stmt("_style = NodeStyle.Alternate") +
(Seq(@"#""") + Star((Seq(@"""""") + Stmt("_parseNeeded = true")) / Set("[^\"]")) + '"'))
) + Stmt("ParseStringValue(false)"), Token);
var BQString2 = Rule("BQString2", Stmt("_parseNeeded = false") +
C('`') + Star(C('\\') + AnyCh + Stmt("_parseNeeded = true") | Set("[^`\\\\\r\n]")) + '`', Private);
var BQString = Rule("BQString", BQString2 + Stmt("ParseBQStringValue()"), Token);
_pg.AddRules(SQString, DQString, TQString, BQString, BQString2);
// Identifiers and symbols
var letterTest = F.Call(F.Dot("#char", "IsLetter"), F.Call(S.Cast, F.Id("LA0"), F.Id(S.Char)));
var lettersOrPunc = Set(@"[0-9a-zA-Z_'#~!%^&*\-+=|<>/\\?:.$]");
Debug.Assert(!((PGIntSet)lettersOrPunc.Set).Contains('`'));
var IdExtLetter = Rule("IdExtLetter",
And(letterTest) + Set("[\u0080-\uFFFC]"), Private);
var NormalId = Rule("NormalId", (Set("[#_a-zA-Z]") | IdExtLetter) +
Star(Set("[#_a-zA-Z0-9']") | IdExtLetter));
var CommentStart = Rule("CommentStart", '/' + (C('/') | '*'), Private);
var FancyId = Rule("FancyId", BQString2 | Plus(AndNot(CommentStart) + lettersOrPunc | IdExtLetter));
var Symbol = Rule("Symbol", Stmt("_parseNeeded = false") +
Seq("@@") + FancyId + Call("ParseSymbolValue"), Token);
var Id = Rule("Id", Stmt("_parseNeeded = false") +
(NormalId | '@' + FancyId + Stmt("_parseNeeded = true")) +
Call("ParseIdValue"), Private);
_pg.AddRules(IdExtLetter, NormalId, CommentStart, FancyId, Symbol, Id);
// Punctuation
var Comma = Rule("Comma", Op(",", "Comma"), Private);
var Semicolon = Rule("Semicolon", Op(";", "Semicolon"), Private);
var At = Rule("At", C('@') + Stmt("_type = TT.At; _value = GSymbol.Empty"), Private);
var ops = Set(@"[~!%^&*\-+=|<>/?:.$]");
var Operator = Rule("Operator", Plus(AndNot(CommentStart) + ops) + Stmt("ParseNormalOp()"), Private);
var BackslashOp = Rule("BackslashOp", '\\' + Opt(FancyId) + Stmt("ParseBackslashOp()"), Private);
_pg.AddRules(Comma, Semicolon, At, Operator, BackslashOp);
// Openers & closers
var LParen = Rule("LParen", C('('), Token);
var RParen = Rule("RParen", C(')'), Token);
var LBrack = Rule("LBrack", C('['), Token);
var RBrack = Rule("RBrack", C(']'), Token);
var LBrace = Rule("LBrace", C('{'), Token);
var RBrace = Rule("RBrace", C('}'), Token);
_pg.AddRules(new[] { LParen, RParen, LBrack, RBrack, LBrace, RBrace });
Rule Number;
_pg.AddRules(NumberParts(out Number));
var Shebang = Rule("Shebang", Seq("#!") + Star(Set("[^\r\n]")) + Opt(Newline));
Alts tokenAlts = (Alts)(
(And(Expr("InputPosition == 0")) + T(Shebang)) /
T(Symbol) /
T(Id) /
T(Spaces) / T(Newline) / DotIndent /
T(SLComment) / T(MLComment) /
T(Number) /
(Stmt("_type = TT.String") + TQString) /
(Stmt("_type = TT.String") + DQString) /
T(SQString) / T(BQString) /
T(Comma) / T(Semicolon) /
T(LParen) / T(LBrack) / T(LBrace) /
T(RParen) / T(RBrack) / T(RBrace) /
T(At) / BackslashOp / Operator);
tokenAlts.DefaultArm = 2; // Id
var token = Rule("Token", tokenAlts, Token, 3);
_pg.AddRules(new[] { token, Shebang });
_pg.FullLLk = true;
//_pg.Verbosity = 3;
var members = _pg.Run(F.File);
members = members.PlusArgs(SymbolsToDeclare.Select(p =>
F.Var(F.Id("Symbol"), p.Key, F.Call(F.Dot("GSymbol", "Get"), F.Literal(p.Value.Name)))));
return(F.Attr(F.Public, F.Id(S.Partial),
F.Call(S.Class, F.Id(_("LesLexer")), F.Tuple(), members)));
}
