protected virtual void InitStartingTokensForRule(Dictionary <string, NonTerminal <IN> > nonTerminals, Rule <IN> rule)
{
if (rule.PossibleLeadingTokens == null || rule.PossibleLeadingTokens.Count == 0)
{
rule.PossibleLeadingTokens = new List <IN>();
if (rule.Clauses.Count > 0)
{
IClause <IN> first = rule.Clauses[0];
if (first is TerminalClause <IN> )
{
TerminalClause <IN> term = first as TerminalClause <IN>;
rule.PossibleLeadingTokens.Add(term.ExpectedToken);
rule.PossibleLeadingTokens = rule.PossibleLeadingTokens.Distinct <IN>().ToList <IN>();
}
else
{
NonTerminalClause <IN> nonterm = first as NonTerminalClause <IN>;
InitStartingTokensForNonTerminal(nonTerminals, nonterm.NonTerminalName);
if (nonTerminals.ContainsKey(nonterm.NonTerminalName))
{
NonTerminal <IN> firstNonTerminal = nonTerminals[nonterm.NonTerminalName];
firstNonTerminal.Rules.ForEach(r =>
{
rule.PossibleLeadingTokens.AddRange(r.PossibleLeadingTokens);
});
rule.PossibleLeadingTokens = rule.PossibleLeadingTokens.Distinct <IN>().ToList <IN>();
}
}
}
}
}
public ParserConfiguration <IN, OUT> ComputeSubRules(ParserConfiguration <IN, OUT> configuration)
{
var newNonTerms = new List <NonTerminal <IN> >();
foreach (var nonTerm in configuration.NonTerminals)
{
foreach (var rule in nonTerm.Value.Rules)
{
var newclauses = new List <IClause <IN> >();
if (rule.ContainsSubRule)
{
foreach (var clause in rule.Clauses)
{
if (clause is GroupClause <IN> group)
{
var newNonTerm = CreateSubRule(group);
newNonTerms.Add(newNonTerm);
var newClause = new NonTerminalClause <IN>(newNonTerm.Name);
newClause.IsGroup = true;
newclauses.Add(newClause);
}
else if (clause is ManyClause <IN> many)
{
if (many.Clause is GroupClause <IN> manyGroup)
{
var newNonTerm = CreateSubRule(manyGroup);
newNonTerms.Add(newNonTerm);
var newInnerNonTermClause = new NonTerminalClause <IN>(newNonTerm.Name);
newInnerNonTermClause.IsGroup = true;
many.Clause = newInnerNonTermClause;
newclauses.Add(many);
}
}
else if (clause is OptionClause <IN> option)
{
if (option.Clause is GroupClause <IN> optionGroup)
{
var newNonTerm = CreateSubRule(optionGroup);
newNonTerms.Add(newNonTerm);
var newInnerNonTermClause = new NonTerminalClause <IN>(newNonTerm.Name);
newInnerNonTermClause.IsGroup = true;
option.Clause = newInnerNonTermClause;
newclauses.Add(option);
}
}
else
{
newclauses.Add(clause);
}
}
rule.Clauses.Clear();
rule.Clauses.AddRange(newclauses);
}
}
}
newNonTerms.ForEach(nonTerminal => configuration.AddNonTerminalIfNotExists(nonTerminal));
return(configuration);
}
protected override void InitStartingTokensForRule(Dictionary <string, NonTerminal <IN> > nonTerminals, Rule <IN> rule)
{
if (rule.PossibleLeadingTokens == null || rule.PossibleLeadingTokens.Count == 0)
{
rule.PossibleLeadingTokens = new List <IN>();
if (rule.Clauses.Count > 0)
{
IClause <IN> first = rule.Clauses[0];
if (first is TerminalClause <IN> )
{
TerminalClause <IN> term = first as TerminalClause <IN>;
InitStartingTokensWithTerminal(rule, term);
}
else if (first is NonTerminalClause <IN> )
{
NonTerminalClause <IN> nonterm = first as NonTerminalClause <IN>;
InitStartingTokensWithNonTerminal(rule, nonterm, nonTerminals);
}
else if (first is ZeroOrMoreClause <IN> )
{
ZeroOrMoreClause <IN> many = first as ZeroOrMoreClause <IN>;
InitStartingTokensWithZeroOrMore(rule, many, nonTerminals);
}
else if (first is OneOrMoreClause <IN> )
{
OneOrMoreClause <IN> many = first as OneOrMoreClause <IN>;
InitStartingTokensWithOneOrMore(rule, many, nonTerminals);
}
}
}
}
private Rule <IN> BuildNonTerminal(Tuple <string, string> ntAndRule)
{
var rule = new Rule <IN>();
var clauses = new List <IClause <IN> >();
var ruleString = ntAndRule.Item2;
var clausesString = ruleString.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var item in clausesString)
{
IClause <IN> clause = null;
var isTerminal = false;
var token = default(IN);
try
{
var tIn = typeof(IN);
var b = Enum.TryParse(item, out token);
if (b)
{
isTerminal = true;
;
}
//token = (IN)Enum.Parse(tIn , item);
//isTerminal = true;
}
catch (ArgumentException)
{
isTerminal = false;
}
if (isTerminal)
{
clause = new TerminalClause <IN>(token);
}
else if (item == "[d]")
{
if (clauses.Last() is TerminalClause <IN> discardedTerminal)
{
discardedTerminal.Discarded = true;
}
}
else
{
clause = new NonTerminalClause <IN>(item);
}
if (clause != null)
{
clauses.Add(clause);
}
}
rule.Clauses = clauses;
//rule.Key = ntAndRule.Item1 + "_" + ntAndRule.Item2.Replace(" ", "_");
return(rule);
}
private void InitStartingTokensWithNonTerminal(Rule <IN> rule, NonTerminalClause <IN> nonterm,
Dictionary <string, NonTerminal <IN> > nonTerminals)
{
InitStartingTokensForNonTerminal(nonTerminals, nonterm.NonTerminalName);
if (nonTerminals.ContainsKey(nonterm.NonTerminalName))
{
var firstNonTerminal = nonTerminals[nonterm.NonTerminalName];
firstNonTerminal.Rules.ForEach(r => { rule.PossibleLeadingTokens.AddRange(r.PossibleLeadingTokens); });
rule.PossibleLeadingTokens = rule.PossibleLeadingTokens.Distinct().ToList();
}
}
private void InitStartingTokensWithOneOrMore(Rule <IN> rule, OneOrMoreClause <IN> manyClause,
Dictionary <string, NonTerminal <IN> > nonTerminals)
{
if (manyClause.Clause is TerminalClause <IN> )
{
TerminalClause <IN> term = manyClause.Clause as TerminalClause <IN>;
InitStartingTokensWithTerminal(rule, term);
}
else if (manyClause.Clause is NonTerminalClause <IN> )
{
NonTerminalClause <IN> nonterm = manyClause.Clause as NonTerminalClause <IN>;
InitStartingTokensWithNonTerminal(rule, nonterm, nonTerminals);
}
}
private IClause <IN> BuildTerminalOrNonTerimal(string name, bool discard = false)
{
IN token = default(IN);
IClause <IN> clause;
bool isTerminal = false;
bool b = Enum.TryParse <IN>(name, out token);
if (b)
{
isTerminal = true;
;
}
if (isTerminal)
{
clause = new TerminalClause <IN>(token, discard);
}
else
{
clause = new NonTerminalClause <IN>(name);
}
return(clause);
}
public SyntaxParseResult <IN> ParseNonTerminal(IList <Token <IN> > tokens, NonTerminalClause <IN> nonTermClause, int currentPosition)
{
int startPosition = currentPosition;
int endingPosition = 0;
NonTerminal <IN> nt = Configuration.NonTerminals[nonTermClause.NonTerminalName];
List <UnexpectedTokenSyntaxError <IN> > errors = new List <UnexpectedTokenSyntaxError <IN> >();
int i = 0;
List <IN> allAcceptableTokens = new List <IN>();
nt.Rules.ForEach(r =>
{
if (r != null && r.PossibleLeadingTokens != null)
{
allAcceptableTokens.AddRange(r.PossibleLeadingTokens);
}
});
allAcceptableTokens = allAcceptableTokens.Distinct <IN>().ToList <IN>();
List <Rule <IN> > rules = nt.Rules
.Where <Rule <IN> >(r => r.PossibleLeadingTokens.Contains(tokens[startPosition].TokenID) || r.MayBeEmpty)
.ToList <Rule <IN> >();
if (rules.Count == 0)
{
errors.Add(new UnexpectedTokenSyntaxError <IN>(tokens[startPosition],
allAcceptableTokens.ToArray <IN>()));
}
List <UnexpectedTokenSyntaxError <IN> > innerRuleErrors = new List <UnexpectedTokenSyntaxError <IN> >();
SyntaxParseResult <IN> okResult = null;
int greaterIndex = 0;
bool allRulesInError = true;
while (i < rules.Count)
{
Rule <IN> innerrule = rules[i];
SyntaxParseResult <IN> innerRuleRes = Parse(tokens, innerrule, startPosition, nonTermClause.NonTerminalName);
if (!innerRuleRes.IsError && okResult == null || (okResult != null && innerRuleRes.EndingPosition > okResult.EndingPosition))
{
okResult = innerRuleRes;
okResult.Errors = innerRuleRes.Errors;
endingPosition = innerRuleRes.EndingPosition;
}
bool other = greaterIndex == 0 && innerRuleRes.EndingPosition == 0;
if ((innerRuleRes.EndingPosition > greaterIndex && innerRuleRes.Errors != null &&
!innerRuleRes.Errors.Any()) || other)
{
greaterIndex = innerRuleRes.EndingPosition;
innerRuleErrors.Clear();
innerRuleErrors.AddRange(innerRuleRes.Errors);
}
innerRuleErrors.AddRange(innerRuleRes.Errors);
allRulesInError = allRulesInError && innerRuleRes.IsError;
i++;
}
errors.AddRange(innerRuleErrors);
SyntaxParseResult <IN> result = new SyntaxParseResult <IN>();
result.Errors = errors;
if (okResult != null)
{
result.Root = okResult.Root;
result.IsError = false;
result.EndingPosition = okResult.EndingPosition;
result.IsEnded = okResult.IsEnded;
result.Errors = errors;
}
else
{
result.IsError = true;
result.Errors = errors;
greaterIndex = errors.Select(e => e.UnexpectedToken.PositionInTokenFlow).Max();
result.EndingPosition = greaterIndex;
}
return(result);
}
public SyntaxParseResult <IN> ParseNonTerminal(IList <Token <IN> > tokens, NonTerminalClause <IN> nonTermClause,
int currentPosition)
{
var startPosition = currentPosition;
var endingPosition = 0;
var nt = Configuration.NonTerminals[nonTermClause.NonTerminalName];
var errors = new List <UnexpectedTokenSyntaxError <IN> >();
var i = 0;
var allAcceptableTokens = new List <IN>();
nt.Rules.ForEach(r =>
{
if (r != null && r.PossibleLeadingTokens != null)
{
allAcceptableTokens.AddRange(r.PossibleLeadingTokens);
}
});
allAcceptableTokens = allAcceptableTokens.Distinct().ToList();
var rules = nt.Rules
.Where(r => startPosition < tokens.Count && r.PossibleLeadingTokens.Contains(tokens[startPosition].TokenID) || r.MayBeEmpty)
.ToList();
if (rules.Count == 0)
{
return(NoMatchingRuleError(tokens, currentPosition, allAcceptableTokens));
}
var innerRuleErrors = new List <UnexpectedTokenSyntaxError <IN> >();
var greaterIndex = 0;
var rulesResults = new List <SyntaxParseResult <IN> >();
while (i < rules.Count)
{
var innerrule = rules[i];
var innerRuleRes = Parse(tokens, innerrule, startPosition, nonTermClause.NonTerminalName);
rulesResults.Add(innerRuleRes);
var other = greaterIndex == 0 && innerRuleRes.EndingPosition == 0;
if (innerRuleRes.EndingPosition > greaterIndex && innerRuleRes.Errors != null &&
!innerRuleRes.Errors.Any() || other)
{
greaterIndex = innerRuleRes.EndingPosition;
//innerRuleErrors.Clear();
innerRuleErrors.AddRange(innerRuleRes.Errors);
}
innerRuleErrors.AddRange(innerRuleRes.Errors);
i++;
}
errors.AddRange(innerRuleErrors);
SyntaxParseResult <IN> max = null;
if (rulesResults.Any())
{
if (rulesResults.Any(x => x.IsOk))
{
max = rulesResults.Where(x => x.IsOk).OrderBy(x => x.EndingPosition).Last();
}
else
{
max = rulesResults.Where(x => !x.IsOk).OrderBy(x => x.EndingPosition).Last();
}
}
else
{
max = new SyntaxParseResult <IN>();
max.IsError = true;
max.Root = null;
max.IsEnded = false;
max.EndingPosition = currentPosition;
}
var result = new SyntaxParseResult <IN>();
result.Errors = errors;
result.Root = max.Root;
result.EndingPosition = max.EndingPosition;
result.IsError = max.IsError;
result.IsEnded = max.IsEnded;
if (rulesResults.Any())
{
var terr = rulesResults.SelectMany(x => x.Errors).ToList();
var unexpected = terr.Cast <UnexpectedTokenSyntaxError <IN> >().ToList();
var expecting = unexpected.SelectMany(x => x.ExpectedTokens).ToList();
result.AddExpectings(expecting);
}
return(result);
}
public SyntaxParseResult <IN> ParseNonTerminal(IList <Token <IN> > tokens, NonTerminalClause <IN> nonTermClause,
int currentPosition)
{
var startPosition = currentPosition;
var endingPosition = 0;
var nt = Configuration.NonTerminals[nonTermClause.NonTerminalName];
var errors = new List <UnexpectedTokenSyntaxError <IN> >();
var i = 0;
var allAcceptableTokens = new List <IN>();
nt.Rules.ForEach(r =>
{
if (r != null && r.PossibleLeadingTokens != null)
{
allAcceptableTokens.AddRange(r.PossibleLeadingTokens);
}
});
allAcceptableTokens = allAcceptableTokens.Distinct().ToList();
var rules = nt.Rules
.Where(r => r.PossibleLeadingTokens.Contains(tokens[startPosition].TokenID) || r.MayBeEmpty)
.ToList();
if (rules.Count == 0)
{
errors.Add(new UnexpectedTokenSyntaxError <IN>(tokens[startPosition],
allAcceptableTokens.ToArray <IN>()));
}
var innerRuleErrors = new List <UnexpectedTokenSyntaxError <IN> >();
SyntaxParseResult <IN> okResult = null;
var greaterIndex = 0;
var allRulesInError = true;
var test = new List <SyntaxParseResult <IN> >();
while (i < rules.Count)
{
var innerrule = rules[i];
var innerRuleRes = Parse(tokens, innerrule, startPosition, nonTermClause.NonTerminalName);
if (!innerRuleRes.IsError && okResult == null ||
okResult != null && innerRuleRes.EndingPosition > okResult.EndingPosition)
{
okResult = innerRuleRes;
okResult.Errors = innerRuleRes.Errors;
endingPosition = innerRuleRes.EndingPosition;
}
else
{
test.Add(innerRuleRes);
}
var other = greaterIndex == 0 && innerRuleRes.EndingPosition == 0;
if (innerRuleRes.EndingPosition > greaterIndex && innerRuleRes.Errors != null &&
!innerRuleRes.Errors.Any() || other)
{
greaterIndex = innerRuleRes.EndingPosition;
innerRuleErrors.Clear();
innerRuleErrors.AddRange(innerRuleRes.Errors);
}
innerRuleErrors.AddRange(innerRuleRes.Errors);
allRulesInError = allRulesInError && innerRuleRes.IsError;
i++;
}
errors.AddRange(innerRuleErrors);
var result = new SyntaxParseResult <IN>();
result.Errors = errors;
if (okResult != null)
{
result.Root = okResult.Root;
result.IsError = false;
result.EndingPosition = okResult.EndingPosition;
result.IsEnded = okResult.IsEnded;
result.Errors = errors;
}
else
{
result.IsError = true;
result.Errors = errors;
greaterIndex = errors.Count > 0 ? errors.Select(e => e.UnexpectedToken.PositionInTokenFlow).Max() : 0;
result.EndingPosition = greaterIndex;
result.AddExpectings(errors.SelectMany(e => e.ExpectedTokens));
}
if (test.Any())
{
var terr = test.SelectMany(x => x.Errors).ToList();
var unexpected = terr.OfType <UnexpectedTokenSyntaxError <IN> >().ToList();
var expecting = unexpected.SelectMany(x => x.ExpectedTokens).ToList();
result.AddExpectings(expecting);
;
}
return(result);
}
