public LanguageCompiler(GrammarData data)
{
Data = data;
Grammar = data.Grammar;
Parser = new Parser(Data);
Scanner = new Scanner(Data);
}
public override void Init(GrammarData grammarData) {
base.Init(grammarData);
string workPattern = @"\G(" + Pattern + ")";
RegexOptions options = (OwnerGrammar.CaseSensitive ? RegexOptions.None : RegexOptions.IgnoreCase);
_expression = new Regex(workPattern, options);
if (this.EditorInfo == null)
this.EditorInfo = new TokenEditorInfo(TokenType.Unknown, TokenColor.Text, TokenTriggers.None);
}
public override void Init(GrammarData grammarData)
{
base.Init(grammarData);
//Check that Parser-scanner link is enabled - this terminal can be used only if this link is enabled
if (Grammar.LanguageFlags.IsSet(LanguageFlags.DisableScannerParserLink))
grammarData.Language.Errors.Add(GrammarErrorLevel.Error, null, Resources.ErrImpliedOpUseParserLink, Name);
//"ImpliedSymbolTerminal cannot be used in grammar with DisableScannerParserLink flag set"
}
internal void Build() {
_grammarData = _language.GrammarData;
CreateAugmentedRoots();
CollectTermsFromGrammar();
InitTermLists();
FillOperatorReportGroup();
CreateProductions();
ComputeNonTerminalsNullability(_grammarData);
ComputeTailsNullability(_grammarData);
ValidateGrammar();
}
private static void ComputeNonTerminalsNullability(GrammarData data)
{
NonTerminalList undecided = data.NonTerminals;
while (undecided.Count > 0) {
NonTerminalList newUndecided = new NonTerminalList();
foreach (NonTerminal nt in undecided)
if (!ComputeNullability(nt))
newUndecided.Add(nt);
if (undecided.Count == newUndecided.Count) return; //we didn't decide on any new, so we're done
undecided = newUndecided;
}//while
}
internal void Build() {
_grammarData = _language.GrammarData = new GrammarData(_language);
_grammarData.AugmentedRoot = new NonTerminal(_grammar.Root.Name + "'");
_grammarData.AugmentedRoot.Rule = _grammar.Root + _grammar.Eof;
CollectTermsFromGrammar();
InitTermLists(_grammarData);
CreateProductions();
ComputeNonTerminalsNullability(_grammarData);
ComputeTailsNullability(_grammarData);
ComputeFirsts(_grammarData);
if (_grammar.FlagIsSet(LanguageFlags.AutoDetectTransient))
DetectTransientNonTerminals(_grammarData);
ValidateGrammar();
}
//computes DirectFirsts, Firsts for non-terminals and productions
private static void ComputeFirsts(GrammarData data)
{
//compute prod direct firsts and initialize NT.Firsts
foreach (var nt in data.NonTerminals) {
foreach (var prod in nt.Productions) {
foreach (var term in prod.RValues) {
prod.DirectFirsts.Add(term);
nt.DirectFirsts.Add(term);
nt.Firsts.Add(term);
if (!term.IsSet(TermOptions.IsNullable)) break; //foreach term
}
}
}//foreach nt
//propagate NT.Firsts
int time = 0;
var done = false;
var newSet = new BnfTermSet();
while (!done) {
done = true;
foreach (var nt in data.NonTerminals) {
newSet.Clear();
foreach (var first in nt.Firsts) {
var ntFirst = first as NonTerminal;
if (ntFirst != null && ntFirst._lastChanged >= nt._lastChecked)
newSet.UnionWith(ntFirst.Firsts);
}
nt._lastChecked = time++;
var oldCount = nt.Firsts.Count;
nt.Firsts.UnionWith(newSet);
if (nt.Firsts.Count > oldCount) {
done = false;
nt._lastChanged = time;
}
}//foreach nt
}//while
//compute prod.Firsts
foreach (var nt in data.NonTerminals) {
foreach (var prod in nt.Productions) {
prod.Firsts.UnionWith(prod.DirectFirsts);
foreach (var directFirst in prod.DirectFirsts) {
var ntDirectFirst = directFirst as NonTerminal;
if (ntDirectFirst != null)
prod.Firsts.UnionWith(ntDirectFirst.Firsts);
}//foreach directFirst
}//foreach prod
}//foreach nt
}
public LanguageCompiler(Grammar grammar)
{
Grammar = grammar;
#if !SILVERLIGHT
Stopwatch sw = new Stopwatch();
sw.Start();
#endif
GrammarDataBuilder bld = new GrammarDataBuilder(grammar);
bld.Build();
Data = bld.Data;
Parser = new Parser(Data);
Scanner = new Scanner(Data);
#if !SILVERLIGHT
sw.Stop();
InitTime = sw.ElapsedMilliseconds;
#endif
}
private static void CheckWrapTailHints(GrammarData data, NonTerminal nonTerminal, BnfTermList operands)
{
//WrapTail hint doesn't make sense in last position, so we start with Count-2
for (int i = operands.Count - 2; i >= 0; i--) {
var hint = operands[i] as GrammarHint;
if (hint == null || hint.HintType != HintType.WrapTail) continue;
//we have WrapTail hint; wrap all operands after this into new non-terminal
var wrapNt = new NonTerminal(nonTerminal.Name + "_tail" + nonTerminal._tailCount++);
wrapNt.SetOption(TermOptions.IsTransient);
wrapNt.Rule = new BnfExpression();
for (int j = i + 1; j < operands.Count; j++) {
wrapNt.Rule.Data[0].Add(operands[j]);
}
operands.RemoveRange(i, operands.Count - i);
operands.Add(wrapNt);
data.AllTerms.Add(wrapNt);
data.NonTerminals.Add(wrapNt);
}//for i
}
}//method
private static void InitTermLists(GrammarData data) {
//Collect terminals and NonTerminals
foreach (BnfTerm term in data.AllTerms) { //remember - we may have hints, so it's not only terminals and non-terminals
if (term is NonTerminal) data.NonTerminals.Add((NonTerminal)term);
if (term is Terminal) data.Terminals.Add((Terminal)term);
}
data.Terminals.Sort(Terminal.ByName);
//Mark keywords - any "word" symbol directly mentioned in the grammar
if (data.Grammar.FlagIsSet(LanguageFlags.AutoDetectKeywords))
foreach (var term in data.Terminals) {
var symTerm = term as SymbolTerminal;
if (symTerm == null) continue;
if (symTerm.Symbol.Length > 0 && char.IsLetter(symTerm.Symbol[0]))
symTerm.SetOption(TermOptions.IsKeyword);
}//foreach term
//Init all terms
foreach (BnfTerm term in data.AllTerms)
term.Init(data);
}//method
private void CheckPrecedenceSettings(GrammarData data, ParseMethod method) {
if(!_grammar.UsePrecedenceRestricted) {
// All terms registered with RegisterOperator method already have IsOperator and UsePrecedence flags set.
// What we need to do is detect non-terminals (like BinOp) that also should be treated as operators
// in the parser input, and set the UsePrecedence flag on them.
// We find all non-terminals having all productions either empty or consisting of a single term which is operator
// It will cover situations when we define non-terminal like 'BinOp.Rule = "+" | "-" | "*" | "/";'
// After reducing lookaheads in NLALR BinOp may become a lookahead, and it must be treated as operator
foreach (NonTerminal nt in data.NonTerminals) {
var isOp = true;
foreach (var prod in nt.Productions) {
isOp &= prod.RValues.Count == 0 || prod.RValues.Count == 1 && prod.RValues[0].IsSet(TermOptions.IsOperator);
if (!isOp) break;
}//foreac prod
if (isOp)
nt.SetOption(TermOptions.UsePrecedence);
}//foreach
}//else
}//method
private static void ComputeTailsNullability(GrammarData data)
{
foreach (var nt in data.NonTerminals)
{
foreach (var prod in nt.Productions)
{
var count = prod.LR0Items.Count;
for (int i = count - 1; i >= 0; i--)
{
var item = prod.LR0Items[i];
item.TailIsNullable = true;
if (item.Current == null)
{
continue;
}
if (!item.Current.IsSet(TermOptions.IsNullable))
{
break; //for i
}
}//for i
}//foreach prod
}
}
} //method
private static void InitTermLists(GrammarData data)
{
//Collect terminals and NonTerminals
foreach (BnfTerm term in data.AllTerms) //remember - we may have hints, so it's not only terminals and non-terminals
{
if (term is NonTerminal)
{
data.NonTerminals.Add((NonTerminal)term);
}
if (term is Terminal)
{
data.Terminals.Add((Terminal)term);
}
}
data.Terminals.Sort(Terminal.ByName);
//Mark keywords - any "word" symbol directly mentioned in the grammar
if (data.Grammar.FlagIsSet(LanguageFlags.AutoDetectKeywords))
{
foreach (var term in data.Terminals)
{
var symTerm = term as SymbolTerminal;
if (symTerm == null)
{
continue;
}
if (symTerm.Symbol.Length > 0 && char.IsLetter(symTerm.Symbol[0]))
{
symTerm.SetOption(TermOptions.IsKeyword);
}
}//foreach term
}
//Init all terms
foreach (BnfTerm term in data.AllTerms)
{
term.Init(data);
}
}//method
private static void CheckWrapTailHints(GrammarData data, NonTerminal nonTerminal, BnfTermList operands)
{
//WrapTail hint doesn't make sense in last position, so we start with Count-2
for (int i = operands.Count - 2; i >= 0; i--)
{
var hint = operands[i] as GrammarHint;
if (hint == null || hint.HintType != HintType.WrapTail)
{
continue;
}
//we have WrapTail hint; wrap all operands after this into new non-terminal
var wrapNt = new NonTerminal(nonTerminal.Name + "_tail" + nonTerminal._tailCount++);
wrapNt.SetOption(TermOptions.IsTransient);
wrapNt.Rule = new BnfExpression();
for (int j = i + 1; j < operands.Count; j++)
{
wrapNt.Rule.Data[0].Add(operands[j]);
}
operands.RemoveRange(i, operands.Count - i);
operands.Add(wrapNt);
data.AllTerms.Add(wrapNt);
data.NonTerminals.Add(wrapNt);
}//for i
}
public Parser(GrammarData data)
{
Data = data;
}
public override void Init(GrammarData grammarData) {
base.Init(grammarData);
//Remove new line chars from whitespace
foreach(char t in LineTerminators)
grammarData.Grammar.WhitespaceChars = grammarData.Grammar.WhitespaceChars.Replace(t.ToString(), string.Empty);
}
internal ScannerDataBuilder(LanguageData language)
{
_language = language;
_grammar = _language.Grammar;
_grammarData = language.GrammarData;
}
public override void Init(GrammarData grammarData)
{
base.Init(grammarData);
}
private static int Main(string[] args)
{
Console.WriteLine("AbstractCode parser automaton compiler utility.");
Console.WriteLine();
string filePath;
if (args.Length > 0)
{
filePath = args[0];
}
else
{
Console.Write("Assembly: ");
filePath = Console.ReadLine();
}
Assembly assembly = null;
if (!PerformAction("Reading assembly", () => assembly = Assembly.LoadFile(filePath.Replace("\"", ""))))
return 1;
string typeName;
if (args.Length > 1)
{
typeName = args[1];
}
else
{
Console.Write("Grammar type: ");
typeName = Console.ReadLine();
}
Type grammarType = null;
if (!PerformAction("Finding type", () => grammarType = assembly.GetType(typeName, true)))
return 1;
Grammar grammar = null;
if (!PerformAction("Creating instance of grammar", () => grammar = (Grammar)Activator.CreateInstance(grammarType)))
return 1;
string outputFile;
if (args.Length > 2)
{
outputFile = args[2];
}
else
{
Console.Write("Output file: ");
outputFile = Console.ReadLine();
}
GrammarData data = null;
if (!PerformAction("Collecting grammar data", () => data = new GrammarData(grammar)))
return 1;
GrammarCompilationResult result = null;
if (!PerformAction("Constructing automaton", () => result = GrammarCompiler.Compile(data)))
return 1;
if (result.CompilationContext.Conflicts.Count > 0)
{
Console.ForegroundColor = ConsoleColor.Yellow;
Console.Write("Warning: ");
Console.ForegroundColor = ConsoleColor.Gray;
Console.WriteLine("{0} conflicts detected! Grammar might not be LALR(1).",
result.CompilationContext.Conflicts.Count);
foreach (var conflict in result.CompilationContext.Conflicts)
{
Console.WriteLine("State: {0}, lookahead: {1}, action 1: {2}, action 2: {3}", conflict.State.Id,
conflict.Lookahead.Name, conflict.Action1, conflict.Action2);
}
}
if (!PerformAction("Serializing automaton", () =>
{
using (var stream = File.Create(outputFile))
{
var serializer = new ParserAutomatonSerializer(data);
serializer.Serialize(stream, result.Automaton);
}
}))
return 1;
return 0;
}
private static void ComputeTailsNullability(GrammarData data)
{
foreach (var nt in data.NonTerminals) {
foreach (var prod in nt.Productions) {
var count = prod.LR0Items.Count;
for (int i = count - 1; i >= 0; i--) {
var item = prod.LR0Items[i];
item.TailIsNullable = true;
if (item.Current == null) continue;
if (!item.Current.Flags.IsSet(TermFlags.IsNullable))
break; //for i
}//for i
}//foreach prod
}
}
public override void Init(GrammarData grammarData) {
base.Init(grammarData);
}
internal ScannerDataBuilder(LanguageData language)
{
_language = language;
_grammar = _language.Grammar;
_grammarData = language.GrammarData;
}
//Used in unit tests
public static LanguageCompiler CreateDummy()
{
GrammarData data = new GrammarData();
data.Grammar = new Grammar();
LanguageCompiler compiler = new LanguageCompiler(data);
return compiler;
}
public Scanner(GrammarData data)
{
_data = data;
_lineTerminators = _data.Grammar.LineTerminators.ToCharArray();
}
//Automatically detect transient non-terminals; these are nonterminals that have rules with single-element productions:
// N.rule = A | B | C;
private static void DetectTransientNonTerminals(GrammarData data)
{
foreach (NonTerminal nt in data.NonTerminals) {
var transient = true;
foreach (var prod in nt.Productions)
if (prod.RValues.Count != 1) {
transient = false;
break;
}
if (transient)
nt.SetOption(TermOptions.IsTransient);
}
}
public override void Init(GrammarData grammarData) {
base.Init(grammarData);
if (string.IsNullOrEmpty(QuickParseTerminators))
QuickParseTerminators = OwnerGrammar.WhitespaceChars + OwnerGrammar.Delimiters;
_defaultFloatTypes = new TypeCode[] { DefaultFloatType };
if (this.EditorInfo == null)
this.EditorInfo = new TokenEditorInfo(TokenType.Literal, TokenColor.Number, TokenTriggers.None);
}
//computes DirectFirsts, Firsts for non-terminals and productions
private static void ComputeFirsts(GrammarData data)
{
//compute prod direct firsts and initialize NT.Firsts
foreach (var nt in data.NonTerminals)
{
foreach (var prod in nt.Productions)
{
foreach (var term in prod.RValues)
{
prod.DirectFirsts.Add(term);
nt.DirectFirsts.Add(term);
nt.Firsts.Add(term);
if (!term.IsSet(TermOptions.IsNullable))
{
break; //foreach term
}
}
}
}//foreach nt
//propagate NT.Firsts
int time = 0;
var done = false;
var newSet = new BnfTermSet();
while (!done)
{
done = true;
foreach (var nt in data.NonTerminals)
{
newSet.Clear();
foreach (var first in nt.Firsts)
{
var ntFirst = first as NonTerminal;
if (ntFirst != null && ntFirst._lastChanged >= nt._lastChecked)
{
newSet.UnionWith(ntFirst.Firsts);
}
}
nt._lastChecked = time++;
var oldCount = nt.Firsts.Count;
nt.Firsts.UnionWith(newSet);
if (nt.Firsts.Count > oldCount)
{
done = false;
nt._lastChanged = time;
}
} //foreach nt
} //while
//compute prod.Firsts
foreach (var nt in data.NonTerminals)
{
foreach (var prod in nt.Productions)
{
prod.Firsts.UnionWith(prod.DirectFirsts);
foreach (var directFirst in prod.DirectFirsts)
{
var ntDirectFirst = directFirst as NonTerminal;
if (ntDirectFirst != null)
{
prod.Firsts.UnionWith(ntDirectFirst.Firsts);
}
} //foreach directFirst
} //foreach prod
} //foreach nt
} //method