public void TestLexerDelegatorRuleOverridesDelegateLeavingNoRules() /*throws Exception*/
{
// M.Tokens has nothing to predict tokens from S. Should
// not include S.Tokens alt in this case?
string slave =
"lexer grammar S;\n" +
"A : 'a' {System.out.println(\"S.A\");} ;\n";
mkdir(tmpdir);
writeFile(tmpdir, "S.g", slave);
string master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a' {System.out.println(\"M.A\");} ;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
writeFile(tmpdir, "M.g", master);
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener(equeue);
AntlrTool antlr = newTool(new string[] { "-lib", tmpdir });
CompositeGrammar composite = new CompositeGrammar();
Grammar g = new Grammar(antlr, tmpdir + "/M.g", composite);
composite.SetDelegationRoot(g);
g.ParseAndBuildAST();
composite.AssignTokenTypes();
composite.DefineGrammarSymbols();
composite.CreateNFAs();
g.CreateLookaheadDFAs(false);
// predict only alts from M not S
string expectingDFA =
".s0-'a'->.s1\n" +
".s0-{'\\n', ' '}->:s3=>2\n" +
".s1-<EOT>->:s2=>1\n";
Antlr3.Analysis.DFA dfa = g.GetLookaheadDFA(1);
FASerializer serializer = new FASerializer(g);
string result = serializer.Serialize(dfa.startState);
assertEquals(expectingDFA, result);
// must not be a "unreachable alt: Tokens" error
assertEquals("unexpected errors: " + equeue, 0, equeue.errors.Count);
}
protected void checkDecision(Grammar g,
int decision,
string expecting,
int[] expectingUnreachableAlts)
//throws Exception
{
Antlr3.AntlrTool tool = new Antlr3.AntlrTool();
// mimic actions of org.antlr.Tool first time for grammar g
if (g.CodeGenerator == null)
{
CodeGenerator generator = new CodeGenerator(tool, g, "Java");
g.CodeGenerator = generator;
g.BuildNFA();
g.CreateLookaheadDFAs(false);
}
DFA dfa = g.GetLookaheadDFA(decision);
assertNotNull("unknown decision #" + decision, dfa);
FASerializer serializer = new FASerializer(g);
string result = serializer.Serialize(dfa.startState);
//System.out.print(result);
var nonDetAlts = dfa.UnreachableAlts;
//System.out.println("alts w/o predict state="+nonDetAlts);
// first make sure nondeterministic alts are as expected
if (expectingUnreachableAlts == null)
{
if (nonDetAlts != null && nonDetAlts.Count != 0)
{
Console.Error.WriteLine("nondeterministic alts (should be empty): " + ((IList)nonDetAlts).ToElementString());
}
assertEquals("unreachable alts mismatch", 0, nonDetAlts != null ? nonDetAlts.Count : 0);
}
else
{
for (int i = 0; i < expectingUnreachableAlts.Length; i++)
{
assertTrue("unreachable alts mismatch",
nonDetAlts != null ? nonDetAlts.Contains(expectingUnreachableAlts[i]) : false);
}
}
assertEquals(expecting, result);
}
public void TestLoopsWithOptimizedOutExitBranches() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'x'* ~'x'+ ;\n");
string expecting =
".s0-'x'->:s1=>1\n" +
".s0-{'\\u0000'..'w', 'y'..'\\uFFFF'}->:s2=>2\n";
checkDecision(g, 1, expecting, null);
// The optimizer yanks out all exit branches from EBNF blocks
// This is ok because we've already verified there are no problems
// with the enter/exit decision
DFAOptimizer optimizer = new DFAOptimizer(g);
optimizer.Optimize();
FASerializer serializer = new FASerializer(g);
DFA dfa = g.GetLookaheadDFA(1);
string result = serializer.Serialize(dfa.startState);
expecting = ".s0-'x'->:s1=>1\n";
assertEquals(expecting, result);
}
protected void checkDecision(Grammar g,
int decision,
string expecting,
int[] expectingUnreachableAlts,
int[] expectingNonDetAlts,
string expectingAmbigInput,
int[] expectingInsufficientPredAlts,
int[] expectingDanglingAlts,
int expectingNumWarnings,
bool hasPredHiddenByAction)
//throws Exception
{
DecisionProbe.verbose = true; // make sure we get all error info
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener(equeue);
CodeGenerator generator = new CodeGenerator(newTool(), g, "Java");
g.CodeGenerator = generator;
// mimic actions of org.antlr.Tool first time for grammar g
if (g.NumberOfDecisions == 0)
{
g.BuildNFA();
g.CreateLookaheadDFAs(false);
}
if (equeue.size() != expectingNumWarnings)
{
Console.Error.WriteLine("Warnings issued: " + equeue);
}
Assert.AreEqual(expectingNumWarnings, equeue.size(), "unexpected number of expected problems");
DFA dfa = g.GetLookaheadDFA(decision);
FASerializer serializer = new FASerializer(g);
string result = serializer.Serialize(dfa.StartState);
//System.out.print(result);
var unreachableAlts = dfa.UnreachableAlts;
// make sure unreachable alts are as expected
if (expectingUnreachableAlts != null)
{
BitSet s = new BitSet();
s.AddAll(expectingUnreachableAlts);
BitSet s2 = new BitSet();
s2.AddAll(unreachableAlts);
Assert.AreEqual(s, s2, "unreachable alts mismatch");
}
else
{
Assert.AreEqual(0, unreachableAlts != null ? unreachableAlts.Count : 0, "unreachable alts mismatch");
}
// check conflicting input
if (expectingAmbigInput != null)
{
// first, find nondet message
Message msg = getNonDeterminismMessage(equeue.warnings);
Assert.IsNotNull(msg, "no nondeterminism warning?");
Assert.IsTrue(msg is GrammarNonDeterminismMessage, "expecting nondeterminism; found " + msg.GetType().Name);
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage(equeue.warnings);
var labels =
nondetMsg.probe.GetSampleNonDeterministicInputSequence(nondetMsg.problemState);
string input = nondetMsg.probe.GetInputSequenceDisplay(labels);
Assert.AreEqual(expectingAmbigInput, input);
}
// check nondet alts
if (expectingNonDetAlts != null)
{
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage(equeue.warnings);
Assert.IsNotNull(nondetMsg, "found no nondet alts; expecting: " + str(expectingNonDetAlts));
var nonDetAlts =
nondetMsg.probe.GetNonDeterministicAltsForState(nondetMsg.problemState);
// compare nonDetAlts with expectingNonDetAlts
BitSet s = new BitSet();
s.AddAll(expectingNonDetAlts);
BitSet s2 = new BitSet();
s2.AddAll(nonDetAlts);
Assert.AreEqual(s, s2, "nondet alts mismatch");
Assert.AreEqual(hasPredHiddenByAction, nondetMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction");
}
else
{
// not expecting any nondet alts, make sure there are none
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage(equeue.warnings);
Assert.IsNull(nondetMsg, "found nondet alts, but expecting none");
}
if (expectingInsufficientPredAlts != null)
{
GrammarInsufficientPredicatesMessage insuffPredMsg =
getGrammarInsufficientPredicatesMessage(equeue.warnings);
Assert.IsNotNull(insuffPredMsg, "found no GrammarInsufficientPredicatesMessage alts; expecting: " + str(expectingNonDetAlts));
var locations = insuffPredMsg.altToLocations;
var actualAlts = locations.Keys;
BitSet s = new BitSet();
s.AddAll(expectingInsufficientPredAlts);
BitSet s2 = new BitSet();
s2.AddAll(actualAlts);
Assert.AreEqual(s, s2, "mismatch between insufficiently covered alts");
Assert.AreEqual(hasPredHiddenByAction, insuffPredMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction");
}
else
{
// not expecting any nondet alts, make sure there are none
GrammarInsufficientPredicatesMessage nondetMsg =
getGrammarInsufficientPredicatesMessage(equeue.warnings);
if (nondetMsg != null)
{
Console.Out.WriteLine(equeue.warnings);
}
Assert.IsNull(nondetMsg, "found insufficiently covered alts, but expecting none");
}
Assert.AreEqual(expecting, result);
}
public void Init()
{
// List of DFA nodes in minimized DFA node
List <CGraphNode> configurationAcc, configurationNonAcc;
// Create configuration (min-DFA node) for accepted nodes
CGraphNode acceptedConf = m_minimizedDFA.CreateGraphNode <CGraphNode>();
// Create a list of initial-DFA nodes that are registered as accepted nodes
configurationAcc = new List <CGraphNode>();
// Store the list for initial-DFA accepted nodes in the min-DFA for accepted nodes
SetNodesInConfiguration(acceptedConf, configurationAcc);
// Create configuration (min-DFA node) for non-accepted nodes
CGraphNode non_acceptedConf = m_minimizedDFA.CreateGraphNode <CGraphNode>();
// Create a list for initial-DFA nodes that are registered as non-accepted nodes
configurationNonAcc = new List <CGraphNode>();
// Store the list of initial-DFA non-accepted node in the min-DFA for accepted nodes
SetNodesInConfiguration(non_acceptedConf, configurationNonAcc);
// Separate accepted from non-accepted nodes into two distinct
// configurations. Iterate over the input DFA and place the
// accepted node into configurationAcc and none accepted nodes
// into configurationNonAcc
CIt_GraphNodes it = new CIt_GraphNodes(m_DFA);
for (it.Begin(); !it.End(); it.Next())
{
if (m_DFA.GetFinalStates().Contains(it.M_CurrentItem))
{
// Record in the input DFA node the in which minimized DFA node
// will be placed
SetNodeConfiguration(it.M_CurrentItem, acceptedConf);
configurationAcc.Add(it.M_CurrentItem);
}
else
{
// Record in the input DFA node the in which minimized DFA node
// will be placed
SetNodeConfiguration(it.M_CurrentItem, non_acceptedConf);
configurationNonAcc.Add(it.M_CurrentItem);
}
}
// ************************* Debug Initialization ***************************
m_DFA.RegisterGraphPrinter(new FATextPrinter(m_DFA));
m_DFA.Generate("HopCroft_InitialDFA_.txt");
m_reporting.SetInitialMinDFAStatesContents(acceptedConf, configurationAcc, non_acceptedConf, configurationNonAcc);
int nodeCount = 0;
int iteration_count = 0;
CIt_GraphNodes minDFA_it = new CIt_GraphNodes(m_minimizedDFA);
// Iterate while the algorithm reaches a fixed point state
while (nodeCount != m_minimizedDFA.M_NumberOfNodes)
{
nodeCount = m_minimizedDFA.M_NumberOfNodes;
// ************************* Debug ***************************
CIterationRecord currentIteration = m_reporting.AddIteration(iteration_count);
for (minDFA_it.Begin(); !minDFA_it.End(); minDFA_it.Next())
{
// ************************* Debug ***************************
currentIteration.M_NodesToInspect.Add(minDFA_it.M_CurrentItem);
currentIteration.M_InitialNodesConfiguration.Add(minDFA_it.M_CurrentItem, new List <CGraphNode>(GetNodesInConfiguration(minDFA_it.M_CurrentItem)));
Split(minDFA_it.M_CurrentItem);
}
}
// Draw the final edges
// Edges between nodes of the initial DFA are mapped to edges between
// configurations in minimized-DFA. Thus, edges are drawn between two
// min-DFA related configurations when their corresponding nodes are
// connected and their transition character set is combined
CIt_GraphNodes minit1 = new CIt_GraphNodes(m_minimizedDFA);
CIt_GraphNodes minit2 = new CIt_GraphNodes(m_minimizedDFA);
List <CGraphNode> confs, conft;
CGraphEdge edge, newedge;
for (minit1.Begin(); !minit1.End(); minit1.Next())
{
for (minit2.Begin(); !minit2.End(); minit2.Next())
{
confs = GetNodesInConfiguration(minit1.M_CurrentItem);
conft = GetNodesInConfiguration(minit2.M_CurrentItem);
foreach (CGraphNode snode in confs)
{
foreach (CGraphNode tnode in conft)
{
edge = m_DFA.Edge(snode, tnode);
if (edge != null)
{
// Disallow duplicate edges between nodes of the minimized DFA
newedge = m_minimizedDFA.Edge(minit1.M_CurrentItem, minit2.M_CurrentItem);
if (newedge == null)
{
newedge = m_minimizedDFA.AddGraphEdge <CGraphEdge, CGraphNode>(minit1.M_CurrentItem, minit2.M_CurrentItem, GraphType.GT_DIRECTED);
}
// Add transition characters
if (GetMinDFAStateTransitionCharacterSet(newedge) == null)
{
SetMinDFAStateTransitionCharacterSet(newedge, new CCharRangeSet(false));
m_minimizedDFA.SetFAEdgeInfo(newedge, new CCharRangeSet(false));
}
CCharRangeSet charset = GetMinDFAStateTransitionCharacterSet(newedge);
m_minimizedDFA.GetFAEdgeInfo(newedge).AddSet(m_DFA.GetFAEdgeInfo(edge));
}
}
}
}
}
// Detect accepted nodes in minimized DFA
CIt_GraphNodes it1 = new CIt_GraphNodes(m_minimizedDFA);
for (it1.Begin(); !it1.End(); it1.Next())
{
List <CGraphNode> configuration = GetNodesInConfiguration(it1.M_CurrentItem);
List <CGraphNode> finals = m_DFA.GetFinalStates();
foreach (CGraphNode node in configuration)
{
if (finals.Contains(node))
{
m_minimizedDFA.SetFinalState(it1.M_CurrentItem);
}
}
}
// Detect initial state of minimized DFA
for (it1.Begin(); !it1.End(); it1.Next())
{
List <CGraphNode> configuration = GetNodesInConfiguration(it1.M_CurrentItem);
CGraphNode initial = m_DFA.M_Initial;
foreach (CGraphNode node in configuration)
{
if (initial == node)
{
m_minimizedDFA.M_Initial = it1.M_CurrentItem;
}
}
}
// Set Final minimized-DFA labels
for (it1.Begin(); !it1.End(); it1.Next())
{
List <CGraphNode> conf = GetNodesInConfiguration(it1.M_CurrentItem);
foreach (CGraphNode iNode in conf)
{
HashSet <string> prefs = m_DFA.GetFANodePrefixLabels(iNode);
foreach (string s in prefs)
{
m_minimizedDFA.SetFANodePrefix(s, it1.M_CurrentItem);
}
foreach (uint dependency in m_DFA.GetFANodeLineDependencies(iNode))
{
m_minimizedDFA.SetFANodeLineDependency(dependency, it1.M_CurrentItem);
}
}
m_minimizedDFA.PrefixElementLabel(m_minimizedDFA.GetFANodePrefix(it1.M_CurrentItem), it1.M_CurrentItem);
}
m_reporting.Report("HOPCROFTReport.txt");
FASerializer serializer = new FASerializer(m_minimizedDFA);
serializer.Print();
m_DFA.RegisterGraphPrinter(new FAGraphVizPrinter(m_minimizedDFA, new UOPCore.Options <ThompsonOptions>()));
m_DFA.Generate(@"minimizedDFA.dot", true);
}
