public void TestPredsUsedAfterRecursionOverflow() /*throws Exception*/
{
// analysis must bail out due to non-LL(*) nature (ovf)
// retries with k=1 (but with LL(*) algorithm not optimized version
// as it has preds)
Grammar g = new Grammar(
"parser grammar P;\n" +
"s : {p1}? e '.' | {p2}? e ':' ;\n" +
"e : '(' e ')' | INT ;\n");
string expecting =
".s0-'('->.s1" + NewLine +
".s0-INT->.s4" + NewLine +
".s1-{p1}?->:s2=>1" + NewLine +
".s1-{p2}?->:s3=>2" + NewLine +
".s4-{p1}?->:s2=>1" + NewLine +
".s4-{p2}?->:s3=>2" + NewLine;
DecisionProbe.verbose = true; // make sure we get all error info
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener(equeue);
CodeGenerator generator = new CodeGenerator(newTool(), g, "Java");
g.CodeGenerator = generator;
if (g.NumberOfDecisions == 0)
{
g.BuildNFA();
g.CreateLookaheadDFAs(false);
}
Assert.AreEqual(0, equeue.size(), "unexpected number of expected problems");
checkDecision(g, 1, expecting, null, null, null, null, null, 0, false);
}
public void TestIdenticalRules() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a' ;\n" +
"B : 'a' ;\n"); // can't reach this
string expecting =
".s0-'a'->.s1\n" +
".s1-<EOT>->:s2=>1\n";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener(equeue);
checkDecision(g, 1, expecting, new int[] { 2 });
assertEquals("unexpected number of expected problems",
1, equeue.size());
Message msg = (Message)equeue.errors[0];
assertTrue("warning must be an unreachable alt",
msg is GrammarUnreachableAltsMessage);
GrammarUnreachableAltsMessage u = (GrammarUnreachableAltsMessage)msg;
assertEquals("[2]", u.alts.ToElementString());
}
public void TestLeftRecursivePred() /*throws Exception*/
{
// No analysis possible. but probably good to fail. Not sure we really want
// left-recursion even if guarded with pred.
Grammar g = new Grammar(
"parser grammar P;\n" +
"s : a ;\n" +
"a : {p1}? a | ID ;\n");
DecisionProbe.verbose = true; // make sure we get all error info
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener(equeue);
CodeGenerator generator = new CodeGenerator(newTool(), g, "Java");
g.CodeGenerator = generator;
if (g.NumberOfDecisions == 0)
{
g.BuildNFA();
g.CreateLookaheadDFAs(false);
}
DFA dfa = g.GetLookaheadDFA(1);
assertEquals(null, dfa); // can't analyze.
/*
* String expecting =
* ".s0-ID->.s1\n" +
* ".s1-{p1}?->:s2=>1\n" +
* ".s1-{true}?->:s3=>2\n";
* String result = serializer.serialize(dfa.startState);
* assertEquals(expecting, result);
*/
assertEquals("unexpected number of expected problems", 1, equeue.size());
Message msg = equeue.errors[0];
assertTrue("warning must be a left recursion msg",
msg is LeftRecursionCyclesMessage);
}
public void TestNonGreedyLoopThatNeverLoops() /*throws Exception*/
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"DUH : (options {greedy=false;}:'x')+ ;"); // loop never matched
string expecting =
":s0=>2" + NewLine;
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener(equeue);
checkDecision(g, 1, expecting, new int[] { 1 });
Assert.AreEqual(1, equeue.size(), "unexpected number of expected problems");
Message msg = (Message)equeue.errors[0];
Assert.IsTrue(msg is GrammarUnreachableAltsMessage, "warning must be an unreachable alt");
GrammarUnreachableAltsMessage u = (GrammarUnreachableAltsMessage)msg;
Assert.AreEqual("[1]", u.alts.ToElementString());
}
public void TestNonGreedyLoopThatNeverLoops()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"DUH : (options {greedy=false;}:'x')+ ;" ); // loop never matched
string expecting =
":s0=>2" + NewLine;
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
checkDecision( g, 1, expecting, new int[] { 1 } );
Assert.AreEqual(1, equeue.size(), "unexpected number of expected problems");
Message msg = (Message)equeue.errors[0];
Assert.IsTrue(msg is GrammarUnreachableAltsMessage, "warning must be an unreachable alt");
GrammarUnreachableAltsMessage u = (GrammarUnreachableAltsMessage)msg;
Assert.AreEqual( "[1]", u.alts.ToElementString() );
}
public void TestIdenticalRules()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a' ;\n" +
"B : 'a' ;\n" ); // can't reach this
string expecting =
".s0-'a'->.s1" + NewLine +
".s1-<EOT>->:s2=>1" + NewLine;
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
checkDecision( g, 1, expecting, new int[] { 2 } );
Assert.AreEqual(1, equeue.size(), "unexpected number of expected problems");
Message msg = (Message)equeue.errors[0];
Assert.IsTrue( msg is GrammarUnreachableAltsMessage, "warning must be an unreachable alt" );
GrammarUnreachableAltsMessage u = (GrammarUnreachableAltsMessage)msg;
Assert.AreEqual( "[2]", u.alts.ToElementString() );
}
protected void checkDecision(Grammar g,
int decision,
string expecting,
int[] expectingUnreachableAlts,
int[] expectingNonDetAlts,
string expectingAmbigInput,
int[] expectingInsufficientPredAlts,
int[] expectingDanglingAlts,
int expectingNumWarnings,
bool hasPredHiddenByAction)
//throws Exception
{
DecisionProbe.verbose = true; // make sure we get all error info
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener(equeue);
CodeGenerator generator = new CodeGenerator(newTool(), g, "Java");
g.CodeGenerator = generator;
// mimic actions of org.antlr.Tool first time for grammar g
if (g.NumberOfDecisions == 0)
{
g.BuildNFA();
g.CreateLookaheadDFAs(false);
}
if (equeue.size() != expectingNumWarnings)
{
Console.Error.WriteLine("Warnings issued: " + equeue);
}
Assert.AreEqual(expectingNumWarnings, equeue.size(), "unexpected number of expected problems");
DFA dfa = g.GetLookaheadDFA(decision);
FASerializer serializer = new FASerializer(g);
string result = serializer.Serialize(dfa.StartState);
//System.out.print(result);
var unreachableAlts = dfa.UnreachableAlts;
// make sure unreachable alts are as expected
if (expectingUnreachableAlts != null)
{
BitSet s = new BitSet();
s.AddAll(expectingUnreachableAlts);
BitSet s2 = new BitSet();
s2.AddAll(unreachableAlts);
Assert.AreEqual(s, s2, "unreachable alts mismatch");
}
else
{
Assert.AreEqual(0, unreachableAlts != null ? unreachableAlts.Count : 0, "unreachable alts mismatch");
}
// check conflicting input
if (expectingAmbigInput != null)
{
// first, find nondet message
Message msg = getNonDeterminismMessage(equeue.warnings);
Assert.IsNotNull(msg, "no nondeterminism warning?");
Assert.IsTrue(msg is GrammarNonDeterminismMessage, "expecting nondeterminism; found " + msg.GetType().Name);
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage(equeue.warnings);
var labels =
nondetMsg.probe.GetSampleNonDeterministicInputSequence(nondetMsg.problemState);
string input = nondetMsg.probe.GetInputSequenceDisplay(labels);
Assert.AreEqual(expectingAmbigInput, input);
}
// check nondet alts
if (expectingNonDetAlts != null)
{
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage(equeue.warnings);
Assert.IsNotNull(nondetMsg, "found no nondet alts; expecting: " + str(expectingNonDetAlts));
var nonDetAlts =
nondetMsg.probe.GetNonDeterministicAltsForState(nondetMsg.problemState);
// compare nonDetAlts with expectingNonDetAlts
BitSet s = new BitSet();
s.AddAll(expectingNonDetAlts);
BitSet s2 = new BitSet();
s2.AddAll(nonDetAlts);
Assert.AreEqual(s, s2, "nondet alts mismatch");
Assert.AreEqual(hasPredHiddenByAction, nondetMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction");
}
else
{
// not expecting any nondet alts, make sure there are none
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage(equeue.warnings);
Assert.IsNull(nondetMsg, "found nondet alts, but expecting none");
}
if (expectingInsufficientPredAlts != null)
{
GrammarInsufficientPredicatesMessage insuffPredMsg =
getGrammarInsufficientPredicatesMessage(equeue.warnings);
Assert.IsNotNull(insuffPredMsg, "found no GrammarInsufficientPredicatesMessage alts; expecting: " + str(expectingNonDetAlts));
var locations = insuffPredMsg.altToLocations;
var actualAlts = locations.Keys;
BitSet s = new BitSet();
s.AddAll(expectingInsufficientPredAlts);
BitSet s2 = new BitSet();
s2.AddAll(actualAlts);
Assert.AreEqual(s, s2, "mismatch between insufficiently covered alts");
Assert.AreEqual(hasPredHiddenByAction, insuffPredMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction");
}
else
{
// not expecting any nondet alts, make sure there are none
GrammarInsufficientPredicatesMessage nondetMsg =
getGrammarInsufficientPredicatesMessage(equeue.warnings);
if (nondetMsg != null)
{
Console.Out.WriteLine(equeue.warnings);
}
Assert.IsNull(nondetMsg, "found insufficiently covered alts, but expecting none");
}
Assert.AreEqual(expecting, result);
}
//throws Exception
protected void checkDecision( Grammar g,
int decision,
string expecting,
int[] expectingUnreachableAlts,
int[] expectingNonDetAlts,
string expectingAmbigInput,
int[] expectingInsufficientPredAlts,
int[] expectingDanglingAlts,
int expectingNumWarnings,
bool hasPredHiddenByAction )
{
DecisionProbe.verbose = true; // make sure we get all error info
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" );
g.CodeGenerator = generator;
// mimic actions of org.antlr.Tool first time for grammar g
if ( g.NumberOfDecisions == 0 )
{
g.BuildNFA();
g.CreateLookaheadDFAs( false );
}
if ( equeue.size() != expectingNumWarnings )
{
Console.Error.WriteLine( "Warnings issued: " + equeue );
}
Assert.AreEqual(expectingNumWarnings, equeue.size(), "unexpected number of expected problems");
DFA dfa = g.GetLookaheadDFA( decision );
FASerializer serializer = new FASerializer( g );
string result = serializer.Serialize( dfa.StartState );
//System.out.print(result);
var unreachableAlts = dfa.UnreachableAlts;
// make sure unreachable alts are as expected
if ( expectingUnreachableAlts != null )
{
BitSet s = new BitSet();
s.AddAll( expectingUnreachableAlts );
BitSet s2 = new BitSet();
s2.AddAll( unreachableAlts );
Assert.AreEqual(s, s2, "unreachable alts mismatch");
}
else
{
Assert.AreEqual(0, unreachableAlts != null ? unreachableAlts.Count : 0, "unreachable alts mismatch");
}
// check conflicting input
if ( expectingAmbigInput != null )
{
// first, find nondet message
Message msg = getNonDeterminismMessage( equeue.warnings );
Assert.IsNotNull(msg, "no nondeterminism warning?");
Assert.IsTrue(msg is GrammarNonDeterminismMessage, "expecting nondeterminism; found " + msg.GetType().Name);
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage( equeue.warnings );
var labels =
nondetMsg.probe.GetSampleNonDeterministicInputSequence( nondetMsg.problemState );
string input = nondetMsg.probe.GetInputSequenceDisplay( labels );
Assert.AreEqual( expectingAmbigInput, input );
}
// check nondet alts
if ( expectingNonDetAlts != null )
{
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage( equeue.warnings );
Assert.IsNotNull(nondetMsg, "found no nondet alts; expecting: " + str(expectingNonDetAlts));
var nonDetAlts =
nondetMsg.probe.GetNonDeterministicAltsForState( nondetMsg.problemState );
// compare nonDetAlts with expectingNonDetAlts
BitSet s = new BitSet();
s.AddAll( expectingNonDetAlts );
BitSet s2 = new BitSet();
s2.AddAll( nonDetAlts );
Assert.AreEqual(s, s2, "nondet alts mismatch");
Assert.AreEqual(hasPredHiddenByAction, nondetMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction");
}
else
{
// not expecting any nondet alts, make sure there are none
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage( equeue.warnings );
Assert.IsNull(nondetMsg, "found nondet alts, but expecting none");
}
if ( expectingInsufficientPredAlts != null )
{
GrammarInsufficientPredicatesMessage insuffPredMsg =
getGrammarInsufficientPredicatesMessage( equeue.warnings );
Assert.IsNotNull(insuffPredMsg, "found no GrammarInsufficientPredicatesMessage alts; expecting: " + str(expectingNonDetAlts));
var locations = insuffPredMsg.altToLocations;
var actualAlts = locations.Keys;
BitSet s = new BitSet();
s.AddAll( expectingInsufficientPredAlts );
BitSet s2 = new BitSet();
s2.AddAll( actualAlts );
Assert.AreEqual(s, s2, "mismatch between insufficiently covered alts");
Assert.AreEqual(hasPredHiddenByAction, insuffPredMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction");
}
else
{
// not expecting any nondet alts, make sure there are none
GrammarInsufficientPredicatesMessage nondetMsg =
getGrammarInsufficientPredicatesMessage( equeue.warnings );
if ( nondetMsg != null )
{
Console.Out.WriteLine( equeue.warnings );
}
Assert.IsNull(nondetMsg, "found insufficiently covered alts, but expecting none");
}
Assert.AreEqual( expecting, result );
}
public void TestPredsUsedAfterRecursionOverflow()
{
// analysis must bail out due to non-LL(*) nature (ovf)
// retries with k=1 (but with LL(*) algorithm not optimized version
// as it has preds)
Grammar g = new Grammar(
"parser grammar P;\n" +
"s : {p1}? e '.' | {p2}? e ':' ;\n" +
"e : '(' e ')' | INT ;\n" );
string expecting =
".s0-'('->.s1" + NewLine +
".s0-INT->.s4" + NewLine +
".s1-{p1}?->:s2=>1" + NewLine +
".s1-{p2}?->:s3=>2" + NewLine +
".s4-{p1}?->:s2=>1" + NewLine +
".s4-{p2}?->:s3=>2" + NewLine;
DecisionProbe.verbose = true; // make sure we get all error info
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" );
g.CodeGenerator = generator;
if ( g.NumberOfDecisions == 0 )
{
g.BuildNFA();
g.CreateLookaheadDFAs( false );
}
Assert.AreEqual(0, equeue.size(), "unexpected number of expected problems");
checkDecision( g, 1, expecting, null, null, null, null, null, 0, false );
}
public void TestLeftRecursivePred()
{
// No analysis possible. but probably good to fail. Not sure we really want
// left-recursion even if guarded with pred.
Grammar g = new Grammar(
"parser grammar P;\n" +
"s : a ;\n" +
"a : {p1}? a | ID ;\n" );
DecisionProbe.verbose = true; // make sure we get all error info
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" );
g.CodeGenerator = generator;
if ( g.NumberOfDecisions == 0 )
{
g.BuildNFA();
g.CreateLookaheadDFAs( false );
}
DFA dfa = g.GetLookaheadDFA( 1 );
Assert.AreEqual( null, dfa ); // can't analyze.
/*
String expecting =
".s0-ID->.s1" + NewLine +
".s1-{p1}?->:s2=>1" + NewLine +
".s1-{true}?->:s3=>2" + NewLine;
String result = serializer.serialize(dfa.startState);
Assert.AreEqual(expecting, result);
*/
Assert.AreEqual(1, equeue.size(), "unexpected number of expected problems");
Message msg = equeue.errors[0];
Assert.IsTrue(msg is LeftRecursionCyclesMessage, "warning must be a left recursion msg");
}
//throws Exception
protected void checkDecision( Grammar g,
int decision,
string expecting,
int[] expectingUnreachableAlts,
int[] expectingNonDetAlts,
string expectingAmbigInput,
int[] expectingDanglingAlts,
int expectingNumWarnings )
{
DecisionProbe.verbose = true; // make sure we get all error info
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
// mimic actions of org.antlr.Tool first time for grammar g
if ( g.NumberOfDecisions == 0 )
{
g.BuildNFA();
g.CreateLookaheadDFAs( false );
}
CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" );
g.CodeGenerator = generator;
if ( equeue.size() != expectingNumWarnings )
{
Console.Error.WriteLine( "Warnings issued: " + equeue );
}
Assert.AreEqual(expectingNumWarnings, equeue.size(), "unexpected number of expected problems");
DFA dfa = g.GetLookaheadDFA( decision );
Assert.IsNotNull( dfa, "no DFA for decision " + decision );
FASerializer serializer = new FASerializer( g );
string result = serializer.Serialize( dfa.StartState );
var unreachableAlts = dfa.UnreachableAlts;
// make sure unreachable alts are as expected
if ( expectingUnreachableAlts != null )
{
BitSet s = new BitSet();
s.AddAll( expectingUnreachableAlts );
BitSet s2 = new BitSet();
s2.AddAll( unreachableAlts );
Assert.AreEqual(s, s2, "unreachable alts mismatch");
}
else
{
Assert.AreEqual(0, unreachableAlts != null ? unreachableAlts.Count : 0, "number of unreachable alts");
}
// check conflicting input
if ( expectingAmbigInput != null )
{
// first, find nondet message
Message msg = (Message)equeue.warnings[0];
Assert.IsTrue(msg is GrammarNonDeterminismMessage, "expecting nondeterminism; found " + msg.GetType().Name);
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage( equeue.warnings );
var labels =
nondetMsg.probe.GetSampleNonDeterministicInputSequence( nondetMsg.problemState );
string input = nondetMsg.probe.GetInputSequenceDisplay( labels );
Assert.AreEqual( expectingAmbigInput, input );
}
// check nondet alts
if ( expectingNonDetAlts != null )
{
RecursionOverflowMessage recMsg = null;
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage( equeue.warnings );
IList<int> nonDetAlts = null;
if ( nondetMsg != null )
{
nonDetAlts =
nondetMsg.probe.GetNonDeterministicAltsForState( nondetMsg.problemState );
}
else
{
recMsg = getRecursionOverflowMessage( equeue.warnings );
if ( recMsg != null )
{
//nonDetAlts = new ArrayList(recMsg.alts);
}
}
// compare nonDetAlts with expectingNonDetAlts
BitSet s = new BitSet();
s.AddAll( expectingNonDetAlts );
BitSet s2 = new BitSet();
s2.AddAll( nonDetAlts );
Assert.AreEqual(s, s2, "nondet alts mismatch");
Assert.IsTrue(nondetMsg != null || recMsg != null, "found no nondet alts; expecting: " + str(expectingNonDetAlts));
}
else
{
// not expecting any nondet alts, make sure there are none
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage( equeue.warnings );
Assert.IsNull(nondetMsg, "found nondet alts, but expecting none");
}
Assert.AreEqual( expecting, result );
}
