public void TestLoopsWithOptimizedOutExitBranches()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'x'* ~'x'+ ;\n" );
string expecting =
".s0-'x'->:s1=>1" + NewLine +
".s0-{'\\u0000'..'w', 'y'..'\\uFFFF'}->:s2=>2" + NewLine;
checkDecision( g, 1, expecting, null );
// The optimizer yanks out all exit branches from EBNF blocks
// This is ok because we've already verified there are no problems
// with the enter/exit decision
DFAOptimizer optimizer = new DFAOptimizer( g );
optimizer.Optimize();
FASerializer serializer = new FASerializer( g );
DFA dfa = g.GetLookaheadDFA( 1 );
string result = serializer.Serialize( dfa.StartState );
expecting = ".s0-'x'->:s1=>1" + NewLine;
Assert.AreEqual( expecting, result );
}
//throws Exception
protected void checkDecision( Grammar g,
int decision,
string expecting,
int[] expectingUnreachableAlts )
{
Antlr3.AntlrTool tool = new Antlr3.AntlrTool();
// mimic actions of org.antlr.Tool first time for grammar g
if ( g.CodeGenerator == null )
{
CodeGenerator generator = new CodeGenerator( tool, g, "Java" );
g.CodeGenerator = generator;
g.BuildNFA();
g.CreateLookaheadDFAs( false );
}
DFA dfa = g.GetLookaheadDFA( decision );
Assert.IsNotNull(dfa, "unknown decision #" + decision);
FASerializer serializer = new FASerializer( g );
string result = serializer.Serialize( dfa.StartState );
//System.out.print(result);
var nonDetAlts = dfa.UnreachableAlts;
//System.out.println("alts w/o predict state="+nonDetAlts);
// first make sure nondeterministic alts are as expected
if ( expectingUnreachableAlts == null )
{
if ( nonDetAlts != null && nonDetAlts.Count != 0 )
{
Console.Error.WriteLine( "nondeterministic alts (should be empty): " + ( (IList)nonDetAlts ).ToElementString() );
}
Assert.AreEqual(0, nonDetAlts != null ? nonDetAlts.Count : 0, "unreachable alts mismatch");
}
else
{
for ( int i = 0; i < expectingUnreachableAlts.Length; i++ )
{
Assert.IsTrue(nonDetAlts != null ? nonDetAlts.Contains(expectingUnreachableAlts[i]) : false, "unreachable alts mismatch");
}
}
Assert.AreEqual( expecting, result );
}
public override string ToString()
{
FASerializer serializer = new FASerializer( Nfa.Grammar );
if ( StartState == null )
{
return "";
}
return serializer.Serialize( StartState, false );
}
//throws Exception
protected void checkDecision( Grammar g,
int decision,
string expecting,
int[] expectingUnreachableAlts,
int[] expectingNonDetAlts,
string expectingAmbigInput,
int[] expectingInsufficientPredAlts,
int[] expectingDanglingAlts,
int expectingNumWarnings,
bool hasPredHiddenByAction )
{
DecisionProbe.verbose = true; // make sure we get all error info
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" );
g.CodeGenerator = generator;
// mimic actions of org.antlr.Tool first time for grammar g
if ( g.NumberOfDecisions == 0 )
{
g.BuildNFA();
g.CreateLookaheadDFAs( false );
}
if ( equeue.size() != expectingNumWarnings )
{
Console.Error.WriteLine( "Warnings issued: " + equeue );
}
Assert.AreEqual(expectingNumWarnings, equeue.size(), "unexpected number of expected problems");
DFA dfa = g.GetLookaheadDFA( decision );
FASerializer serializer = new FASerializer( g );
string result = serializer.Serialize( dfa.StartState );
//System.out.print(result);
var unreachableAlts = dfa.UnreachableAlts;
// make sure unreachable alts are as expected
if ( expectingUnreachableAlts != null )
{
BitSet s = new BitSet();
s.AddAll( expectingUnreachableAlts );
BitSet s2 = new BitSet();
s2.AddAll( unreachableAlts );
Assert.AreEqual(s, s2, "unreachable alts mismatch");
}
else
{
Assert.AreEqual(0, unreachableAlts != null ? unreachableAlts.Count : 0, "unreachable alts mismatch");
}
// check conflicting input
if ( expectingAmbigInput != null )
{
// first, find nondet message
Message msg = getNonDeterminismMessage( equeue.warnings );
Assert.IsNotNull(msg, "no nondeterminism warning?");
Assert.IsTrue(msg is GrammarNonDeterminismMessage, "expecting nondeterminism; found " + msg.GetType().Name);
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage( equeue.warnings );
var labels =
nondetMsg.probe.GetSampleNonDeterministicInputSequence( nondetMsg.problemState );
string input = nondetMsg.probe.GetInputSequenceDisplay( labels );
Assert.AreEqual( expectingAmbigInput, input );
}
// check nondet alts
if ( expectingNonDetAlts != null )
{
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage( equeue.warnings );
Assert.IsNotNull(nondetMsg, "found no nondet alts; expecting: " + str(expectingNonDetAlts));
var nonDetAlts =
nondetMsg.probe.GetNonDeterministicAltsForState( nondetMsg.problemState );
// compare nonDetAlts with expectingNonDetAlts
BitSet s = new BitSet();
s.AddAll( expectingNonDetAlts );
BitSet s2 = new BitSet();
s2.AddAll( nonDetAlts );
Assert.AreEqual(s, s2, "nondet alts mismatch");
Assert.AreEqual(hasPredHiddenByAction, nondetMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction");
}
else
{
// not expecting any nondet alts, make sure there are none
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage( equeue.warnings );
Assert.IsNull(nondetMsg, "found nondet alts, but expecting none");
}
if ( expectingInsufficientPredAlts != null )
{
GrammarInsufficientPredicatesMessage insuffPredMsg =
getGrammarInsufficientPredicatesMessage( equeue.warnings );
Assert.IsNotNull(insuffPredMsg, "found no GrammarInsufficientPredicatesMessage alts; expecting: " + str(expectingNonDetAlts));
var locations = insuffPredMsg.altToLocations;
var actualAlts = locations.Keys;
BitSet s = new BitSet();
s.AddAll( expectingInsufficientPredAlts );
BitSet s2 = new BitSet();
s2.AddAll( actualAlts );
Assert.AreEqual(s, s2, "mismatch between insufficiently covered alts");
Assert.AreEqual(hasPredHiddenByAction, insuffPredMsg.problemState.Dfa.HasPredicateBlockedByAction, "mismatch between expected hasPredHiddenByAction");
}
else
{
// not expecting any nondet alts, make sure there are none
GrammarInsufficientPredicatesMessage nondetMsg =
getGrammarInsufficientPredicatesMessage( equeue.warnings );
if ( nondetMsg != null )
{
Console.Out.WriteLine( equeue.warnings );
}
Assert.IsNull(nondetMsg, "found insufficiently covered alts, but expecting none");
}
Assert.AreEqual( expecting, result );
}
public virtual void CreateLookaheadDFAs( bool wackTempStructures )
{
if ( nfa == null )
{
BuildNFA();
}
// CHECK FOR LEFT RECURSION; Make sure we can actually do analysis
CheckAllRulesForLeftRecursion();
/*
// was there a severe problem while sniffing the grammar?
if ( ErrorManager.doNotAttemptAnalysis() ) {
return;
}
*/
DateTime start = DateTime.Now;
//[email protected]("### create DFAs");
int numDecisions = NumberOfDecisions;
if ( NFAToDFAConverter.SINGLE_THREADED_NFA_CONVERSION )
{
for ( int decision = 1; decision <= numDecisions; decision++ )
{
NFAState decisionStartState = GetDecisionNFAStartState( decision );
if ( leftRecursiveRules.Contains( decisionStartState.enclosingRule ) )
{
// don't bother to process decisions within left recursive rules.
if ( composite.WatchNFAConversion )
{
Console.Out.WriteLine( "ignoring decision " + decision +
" within left-recursive rule " + decisionStartState.enclosingRule.Name );
}
continue;
}
if ( !externalAnalysisAbort && decisionStartState.NumberOfTransitions > 1 )
{
Rule r = decisionStartState.enclosingRule;
if ( r.IsSynPred && !synPredNamesUsedInDFA.Contains( r.Name ) )
{
continue;
}
DFA dfa = null;
// if k=* or k=1, try LL(1)
if ( GetUserMaxLookahead( decision ) == 0 ||
GetUserMaxLookahead( decision ) == 1 )
{
dfa = CreateLL_1_LookaheadDFA( decision );
}
if ( dfa == null )
{
if ( composite.WatchNFAConversion )
{
Console.Out.WriteLine( "decision " + decision +
" not suitable for LL(1)-optimized DFA analysis" );
}
dfa = CreateLookaheadDFA( decision, wackTempStructures );
}
if ( dfa.StartState == null )
{
// something went wrong; wipe out DFA
SetLookaheadDFA( decision, null );
}
if ( Tool.internalOption_PrintDFA )
{
Console.Out.WriteLine( "DFA d=" + decision );
FASerializer serializer = new FASerializer( nfa.Grammar );
string result = serializer.Serialize( dfa.StartState );
Console.Out.WriteLine( result );
}
}
}
}
else
{
ErrorManager.Info( "two-threaded DFA conversion" );
// create a barrier expecting n DFA and this main creation thread
Barrier barrier = new Barrier( 3 );
// assume 2 CPU for now
int midpoint = numDecisions / 2;
NFAConversionThread t1 =
new NFAConversionThread( this, barrier, 1, midpoint );
new System.Threading.Thread( t1.Run ).Start();
//new Thread( t1 ).start();
if ( midpoint == ( numDecisions / 2 ) )
{
midpoint++;
}
NFAConversionThread t2 =
new NFAConversionThread( this, barrier, midpoint, numDecisions );
new System.Threading.Thread( t2.Run ).Start();
//new Thread( t2 ).start();
// wait for these two threads to finish
try
{
barrier.WaitForRelease();
}
//catch ( InterruptedException e )
//{
// ErrorManager.internalError( "what the hell? DFA interruptus", e );
//}
catch
{
throw new System.NotImplementedException();
}
}
DateTime stop = DateTime.Now;
DFACreationWallClockTimeInMS = stop - start;
// indicate that we've finished building DFA (even if #decisions==0)
allDecisionDFACreated = true;
}
//throws Exception
protected void checkDecision( Grammar g,
int decision,
string expecting,
int[] expectingUnreachableAlts,
int[] expectingNonDetAlts,
string expectingAmbigInput,
int[] expectingDanglingAlts,
int expectingNumWarnings )
{
DecisionProbe.verbose = true; // make sure we get all error info
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
// mimic actions of org.antlr.Tool first time for grammar g
if ( g.NumberOfDecisions == 0 )
{
g.BuildNFA();
g.CreateLookaheadDFAs( false );
}
CodeGenerator generator = new CodeGenerator( newTool(), g, "Java" );
g.CodeGenerator = generator;
if ( equeue.size() != expectingNumWarnings )
{
Console.Error.WriteLine( "Warnings issued: " + equeue );
}
Assert.AreEqual(expectingNumWarnings, equeue.size(), "unexpected number of expected problems");
DFA dfa = g.GetLookaheadDFA( decision );
Assert.IsNotNull( dfa, "no DFA for decision " + decision );
FASerializer serializer = new FASerializer( g );
string result = serializer.Serialize( dfa.StartState );
var unreachableAlts = dfa.UnreachableAlts;
// make sure unreachable alts are as expected
if ( expectingUnreachableAlts != null )
{
BitSet s = new BitSet();
s.AddAll( expectingUnreachableAlts );
BitSet s2 = new BitSet();
s2.AddAll( unreachableAlts );
Assert.AreEqual(s, s2, "unreachable alts mismatch");
}
else
{
Assert.AreEqual(0, unreachableAlts != null ? unreachableAlts.Count : 0, "number of unreachable alts");
}
// check conflicting input
if ( expectingAmbigInput != null )
{
// first, find nondet message
Message msg = (Message)equeue.warnings[0];
Assert.IsTrue(msg is GrammarNonDeterminismMessage, "expecting nondeterminism; found " + msg.GetType().Name);
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage( equeue.warnings );
var labels =
nondetMsg.probe.GetSampleNonDeterministicInputSequence( nondetMsg.problemState );
string input = nondetMsg.probe.GetInputSequenceDisplay( labels );
Assert.AreEqual( expectingAmbigInput, input );
}
// check nondet alts
if ( expectingNonDetAlts != null )
{
RecursionOverflowMessage recMsg = null;
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage( equeue.warnings );
IList<int> nonDetAlts = null;
if ( nondetMsg != null )
{
nonDetAlts =
nondetMsg.probe.GetNonDeterministicAltsForState( nondetMsg.problemState );
}
else
{
recMsg = getRecursionOverflowMessage( equeue.warnings );
if ( recMsg != null )
{
//nonDetAlts = new ArrayList(recMsg.alts);
}
}
// compare nonDetAlts with expectingNonDetAlts
BitSet s = new BitSet();
s.AddAll( expectingNonDetAlts );
BitSet s2 = new BitSet();
s2.AddAll( nonDetAlts );
Assert.AreEqual(s, s2, "nondet alts mismatch");
Assert.IsTrue(nondetMsg != null || recMsg != null, "found no nondet alts; expecting: " + str(expectingNonDetAlts));
}
else
{
// not expecting any nondet alts, make sure there are none
GrammarNonDeterminismMessage nondetMsg =
getNonDeterminismMessage( equeue.warnings );
Assert.IsNull(nondetMsg, "found nondet alts, but expecting none");
}
Assert.AreEqual( expecting, result );
}
public void TestLexerDelegatorRuleOverridesDelegateLeavingNoRules()
{
// M.Tokens has nothing to predict tokens from S. Should
// not include S.Tokens alt in this case?
string slave =
"lexer grammar S;\n" +
"A : 'a' {System.out.println(\"S.A\");} ;\n";
mkdir( tmpdir );
writeFile( tmpdir, "S.g", slave );
string master =
"lexer grammar M;\n" +
"import S;\n" +
"A : 'a' {System.out.println(\"M.A\");} ;\n" +
"WS : (' '|'\\n') {skip();} ;\n";
writeFile( tmpdir, "M.g", master );
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
CompositeGrammar composite = new CompositeGrammar();
Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
composite.SetDelegationRoot( g );
g.ParseAndBuildAST();
composite.AssignTokenTypes();
composite.DefineGrammarSymbols();
composite.CreateNFAs();
g.CreateLookaheadDFAs( false );
// predict only alts from M not S
string expectingDFA =
".s0-'a'->.s1\n" +
".s0-{'\\n', ' '}->:s3=>2\n" +
".s1-<EOT>->:s2=>1\n";
Antlr3.Analysis.DFA dfa = g.GetLookaheadDFA( 1 );
FASerializer serializer = new FASerializer( g );
string result = serializer.Serialize( dfa.startState );
assertEquals( expectingDFA, result );
// must not be a "unreachable alt: Tokens" error
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
}