public GrammarSemanticsMessage( int msgID,
Grammar g,
IToken offendingToken,
object arg )
: this(msgID, g, offendingToken, arg, null)
{
}
public GrammarSyntaxMessage( int msgID,
Grammar grammar,
IToken offendingToken,
RecognitionException exception )
: this(msgID, grammar, offendingToken, null, exception)
{
}
public void TestMessageStringificationIsConsistent()
{
string action = "$other.tree = null;";
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar a;\n" +
"options { output = AST;}" +
"otherrule\n" +
" : 'y' ;" +
"rule\n" +
" : other=otherrule {" + action + "}\n" +
" ;" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
ActionTranslator translator = new ActionTranslator( generator,
"rule",
new CommonToken( ANTLRParser.ACTION, action ), 1 );
string rawTranslation =
translator.Translate();
int expectedMsgID = ErrorManager.MSG_WRITE_TO_READONLY_ATTR;
object expectedArg = "other";
object expectedArg2 = "tree";
GrammarSemanticsMessage expectedMessage =
new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg, expectedArg2 );
string expectedMessageString = expectedMessage.ToString();
Assert.AreEqual( expectedMessageString, expectedMessage.ToString() );
}
public TreeToNFAConverter( Grammar g, NFA nfa, NFAFactory factory, ITreeNodeStream input )
: this(input)
{
this.grammar = g;
this.nfa = nfa;
this.factory = factory;
}
public void TestCompleteBuffer()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"ID : 'a'..'z'+;\n" +
"INT : '0'..'9'+;\n" +
"SEMI : ';';\n" +
"ASSIGN : '=';\n" +
"PLUS : '+';\n" +
"MULT : '*';\n" +
"WS : ' '+;\n");
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
ICharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
Interpreter lexEngine = new Interpreter(g, input);
BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
int i = 1;
IToken t = tokens.LT(i);
while (t.Type != CharStreamConstants.EndOfFile)
{
i++;
t = tokens.LT(i);
}
tokens.LT(i++); // push it past end
tokens.LT(i++);
string result = tokens.ToString();
string expecting = "x = 3 * 0 + 2 * 0;";
Assert.AreEqual(expecting, result);
}
public void TestCannotHaveSpaceAfterDot()
{
string action = "%x. y = z;";
//String expecting = null;
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
Grammar g = new Grammar(
"grammar t;\n" +
"options {\n" +
" output=template;\n" +
"}\n" +
"\n" +
"a : ID {" + action + "}\n" +
" ;\n" +
"\n" +
"ID : 'a';\n" );
AntlrTool antlr = newTool();
CodeGenerator generator = new CodeGenerator( antlr, g, "Java" );
g.CodeGenerator = generator;
generator.GenRecognizer(); // forces load of templates
int expectedMsgID = ErrorManager.MSG_INVALID_TEMPLATE_ACTION;
object expectedArg = "%x.";
GrammarSemanticsMessage expectedMessage =
new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg );
checkError( equeue, expectedMessage );
}
public DgmlGenerator(Grammar grammar)
{
if (grammar == null)
throw new ArgumentNullException("grammar");
_grammar = grammar;
}
public ActionAnalysisLexer(Grammar grammar, string ruleName, GrammarAST actionAST)
: this(new ANTLRStringStream(actionAST.Token.Text))
{
this.grammar = grammar;
this.enclosingRule = grammar.GetLocallyDefinedRule(ruleName);
this.actionToken = actionAST.Token;
this.outerAltNum = actionAST.outerAltNum;
}
public void TestA()
{
Grammar g = new Grammar(
"parser grammar t;\n" +
"a : A C | B;" );
string expecting =
".s0-A->:s1=>1" + NewLine +
".s0-B->:s2=>2" + NewLine;
checkDecision( g, 1, expecting, null, null, null, null, 0 );
}
public LeftRecursiveRuleAnalyzer(ITreeNodeStream input, Grammar g, string ruleName)
: base(input)
{
this.g = g;
this.ruleName = ruleName;
language = (string)g.GetOption("language");
generator = new CodeGenerator(g.Tool, g, language);
generator.LoadTemplates(language);
recRuleTemplates = LoadPrecRuleTemplates(g.Tool);
}
public GrammarSemanticsMessage( int msgID,
Grammar g,
IToken offendingToken,
object arg,
object arg2 )
: base(msgID, arg, arg2)
{
this.g = g;
this.offendingToken = offendingToken;
}
public void TestAB_or_AC()
{
Grammar g = new Grammar(
"parser grammar t;\n" +
"a : A B | A C;" );
string expecting =
".s0-A->.s1" + NewLine +
".s1-B->:s2=>1" + NewLine +
".s1-C->:s3=>2" + NewLine;
checkDecision( g, 1, expecting, null, null, null, null, 0 );
}
public void TestAndPredicates()
{
Grammar g = new Grammar(
"parser grammar P;\n" +
"a : {p1}? {p1a}? A | {p2}? A ;" );
string expecting =
".s0-A->.s1" + NewLine +
".s1-{(p1&&p1a)}?->:s2=>1" + NewLine +
".s1-{p2}?->:s3=>2" + NewLine;
checkDecision( g, 1, expecting, null, null, null, null, null, 0, false );
}
public GrammarSyntaxMessage( int msgID,
Grammar grammar,
IToken offendingToken,
Object arg,
RecognitionException exception )
: base(msgID, arg, null)
{
this.offendingToken = offendingToken;
this.exception = exception;
this.g = grammar;
}
public void TestAdjacentNotCharLoops()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : (~'r')+ ;\n" +
"B : (~'s')+ ;\n" );
string expecting =
".s0-'r'->:s3=>2" + NewLine +
".s0-'s'->:s2=>1" + NewLine +
".s0-{'\\u0000'..'q', 't'..'\\uFFFF'}->.s1" + NewLine +
".s1-'r'->:s3=>2" + NewLine +
".s1-<EOT>->:s2=>1" + NewLine +
".s1-{'\\u0000'..'q', 't'..'\\uFFFF'}->.s1" + NewLine;
checkDecision( g, 3, expecting, null );
}
public void TestBadGrammarOption()
{
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue ); // unique listener per thread
AntlrTool antlr = newTool();
Grammar g = new Grammar( antlr,
"grammar t;\n" +
"options {foo=3; language=Java;}\n" +
"a : 'a';\n" );
object expectedArg = "foo";
int expectedMsgID = ErrorManager.MSG_ILLEGAL_OPTION;
GrammarSemanticsMessage expectedMessage =
new GrammarSemanticsMessage( expectedMsgID, g, null, expectedArg );
checkGrammarSemanticsError( equeue, expectedMessage );
}
public void Test3LevelImport()
{
ErrorQueue equeue = new ErrorQueue();
ErrorManager.SetErrorListener( equeue );
string slave =
"parser grammar T;\n" +
"a : T ;\n";
mkdir( tmpdir );
writeFile( tmpdir, "T.g", slave );
string slave2 =
"parser grammar S;\n" + // A, B, C token type order
"import T;\n" +
"a : S ;\n";
mkdir( tmpdir );
writeFile( tmpdir, "S.g", slave2 );
string master =
"grammar M;\n" +
"import S;\n" +
"a : M ;\n";
writeFile( tmpdir, "M.g", master );
AntlrTool antlr = newTool( new string[] { "-lib", tmpdir } );
CompositeGrammar composite = new CompositeGrammar();
Grammar g = new Grammar( antlr, tmpdir + "/M.g", composite );
composite.SetDelegationRoot( g );
g.ParseAndBuildAST();
g.composite.AssignTokenTypes();
g.composite.DefineGrammarSymbols();
string expectedTokenIDToTypeMap = "[M=6, S=5, T=4]";
string expectedStringLiteralToTypeMap = "{}";
string expectedTypeToTokenList = "[T, S, M]";
assertEquals( expectedTokenIDToTypeMap,
realElements( g.composite.tokenIDToTypeMap ).ToElementString() );
assertEquals( expectedStringLiteralToTypeMap, g.composite.stringLiteralToTypeMap.ToElementString() );
assertEquals( expectedTypeToTokenList,
realElements( g.composite.typeToTokenList ).ToElementString() );
assertEquals( "unexpected errors: " + equeue, 0, equeue.errors.Count );
bool ok =
rawGenerateAndBuildRecognizer( "M.g", master, "MParser", null, false );
bool expecting = true; // should be ok
assertEquals( expecting, ok );
}
protected internal override void DefineTokens( Grammar root )
{
//System.Console.Out.WriteLine( "stringLiterals=" + stringLiterals );
//System.Console.Out.WriteLine( "tokens=" + tokens );
//System.Console.Out.WriteLine( "aliases=" + aliases );
//System.Console.Out.WriteLine( "aliasesReverseIndex=" + aliasesReverseIndex );
AssignTokenIDTypes( root );
AliasTokenIDsAndLiterals( root );
AssignStringTypes( root );
//System.Console.Out.WriteLine( "stringLiterals=" + stringLiterals );
//System.Console.Out.WriteLine( "tokens=" + tokens );
//System.Console.Out.WriteLine( "aliases=" + aliases );
DefineTokenNamesAndLiteralsInGrammar( root );
}
public virtual CompositeGrammarTree FindNode( Grammar g )
{
if ( g == null )
{
return null;
}
if ( this.grammar == g )
{
return this;
}
CompositeGrammarTree n = null;
for ( int i = 0; n == null && children != null && i < children.Count; i++ )
{
CompositeGrammarTree child = children[i];
n = child.FindNode( g );
}
return n;
}
internal virtual void Stats(Grammar g, StringBuilder buf)
{
int numDec = g.NumberOfDecisions;
for (int decision = 1; decision <= numDec; decision++)
{
Grammar.Decision d = g.GetDecision(decision);
if (d.dfa == null)
{ // unusued decisions in auto synpreds
//System.err.println("no decision "+decision+" dfa for "+d.blockAST.toStringTree());
continue;
}
int k = d.dfa.MaxLookaheadDepth;
Rule enclosingRule = d.dfa.NFADecisionStartState.enclosingRule;
if (enclosingRule.IsSynPred)
continue; // don't count synpred rules
buf.Append(g.name + "." + enclosingRule.Name + ":" +
"");
GrammarAST decisionAST =
d.dfa.NFADecisionStartState.associatedASTNode;
buf.Append(decisionAST.Line);
buf.Append(":");
buf.Append(decisionAST.CharPositionInLine);
buf.Append(" decision " + decision + ":");
if (d.dfa.IsCyclic)
buf.Append(" cyclic");
if (k != int.MaxValue)
buf.Append(" k=" + k); // fixed, no sempreds
if (d.dfa.HasSynPred)
buf.Append(" backtracks"); // isolated synpred not gated
if (d.dfa.HasSemPred)
buf.Append(" sempred"); // user-defined sempred
// else {
// buf.append("undefined");
// FASerializer serializer = new FASerializer(g);
// String result = serializer.serialize(d.dfa.startState);
// System.err.println(result);
// }
buf.AppendLine();
}
}
public void Test2ndToken()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"ID : 'a'..'z'+;\n" +
"INT : '0'..'9'+;\n" +
"SEMI : ';';\n" +
"ASSIGN : '=';\n" +
"PLUS : '+';\n" +
"MULT : '*';\n" +
"WS : ' '+;\n");
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
ICharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
Interpreter lexEngine = new Interpreter(g, input);
BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
string result = tokens.LT(2).Text;
string expecting = " ";
Assert.AreEqual(expecting, result);
}
/** This aspect is associated with a grammar */
public DOTGenerator( Grammar grammar )
{
this.grammar = grammar;
this.dfaTemplateDirectoryName = Path.Combine(AntlrTool.ToolPathRoot, @"Tool\Templates\dot");
}
public void TestAltConflictsWithLoopThenExit()
{
// \" predicts alt 1, but wildcard then " can predict exit also
Grammar g = new Grammar(
"lexer grammar t;\n" +
"STRING : '\"' (options {greedy=false;}: '\\\\\"' | .)* '\"' ;\n"
);
string expecting =
".s0-'\"'->:s1=>3" + NewLine +
".s0-'\\\\'->.s2" + NewLine +
".s0-{'\\u0000'..'!', '#'..'[', ']'..'\\uFFFF'}->:s4=>2" + NewLine +
".s2-'\"'->:s3=>1" + NewLine +
".s2-{'\\u0000'..'!', '#'..'\\uFFFF'}->:s4=>2" + NewLine;
checkDecision( g, 1, expecting, null );
}
//throws Exception
protected void checkDecision( Grammar g,
int decision,
string expecting,
int[] expectingUnreachableAlts )
{
Antlr3.AntlrTool tool = new Antlr3.AntlrTool();
// mimic actions of org.antlr.Tool first time for grammar g
if ( g.CodeGenerator == null )
{
CodeGenerator generator = new CodeGenerator( tool, g, "Java" );
g.CodeGenerator = generator;
g.BuildNFA();
g.CreateLookaheadDFAs( false );
}
DFA dfa = g.GetLookaheadDFA( decision );
Assert.IsNotNull(dfa, "unknown decision #" + decision);
FASerializer serializer = new FASerializer( g );
string result = serializer.Serialize( dfa.StartState );
//System.out.print(result);
var nonDetAlts = dfa.UnreachableAlts;
//System.out.println("alts w/o predict state="+nonDetAlts);
// first make sure nondeterministic alts are as expected
if ( expectingUnreachableAlts == null )
{
if ( nonDetAlts != null && nonDetAlts.Count != 0 )
{
Console.Error.WriteLine( "nondeterministic alts (should be empty): " + ( (IList)nonDetAlts ).ToElementString() );
}
Assert.AreEqual(0, nonDetAlts != null ? nonDetAlts.Count : 0, "unreachable alts mismatch");
}
else
{
for ( int i = 0; i < expectingUnreachableAlts.Length; i++ )
{
Assert.IsTrue(nonDetAlts != null ? nonDetAlts.Contains(expectingUnreachableAlts[i]) : false, "unreachable alts mismatch");
}
}
Assert.AreEqual( expecting, result );
}
/*throws Exception*/
// S U P P O R T
public void _template()
{
Grammar g = new Grammar(
"grammar T;\n" +
"a : A | B;" );
string expecting =
"\n";
checkDecision( g, 1, expecting, null );
}
public void TestSynPredInLexer()
{
Grammar g = new Grammar(
"lexer grammar T;\n" +
"LT: '<' ' '*\n" +
" | ('<' IDENT) => '<' IDENT '>'\n" + // this was causing syntax error
" ;\n" +
"IDENT: 'a'+;\n" );
// basically, Tokens rule should not do set compression test
string expecting =
".s0-'<'->:s1=>1" + NewLine +
".s0-'a'->:s2=>2" + NewLine;
checkDecision( g, 4, expecting, null ); // 4 is Tokens rule
}
public void TestSimpleRangeVersusChar()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"A : 'a'..'z' '@' | 'k' '$' ;" );
g.CreateLookaheadDFAs();
string expecting =
".s0-'k'->.s1" + NewLine +
".s0-{'a'..'j', 'l'..'z'}->:s2=>1" + NewLine +
".s1-'$'->:s3=>2" + NewLine +
".s1-'@'->:s2=>1" + NewLine;
checkDecision( g, 1, expecting, null );
}
public void TestSetCallsRuleWithNot()
{
Grammar g = new Grammar(
"lexer grammar A;\n" +
"T : ~'x' ;\n" +
"S : 'x' (T | 'x') ;\n" );
string expecting =
".s0-'x'->:s2=>2" + NewLine +
".s0-{'\\u0000'..'w', 'y'..'\\uFFFF'}->:s1=>1" + NewLine;
checkDecision( g, 1, expecting, null );
}
public void TestRecursive2()
{
// this is also cool because it resolves \\ to be ESC alt; it's just
// less efficient of a DFA
Grammar g = new Grammar(
"lexer grammar duh;\n" +
"SUBTEMPLATE\n" +
" : '{'\n" +
" ( SUBTEMPLATE\n" +
" | ESC\n" +
" | ~('}'|'{')\n" +
" )*\n" +
" '}'\n" +
" ;\n" +
"fragment\n" +
"ESC : '\\\\' . ;" );
g.CreateLookaheadDFAs();
string expecting =
".s0-'\\\\'->.s3" + NewLine +
".s0-'{'->:s2=>1" + NewLine +
".s0-'}'->:s1=>4" + NewLine +
".s0-{'\\u0000'..'[', ']'..'z', '|', '~'..'\\uFFFF'}->:s5=>3" + NewLine +
".s3-'\\\\'->:s8=>2" + NewLine +
".s3-'{'->:s7=>2" + NewLine +
".s3-'}'->.s4" + NewLine +
".s3-{'\\u0000'..'[', ']'..'z', '|', '~'..'\\uFFFF'}->:s6=>2" + NewLine +
".s4-'\\u0000'..'\\uFFFF'->:s6=>2" + NewLine +
".s4-<EOT>->:s5=>3" + NewLine;
checkDecision( g, 1, expecting, null );
}
public void TestRecursive()
{
// this is cool because the 3rd alt includes !(all other possibilities)
Grammar g = new Grammar(
"lexer grammar duh;\n" +
"SUBTEMPLATE\n" +
" : '{'\n" +
" ( SUBTEMPLATE\n" +
" | ESC\n" +
" | ~('}'|'\\\\'|'{')\n" +
" )*\n" +
" '}'\n" +
" ;\n" +
"fragment\n" +
"ESC : '\\\\' . ;" );
g.CreateLookaheadDFAs();
string expecting =
".s0-'\\\\'->:s2=>2" + NewLine +
".s0-'{'->:s1=>1" + NewLine +
".s0-'}'->:s4=>4" + NewLine +
".s0-{'\\u0000'..'[', ']'..'z', '|', '~'..'\\uFFFF'}->:s3=>3" + NewLine;
checkDecision( g, 1, expecting, null );
}