public static void ParseProgramOrClass(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> codeElementsLines, TypeCobolOptions compilerOptions, SymbolTable customSymbols, out Program newProgram, out Class newClass, out IList<ParserDiagnostic> diagnostics)
{
// Create an Antlr compatible token source on top a the token iterator
CodeElementsLinesTokenSource tokenSource = new CodeElementsLinesTokenSource(
textSourceInfo.Name,
codeElementsLines);
// Init parser
ITokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
ProgramClassParser cobolParser = new ProgramClassParser(tokenStream);
// -> activate full ambiguities detection
//parser.Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection;
// Register all parse errors in a list in memory
DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener();
cobolParser.RemoveErrorListeners();
cobolParser.AddErrorListener(errorListener);
// Try to parse a Cobol program or class
ProgramClassParser.CobolCompilationUnitContext codeElementParseTree = cobolParser.cobolCompilationUnit();
// Visit the parse tree to build a first class object representing a Cobol program or class
ParseTreeWalker walker = new ParseTreeWalker();
CobolNodeBuilder programClassBuilder = new CobolNodeBuilder();
programClassBuilder.CustomSymbols = customSymbols;
programClassBuilder.Dispatcher = new NodeDispatcher();
programClassBuilder.Dispatcher.CreateListeners();
walker.Walk(programClassBuilder, codeElementParseTree);
// Register compiler results
newProgram = programClassBuilder.Program;
newClass = programClassBuilder.Class;
diagnostics = errorListener.Diagnostics;
}
static void Main(string[] args)
{
string lQuery = "SELECT Jmeno, Prijmeni FROM Osoba, Film WHERE Stat LIKE 'Kanada' OR Rok_natoceni >1995 UNION " +
"(SELECT Jmeno, Prijmeni FROM Uzivatel WHERE Jmeno = 'ads' AND Prijmeni = 'asd');";
string lQuery2 = "SElect jmeno, nazev_cz FROM Osoba os JOIN Osoba_Film osf ON os.idO = osf.Osoba_idO JOIN Film f ON f.idF = osf.Film_idF Where f.Rok_natoceni > 1995;";
string lQuery1 = "select * from table1 t1, t2 where t1.neco = 1";
string text = ReadFile($@"C:\Users\Lukáš\Desktop\doc\Workload_bp.txt");
StringReader reader = new StringReader(text);
AntlrInputStream input = new AntlrInputStream(reader);
TSqlLexer lexer = new TSqlLexer(new CaseChangingCharStream(input, true));
CommonTokenStream tokens = new CommonTokenStream(lexer);
TSqlParser parser = new TSqlParser(tokens);
TSqlParser.Tsql_fileContext Tsql_fileContext1 = parser.tsql_file();
//Console.WriteLine("Tsql_fileContext1.ChildCount = " + Tsql_fileContext1.ChildCount.ToString());
Antlr4.Runtime.Tree.ParseTreeWalker walker = new Antlr4.Runtime.Tree.ParseTreeWalker();
AntlrTsqlListener listener = new AntlrTsqlListener();
walker.Walk(listener, Tsql_fileContext1);
foreach (var nTable in listener.AnalyzedWorkload)
{
Console.WriteLine("Tabulka " + nTable.Name);
foreach (var nColumn in nTable.Columns)
{
foreach (var nCondition in nColumn.Conditions)
{
Console.WriteLine($" col { nCondition.ColumnName} operator { nCondition.Operator} val {nCondition.Value}");
}
}
}
Console.ReadKey();
}
public static void Main()
{
var s = new Antlr4.Runtime.AntlrInputStream(File.OpenRead("Program.m"));
var l = new MethLab.Parser.ANTLR.MATLABLexer(s);
var p = new MethLab.Parser.ANTLR.MATLABParser(new CommonTokenStream(l));
var o = new MatlabListener();
var w = new ParseTreeWalker();
w.Walk(o, p.file());
Console.ReadLine();
}
public InterfaceExtruder(string objcFileContent)
{
content = objcFileContent;
var input = new AntlrInputStream (content);
lexer = new ObjCLexer (input);
tokens = new CommonTokenStream (lexer);
parser = new ObjCParser (tokens);
listner = new ObjCListnerImpl ();
walker = new ParseTreeWalker ();
}
public static void Main(string[] args) {
if (args.Length == 0) {
Console.WriteLine("No arguments given!");
Console.WriteLine("Usage: input.csv [-json | -xml | -excel] [output.json | output.xml | output.xlsx]");
return;
}
ParseArgs(args);
StreamReader stream = null;
try {
stream = new StreamReader(csvInputFile);
} catch (ArgumentException) {
Console.WriteLine("Input filename not set!");
return;
}
AntlrInputStream antlrStream = new AntlrInputStream(stream.ReadToEnd());
CSVLexer lexer = new CSVLexer(antlrStream);
CommonTokenStream tokens = new CommonTokenStream(lexer);
CSVParser parser = new CSVParser(tokens);
IParseTree parseTree = parser.file();
ParseTreeWalker walker = new ParseTreeWalker();
LoaderCSV load = new LoaderCSV();
walker.Walk(load, parseTree);
if (jsonSwitch) {
if (jsonOutputFile == string.Empty) {
Console.WriteLine("[JSON] Output filename not set!");
return;
}
SaveToJSON(jsonOutputFile, load);
}
if (xmlSwitch) {
if (xmlOutputFile == string.Empty) {
Console.WriteLine("[XML] Output filename not set!");
return;
}
SaveToXML(xmlOutputFile, load);
}
if (excelSwitch) {
if (excelOutputFile == string.Empty) {
Console.WriteLine("[Excel] Output filename not set!");
return;
}
SaveToExcel(excelOutputFile, load);
}
}
public VBComponentParseResult(VBComponent component, IParseTree parseTree, IEnumerable<CommentNode> comments, ITokenStream tokenStream)
{
_component = component;
_qualifiedName = new QualifiedModuleName(component);
_parseTree = parseTree;
_comments = comments;
_tokenStream = tokenStream;
var listener = new DeclarationSymbolsListener(_qualifiedName, Accessibility.Implicit, _component.Type);
var walker = new ParseTreeWalker();
walker.Walk(listener, _parseTree);
_declarations.AddRange(listener.Declarations.Items);
}
public static string Compile(string input, BaseErrorListener errorListener)
{
AntlrInputStream stream = new AntlrInputStream(input);
HorseshoeLexer lexer = new HorseshoeLexer(stream);
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
HorseshoeParser parser = new HorseshoeParser(tokenStream);
if (errorListener != null)
{
parser.RemoveErrorListeners();
parser.AddErrorListener(errorListener);
}
var context = parser.document();
HorseshoeTranslationListener listener = new HorseshoeTranslationListener();
ParseTreeWalker walker = new ParseTreeWalker();
walker.Walk(listener, context);
return listener.Result;
}
public static SelectStmtInfo ParseSQL(string sql)
{
Antlr4.Runtime.AntlrInputStream input = new Antlr4.Runtime.AntlrInputStream(sql);
SelectSQLLexer lexer = new SelectSQLLexer(input);
Antlr4.Runtime.UnbufferedTokenStream tokens = new Antlr4.Runtime.UnbufferedTokenStream(lexer);
SelectSQLParser parser = new SelectSQLParser(tokens);
var tree = parser.compileUnit();
ParseTreeWalker walker = new ParseTreeWalker();
SelectSQLTreeListener lsn = new SelectSQLTreeListener();
walker.Walk(lsn, tree);
return lsn.SelectStmt;
}
public static ParseResult Parse(string sentence)
{
Antlr4.Runtime.AntlrInputStream input = new Antlr4.Runtime.AntlrInputStream(sentence);
WeatherRuleLexer lexer = new WeatherRuleLexer(input);
Antlr4.Runtime.UnbufferedTokenStream tokens = new Antlr4.Runtime.UnbufferedTokenStream(lexer);
WeatherRuleParser parser = new WeatherRuleParser(tokens);
parser.RemoveErrorListeners();
var tree = parser.compileUnit();
ParseTreeWalker walker = new ParseTreeWalker();
WeatherListener lsn = new WeatherListener();
walker.Walk(lsn, tree);
return lsn.Result;
}
public ExpressionLocator(String Name, int Order, Scope Scope, String Expression)
: base(Name, Order, Scope)
{
_description = String.Format("Expression Locator: Order: {0} Scope: [{1}] Expression: [{2}]", Order, Scope, Expression);
PrevailLexer lexer = new PrevailLexer(new AntlrInputStream(Expression));
PrevailParser parser = new PrevailParser(new CommonTokenStream(lexer));
IParseTree tree = parser.expression();
ParseTreeWalker walker = new ParseTreeWalker();
PrevailListener prevailListener = new PrevailListener();
walker.Walk(prevailListener, tree);
_condition = prevailListener.Result;
}
public void Resolve()
{
foreach (var componentParseResult in _parseResults)
{
OnProgress(componentParseResult);
try
{
var resolver = new IdentifierReferenceResolver(componentParseResult.QualifiedName, _declarations);
var listener = new IdentifierReferenceListener(resolver);
var walker = new ParseTreeWalker();
walker.Walk(listener, componentParseResult.ParseTree);
}
catch (InvalidOperationException)
{
// could not resolve all identifier references in this module.
}
}
}
private static void Main(string[] args)
{
try
{
AntlrInputStream input = new AntlrInputStream("X = a + b / c\r\n");
var lexer = new NireLangLexer(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);
var parser = new NireLangParser(tokens);
var tree = parser.stat();
ParseTreeWalker walker = new ParseTreeWalker();
walker.Walk(new Foo(), tree);
Console.WriteLine(tree.ToStringTree(parser));
ShowVersion();
var cle = new CommandLineExecutor();
if (args.Length == 0)
{
//args[0] = "..\\..\\
}
if (cle.ParseCommandLine(args))
{
cle.CompileAndExecute();
}
else
{
Console.WriteLine("No command line arguments\n usage NireLang <path>\\.vm file");
Console.Read();
}
}
catch (System.MethodAccessException mae)
{
Console.WriteLine(mae.ToString());
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
}
public static XcodeProject Parse(string projectDump)
{
var input = new AntlrInputStream (projectDump);
var lexer = new XcodeProjectLexer (input);
lexer.RemoveErrorListeners ();
lexer.AddErrorListener (LexerErrorListener.ErrorHandler);
var tokens = new CommonTokenStream (lexer);
var parser = new XcodeProjectParser (tokens) {
BuildParseTree = true
};
parser.RemoveErrorListeners ();
parser.AddErrorListener (ParsingErrorListener.ErrorHandler);
IParseTree tree = parser.project ();
var visitor = new ParseTreeWalker ();
var parseListener = new ProjectParserListener ();
visitor.Walk (parseListener, tree);
return parseListener.ProjectModel;
}
private void Parse_Click(object sender, RoutedEventArgs e)
{
AntlrInputStream stream = new AntlrInputStream(this._tbInput.Text);
QSLexer lex = new QSLexer(stream);
CommonTokenStream tokens = new CommonTokenStream(lex);
QSParser parser = new QSParser(tokens);
IParseTree tree = parser.form();
MyListener listener = new MyListener(parser);
ParseTreeWalker walker = new ParseTreeWalker();
walker.Walk(listener, tree);
IVisitor<bool> check = TypeChecker.StartVisit(listener.Root);
if (check.Result)
{
GUIBuilder builder = GUIBuilder.BuildGUI(listener.Root);
System.Windows.Window wind = new Window();
wind.Width = 500;
wind.Content = builder.Result;
wind.Show();
}
else
this._tbOutput.Text = ((TypeChecker)check).Output.ToString();
}
private Form init_parse(string text)
{
AntlrInputStream stream = new AntlrInputStream(text);
QSLexer lex = new QSLexer(stream);
CommonTokenStream tokens = new CommonTokenStream(lex);
QSParser parser = new QSParser(tokens);
IParseTree tree = parser.form();
MyListener listener = new MyListener(parser);
ParseTreeWalker walker = new ParseTreeWalker();
walker.Walk(listener, tree);
return listener.Root;
}
public void SecondPass()
{
if (Vectors == null)
{
ErrorMessage error = new ErrorMessage();
error.SourceFile = SourceFilePath;
error.Message = "Required vectors statement not found.";
Errors.Add(error);
}
ParseTreeWalker walker = new ParseTreeWalker();
walker.Walk(new SecondCpuPass(this), _rootTree);
if (Errors.Count > 0)
{
throw new CompilerErrorException();
}
}
private void ResolveReferences(DeclarationFinder finder, VBComponent component, IParseTree tree)
{
var state = _state.GetModuleState(component);
if (_state.Status == ParserState.ResolverError || (state != ParserState.Parsed))
{
return;
}
Debug.WriteLine("Resolving '{0}'... (thread {1})", component.Name, Thread.CurrentThread.ManagedThreadId);
var qualifiedName = new QualifiedModuleName(component);
var resolver = new IdentifierReferenceResolver(qualifiedName, finder);
var listener = new IdentifierReferenceListener(resolver);
if (!string.IsNullOrWhiteSpace(tree.GetText().Trim()))
{
var walker = new ParseTreeWalker();
try
{
walker.Walk(listener, tree);
state = ParserState.Ready;
}
catch (Exception exception)
{
Debug.Print("Exception thrown resolving '{0}' (thread {2}): {1}", component.Name, exception, Thread.CurrentThread.ManagedThreadId);
state = ParserState.ResolverError;
}
}
_state.SetModuleState(component, state);
Debug.Print("'{0}' is {1}. Resolver took {2}ms to complete (thread {3})", component.Name, _state.GetModuleState(component), /*_resolverTimer[component].ElapsedMilliseconds*/0, Thread.CurrentThread.ManagedThreadId);
}
private void runToolStripMenuItem_Click(object sender, EventArgs e)
{
AntlrInputStream inputCharStream4 = new AntlrInputStream(CodeTB.Text);
AntlrInputStream inputCharStream3 = new AntlrInputStream(CodeTB.Text);
AntlrInputStream inputCharStream2 = new AntlrInputStream(CodeTB.Text);
AntlrInputStream inputCharStream = new AntlrInputStream(CodeTB.Text);
NewDecafLexer tokenCreator4 = new NewDecafLexer(inputCharStream4);
NewDecafLexer tokenCreator3 = new NewDecafLexer(inputCharStream3);
NewDecafLexer tokenCreator2 = new NewDecafLexer(inputCharStream2);
DecafLexer tokenCreator = new DecafLexer(inputCharStream);
CommonTokenStream inputTokenStream4 = new CommonTokenStream(tokenCreator4);
CommonTokenStream inputTokenStream3 = new CommonTokenStream(tokenCreator3);
CommonTokenStream inputTokenStream2 = new CommonTokenStream(tokenCreator2);
CommonTokenStream inputTokenStream = new CommonTokenStream(tokenCreator);
NewDecafParser mainParser4 = new NewDecafParser(inputTokenStream4);
NewDecafParser mainParser3 = new NewDecafParser(inputTokenStream3);
NewDecafParser mainParser2 = new NewDecafParser(inputTokenStream2);
DecafParser mainParser = new DecafParser(inputTokenStream);
errorTB.Text = "";
mainParser.RemoveErrorListeners();
mainParser.AddErrorListener(new ParsingErrorDetector(errorTB));
NewDecafParser.ProgramContext AbstractSyntaxTree2 = mainParser2.program();
DecafParser.ProgramContext AbstractSyntaxTree = mainParser.program();
NewDecafParser.ProgramContext AbstractSyntaxTree3 = mainParser3.program();
NewDecafParser.ProgramContext AbstractSyntaxTree4 = mainParser4.program();
//TreeConstructingVisitor visitor = new TreeConstructingVisitor(TreeVisualizer);
//visitor.Visit(AbstractSyntaxTree);
ParseTreeWalker walker2 = new ParseTreeWalker();
ParseTreeWalker walker = new ParseTreeWalker();
ParseTreeWalker walker3 = new ParseTreeWalker();
ParseTreeWalker walker4 = new ParseTreeWalker();
//SymbolTableConstructor theConstructor = new SymbolTableConstructor();
//SymbolTable finalTable = theConstructor.finalTable;
//string generated = "";
//ILGenerator theGenerator = new ILGenerator(generated);
DefPhase theDefinition = new DefPhase();
//try
//{
// walker2.Walk(theGenerator, AbstractSyntaxTree2);
//walker.Walk(theConstructor, AbstractSyntaxTree);
walker3.Walk(theDefinition, AbstractSyntaxTree3);
STBasedGenerator newGenreator = new STBasedGenerator(theDefinition.scopes);
walker3.Walk(newGenreator, AbstractSyntaxTree3);
string code = newGenreator.mainTemplate.Render();
//}
//catch (Exception ex)
//{
// errorTB.Text += ex.Message + Environment.NewLine;
// if (theConstructor.beginningLine == theConstructor.endingLine)
// errorTB.Text += "Error found in line " + theConstructor.beginningLine;
// else
// errorTB.Text += "Error found between line " + theConstructor.beginningLine + " and line " + theConstructor.endingLine;
//}
if (errorTB.Text == string.Empty)
errorTB.Text = "Congrats! No errors were found!";
FileStream myStream;
if (saveFileDialog1.ShowDialog() == DialogResult.OK)
{
using (StreamWriter outfile = new StreamWriter(saveFileDialog1.FileName))
{
outfile.Write(code);
}
}
MessageBox.Show(code);
}
/// <summary>
/// Incremental parsing of a set of processed tokens lines changes
/// </summary>
internal static IList<DocumentChange<ICodeElementsLine>> ParseProcessedTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> documentLines, IList<DocumentChange<IProcessedTokensLine>> processedTokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions)
{
// Collect all changes applied to the processed tokens lines during the incremental scan
IList<DocumentChange<ICodeElementsLine>> codeElementsLinesChanges = new List<DocumentChange<ICodeElementsLine>>();
// There are 2 reasons to re-parse a tokens line after a change :
// 1. The tokens line changed : these lines were already reset during the previous steps
// 2. If a tokens line that changed was involved in the parsing of a multiline code element, the whole group of lines must be parsed again
// --- PREPARATION PHASE : identify all parse sections where code elements need to be refreshed ---
IList<ParseSection> refreshParseSections = null;
// Iterate over all processed tokens changes detected by the PreprocessorStep :
// - refresh all the adjacent lines participating in a CodeElement
// - register the start and stop token for all sections of the document which need to be parsed again
if (processedTokensLinesChanges != null)
{
// If the document was cleared, everything must be parsed again
if (processedTokensLinesChanges[0].Type != DocumentChangeType.DocumentCleared)
{
refreshParseSections = new List<ParseSection>();
ParseSection lastParseSection = null;
foreach (DocumentChange<IProcessedTokensLine> tokensChange in processedTokensLinesChanges)
{
if (lastParseSection == null || tokensChange.LineIndex > lastParseSection.StopLineIndex)
{
lastParseSection = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, codeElementsLinesChanges, lastParseSection);
refreshParseSections.Add(lastParseSection);
}
}
}
}
// --- INITIALIZE ANTLR CodeElements parser ---
// Create a token iterator on top of pre-processed tokens lines
ITokensLinesIterator tokensIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines);
// Create an Antlr compatible token source on top of the token iterator
TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
textSourceInfo.Name,
tokensIterator);
// Init parser
TokensLinesTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
TracingCobolParser cobolParser = new TracingCobolParser(tokenStream);
// -> activate full ambiguities detection
//parser.Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection;
IAntlrErrorStrategy cobolErrorStrategy = new CobolErrorStrategy();
cobolParser.ErrorHandler = cobolErrorStrategy;
// Register all parse errors in a list in memory
DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener();
cobolParser.RemoveErrorListeners();
cobolParser.AddErrorListener(errorListener);
// Prepare to analyze the parse tree
ParseTreeWalker walker = new ParseTreeWalker();
CodeElementBuilder codeElementBuilder = new CodeElementBuilder();
codeElementBuilder.Dispatcher = new CodeElementDispatcher();
codeElementBuilder.Dispatcher.CreateListeners();
// --- INCREMENTAL PARSING ---
// In case of incremental parsing, parse only the code sections we need to refresh
IEnumerator<ParseSection> parseSectionsEnumerator = null;
ParseSection currentParseSection = null;
if (refreshParseSections != null)
{
// Get the first code section we need to refresh
parseSectionsEnumerator = refreshParseSections.GetEnumerator();
parseSectionsEnumerator.MoveNext();
currentParseSection = parseSectionsEnumerator.Current;
// Seek just before the next code element starting token
tokenStream.SeekToToken(currentParseSection.StartToken);
tokenStream.StartLookingForStopToken(currentParseSection.StopToken);
}
// Parse one CodeElement at a time while advancing in the underlying token stream
do
{
// Reset parsing error diagnostics
//cobolErrorStrategy.Reset(cobolParser);
errorListener.Diagnostics.Clear();
// Reset parser traces (consumed tokens)
cobolParser.ResetTraces();
// Try to parse a code element starting with the current token
CodeElementsParser.CodeElementContext codeElementParseTree = cobolParser.codeElement();
// If the parse tree is not empty
if (codeElementParseTree.Start.Type > 0)
{
// Get the first line that was parsed
var tokenStart = (Token)codeElementParseTree.Start;
CodeElementsLine codeElementsLine;
var importedToken = tokenStart as ImportedToken;
if (importedToken != null)
{
codeElementsLine = (CodeElementsLine) importedToken.CopyDirective.TextNameSymbol.TokensLine;
}
else
{
codeElementsLine = ((CodeElementsLine) tokenStart.TokensLine);
}
// Register that this line was updated
// COMMENTED FOR THE SAKE OF PERFORMANCE -- SEE ISSUE #160
//int updatedLineIndex = documentLines.IndexOf(codeElementsLine, codeElementsLine.InitialLineIndex);
//codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, updatedLineIndex, codeElementsLine));
codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, codeElementsLine.InitialLineIndex, codeElementsLine));
// Visit the parse tree to build a first class object representing the code elements
try { walker.Walk(codeElementBuilder, codeElementParseTree); }
catch (Exception ex) {
var code = Diagnostics.MessageCode.ImplementationError;
int line = 0; int start = 0; int stop = 0;
if (codeElementsLine.SourceTokens != null && codeElementsLine.SourceTokens.Count > 0){
start = codeElementsLine.SourceTokens[0].StartIndex;
stop = codeElementsLine.SourceTokens[codeElementsLine.SourceTokens.Count-1].StopIndex;
}
codeElementsLine.AddParserDiagnostic(new ParserDiagnostic(ex.ToString(), start,stop,line, null, code));
}
CodeElement codeElement = codeElementBuilder.CodeElement;
if (codeElement != null)
{
// Attach consumed tokens and main document line numbers information to the code element
codeElement.ConsumedTokens = cobolParser.ConsumedTokens;
if (codeElement.ConsumedTokens == null) {// ISSUE #204:
if (tokenStream.Lt(1) != null) {// if not end of file,
// add next token to ConsumedTokens to know where is the CodeElement in error
codeElement.ConsumedTokens.Add((Token)tokenStream.Lt(1));
// this alter CodeElements semantics: in addition to matched tokens,
// it includes the first token in error if no token has been matched
}
}
//TODO Issue #384 to discuss if this code should stay here:
//This should be in a Checker, but "codeElement.ConsumedTokens" is only set after all the checkers have been called
//Rule TCLIMITATION_NO_CE_ACROSS_SOURCES
if (codeElement.IsAcrossSourceFile()) {
DiagnosticUtils.AddError(codeElement, "A Cobol statement cannot be across 2 sources files (eg. Main program and a COPY)", MessageCode.TypeCobolParserLimitation);
}
// Add code element to the list
codeElementsLine.AddCodeElement(codeElement);
foreach (ParserDiagnostic d in errorListener.Diagnostics)
{
codeElement.Diagnostics.Add(d);
}
foreach (Diagnostic d in codeElement.Diagnostics)
{
codeElementsLine.AddParserDiagnostic(d);
}
}
else
{
foreach (ParserDiagnostic d in errorListener.Diagnostics)
{
codeElementsLine.AddParserDiagnostic(d);
}
}
}
// In case of incremental parsing, directly jump to next parse section in the token stream
// Else, simply start parsing the next CodeElement beginning with the next token
if (currentParseSection != null)
{
// Check if we reached the end of the current ParseSection
if (tokenStream.StreamReachedStopToken)
{
// Adavance to the next ParseSection
if (parseSectionsEnumerator.MoveNext())
{
currentParseSection = parseSectionsEnumerator.Current;
tokenStream.SeekToToken(currentParseSection.StartToken);
tokenStream.StartLookingForStopToken(currentParseSection.StopToken);
}
// No more section to parse
else
{
break;
}
}
}
}
while (tokenStream.La(1) >= 0);
return codeElementsLinesChanges;
}
public void Parse()
{
_rootTree = _parser.root();
ParseTreeWalker walker = new ParseTreeWalker();
walker.Walk(new ParseCpuPass(this), _rootTree);
if (Errors.Count > 0)
{
throw new CompilerErrorException();
}
resolveLabels();
verifyBranchLength();
if (Errors.Count > 0)
{
throw new CompilerErrorException();
}
}
/// <summary>
/// Incremental preprocessing of a set of tokens lines changes
/// </summary>
internal static IList<DocumentChange<IProcessedTokensLine>> ProcessTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<ProcessedTokensLine> documentLines, IList<DocumentChange<ITokensLine>> tokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider)
{
// Collect all changes applied to the processed tokens lines during the incremental scan
IList<DocumentChange<IProcessedTokensLine>> processedTokensLinesChanges = new List<DocumentChange<IProcessedTokensLine>>();
// There are 2 reasons to a preprocess a tokens line after a change :
// 1. A tokens line changed : these lines were already reset during the previous steps
// 2. If a tokens line that changed was involved in the parsing of a multiline compiler directive, the whole group of lines must be parsed again
// Then, if a new COPY directive was parsed : the CompilationDocument to include must be prepared
// --- PREPARATION PHASE : reset all processed tokens lines which were involved in a multiline compiler directive where at least one line changed ---
// Iterate over all tokens changes detected by the ScannerStep :
// refresh all the adjacent lines participating in a ContinuationTokensGroup
if (tokensLinesChanges != null)
{
int lastLineIndexReset = -1;
foreach (DocumentChange<ITokensLine> tokensChange in tokensLinesChanges)
{
processedTokensLinesChanges.Add(new DocumentChange<IProcessedTokensLine>(tokensChange.Type, tokensChange.LineIndex, (IProcessedTokensLine)tokensChange.NewLine));
if (tokensChange.LineIndex > lastLineIndexReset)
{
lastLineIndexReset = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, processedTokensLinesChanges, lastLineIndexReset);
}
}
}
// --- COMPILER DIRECTIVES PHASE : Find and parse all compiler directives ---
// Init. Prepare a compiler directive parser
// Create a token iterator on top of tokens lines
TokensLinesIterator tokensIterator = new TokensLinesIterator(
textSourceInfo.Name,
documentLines,
null,
Token.CHANNEL_SourceTokens);
// Crate an Antlr compatible token source on top a the token iterator
TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
textSourceInfo.Name,
tokensIterator);
// Init a compiler directive parser
CommonTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
CobolCompilerDirectivesParser directivesParser = new CobolCompilerDirectivesParser(tokenStream);
IAntlrErrorStrategy compilerDirectiveErrorStrategy = new CompilerDirectiveErrorStrategy();
directivesParser.ErrorHandler = compilerDirectiveErrorStrategy;
// Register all parse errors in a list in memory
DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener();
directivesParser.RemoveErrorListeners();
directivesParser.AddErrorListener(errorListener);
// Prepare to analyze the parse tree
ParseTreeWalker walker = new ParseTreeWalker();
CompilerDirectiveBuilder directiveBuilder = new CompilerDirectiveBuilder();
// 1. Iterate over all compiler directive starting tokens found in the lines which were updated
foreach (Token compilerDirectiveStartingToken in documentLines
.Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing)
.SelectMany(line => line.SourceTokens)
.Where(token => token.TokenFamily == TokenFamily.CompilerDirectiveStartingKeyword))
{
// 2. Reset the compiler directive parser state
// Reset tokens iterator position before parsing
// -> seek just before the compiler directive starting token
tokensIterator.SeekToToken(compilerDirectiveStartingToken);
tokensIterator.PreviousToken();
// Special case : for efficiency reasons, in EXEC SQL INCLUDE directives
// only the third token INCLUDE is recognized as a compiler directive
// starting keyword by the scanner. In this case, we must rewind the
// iterator two tokens backwards to start parsing just before the EXEC token.
if (compilerDirectiveStartingToken.TokenType == TokenType.EXEC_SQL_INCLUDE)
{
tokensIterator.PreviousToken();
tokensIterator.PreviousToken();
}
// Reset Antlr BufferedTokenStream position
tokenStream.SetTokenSource(tokenSource);
// Reset parsing error diagnostics
compilerDirectiveErrorStrategy.Reset(directivesParser);
errorListener.Diagnostics.Clear();
// 3. Try to parse a compiler directive starting with the current token
CobolCompilerDirectivesParser.CompilerDirectingStatementContext directiveParseTree = directivesParser.compilerDirectingStatement();
// 4. Visit the parse tree to build a first class object representing the compiler directive
walker.Walk(directiveBuilder, directiveParseTree);
CompilerDirective compilerDirective = directiveBuilder.CompilerDirective;
bool errorFoundWhileParsingDirective = errorListener.Diagnostics.Count > 0 || directiveBuilder.Diagnostics.Count > 0;
// 5. Get all tokens consumed while parsing the compiler directive
// and partition them line by line
Token startToken = (Token)directiveParseTree.Start;
Token stopToken = (Token)directiveParseTree.Stop;
if (stopToken == null) stopToken = startToken;
MultilineTokensGroupSelection tokensSelection = tokensIterator.SelectAllTokensBetween(startToken, stopToken);
if (compilerDirective != null)
{
// 6. Replace all matched tokens by :
// - a CompilerDirectiveToken on the first line
ProcessedTokensLine firstProcessedTokensLine = documentLines[tokensSelection.FirstLineIndex];
if (tokensSelection.SelectedTokensOnSeveralLines.Length == 1)
{
firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
tokensSelection.TokensOnFirstLineBeforeStartToken,
compilerDirective, errorFoundWhileParsingDirective,
tokensSelection.SelectedTokensOnSeveralLines[0],
tokensSelection.TokensOnLastLineAfterStopToken, false);
}
else
{
TokensGroup continuedTokensGroup = firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
tokensSelection.TokensOnFirstLineBeforeStartToken,
compilerDirective, errorFoundWhileParsingDirective,
tokensSelection.SelectedTokensOnSeveralLines[0],
null, true);
// - a ContinuationTokensGroup on the following lines
int selectionLineIndex = 1;
int lastLineIndex = tokensSelection.FirstLineIndex + tokensSelection.SelectedTokensOnSeveralLines.Length - 1;
for (int nextLineIndex = tokensSelection.FirstLineIndex + 1; nextLineIndex <= lastLineIndex; nextLineIndex++, selectionLineIndex++)
{
IList<Token> compilerDirectiveTokensOnNextLine = tokensSelection.SelectedTokensOnSeveralLines[selectionLineIndex];
if (compilerDirectiveTokensOnNextLine.Count > 0)
{
ProcessedTokensLine nextProcessedTokensLine = documentLines[nextLineIndex];
if (nextLineIndex != lastLineIndex)
{
continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
continuedTokensGroup,
compilerDirectiveTokensOnNextLine,
null, true);
}
else
{
continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
continuedTokensGroup,
compilerDirectiveTokensOnNextLine,
tokensSelection.TokensOnLastLineAfterStopToken, false);
}
}
}
}
}
// 7. Register compiler directive parse errors
if (errorFoundWhileParsingDirective)
{
ProcessedTokensLine compilerDirectiveLine = documentLines[tokensSelection.FirstLineIndex];
foreach (ParserDiagnostic parserDiag in errorListener.Diagnostics)
{
compilerDirectiveLine.AddDiagnostic(parserDiag);
}
foreach (Diagnostic directiveDiag in directiveBuilder.Diagnostics)
{
compilerDirectiveLine.AddDiagnostic(directiveDiag);
}
}
}
// 8. Advance the state off all ProcessedTokensLines :
// NeedsCompilerDirectiveParsing => NeedsCopyDirectiveProcessing if it contains a COPY directive
IList<ProcessedTokensLine> parsedLinesWithCopyDirectives = null;
// NeedsCompilerDirectiveParsing => Ready otherwise
foreach (ProcessedTokensLine parsedLine in documentLines
.Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing))
{
if (parsedLine.ImportedDocuments != null)
{
if(parsedLinesWithCopyDirectives == null)
{
parsedLinesWithCopyDirectives = new List<ProcessedTokensLine>();
}
parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.NeedsCopyDirectiveProcessing;
parsedLinesWithCopyDirectives.Add(parsedLine);
}
else
{
parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
}
}
// --- COPY IMPORT PHASE : Process COPY (REPLACING) directives ---
// 1. Iterate over all updated lines containing a new COPY directive
if (parsedLinesWithCopyDirectives != null)
{
foreach (ProcessedTokensLine tokensLineWithCopyDirective in parsedLinesWithCopyDirectives)
{
// Iterate over all COPY directives found on one updated line
foreach (CopyDirective copyDirective in tokensLineWithCopyDirective.ImportedDocuments.Keys.ToArray<CopyDirective>())
{
try
{
// Load (or retrieve in cache) the document referenced by the COPY directive
ProcessedTokensDocument importedDocumentSource = processedTokensDocumentProvider.GetProcessedTokensDocument(copyDirective.LibraryName, copyDirective.TextName);
// Store it on the current line after appying the REPLACING directive
ImportedTokensDocument importedDocument = new ImportedTokensDocument(copyDirective, importedDocumentSource);
tokensLineWithCopyDirective.ImportedDocuments[copyDirective] = importedDocument;
}
catch (Exception e)
{
// Text name refenced by COPY directive was not found
// => register a preprocessor error on this line
Token failedDirectiveToken = tokensLineWithCopyDirective.TokensWithCompilerDirectives
.Where(token => token.TokenType == TokenType.CopyImportDirective && ((CompilerDirectiveToken)token).CompilerDirective == copyDirective)
.First();
Diagnostic diag = new Diagnostic(
MessageCode.FailedToLoadTextDocumentReferencedByCopyDirective,
failedDirectiveToken.Column, failedDirectiveToken.EndColumn,
e.Message);
tokensLineWithCopyDirective.AddDiagnostic(diag);
}
}
// Advance processing status of the line
tokensLineWithCopyDirective.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
}
}
// --- REPLACE PHASE : REPLACE directives are implemented in ReplaceTokensLinesIterator ---
/* Algorithm :
*
* one REPLACE directive can express several replacement operations
* one replacement operation can be of several types (distinguished for optimization purposes)
* - SimpleTokenReplace : one source token / zero or one replacement token
* - PartialWordReplace : one pure partial word / zero or one replacement token
* - SimpleToMultipleTokenReplace : one source token / several replacement tokens
* - MultipleTokenReplace : one first + several following source tokens / zero to many replacement tokens
*
* an iterator maintains a current set of replacement operations
*
* if nextToken is replace directive
* the current set of replacement operations is updated
* else
* nextToken is compared to each replacement operation in turn
* if single -> single source token operation : return replacement token
* if single -> multiple operation : setup a secondary iterator with the list of replacement tokens
* if multiple -> multiple operation
* snapshot of the underlying iterator
* try to match all of the following source tokens
* if failure : restore snapshot and try next operation
* if success : setup a secondary iterator
*
* token comparison sourceToken / replacementCandidate :
* 1. Compare Token type
* 2. If same token type and for families
* AlphanumericLiteral
* NumericLiteral
* SyntaxLiteral
* Symbol
* => compare also Token text
*
* PartialCobolWord replacement :
* p535 : The COPY statement with REPLACING phrase can be used to replace parts of words.
* By inserting a dummy operand delimited by colons into the program text,
* the compiler will replace the dummy operand with the required text.
* Example 3 shows how this is used with the dummy operand :TAG:.
* The colons serve as separators and make TAG a stand-alone operand.
*/
return processedTokensLinesChanges;
}
public static CsharpParseResults InvokeParse(string fileName)
{
if (string.IsNullOrWhiteSpace(fileName))
return null;
if (!System.IO.File.Exists(fileName))
return null;
var tr = System.IO.File.OpenRead(fileName);
var input = new AntlrInputStream(tr);
var lexer = new CSharp4Lexer(input);
var tokens = new CommonTokenStream(lexer);
var parser = new CSharp4Parser(tokens);
var tree = parser.compilation_unit();
var walker = new ParseTreeWalker();
var loader = new CsharpParseTree();
walker.Walk(loader, tree);
var results = loader.Results;
tr.Close();
return results;
}
public static HtmlParseResults InvokeParse(Stream stream)
{
var input = new AntlrInputStream(stream);
var lexer = new HTMLLexer(input);
var tokens = new CommonTokenStream(lexer);
var parser = new HTMLParser(tokens);
var tree = parser.htmlDocument();
var walker = new ParseTreeWalker();
var loader = new AspNetParseTree();
walker.Walk(loader, tree);
return loader.Results;
}
public static void Work(string java, string csharp)
{
using (var fs = new FileStream(java, FileMode.Open))
{
var input = new AntlrInputStream(fs);
var lexer = new MyJavaLexer(input);
var tokens = new CommonTokenStream(lexer);
var parser = new MyJavaParser(tokens);
// use the first rule of the parser
// MyJavaParser.CompilationUnitContext tree = parser.compilationUnit();
var my_action = new Java2CSharp();
var tree = parser.compilationUnit();
var walker = new ParseTreeWalker();
walker.Walk(my_action, tree);
GenerateCSharpCode(csharp, my_action.compileUnit);
}
}
