public static void ParseProgramOrClass(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> codeElementsLines, TypeCobolOptions compilerOptions, SymbolTable customSymbols, out Program newProgram, out Class newClass, out IList<ParserDiagnostic> diagnostics) { // Create an Antlr compatible token source on top a the token iterator CodeElementsLinesTokenSource tokenSource = new CodeElementsLinesTokenSource( textSourceInfo.Name, codeElementsLines); // Init parser ITokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens); ProgramClassParser cobolParser = new ProgramClassParser(tokenStream); // -> activate full ambiguities detection //parser.Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection; // Register all parse errors in a list in memory DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener(); cobolParser.RemoveErrorListeners(); cobolParser.AddErrorListener(errorListener); // Try to parse a Cobol program or class ProgramClassParser.CobolCompilationUnitContext codeElementParseTree = cobolParser.cobolCompilationUnit(); // Visit the parse tree to build a first class object representing a Cobol program or class ParseTreeWalker walker = new ParseTreeWalker(); CobolNodeBuilder programClassBuilder = new CobolNodeBuilder(); programClassBuilder.CustomSymbols = customSymbols; programClassBuilder.Dispatcher = new NodeDispatcher(); programClassBuilder.Dispatcher.CreateListeners(); walker.Walk(programClassBuilder, codeElementParseTree); // Register compiler results newProgram = programClassBuilder.Program; newClass = programClassBuilder.Class; diagnostics = errorListener.Diagnostics; }
static void Main(string[] args) { string lQuery = "SELECT Jmeno, Prijmeni FROM Osoba, Film WHERE Stat LIKE 'Kanada' OR Rok_natoceni >1995 UNION " + "(SELECT Jmeno, Prijmeni FROM Uzivatel WHERE Jmeno = 'ads' AND Prijmeni = 'asd');"; string lQuery2 = "SElect jmeno, nazev_cz FROM Osoba os JOIN Osoba_Film osf ON os.idO = osf.Osoba_idO JOIN Film f ON f.idF = osf.Film_idF Where f.Rok_natoceni > 1995;"; string lQuery1 = "select * from table1 t1, t2 where t1.neco = 1"; string text = ReadFile($@"C:\Users\Lukáš\Desktop\doc\Workload_bp.txt"); StringReader reader = new StringReader(text); AntlrInputStream input = new AntlrInputStream(reader); TSqlLexer lexer = new TSqlLexer(new CaseChangingCharStream(input, true)); CommonTokenStream tokens = new CommonTokenStream(lexer); TSqlParser parser = new TSqlParser(tokens); TSqlParser.Tsql_fileContext Tsql_fileContext1 = parser.tsql_file(); //Console.WriteLine("Tsql_fileContext1.ChildCount = " + Tsql_fileContext1.ChildCount.ToString()); Antlr4.Runtime.Tree.ParseTreeWalker walker = new Antlr4.Runtime.Tree.ParseTreeWalker(); AntlrTsqlListener listener = new AntlrTsqlListener(); walker.Walk(listener, Tsql_fileContext1); foreach (var nTable in listener.AnalyzedWorkload) { Console.WriteLine("Tabulka " + nTable.Name); foreach (var nColumn in nTable.Columns) { foreach (var nCondition in nColumn.Conditions) { Console.WriteLine($" col { nCondition.ColumnName} operator { nCondition.Operator} val {nCondition.Value}"); } } } Console.ReadKey(); }
public static void Main() { var s = new Antlr4.Runtime.AntlrInputStream(File.OpenRead("Program.m")); var l = new MethLab.Parser.ANTLR.MATLABLexer(s); var p = new MethLab.Parser.ANTLR.MATLABParser(new CommonTokenStream(l)); var o = new MatlabListener(); var w = new ParseTreeWalker(); w.Walk(o, p.file()); Console.ReadLine(); }
public static void Main(string[] args) { if (args.Length == 0) { Console.WriteLine("No arguments given!"); Console.WriteLine("Usage: input.csv [-json | -xml | -excel] [output.json | output.xml | output.xlsx]"); return; } ParseArgs(args); StreamReader stream = null; try { stream = new StreamReader(csvInputFile); } catch (ArgumentException) { Console.WriteLine("Input filename not set!"); return; } AntlrInputStream antlrStream = new AntlrInputStream(stream.ReadToEnd()); CSVLexer lexer = new CSVLexer(antlrStream); CommonTokenStream tokens = new CommonTokenStream(lexer); CSVParser parser = new CSVParser(tokens); IParseTree parseTree = parser.file(); ParseTreeWalker walker = new ParseTreeWalker(); LoaderCSV load = new LoaderCSV(); walker.Walk(load, parseTree); if (jsonSwitch) { if (jsonOutputFile == string.Empty) { Console.WriteLine("[JSON] Output filename not set!"); return; } SaveToJSON(jsonOutputFile, load); } if (xmlSwitch) { if (xmlOutputFile == string.Empty) { Console.WriteLine("[XML] Output filename not set!"); return; } SaveToXML(xmlOutputFile, load); } if (excelSwitch) { if (excelOutputFile == string.Empty) { Console.WriteLine("[Excel] Output filename not set!"); return; } SaveToExcel(excelOutputFile, load); } }
public VBComponentParseResult(VBComponent component, IParseTree parseTree, IEnumerable<CommentNode> comments, ITokenStream tokenStream) { _component = component; _qualifiedName = new QualifiedModuleName(component); _parseTree = parseTree; _comments = comments; _tokenStream = tokenStream; var listener = new DeclarationSymbolsListener(_qualifiedName, Accessibility.Implicit, _component.Type); var walker = new ParseTreeWalker(); walker.Walk(listener, _parseTree); _declarations.AddRange(listener.Declarations.Items); }
public static string Compile(string input, BaseErrorListener errorListener) { AntlrInputStream stream = new AntlrInputStream(input); HorseshoeLexer lexer = new HorseshoeLexer(stream); CommonTokenStream tokenStream = new CommonTokenStream(lexer); HorseshoeParser parser = new HorseshoeParser(tokenStream); if (errorListener != null) { parser.RemoveErrorListeners(); parser.AddErrorListener(errorListener); } var context = parser.document(); HorseshoeTranslationListener listener = new HorseshoeTranslationListener(); ParseTreeWalker walker = new ParseTreeWalker(); walker.Walk(listener, context); return listener.Result; }
public static SelectStmtInfo ParseSQL(string sql) { Antlr4.Runtime.AntlrInputStream input = new Antlr4.Runtime.AntlrInputStream(sql); SelectSQLLexer lexer = new SelectSQLLexer(input); Antlr4.Runtime.UnbufferedTokenStream tokens = new Antlr4.Runtime.UnbufferedTokenStream(lexer); SelectSQLParser parser = new SelectSQLParser(tokens); var tree = parser.compileUnit(); ParseTreeWalker walker = new ParseTreeWalker(); SelectSQLTreeListener lsn = new SelectSQLTreeListener(); walker.Walk(lsn, tree); return lsn.SelectStmt; }
public void Resolve() { foreach (var componentParseResult in _parseResults) { OnProgress(componentParseResult); try { var resolver = new IdentifierReferenceResolver(componentParseResult.QualifiedName, _declarations); var listener = new IdentifierReferenceListener(resolver); var walker = new ParseTreeWalker(); walker.Walk(listener, componentParseResult.ParseTree); } catch (InvalidOperationException) { // could not resolve all identifier references in this module. } } }
public static ParseResult Parse(string sentence) { Antlr4.Runtime.AntlrInputStream input = new Antlr4.Runtime.AntlrInputStream(sentence); WeatherRuleLexer lexer = new WeatherRuleLexer(input); Antlr4.Runtime.UnbufferedTokenStream tokens = new Antlr4.Runtime.UnbufferedTokenStream(lexer); WeatherRuleParser parser = new WeatherRuleParser(tokens); parser.RemoveErrorListeners(); var tree = parser.compileUnit(); ParseTreeWalker walker = new ParseTreeWalker(); WeatherListener lsn = new WeatherListener(); walker.Walk(lsn, tree); return lsn.Result; }
public ExpressionLocator(String Name, int Order, Scope Scope, String Expression) : base(Name, Order, Scope) { _description = String.Format("Expression Locator: Order: {0} Scope: [{1}] Expression: [{2}]", Order, Scope, Expression); PrevailLexer lexer = new PrevailLexer(new AntlrInputStream(Expression)); PrevailParser parser = new PrevailParser(new CommonTokenStream(lexer)); IParseTree tree = parser.expression(); ParseTreeWalker walker = new ParseTreeWalker(); PrevailListener prevailListener = new PrevailListener(); walker.Walk(prevailListener, tree); _condition = prevailListener.Result; }
private static void Main(string[] args) { try { AntlrInputStream input = new AntlrInputStream("X = a + b / c\r\n"); var lexer = new NireLangLexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); var parser = new NireLangParser(tokens); var tree = parser.stat(); ParseTreeWalker walker = new ParseTreeWalker(); walker.Walk(new Foo(), tree); Console.WriteLine(tree.ToStringTree(parser)); ShowVersion(); var cle = new CommandLineExecutor(); if (args.Length == 0) { //args[0] = "..\\..\\ } if (cle.ParseCommandLine(args)) { cle.CompileAndExecute(); } else { Console.WriteLine("No command line arguments\n usage NireLang <path>\\.vm file"); Console.Read(); } } catch (System.MethodAccessException mae) { Console.WriteLine(mae.ToString()); } catch (Exception e) { Console.WriteLine(e.ToString()); } }
public static XcodeProject Parse(string projectDump) { var input = new AntlrInputStream (projectDump); var lexer = new XcodeProjectLexer (input); lexer.RemoveErrorListeners (); lexer.AddErrorListener (LexerErrorListener.ErrorHandler); var tokens = new CommonTokenStream (lexer); var parser = new XcodeProjectParser (tokens) { BuildParseTree = true }; parser.RemoveErrorListeners (); parser.AddErrorListener (ParsingErrorListener.ErrorHandler); IParseTree tree = parser.project (); var visitor = new ParseTreeWalker (); var parseListener = new ProjectParserListener (); visitor.Walk (parseListener, tree); return parseListener.ProjectModel; }
private void Parse_Click(object sender, RoutedEventArgs e) { AntlrInputStream stream = new AntlrInputStream(this._tbInput.Text); QSLexer lex = new QSLexer(stream); CommonTokenStream tokens = new CommonTokenStream(lex); QSParser parser = new QSParser(tokens); IParseTree tree = parser.form(); MyListener listener = new MyListener(parser); ParseTreeWalker walker = new ParseTreeWalker(); walker.Walk(listener, tree); IVisitor<bool> check = TypeChecker.StartVisit(listener.Root); if (check.Result) { GUIBuilder builder = GUIBuilder.BuildGUI(listener.Root); System.Windows.Window wind = new Window(); wind.Width = 500; wind.Content = builder.Result; wind.Show(); } else this._tbOutput.Text = ((TypeChecker)check).Output.ToString(); }
/// <summary> /// Incremental parsing of a set of processed tokens lines changes /// </summary> internal static IList<DocumentChange<ICodeElementsLine>> ParseProcessedTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> documentLines, IList<DocumentChange<IProcessedTokensLine>> processedTokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions) { // Collect all changes applied to the processed tokens lines during the incremental scan IList<DocumentChange<ICodeElementsLine>> codeElementsLinesChanges = new List<DocumentChange<ICodeElementsLine>>(); // There are 2 reasons to re-parse a tokens line after a change : // 1. The tokens line changed : these lines were already reset during the previous steps // 2. If a tokens line that changed was involved in the parsing of a multiline code element, the whole group of lines must be parsed again // --- PREPARATION PHASE : identify all parse sections where code elements need to be refreshed --- IList<ParseSection> refreshParseSections = null; // Iterate over all processed tokens changes detected by the PreprocessorStep : // - refresh all the adjacent lines participating in a CodeElement // - register the start and stop token for all sections of the document which need to be parsed again if (processedTokensLinesChanges != null) { // If the document was cleared, everything must be parsed again if (processedTokensLinesChanges[0].Type != DocumentChangeType.DocumentCleared) { refreshParseSections = new List<ParseSection>(); ParseSection lastParseSection = null; foreach (DocumentChange<IProcessedTokensLine> tokensChange in processedTokensLinesChanges) { if (lastParseSection == null || tokensChange.LineIndex > lastParseSection.StopLineIndex) { lastParseSection = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, codeElementsLinesChanges, lastParseSection); refreshParseSections.Add(lastParseSection); } } } } // --- INITIALIZE ANTLR CodeElements parser --- // Create a token iterator on top of pre-processed tokens lines ITokensLinesIterator tokensIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines); // Create an Antlr compatible token source on top of the token iterator TokensLinesTokenSource tokenSource = new TokensLinesTokenSource( textSourceInfo.Name, tokensIterator); // Init parser TokensLinesTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens); TracingCobolParser cobolParser = new TracingCobolParser(tokenStream); // -> activate full ambiguities detection //parser.Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection; IAntlrErrorStrategy cobolErrorStrategy = new CobolErrorStrategy(); cobolParser.ErrorHandler = cobolErrorStrategy; // Register all parse errors in a list in memory DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener(); cobolParser.RemoveErrorListeners(); cobolParser.AddErrorListener(errorListener); // Prepare to analyze the parse tree ParseTreeWalker walker = new ParseTreeWalker(); CodeElementBuilder codeElementBuilder = new CodeElementBuilder(); codeElementBuilder.Dispatcher = new CodeElementDispatcher(); codeElementBuilder.Dispatcher.CreateListeners(); // --- INCREMENTAL PARSING --- // In case of incremental parsing, parse only the code sections we need to refresh IEnumerator<ParseSection> parseSectionsEnumerator = null; ParseSection currentParseSection = null; if (refreshParseSections != null) { // Get the first code section we need to refresh parseSectionsEnumerator = refreshParseSections.GetEnumerator(); parseSectionsEnumerator.MoveNext(); currentParseSection = parseSectionsEnumerator.Current; // Seek just before the next code element starting token tokenStream.SeekToToken(currentParseSection.StartToken); tokenStream.StartLookingForStopToken(currentParseSection.StopToken); } // Parse one CodeElement at a time while advancing in the underlying token stream do { // Reset parsing error diagnostics //cobolErrorStrategy.Reset(cobolParser); errorListener.Diagnostics.Clear(); // Reset parser traces (consumed tokens) cobolParser.ResetTraces(); // Try to parse a code element starting with the current token CodeElementsParser.CodeElementContext codeElementParseTree = cobolParser.codeElement(); // If the parse tree is not empty if (codeElementParseTree.Start.Type > 0) { // Get the first line that was parsed var tokenStart = (Token)codeElementParseTree.Start; CodeElementsLine codeElementsLine; var importedToken = tokenStart as ImportedToken; if (importedToken != null) { codeElementsLine = (CodeElementsLine) importedToken.CopyDirective.TextNameSymbol.TokensLine; } else { codeElementsLine = ((CodeElementsLine) tokenStart.TokensLine); } // Register that this line was updated // COMMENTED FOR THE SAKE OF PERFORMANCE -- SEE ISSUE #160 //int updatedLineIndex = documentLines.IndexOf(codeElementsLine, codeElementsLine.InitialLineIndex); //codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, updatedLineIndex, codeElementsLine)); codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, codeElementsLine.InitialLineIndex, codeElementsLine)); // Visit the parse tree to build a first class object representing the code elements try { walker.Walk(codeElementBuilder, codeElementParseTree); } catch (Exception ex) { var code = Diagnostics.MessageCode.ImplementationError; int line = 0; int start = 0; int stop = 0; if (codeElementsLine.SourceTokens != null && codeElementsLine.SourceTokens.Count > 0){ start = codeElementsLine.SourceTokens[0].StartIndex; stop = codeElementsLine.SourceTokens[codeElementsLine.SourceTokens.Count-1].StopIndex; } codeElementsLine.AddParserDiagnostic(new ParserDiagnostic(ex.ToString(), start,stop,line, null, code)); } CodeElement codeElement = codeElementBuilder.CodeElement; if (codeElement != null) { // Attach consumed tokens and main document line numbers information to the code element codeElement.ConsumedTokens = cobolParser.ConsumedTokens; if (codeElement.ConsumedTokens == null) {// ISSUE #204: if (tokenStream.Lt(1) != null) {// if not end of file, // add next token to ConsumedTokens to know where is the CodeElement in error codeElement.ConsumedTokens.Add((Token)tokenStream.Lt(1)); // this alter CodeElements semantics: in addition to matched tokens, // it includes the first token in error if no token has been matched } } //TODO Issue #384 to discuss if this code should stay here: //This should be in a Checker, but "codeElement.ConsumedTokens" is only set after all the checkers have been called //Rule TCLIMITATION_NO_CE_ACROSS_SOURCES if (codeElement.IsAcrossSourceFile()) { DiagnosticUtils.AddError(codeElement, "A Cobol statement cannot be across 2 sources files (eg. Main program and a COPY)", MessageCode.TypeCobolParserLimitation); } // Add code element to the list codeElementsLine.AddCodeElement(codeElement); foreach (ParserDiagnostic d in errorListener.Diagnostics) { codeElement.Diagnostics.Add(d); } foreach (Diagnostic d in codeElement.Diagnostics) { codeElementsLine.AddParserDiagnostic(d); } } else { foreach (ParserDiagnostic d in errorListener.Diagnostics) { codeElementsLine.AddParserDiagnostic(d); } } } // In case of incremental parsing, directly jump to next parse section in the token stream // Else, simply start parsing the next CodeElement beginning with the next token if (currentParseSection != null) { // Check if we reached the end of the current ParseSection if (tokenStream.StreamReachedStopToken) { // Adavance to the next ParseSection if (parseSectionsEnumerator.MoveNext()) { currentParseSection = parseSectionsEnumerator.Current; tokenStream.SeekToToken(currentParseSection.StartToken); tokenStream.StartLookingForStopToken(currentParseSection.StopToken); } // No more section to parse else { break; } } } } while (tokenStream.La(1) >= 0); return codeElementsLinesChanges; }
private Form init_parse(string text) { AntlrInputStream stream = new AntlrInputStream(text); QSLexer lex = new QSLexer(stream); CommonTokenStream tokens = new CommonTokenStream(lex); QSParser parser = new QSParser(tokens); IParseTree tree = parser.form(); MyListener listener = new MyListener(parser); ParseTreeWalker walker = new ParseTreeWalker(); walker.Walk(listener, tree); return listener.Root; }
public void SecondPass() { if (Vectors == null) { ErrorMessage error = new ErrorMessage(); error.SourceFile = SourceFilePath; error.Message = "Required vectors statement not found."; Errors.Add(error); } ParseTreeWalker walker = new ParseTreeWalker(); walker.Walk(new SecondCpuPass(this), _rootTree); if (Errors.Count > 0) { throw new CompilerErrorException(); } }
public void Parse() { _rootTree = _parser.root(); ParseTreeWalker walker = new ParseTreeWalker(); walker.Walk(new ParseCpuPass(this), _rootTree); if (Errors.Count > 0) { throw new CompilerErrorException(); } resolveLabels(); verifyBranchLength(); if (Errors.Count > 0) { throw new CompilerErrorException(); } }
private void ResolveReferences(DeclarationFinder finder, VBComponent component, IParseTree tree) { var state = _state.GetModuleState(component); if (_state.Status == ParserState.ResolverError || (state != ParserState.Parsed)) { return; } Debug.WriteLine("Resolving '{0}'... (thread {1})", component.Name, Thread.CurrentThread.ManagedThreadId); var qualifiedName = new QualifiedModuleName(component); var resolver = new IdentifierReferenceResolver(qualifiedName, finder); var listener = new IdentifierReferenceListener(resolver); if (!string.IsNullOrWhiteSpace(tree.GetText().Trim())) { var walker = new ParseTreeWalker(); try { walker.Walk(listener, tree); state = ParserState.Ready; } catch (Exception exception) { Debug.Print("Exception thrown resolving '{0}' (thread {2}): {1}", component.Name, exception, Thread.CurrentThread.ManagedThreadId); state = ParserState.ResolverError; } } _state.SetModuleState(component, state); Debug.Print("'{0}' is {1}. Resolver took {2}ms to complete (thread {3})", component.Name, _state.GetModuleState(component), /*_resolverTimer[component].ElapsedMilliseconds*/0, Thread.CurrentThread.ManagedThreadId); }
private void runToolStripMenuItem_Click(object sender, EventArgs e) { AntlrInputStream inputCharStream4 = new AntlrInputStream(CodeTB.Text); AntlrInputStream inputCharStream3 = new AntlrInputStream(CodeTB.Text); AntlrInputStream inputCharStream2 = new AntlrInputStream(CodeTB.Text); AntlrInputStream inputCharStream = new AntlrInputStream(CodeTB.Text); NewDecafLexer tokenCreator4 = new NewDecafLexer(inputCharStream4); NewDecafLexer tokenCreator3 = new NewDecafLexer(inputCharStream3); NewDecafLexer tokenCreator2 = new NewDecafLexer(inputCharStream2); DecafLexer tokenCreator = new DecafLexer(inputCharStream); CommonTokenStream inputTokenStream4 = new CommonTokenStream(tokenCreator4); CommonTokenStream inputTokenStream3 = new CommonTokenStream(tokenCreator3); CommonTokenStream inputTokenStream2 = new CommonTokenStream(tokenCreator2); CommonTokenStream inputTokenStream = new CommonTokenStream(tokenCreator); NewDecafParser mainParser4 = new NewDecafParser(inputTokenStream4); NewDecafParser mainParser3 = new NewDecafParser(inputTokenStream3); NewDecafParser mainParser2 = new NewDecafParser(inputTokenStream2); DecafParser mainParser = new DecafParser(inputTokenStream); errorTB.Text = ""; mainParser.RemoveErrorListeners(); mainParser.AddErrorListener(new ParsingErrorDetector(errorTB)); NewDecafParser.ProgramContext AbstractSyntaxTree2 = mainParser2.program(); DecafParser.ProgramContext AbstractSyntaxTree = mainParser.program(); NewDecafParser.ProgramContext AbstractSyntaxTree3 = mainParser3.program(); NewDecafParser.ProgramContext AbstractSyntaxTree4 = mainParser4.program(); //TreeConstructingVisitor visitor = new TreeConstructingVisitor(TreeVisualizer); //visitor.Visit(AbstractSyntaxTree); ParseTreeWalker walker2 = new ParseTreeWalker(); ParseTreeWalker walker = new ParseTreeWalker(); ParseTreeWalker walker3 = new ParseTreeWalker(); ParseTreeWalker walker4 = new ParseTreeWalker(); //SymbolTableConstructor theConstructor = new SymbolTableConstructor(); //SymbolTable finalTable = theConstructor.finalTable; //string generated = ""; //ILGenerator theGenerator = new ILGenerator(generated); DefPhase theDefinition = new DefPhase(); //try //{ // walker2.Walk(theGenerator, AbstractSyntaxTree2); //walker.Walk(theConstructor, AbstractSyntaxTree); walker3.Walk(theDefinition, AbstractSyntaxTree3); STBasedGenerator newGenreator = new STBasedGenerator(theDefinition.scopes); walker3.Walk(newGenreator, AbstractSyntaxTree3); string code = newGenreator.mainTemplate.Render(); //} //catch (Exception ex) //{ // errorTB.Text += ex.Message + Environment.NewLine; // if (theConstructor.beginningLine == theConstructor.endingLine) // errorTB.Text += "Error found in line " + theConstructor.beginningLine; // else // errorTB.Text += "Error found between line " + theConstructor.beginningLine + " and line " + theConstructor.endingLine; //} if (errorTB.Text == string.Empty) errorTB.Text = "Congrats! No errors were found!"; FileStream myStream; if (saveFileDialog1.ShowDialog() == DialogResult.OK) { using (StreamWriter outfile = new StreamWriter(saveFileDialog1.FileName)) { outfile.Write(code); } } MessageBox.Show(code); }
/// <summary> /// Incremental preprocessing of a set of tokens lines changes /// </summary> internal static IList<DocumentChange<IProcessedTokensLine>> ProcessTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<ProcessedTokensLine> documentLines, IList<DocumentChange<ITokensLine>> tokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider) { // Collect all changes applied to the processed tokens lines during the incremental scan IList<DocumentChange<IProcessedTokensLine>> processedTokensLinesChanges = new List<DocumentChange<IProcessedTokensLine>>(); // There are 2 reasons to a preprocess a tokens line after a change : // 1. A tokens line changed : these lines were already reset during the previous steps // 2. If a tokens line that changed was involved in the parsing of a multiline compiler directive, the whole group of lines must be parsed again // Then, if a new COPY directive was parsed : the CompilationDocument to include must be prepared // --- PREPARATION PHASE : reset all processed tokens lines which were involved in a multiline compiler directive where at least one line changed --- // Iterate over all tokens changes detected by the ScannerStep : // refresh all the adjacent lines participating in a ContinuationTokensGroup if (tokensLinesChanges != null) { int lastLineIndexReset = -1; foreach (DocumentChange<ITokensLine> tokensChange in tokensLinesChanges) { processedTokensLinesChanges.Add(new DocumentChange<IProcessedTokensLine>(tokensChange.Type, tokensChange.LineIndex, (IProcessedTokensLine)tokensChange.NewLine)); if (tokensChange.LineIndex > lastLineIndexReset) { lastLineIndexReset = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, processedTokensLinesChanges, lastLineIndexReset); } } } // --- COMPILER DIRECTIVES PHASE : Find and parse all compiler directives --- // Init. Prepare a compiler directive parser // Create a token iterator on top of tokens lines TokensLinesIterator tokensIterator = new TokensLinesIterator( textSourceInfo.Name, documentLines, null, Token.CHANNEL_SourceTokens); // Crate an Antlr compatible token source on top a the token iterator TokensLinesTokenSource tokenSource = new TokensLinesTokenSource( textSourceInfo.Name, tokensIterator); // Init a compiler directive parser CommonTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens); CobolCompilerDirectivesParser directivesParser = new CobolCompilerDirectivesParser(tokenStream); IAntlrErrorStrategy compilerDirectiveErrorStrategy = new CompilerDirectiveErrorStrategy(); directivesParser.ErrorHandler = compilerDirectiveErrorStrategy; // Register all parse errors in a list in memory DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener(); directivesParser.RemoveErrorListeners(); directivesParser.AddErrorListener(errorListener); // Prepare to analyze the parse tree ParseTreeWalker walker = new ParseTreeWalker(); CompilerDirectiveBuilder directiveBuilder = new CompilerDirectiveBuilder(); // 1. Iterate over all compiler directive starting tokens found in the lines which were updated foreach (Token compilerDirectiveStartingToken in documentLines .Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing) .SelectMany(line => line.SourceTokens) .Where(token => token.TokenFamily == TokenFamily.CompilerDirectiveStartingKeyword)) { // 2. Reset the compiler directive parser state // Reset tokens iterator position before parsing // -> seek just before the compiler directive starting token tokensIterator.SeekToToken(compilerDirectiveStartingToken); tokensIterator.PreviousToken(); // Special case : for efficiency reasons, in EXEC SQL INCLUDE directives // only the third token INCLUDE is recognized as a compiler directive // starting keyword by the scanner. In this case, we must rewind the // iterator two tokens backwards to start parsing just before the EXEC token. if (compilerDirectiveStartingToken.TokenType == TokenType.EXEC_SQL_INCLUDE) { tokensIterator.PreviousToken(); tokensIterator.PreviousToken(); } // Reset Antlr BufferedTokenStream position tokenStream.SetTokenSource(tokenSource); // Reset parsing error diagnostics compilerDirectiveErrorStrategy.Reset(directivesParser); errorListener.Diagnostics.Clear(); // 3. Try to parse a compiler directive starting with the current token CobolCompilerDirectivesParser.CompilerDirectingStatementContext directiveParseTree = directivesParser.compilerDirectingStatement(); // 4. Visit the parse tree to build a first class object representing the compiler directive walker.Walk(directiveBuilder, directiveParseTree); CompilerDirective compilerDirective = directiveBuilder.CompilerDirective; bool errorFoundWhileParsingDirective = errorListener.Diagnostics.Count > 0 || directiveBuilder.Diagnostics.Count > 0; // 5. Get all tokens consumed while parsing the compiler directive // and partition them line by line Token startToken = (Token)directiveParseTree.Start; Token stopToken = (Token)directiveParseTree.Stop; if (stopToken == null) stopToken = startToken; MultilineTokensGroupSelection tokensSelection = tokensIterator.SelectAllTokensBetween(startToken, stopToken); if (compilerDirective != null) { // 6. Replace all matched tokens by : // - a CompilerDirectiveToken on the first line ProcessedTokensLine firstProcessedTokensLine = documentLines[tokensSelection.FirstLineIndex]; if (tokensSelection.SelectedTokensOnSeveralLines.Length == 1) { firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine( tokensSelection.TokensOnFirstLineBeforeStartToken, compilerDirective, errorFoundWhileParsingDirective, tokensSelection.SelectedTokensOnSeveralLines[0], tokensSelection.TokensOnLastLineAfterStopToken, false); } else { TokensGroup continuedTokensGroup = firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine( tokensSelection.TokensOnFirstLineBeforeStartToken, compilerDirective, errorFoundWhileParsingDirective, tokensSelection.SelectedTokensOnSeveralLines[0], null, true); // - a ContinuationTokensGroup on the following lines int selectionLineIndex = 1; int lastLineIndex = tokensSelection.FirstLineIndex + tokensSelection.SelectedTokensOnSeveralLines.Length - 1; for (int nextLineIndex = tokensSelection.FirstLineIndex + 1; nextLineIndex <= lastLineIndex; nextLineIndex++, selectionLineIndex++) { IList<Token> compilerDirectiveTokensOnNextLine = tokensSelection.SelectedTokensOnSeveralLines[selectionLineIndex]; if (compilerDirectiveTokensOnNextLine.Count > 0) { ProcessedTokensLine nextProcessedTokensLine = documentLines[nextLineIndex]; if (nextLineIndex != lastLineIndex) { continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine( continuedTokensGroup, compilerDirectiveTokensOnNextLine, null, true); } else { continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine( continuedTokensGroup, compilerDirectiveTokensOnNextLine, tokensSelection.TokensOnLastLineAfterStopToken, false); } } } } } // 7. Register compiler directive parse errors if (errorFoundWhileParsingDirective) { ProcessedTokensLine compilerDirectiveLine = documentLines[tokensSelection.FirstLineIndex]; foreach (ParserDiagnostic parserDiag in errorListener.Diagnostics) { compilerDirectiveLine.AddDiagnostic(parserDiag); } foreach (Diagnostic directiveDiag in directiveBuilder.Diagnostics) { compilerDirectiveLine.AddDiagnostic(directiveDiag); } } } // 8. Advance the state off all ProcessedTokensLines : // NeedsCompilerDirectiveParsing => NeedsCopyDirectiveProcessing if it contains a COPY directive IList<ProcessedTokensLine> parsedLinesWithCopyDirectives = null; // NeedsCompilerDirectiveParsing => Ready otherwise foreach (ProcessedTokensLine parsedLine in documentLines .Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing)) { if (parsedLine.ImportedDocuments != null) { if(parsedLinesWithCopyDirectives == null) { parsedLinesWithCopyDirectives = new List<ProcessedTokensLine>(); } parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.NeedsCopyDirectiveProcessing; parsedLinesWithCopyDirectives.Add(parsedLine); } else { parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready; } } // --- COPY IMPORT PHASE : Process COPY (REPLACING) directives --- // 1. Iterate over all updated lines containing a new COPY directive if (parsedLinesWithCopyDirectives != null) { foreach (ProcessedTokensLine tokensLineWithCopyDirective in parsedLinesWithCopyDirectives) { // Iterate over all COPY directives found on one updated line foreach (CopyDirective copyDirective in tokensLineWithCopyDirective.ImportedDocuments.Keys.ToArray<CopyDirective>()) { try { // Load (or retrieve in cache) the document referenced by the COPY directive ProcessedTokensDocument importedDocumentSource = processedTokensDocumentProvider.GetProcessedTokensDocument(copyDirective.LibraryName, copyDirective.TextName); // Store it on the current line after appying the REPLACING directive ImportedTokensDocument importedDocument = new ImportedTokensDocument(copyDirective, importedDocumentSource); tokensLineWithCopyDirective.ImportedDocuments[copyDirective] = importedDocument; } catch (Exception e) { // Text name refenced by COPY directive was not found // => register a preprocessor error on this line Token failedDirectiveToken = tokensLineWithCopyDirective.TokensWithCompilerDirectives .Where(token => token.TokenType == TokenType.CopyImportDirective && ((CompilerDirectiveToken)token).CompilerDirective == copyDirective) .First(); Diagnostic diag = new Diagnostic( MessageCode.FailedToLoadTextDocumentReferencedByCopyDirective, failedDirectiveToken.Column, failedDirectiveToken.EndColumn, e.Message); tokensLineWithCopyDirective.AddDiagnostic(diag); } } // Advance processing status of the line tokensLineWithCopyDirective.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready; } } // --- REPLACE PHASE : REPLACE directives are implemented in ReplaceTokensLinesIterator --- /* Algorithm : * * one REPLACE directive can express several replacement operations * one replacement operation can be of several types (distinguished for optimization purposes) * - SimpleTokenReplace : one source token / zero or one replacement token * - PartialWordReplace : one pure partial word / zero or one replacement token * - SimpleToMultipleTokenReplace : one source token / several replacement tokens * - MultipleTokenReplace : one first + several following source tokens / zero to many replacement tokens * * an iterator maintains a current set of replacement operations * * if nextToken is replace directive * the current set of replacement operations is updated * else * nextToken is compared to each replacement operation in turn * if single -> single source token operation : return replacement token * if single -> multiple operation : setup a secondary iterator with the list of replacement tokens * if multiple -> multiple operation * snapshot of the underlying iterator * try to match all of the following source tokens * if failure : restore snapshot and try next operation * if success : setup a secondary iterator * * token comparison sourceToken / replacementCandidate : * 1. Compare Token type * 2. If same token type and for families * AlphanumericLiteral * NumericLiteral * SyntaxLiteral * Symbol * => compare also Token text * * PartialCobolWord replacement : * p535 : The COPY statement with REPLACING phrase can be used to replace parts of words. * By inserting a dummy operand delimited by colons into the program text, * the compiler will replace the dummy operand with the required text. * Example 3 shows how this is used with the dummy operand :TAG:. * The colons serve as separators and make TAG a stand-alone operand. */ return processedTokensLinesChanges; }
public static CsharpParseResults InvokeParse(string fileName) { if (string.IsNullOrWhiteSpace(fileName)) return null; if (!System.IO.File.Exists(fileName)) return null; var tr = System.IO.File.OpenRead(fileName); var input = new AntlrInputStream(tr); var lexer = new CSharp4Lexer(input); var tokens = new CommonTokenStream(lexer); var parser = new CSharp4Parser(tokens); var tree = parser.compilation_unit(); var walker = new ParseTreeWalker(); var loader = new CsharpParseTree(); walker.Walk(loader, tree); var results = loader.Results; tr.Close(); return results; }
public static HtmlParseResults InvokeParse(Stream stream) { var input = new AntlrInputStream(stream); var lexer = new HTMLLexer(input); var tokens = new CommonTokenStream(lexer); var parser = new HTMLParser(tokens); var tree = parser.htmlDocument(); var walker = new ParseTreeWalker(); var loader = new AspNetParseTree(); walker.Walk(loader, tree); return loader.Results; }
public static void Work(string java, string csharp) { using (var fs = new FileStream(java, FileMode.Open)) { var input = new AntlrInputStream(fs); var lexer = new MyJavaLexer(input); var tokens = new CommonTokenStream(lexer); var parser = new MyJavaParser(tokens); // use the first rule of the parser // MyJavaParser.CompilationUnitContext tree = parser.compilationUnit(); var my_action = new Java2CSharp(); var tree = parser.compilationUnit(); var walker = new ParseTreeWalker(); walker.Walk(my_action, tree); GenerateCSharpCode(csharp, my_action.compileUnit); } }