private void checkMissingKeyword(object endToken, ParserRuleContext context, string msg) { if (endToken == null) { // if there is an error inside the start .. end then there is no need to report // a missing end int start = context.Start.StartIndex; int end = context.Stop.StopIndex; bool haserrorinblock = false; foreach (var error in _parseErrors) { if (error.Node.Position > start && error.Node.Position < end) { haserrorinblock = true; break; } } if (!haserrorinblock) { var err = ErrorCode.ERR_SyntaxError; IToken anchor = context.Stop; if (anchor == null) { anchor = context.Start; } var errdata = new ParseErrorData(anchor, err, msg); _parseErrors.Add(errdata); } } return; }
public override void ExitLiteralValue([NotNull] XSharpParser.LiteralValueContext context) { if (context.Token.Type == XSharpLexer.INCOMPLETE_STRING_CONST) { var err = ErrorCode.ERR_UnterminatedStringLit; IToken anchor = context.Stop; if (anchor == null) { anchor = context.Start; } var errdata = new ParseErrorData(anchor, err); _parseErrors.Add(errdata); } else if (context.Token.Type == XSharpLexer.INVALID_NUMBER) { var err = ErrorCode.ERR_InvalidNumber; IToken anchor = context.Stop; if (anchor == null) { anchor = context.Start; } var errdata = new ParseErrorData(anchor, err); _parseErrors.Add(errdata); } }
internal void AddError(ParseErrorData e) { if (ErrorData == null) { ErrorData = new List <ParseErrorData>(); } ErrorData.Add(e); }
public override void ExitBinaryExpression([NotNull] XSharpParser.BinaryExpressionContext context) { if (context.Left != null && context.Right == null) { var err = ErrorCode.ERR_SyntaxError; IToken anchor = context.Stop; if (anchor == null) { anchor = context.Start; } var errdata = new ParseErrorData(anchor, err, "Expression after '" + context.Op.Text + "' operator"); _parseErrors.Add(errdata); } }
public static bool Lex(string sourceText, string fileName, CSharpParseOptions options, IErrorListener listener, out ITokenStream tokens) { tokens = null; var parseErrors = ParseErrorData.NewBag(); try { var lexer = XSharpLexer.Create(sourceText, fileName, options); lexer.Options = options; var tokenStream = lexer.GetTokenStream(); tokenStream.Fill(); tokens = tokenStream; GetLexerErrors(lexer, tokenStream, parseErrors); #region Determine if we need to preprocess bool mustPreprocess = true; if (options.NoStdDef) { mustPreprocess = lexer.MustBeProcessed || lexer.HasPreprocessorTokens; } #endregion XSharpPreprocessor pp = null; BufferedTokenStream ppStream = null; pp = new XSharpPreprocessor(lexer, tokenStream, options, fileName, Encoding.Unicode, SourceHashAlgorithm.None, parseErrors); if (mustPreprocess) { var ppTokens = pp.PreProcess(); ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, ppTokens)); } else { // No Standard Defs and no preprocessor tokens in the lexer // so we bypass the preprocessor and use the lexer token stream ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, tokenStream.GetTokens())); } ppStream.Fill(); } catch (Exception) { } ReportErrors(parseErrors, listener); return(tokens != null); }
public override void VisitErrorNode([NotNull] IErrorNode node) { if (node.Symbol.Type == XSharpLexer.INCOMPLETE_STRING_CONST) { var err = ErrorCode.ERR_UnterminatedStringLit; IToken anchor = node.Symbol; var errdata = new ParseErrorData(anchor, err); _parseErrors.Add(errdata); } else if (node.Symbol.Type == XSharpLexer.INVALID_NUMBER) { var err = ErrorCode.ERR_InvalidNumber; IToken anchor = node.Symbol; var errdata = new ParseErrorData(anchor, err); _parseErrors.Add(errdata); } //else //{ // _parseErrors.Add(new ParseErrorData(node, ErrorCode.ERR_SyntaxError, node)); //} }
private void checkMissingToken(IToken l, IToken r, ParserRuleContext context) { if (l != null && r == null) { ErrorCode err = ErrorCode.ERR_SyntaxError; object par = null; switch (l.Type) { case XSharpLexer.LPAREN: err = ErrorCode.ERR_CloseParenExpected; break; case XSharpLexer.LCURLY: err = ErrorCode.ERR_RbraceExpected; break; case XSharpLexer.LBRKT: err = ErrorCode.ERR_SyntaxError; par = ']'; break; } IToken anchor = context.Stop; if (anchor == null) { anchor = l; } ParseErrorData errdata; if (par != null) { errdata = new ParseErrorData(anchor, err, par); } else { errdata = new ParseErrorData(anchor, err); } _parseErrors.Add(errdata); } }
private static void Parse(string fileName) { ITokenStream stream; IList <ParseErrorData> parseErrors = ParseErrorData.NewBag(); var filestream = new AntlrFileStream(fileName); var lexer = new XSharpLexer(filestream); lexer.TokenFactory = XSharpTokenFactory.Default; stream = new CommonTokenStream(lexer, Lexer.DefaultTokenChannel); var parser = new XSharpParser(stream); parser.IsScript = false; parser.AllowFunctionInsideClass = false; parser.AllowNamedArgs = false; parser.AllowXBaseVariables = false; parser.RemoveErrorListeners(); parser.Interpreter.PredictionMode = PredictionMode.Sll; parser.Interpreter.reportAmbiguities = true; parser.Interpreter.enable_global_context_dfa = true; // default false parser.Interpreter.optimize_tail_calls = true; parser.Interpreter.tail_call_preserves_sll = true; //parser.Interpreter.userWantsCtxSensitive = true; // default true parser.ErrorHandler = new XSharpErrorStrategy(); parser.AddErrorListener(new XSharpErrorListener(fileName, parseErrors, true)); XSharpParserRuleContext tree; try { tree = parser.source(); } catch (ParseCanceledException) { Console.WriteLine("Parse error, Errors from SLL mode"); showErrors(parseErrors); parseErrors.Clear(); parser.ErrorHandler = new XSharpErrorStrategy(); parser.AddErrorListener(new XSharpErrorListener(fileName, parseErrors, true)); parser.Interpreter.PredictionMode = PredictionMode.Ll; parser.Interpreter.force_global_context = true; parser.Interpreter.optimize_ll1 = false; parser.Interpreter.reportAmbiguities = true; parser.Reset(); try { tree = parser.source(); } catch (Exception e) { tree = null; Console.WriteLine(e.Message); } } // find parser errors (missing tokens etc) foreach (var e in lexer.LexErrors) { parseErrors.Add(e); } var walker = new ParseTreeWalker(); var errchecker = new XSharpParseErrorAnalysis(parser, parseErrors); if (tree != null) { walker.Walk(errchecker, tree); } Console.WriteLine("Parse error, Errors:"); showErrors(parseErrors); }
internal CompilationUnitSyntax ParseCompilationUnitCore() { #if DEBUG && DUMP_TIMES DateTime t = DateTime.Now; #endif if (_options.ShowIncludes) { _options.ConsoleOutput.WriteLine("Compiling {0}", _fileName); } var sourceText = _text.ToString(); XSharpLexer lexer = null; XSharpPreprocessor pp = null; XSharpParserRuleContext tree = new XSharpParserRuleContext(); XSharpParser parser = null; var parseErrors = ParseErrorData.NewBag(); try { lexer = XSharpLexer.Create(sourceText, _fileName, _options); lexer.Options = _options; _lexerTokenStream = lexer.GetTokenStream(); } catch (Exception e) { // Exception during Lexing parseErrors.Add(new ParseErrorData(_fileName, ErrorCode.ERR_Internal, e.Message, e.StackTrace)); // create empty token stream so we can continue the rest of the code _lexerTokenStream = new BufferedTokenStream(new XSharpListTokenSource(lexer, new List <IToken>())); } #if DEBUG && DUMP_TIMES { var ts = DateTime.Now - t; t += ts; Debug.WriteLine("Lexing completed in {0}", ts); } #endif // do not pre-process when there were lexer exceptions if (lexer != null && parseErrors.Count == 0) { foreach (var e in lexer.LexErrors) { parseErrors.Add(e); } BufferedTokenStream ppStream = null; try { // Check for #pragma in the lexerTokenStream _lexerTokenStream.Fill(); if (!_options.MacroScript) { pp = new XSharpPreprocessor(lexer, _lexerTokenStream, _options, _fileName, _text.Encoding, _text.ChecksumAlgorithm, parseErrors); } var mustPreprocess = !_options.MacroScript && (lexer.HasPreprocessorTokens || !_options.NoStdDef); if (mustPreprocess) { var ppTokens = pp.PreProcess(); ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, ppTokens)); } else { // No Standard Defs and no preprocessor tokens in the lexer // so we bypass the preprocessor and use the lexer tokenstream // but if a .ppo is required we must use the preprocessor to // write the source text to the .ppo file if (_options.PreprocessorOutput && pp != null) { pp.writeToPPO(sourceText, false); } BufferedTokenStream ts = (BufferedTokenStream)_lexerTokenStream; var tokens = ts.GetTokens(); // commontokenstream filters on tokens on the default channel. All other tokens are ignored ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, tokens)); } ppStream.Fill(); _preprocessorTokenStream = ppStream; } catch (Exception e) { // Exception during Preprocessing parseErrors.Add(new ParseErrorData(_fileName, ErrorCode.ERR_Internal, e.Message, e.StackTrace)); // create empty token stream so we can continue the rest of the code _preprocessorTokenStream = new BufferedTokenStream(new XSharpListTokenSource(lexer, new List <IToken>())); } } #if DEBUG && DUMP_TIMES { var ts = DateTime.Now - t; t += ts; Debug.WriteLine("Preprocessing completed in {0}", ts); } #endif parser = new XSharpParser(_preprocessorTokenStream) { Options = _options }; tree = new XSharpParserRuleContext(); if (_options.ParseLevel != ParseLevel.Lex) { // When parsing in Sll mode we do not record any parser errors. // When this fails, then we try again with LL mode and then we record errors parser.RemoveErrorListeners(); parser.Interpreter.PredictionMode = PredictionMode.Sll; // some options to have FAST parsing parser.Interpreter.tail_call_preserves_sll = false; parser.Interpreter.treat_sllk1_conflict_as_ambiguity = true; parser.ErrorHandler = new BailErrorStrategy(); try { tree = buildTree(parser); } catch (ParseCanceledException e) { if (_options.Verbose) { string msg = _GetInnerExceptionMessage(e); _options.ConsoleOutput.WriteLine("Antlr: SLL parsing failed with failure: " + msg + ". Trying again in LL mode."); } var errorListener = new XSharpErrorListener(_fileName, parseErrors); parser.AddErrorListener(errorListener); parser.ErrorHandler = new XSharpErrorStrategy(); // we need to set force_global_context to get proper error messages. This makes parsing slower // but gives better messages parser.Interpreter.treat_sllk1_conflict_as_ambiguity = false; parser.Interpreter.force_global_context = true; parser.Interpreter.enable_global_context_dfa = true; parser.Interpreter.PredictionMode = PredictionMode.Ll; _preprocessorTokenStream.Reset(); if (_options.Verbose && pp != null) { pp.DumpStats(); } if (pp != null) { pp.Close(); } parser.Reset(); try { tree = buildTree(parser); } catch (Exception e1) { // Cannot parse again. Must be a syntax error. if (_options.Verbose) { string msg = _GetInnerExceptionMessage(e1); _options.ConsoleOutput.WriteLine("Antlr: LL parsing also failed with failure: " + msg); } } } }// _options.ParseLevel < Complete #if DEBUG && DUMP_TIMES { var ts = DateTime.Now - t; t += ts; Debug.WriteLine("Parsing completed in {0}", ts); } #endif if (_options.DumpAST && tree != null) { string strTree = tree.ToStringTree(); string file = System.IO.Path.ChangeExtension(_fileName, "ast"); strTree = strTree.Replace(@"\r\n)))))", @"\r\n*)))))" + "\r\n"); strTree = strTree.Replace(@"\r\n))))", @"\r\n*)))" + "\r\n"); strTree = strTree.Replace(@"\r\n)))", @"\r\n*)))" + "\r\n"); strTree = strTree.Replace(@"\r\n))", @"\r\n*))" + "\r\n"); strTree = strTree.Replace(@"\r\n)", @"\r\n*)" + "\r\n"); strTree = strTree.Replace(@"\r\n*)", @"\r\n)"); System.IO.File.WriteAllText(file, strTree); } var walker = new ParseTreeWalker(); if (_options.ParseLevel == ParseLevel.Complete) { // check for parser errors, such as missing tokens // This adds items to the parseErrors list for missing // tokens and missing keywords try { var errchecker = new XSharpParseErrorAnalysis(parser, parseErrors, _options); walker.Walk(errchecker, tree); } catch (Exception e) { parseErrors.Add(new ParseErrorData(_fileName, ErrorCode.ERR_Internal, e.Message, e.StackTrace)); } } var treeTransform = CreateTransform(parser, _options, _pool, _syntaxFactory, _fileName); bool hasErrors = false; SyntaxToken eof = null; try { if (_options.ParseLevel < ParseLevel.Complete || parser.NumberOfSyntaxErrors != 0 || (parseErrors.Count != 0 && parseErrors.Contains(p => !ErrorFacts.IsWarning(p.Code)))) { eof = SyntaxFactory.Token(SyntaxKind.EndOfFileToken); eof = AddLeadingSkippedSyntax(eof, ParserErrorsAsTrivia(parseErrors, pp.IncludedFiles)); if (tree != null) { eof.XNode = new XTerminalNodeImpl(tree.Stop); } else { eof.XNode = new XTerminalNodeImpl(_lexerTokenStream.Get(_lexerTokenStream.Size - 1)); } hasErrors = true; } if (!hasErrors) { try { walker.Walk(treeTransform, tree); } catch (Exception e) { parseErrors.Add(new ParseErrorData(_fileName, ErrorCode.ERR_Internal, e.Message, e.StackTrace)); } eof = SyntaxFactory.Token(SyntaxKind.EndOfFileToken); if (!parseErrors.IsEmpty()) { eof = AddLeadingSkippedSyntax(eof, ParserErrorsAsTrivia(parseErrors, pp.IncludedFiles)); } } var result = _syntaxFactory.CompilationUnit( treeTransform.GlobalEntities.Externs, treeTransform.GlobalEntities.Usings, treeTransform.GlobalEntities.Attributes, treeTransform.GlobalEntities.Members, eof); result.XNode = tree; tree.CsNode = result; result.XTokens = _lexerTokenStream; result.XPPTokens = _preprocessorTokenStream; result.HasDocComments = lexer.HasDocComments; if (!_options.MacroScript && !hasErrors) { result.InitProcedures = treeTransform.GlobalEntities.InitProcedures; result.Globals = treeTransform.GlobalEntities.Globals; result.PragmaWarnings = treeTransform.GlobalEntities.PragmaWarnings; result.PragmaOptions = treeTransform.GlobalEntities.PragmaOptions; result.IncludedFiles = pp?.IncludedFiles; result.FileWidePublics = treeTransform.GlobalEntities.FileWidePublics; result.HasPCall = treeTransform.GlobalEntities.HasPCall; result.NeedsProcessing = treeTransform.GlobalEntities.NeedsProcessing; if (_options.HasRuntime) { result.LiteralSymbols = ((XSharpTreeTransformationRT)treeTransform).LiteralSymbols; result.LiteralPSZs = ((XSharpTreeTransformationRT)treeTransform).LiteralPSZs; } } return(result); } finally { #if DEBUG && DUMP_TIMES { var ts = DateTime.Now - t; t += ts; Debug.WriteLine("Tree transform completed in {0}", ts); } #endif treeTransform.Free(); if (pp != null) { pp.Close(); } } }
public override void ExitPragma([NotNull] XSharpParser.PragmaContext context) { ErrorCode error = ErrorCode.Unknown; var tokens = context._Tokens; IToken i1 = null; IToken i2 = null; var numbers = new List <IToken>(); Pragmastate state = Pragmastate.Default; IToken errortoken = null; bool isWarning = false; if (tokens.Count > 0) { i1 = tokens[0]; if (tokens.Count > 1) { i2 = tokens[1]; // 0 1 2 3 4 5 // #pragma warnings ( 123 , off ) // #pragma warnings ( pop ) if (i2.Type == XSharpParser.LPAREN) { if (tokens.Count >= 6 && tokens[3].Type == XSharpParser.COMMA && tokens[5].Type == XSharpParser.RPAREN) { i2 = tokens[4]; numbers.Add(tokens[2]); } else if (tokens.Count >= 4 && tokens[3].Type == XSharpParser.RPAREN) { i2 = tokens[2]; } } else { // 0 1 2 3 4 5 6 // #pragma warnings disable 1 , 2 , 3 if (tokens.Count > 2) { for (int i = 2; i < context._Tokens.Count; i++) { if (tokens[i].Type != XSharpParser.COMMA) { numbers.Add(tokens[i]); } } } } } } if (i1 == null) { error = ErrorCode.WRN_IllegalPragma; errortoken = context.P; } if (error == ErrorCode.Unknown) { switch (i1.Text.ToLower()) { case "options": isWarning = false; break; case "warning": case "warnings": isWarning = true; break; default: error = ErrorCode.WRN_IllegalPragma; errortoken = i1; break; } } if (error == ErrorCode.Unknown) { if (i2 != null) { switch (i2.Text.ToLower()) { case "on": state = Pragmastate.On; break; case "disable": case "off": state = Pragmastate.Off; break; case "restore": case "default": case "pop": state = Pragmastate.Default; break; default: error = ErrorCode.WRN_IllegalPPWarning; errortoken = i2; break; } } else { error = ErrorCode.WRN_IllegalPPWarning; errortoken = i1; } } if (error == ErrorCode.Unknown) { if (isWarning) { context.Pragma = new PragmaWarning(context, state, numbers, i1, i2); } else if (numbers.Count == 0) { // options pop if (i2.Text.ToLower() == "pop") { context.Pragma = new PragmaOption(context, Pragmastate.Default, CompilerOption.All); } else { error = ErrorCode.WRN_IllegalPPWarning; errortoken = i2; } } else { var token = numbers[0]; var opt = token.Text.ToLower(); if (token.Type == XSharpParser.STRING_CONST && opt.StartsWith("\"") && opt.EndsWith("\"") && opt.Length > 2) { opt = opt.Substring(1, opt.Length - 2); switch (opt) { case "az": case "initlocals": case "lb": case "memvar": case "memvars": case "undeclared": case "vo2": // Initialize string variables with empty strings case "vo4": // SignedUnsignedConversion case "vo5": // Implicit Clipper Calling convention case "vo6": // ResolveTypedFunctionPointersToPtr case "vo7": // Implicit Casts and Conversions case "vo9": // Allow missing return statements or missing return values case "vo10": // Compatible IIF case "vo11": // ArithmeticConversions case "vo12": // Clipper Integer divisions case "vo13": // StringComparisons case "vo14": // Embed real constants as float context.Pragma = new PragmaOption(context, state, CompilerOptionDecoder.Decode(opt)); break; default: error = ErrorCode.WRN_IllegalPPOption; errortoken = numbers[0]; break; } } } } // C# does not validate the error codes, so we will not do that either. if (error != ErrorCode.Unknown) { var errdata = new ParseErrorData(errortoken, error, errortoken.Text); _parseErrors.Add(errdata); } }
public static bool Parse(string sourceText, string fileName, CSharpParseOptions options, IErrorListener listener, out ITokenStream tokens, out XSharpParser.SourceContext tree) { tree = null; tokens = null; var parseErrors = ParseErrorData.NewBag(); try { var lexer = XSharpLexer.Create(sourceText, fileName, options); lexer.Options = options; BufferedTokenStream tokenStream = lexer.GetTokenStream(); tokenStream.Fill(); tokens = (ITokenStream)tokenStream; GetLexerErrors(lexer, tokenStream, parseErrors); // do we need to preprocess #region Determine if we really need the preprocessor bool mustPreprocess = true; if (lexer.HasPreprocessorTokens || !options.NoStdDef) { // no need to pre process in partial compilation // if lexer does not contain UDCs, Messages or Includes mustPreprocess = lexer.MustBeProcessed; } else { mustPreprocess = false; } #endregion XSharpPreprocessor pp = null; BufferedTokenStream ppStream = null; pp = new XSharpPreprocessor(lexer, tokenStream, options, fileName, Encoding.Unicode, SourceHashAlgorithm.None, parseErrors); if (mustPreprocess) { var ppTokens = pp.PreProcess(); ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, ppTokens)); } else { // No Standard Defs and no preprocessor tokens in the lexer // so we bypass the preprocessor and use the lexer token stream ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, tokenStream.GetTokens())); } ppStream.Fill(); var parser = new XSharpParser(ppStream); parser.Interpreter.tail_call_preserves_sll = false; // default = true Setting to FALSE will reduce memory used by parser parser.Options = options; tree = null; parser.RemoveErrorListeners(); parser.Interpreter.PredictionMode = PredictionMode.Sll; parser.ErrorHandler = new BailErrorStrategy(); try { tree = parser.source(); } catch (Exception) { var errorListener = new XSharpErrorListener(fileName, parseErrors); parser.AddErrorListener(errorListener); parser.ErrorHandler = new XSharpErrorStrategy(); parser.Interpreter.PredictionMode = PredictionMode.Ll; ppStream.Reset(); parser.Reset(); try { tree = parser.source(); } catch (Exception) { tree = null; } } } catch (Exception) { tree = null; } ReportErrors(parseErrors, listener); return(tree != null); }