public static bool Lex(string sourceText, string fileName, CSharpParseOptions options, IErrorListener listener, out ITokenStream tokens) { tokens = null; var parseErrors = ParseErrorData.NewBag(); try { var lexer = XSharpLexer.Create(sourceText, fileName, options); lexer.Options = options; var tokenStream = lexer.GetTokenStream(); tokenStream.Fill(); tokens = tokenStream; GetLexerErrors(lexer, tokenStream, parseErrors); #region Determine if we need to preprocess bool mustPreprocess = true; if (options.NoStdDef) { mustPreprocess = lexer.MustBeProcessed || lexer.HasPreprocessorTokens; } #endregion XSharpPreprocessor pp = null; BufferedTokenStream ppStream = null; pp = new XSharpPreprocessor(lexer, tokenStream, options, fileName, Encoding.Unicode, SourceHashAlgorithm.None, parseErrors); if (mustPreprocess) { var ppTokens = pp.PreProcess(); ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, ppTokens)); } else { // No Standard Defs and no preprocessor tokens in the lexer // so we bypass the preprocessor and use the lexer token stream ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, tokenStream.GetTokens())); } ppStream.Fill(); } catch (Exception) { } ReportErrors(parseErrors, listener); return(tokens != null); }
public static bool Parse(string sourceText, string fileName, CSharpParseOptions options, IErrorListener listener, out ITokenStream tokens, out XSharpParser.SourceContext tree) { tree = null; tokens = null; var parseErrors = ParseErrorData.NewBag(); try { var lexer = XSharpLexer.Create(sourceText, fileName, options); lexer.Options = options; BufferedTokenStream tokenStream = lexer.GetTokenStream(); tokenStream.Fill(); tokens = (ITokenStream)tokenStream; GetLexerErrors(lexer, tokenStream, parseErrors); // do we need to preprocess #region Determine if we really need the preprocessor bool mustPreprocess = true; if (lexer.HasPreprocessorTokens || !options.NoStdDef) { // no need to pre process in partial compilation // if lexer does not contain UDCs, Messages or Includes mustPreprocess = lexer.MustBeProcessed; } else { mustPreprocess = false; } #endregion XSharpPreprocessor pp = null; BufferedTokenStream ppStream = null; pp = new XSharpPreprocessor(lexer, tokenStream, options, fileName, Encoding.Unicode, SourceHashAlgorithm.None, parseErrors); if (mustPreprocess) { var ppTokens = pp.PreProcess(); ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, ppTokens)); } else { // No Standard Defs and no preprocessor tokens in the lexer // so we bypass the preprocessor and use the lexer token stream ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, tokenStream.GetTokens())); } ppStream.Fill(); var parser = new XSharpParser(ppStream); parser.Interpreter.tail_call_preserves_sll = false; // default = true Setting to FALSE will reduce memory used by parser parser.Options = options; tree = null; parser.RemoveErrorListeners(); parser.Interpreter.PredictionMode = PredictionMode.Sll; parser.ErrorHandler = new BailErrorStrategy(); try { tree = parser.source(); } catch (Exception) { var errorListener = new XSharpErrorListener(fileName, parseErrors); parser.AddErrorListener(errorListener); parser.ErrorHandler = new XSharpErrorStrategy(); parser.Interpreter.PredictionMode = PredictionMode.Ll; ppStream.Reset(); parser.Reset(); try { tree = parser.source(); } catch (Exception) { tree = null; } } } catch (Exception) { tree = null; } ReportErrors(parseErrors, listener); return(tree != null); }