private static Parser CreateParserWorker(CompilerContext context, PythonOptions options, bool verbatim) {
ContractUtils.RequiresNotNull(context, "context");
ContractUtils.RequiresNotNull(options, "options");
PythonCompilerOptions compilerOptions = context.Options as PythonCompilerOptions;
if (options == null) {
throw new ArgumentException(Resources.PythonContextRequired);
}
SourceCodeReader reader;
try {
reader = context.SourceUnit.GetReader();
if (compilerOptions.SkipFirstLine) {
reader.ReadLine();
}
} catch (IOException e) {
context.Errors.Add(context.SourceUnit, e.Message, SourceSpan.Invalid, 0, Severity.Error);
throw;
}
Tokenizer tokenizer = new Tokenizer(context.Errors, compilerOptions, verbatim);
tokenizer.Initialize(null, reader, context.SourceUnit, SourceLocation.MinValue);
tokenizer.IndentationInconsistencySeverity = options.IndentationInconsistencySeverity;
Parser result = new Parser(tokenizer, context.Errors, context.ParserSink, compilerOptions.LanguageFeatures);
result._sourceReader = reader;
return result;
}
/// <summary>
/// Creates a new PythonAst without a body. ParsingFinished should be called afterwards to set
/// the body.
/// </summary>
public PythonAst(bool isModule, ModuleOptions languageFeatures, bool printExpressions, CompilerContext context)
{
_isModule = isModule;
_printExpressions = printExpressions;
_languageFeatures = languageFeatures;
_mode = ((PythonCompilerOptions)context.Options).CompilationMode ?? GetCompilationMode(context);
_compilerContext = context;
FuncCodeExpr = _functionCode;
PythonCompilerOptions pco = context.Options as PythonCompilerOptions;
Debug.Assert(pco != null);
string name;
if (!context.SourceUnit.HasPath || (pco.Module & ModuleOptions.ExecOrEvalCode) != 0)
{
name = "<module>";
}
else
{
name = context.SourceUnit.Path;
}
_name = name;
Debug.Assert(_name != null);
PythonOptions po = ((PythonContext)context.SourceUnit.LanguageContext).PythonOptions;
if (po.EnableProfiler && _mode != CompilationMode.ToDisk)
{
_profiler = Profiler.GetProfiler(PyContext);
}
_document = context.SourceUnit.Document ?? Ast.SymbolDocument(name, PyContext.LanguageGuid, PyContext.VendorGuid);
}
public PythonParser()
{
pythonEngine = Python.CreateEngine();
var langContext = HostingHelpers.GetLanguageContext(pythonEngine);
compilerOptions = langContext.GetCompilerOptions();
langOptions = (PythonOptions)langContext.Options;
walker = new CustomPythonWalker();
}
private PythonAst ParsePythonFile(string path)
{
var pythonEngine = Python.CreateEngine();
var pythonSource = pythonEngine.CreateScriptSourceFromFile(path);
var pythonSourceUnit = HostingHelpers.GetSourceUnit(pythonSource);
var context = new CompilerContext(pythonSourceUnit, pythonEngine.GetCompilerOptions(), ErrorSink.Default);
var options = new PythonOptions();
var parser = Parser.CreateParser(context, options);
return(parser.ParseFile(false));
}
public FromImportStatement ParseStatement(string text)
{
ScriptEngine engine = Python.CreateEngine();
PythonContext context = HostingHelpers.GetLanguageContext(engine) as PythonContext;
StringTextContentProvider textProvider = new StringTextContentProvider(text);
SourceUnit source = context.CreateSourceUnit(textProvider, String.Empty, SourceCodeKind.SingleStatement);
PythonCompilerSink sink = new PythonCompilerSink();
CompilerContext compilerContext = new CompilerContext(source, new PythonCompilerOptions(), sink);
PythonOptions options = new PythonOptions();
using (Parser parser = Parser.CreateParser(compilerContext, options)) {
return(parser.ParseSingleStatement().Body as FromImportStatement);
}
}
internal PythonScript ParseScript(string filename, string content)
{
var sourceUnit =
HostingHelpers.GetSourceUnit(
_engine.CreateScriptSourceFromString(content, filename, SourceCodeKind.File));
var compilerContext = new CompilerContext(sourceUnit, _compilerOptions, ErrorSink.Default);
var options = new PythonOptions();
var parser = Parser.CreateParser(compilerContext, options);
PythonAst ast;
try
{
ast = parser.ParseFile(false);
}
catch (SyntaxErrorException e)
{
throw new ArgumentException($"Failed to parse file '{filename}' at line {e.Line}.", e);
}
return(new PythonScript(filename, content, ast));
}
private PythonAst ParseSnippet(string snippet)
{
var sourceUnit =
HostingHelpers.GetSourceUnit(
_engine.CreateScriptSourceFromString(snippet, SourceCodeKind.SingleStatement));
var compilerContext = new CompilerContext(sourceUnit, _compilerOptions, ErrorSink.Default);
var options = new PythonOptions();
var parser = Parser.CreateParser(compilerContext, options);
PythonAst ast;
try
{
ast = parser.ParseSingleStatement();
}
catch (SyntaxErrorException e)
{
throw new ArgumentException($"Failed to parse snippet '{snippet}'.", e);
}
return(ast);
}
public void TestAnalyzeStdLib()
{
//string dir = Path.Combine(Environment.GetEnvironmentVariable("ProgramFiles"), "IronPython 2.6 for .NET 4.0 RC\\Lib");
string dir = Path.Combine("C:\\Python27\\Lib");
List <string> files = new List <string>();
CollectFiles(dir, files);
List <SourceUnit> sourceUnits = new List <SourceUnit>();
foreach (string file in files)
{
sourceUnits.Add(
new SourceUnit(
HostingHelpers.GetLanguageContext(_engine),
new FileTextContentProvider(new FileStreamContentProvider(file)),
Path.GetFileNameWithoutExtension(file),
SourceCodeKind.File
)
);
}
Stopwatch sw = new Stopwatch();
sw.Start();
long start0 = sw.ElapsedMilliseconds;
var projectState = new ProjectState(_engine);
List <ProjectEntry> modules = new List <ProjectEntry>();
PythonOptions EmptyOptions = new PythonOptions();
foreach (var sourceUnit in sourceUnits)
{
modules.Add(projectState.AddModule(Path.GetFileNameWithoutExtension(sourceUnit.Path), sourceUnit.Path, null));
}
long start1 = sw.ElapsedMilliseconds;
Console.WriteLine("AddSourceUnit: {0} ms", start1 - start0);
List <PythonAst> nodes = new List <PythonAst>();
for (int i = 0; i < modules.Count; i++)
{
PythonAst ast = null;
try {
var sourceUnit = sourceUnits[i];
var context = new CompilerContext(sourceUnit, HostingHelpers.GetLanguageContext(_engine).GetCompilerOptions(), ErrorSink.Null);
ast = Parser.CreateParser(context, EmptyOptions).ParseFile(false);
} catch (Exception) {
}
nodes.Add(ast);
}
long start2 = sw.ElapsedMilliseconds;
Console.WriteLine("Parse: {0} ms", start2 - start1);
for (int i = 0; i < modules.Count; i++)
{
var ast = nodes[i];
if (ast != null)
{
modules[i].UpdateTree(ast, null);
}
}
long start3 = sw.ElapsedMilliseconds;
for (int i = 0; i < modules.Count; i++)
{
Console.WriteLine("Analyzing {1}: {0} ms", sw.ElapsedMilliseconds - start3, sourceUnits[i].Path);
var ast = nodes[i];
if (ast != null)
{
modules[i].Analyze();
}
}
long start4 = sw.ElapsedMilliseconds;
Console.WriteLine("Analyze: {0} ms", start4 - start3);
Console.ReadLine();
}
public static Parser CreateParser(CompilerContext context, PythonOptions options, bool verbatim) {
return CreateParserWorker(context, options, verbatim);
}
public static Parser CreateParser(CompilerContext context, PythonOptions options) {
return CreateParserWorker(context, options, false);
}
public ScannerService(PythonService python, IOptionsMonitor <PythonOptions> pythonOptions)
{
_python = python;
_pythonOptions = pythonOptions.CurrentValue;
}