internal static AST BuildAst(CodeContext context, SourceUnit sourceUnit, PythonCompilerOptions opts, string mode) {
Parser parser = Parser.CreateParser(
new CompilerContext(sourceUnit, opts, ThrowingErrorSink.Default),
(PythonOptions)context.LanguageContext.Options);
PythonAst ast = parser.ParseFile(true);
return ConvertToAST(ast, mode);
}
void CreateTokenizer(SourceUnit source)
{
PythonCompilerSink sink = new PythonCompilerSink();
PythonCompilerOptions options = new PythonCompilerOptions();
tokenizer = new Tokenizer(sink, options);
tokenizer.Initialize(source);
}
public Tokenizer(ErrorSink errorSink, PythonCompilerOptions options) {
ContractUtils.RequiresNotNull(errorSink, "errorSink");
ContractUtils.RequiresNotNull(options, "options");
_errors = errorSink;
_verbatim = options.Verbatim;
_state = new State(null);
_dontImplyDedent = options.DontImplyDedent;
_printFunction = options.PrintFunction;
_python26 = options.Python26;
}
static PythonEvaluator()
{
//TODO: bring back configuration via config file
var options = new Dictionary<string, object>();
options["LightweightScopes"] = false;
options["Optimize"] = true;
Engine = IronPython.Hosting.Python.CreateEngine(options);
// Allow Optimized option to be set. We are caching compiled code anyway, so pinning for the sake of performance is ok.
DefaultCompilerOptions = new PythonCompilerOptions(ModuleOptions.Optimized | ModuleOptions.ModuleBuiltins);
CodeCache = new Dictionary<string, CompiledCode>();
}
public IPyCompiler.Tokenizer CreateTokenizer(string text)
{
ScriptEngine engine = Python.CreateEngine();
PythonContext context = HostingHelpers.GetLanguageContext(engine) as PythonContext;
StringTextContentProvider textProvider = new StringTextContentProvider(text);
SourceUnit source = context.CreateSourceUnit(textProvider, String.Empty, SourceCodeKind.SingleStatement);
PythonCompilerSink sink = new PythonCompilerSink();
IPyCompiler.PythonCompilerOptions options = new IPyCompiler.PythonCompilerOptions();
tokenizer = new IPyCompiler.Tokenizer(sink, options);
tokenizer.Initialize(source);
return tokenizer;
}
public IPyCompiler.Tokenizer CreateTokenizer(string text)
{
ScriptEngine engine = Python.CreateEngine();
PythonContext context = HostingHelpers.GetLanguageContext(engine) as PythonContext;
StringTextContentProvider textProvider = new StringTextContentProvider(text);
SourceUnit source = context.CreateSourceUnit(textProvider, String.Empty, SourceCodeKind.SingleStatement);
PythonCompilerSink sink = new PythonCompilerSink();
IPyCompiler.PythonCompilerOptions options = new IPyCompiler.PythonCompilerOptions();
tokenizer = new IPyCompiler.Tokenizer(sink, options);
tokenizer.Initialize(source);
return(tokenizer);
}
internal static PythonAst ConvertToPythonAst(CodeContext codeContext, AST source ) {
Statement stmt;
PythonCompilerOptions options = new PythonCompilerOptions(ModuleOptions.ExecOrEvalCode);
SourceUnit unit = new SourceUnit(codeContext.LanguageContext, NullTextContentProvider.Null, "", SourceCodeKind.AutoDetect);
CompilerContext compilerContext = new CompilerContext(unit, options, ErrorSink.Default);
bool printExpression = false;
if (source is Expression) {
Expression exp = (Expression)source;
stmt = new ReturnStatement(expr.Revert(exp.body));
} else if (source is Module) {
Module module = (Module)source;
stmt = _ast.stmt.RevertStmts(module.body);
} else if (source is Interactive) {
Interactive interactive = (Interactive)source;
stmt = _ast.stmt.RevertStmts(interactive.body);
printExpression = true;
} else
throw PythonOps.TypeError("unsupported type of AST: {0}",(source.GetType()));
return new PythonAst(stmt, false, ModuleOptions.ExecOrEvalCode, printExpression, compilerContext, new int[] {} );
}
/// <summary>
/// Creates a FunctionCode object for exec/eval/execfile'd/compile'd code.
///
/// The code is then executed in a specific CodeContext by calling the .Call method.
///
/// If the code is being used for compile (vs. exec/eval/execfile) then it needs to be
/// registered incase our tracing mode changes.
/// </summary>
internal static FunctionCode FromSourceUnit(SourceUnit sourceUnit, PythonCompilerOptions options, bool register) {
var code = PythonContext.CompilePythonCode(sourceUnit, options, ThrowingErrorSink.Default);
return ((RunnableScriptCode)code).GetFunctionCode(register);
}
/// <summary>
/// Creates a FunctionCode object for exec/eval/execfile'd/compile'd code.
///
/// The code is then executed in a specific CodeContext by calling the .Call method.
///
/// If the code is being used for compile (vs. exec/eval/execfile) then it needs to be
/// registered in case our tracing mode changes.
/// </summary>
internal static FunctionCode FromSourceUnit(SourceUnit sourceUnit, PythonCompilerOptions options, bool register) {
var code = PythonContext.CompilePythonCode(sourceUnit, options, ThrowingErrorSink.Default);
if (sourceUnit.CodeProperties == ScriptCodeParseResult.Empty) {
// source span is made up
throw new SyntaxErrorException("unexpected EOF while parsing",
sourceUnit, new SourceSpan(new SourceLocation(0, 1, 1), new SourceLocation(0, 1, 1)), 0, Severity.Error);
}
return ((RunnableScriptCode)code).GetFunctionCode(register);
}
/// <summary>
/// Used to support legacy CreateParser API.
/// </summary>
internal Tokenizer(ErrorSink errorSink, PythonCompilerOptions options, bool verbatim)
: this(errorSink, options) {
_verbatim = verbatim || options.Verbatim;
}
public void Initialize(object state, TextReader reader, SourceUnit sourceUnit, SourceLocation initialLocation, int bufferCapacity, PythonCompilerOptions compilerOptions) {
ContractUtils.RequiresNotNull(reader, "reader");
if (state != null) {
if (!(state is State)) throw new ArgumentException();
_state = new State((State)state);
} else {
_state = new State(null);
}
if (compilerOptions != null && compilerOptions.InitialIndent != null) {
_state.Indent = (int[])compilerOptions.InitialIndent.Clone();
}
_sourceUnit = sourceUnit;
_disableLineFeedLineSeparator = reader is NoLineFeedSourceContentProvider.Reader;
// TODO: we can reuse the buffer if there is enough free space:
_buffer = new TokenizerBuffer(reader, initialLocation, bufferCapacity, !_disableLineFeedLineSeparator);
DumpBeginningOfUnit();
}
/// <summary>
/// Creates a FunctionCode object for exec/eval/execfile'd/compile'd code.
///
/// The code is then executed in a specific CodeContext by calling the .Call method.
/// </summary>
internal static FunctionCode FromSourceUnit(SourceUnit sourceUnit, PythonCompilerOptions options) {
var code = ((PythonContext)sourceUnit.LanguageContext).CompilePythonCode(Compiler.CompilationMode.Lookup, sourceUnit, options, ThrowingErrorSink.Default);
return ((RunnableScriptCode)code).GetFunctionCode();
}