public PythonCompiler(IList<string> sourceFiles, IList<ResourceFile> resourceFiles, string outputAssembly, CompilerSink compilerSink)
{
this.sourceFiles = sourceFiles;
this.resourceFiles = resourceFiles;
this.outputAssembly = outputAssembly;
this.compilerSink = compilerSink;
this.state = new SystemState();
}
static Analyzer()
{
state = new SystemState();
PythonFile empty = new PythonFile(System.IO.Stream.Null, System.Text.Encoding.Default, "rw");
state.__stderr__ = empty;
state.__stdin__ = empty;
state.__stdout__ = empty;
}
internal EngineModule(string moduleName, IDictionary<string, object> globalsDict, SystemState systemState)
{
Debug.Assert(moduleName != null);
globals = globalsDict;
if (globals is IAttributesDictionary)
globalsAdapter = globals as IAttributesDictionary;
else
globalsAdapter = new StringDictionaryAdapterDict(globalsDict);
PythonModule pythonModule = new PythonModule(moduleName, globalsAdapter, systemState);
defaultModuleScope = new ModuleScope(pythonModule);
}
IList<ClassificationSpan> IClassifier.GetClassificationSpans(SnapshotSpan span)
{
var classifications = new List<ClassificationSpan>();
using (var systemState = new SystemState())
{
int startIndex, endIndex;
if (span != null)
{
string spanText = span.GetText();
System.Diagnostics.Debug.WriteLine((spanText != null) ? spanText : string.Empty);
}
// Execute the IPy tokenizer
var tokenizer = new Tokenizer(span.GetText().ToCharArray(), true, systemState, new CompilerContext(string.Empty, new QuietCompilerSink()));
var token = tokenizer.Next();
// Iterate the tokens
while (token.Kind != TokenKind.EndOfFile)
{
// Determine the bounds of the classfication span
startIndex = span.Snapshot.GetLineFromLineNumber(tokenizer.StartLocation.Line - 1 + span.Start.GetContainingLine().LineNumber).Start.Position + tokenizer.StartLocation.Column;
endIndex = span.Snapshot.GetLineFromLineNumber(tokenizer.EndLocation.Line - 1 + span.Start.GetContainingLine().LineNumber).Start.Position + tokenizer.EndLocation.Column;
if (endIndex > span.Snapshot.GetText().Length)
endIndex = span.Snapshot.GetText().Length;
if (endIndex > startIndex && !span.Snapshot.TextBuffer.IsReadOnly(new Span(startIndex, endIndex - startIndex)))
{
// Add the classfication span
classifications.Add(new ClassificationSpan(new SnapshotSpan(span.Snapshot, startIndex, endIndex - startIndex), GetClassificationType(token)));
}
// Get next token
token = tokenizer.Next();
}
}
foreach (var region in span.Snapshot.TextBuffer.GetReadOnlyExtents(span))
{
// Add classfication for read only regions
classifications.Add(new ClassificationSpan(new SnapshotSpan(span.Snapshot, region), classificationRegistryService.GetClassificationType("PythonReadOnlyRegion")));
}
return classifications;
}
public PythonModule(string name, IDictionary <object, object> dict, SystemState state)
: this(name, dict, state, null, CallerContextFlags.None)
{
}
private ReflectedPackage MakePackage(SystemState state, string fullName, string name, bool isolated)
{
ReflectedPackage rp = new ReflectedPackage();
object mod;
PythonModule pmod;
if (isolated || !Importer.TryGetExistingModule(state, name, out mod)) {
// no collisions (yet), create a new module for the package.
pmod = new PythonModule(name, new Dict(), state);
pmod.PackageImported = true;
} else {
// there's already a module by this name. We'll just
// set the InnerModule but not make it visible until
// the user does an import (and we set PackageImported).
pmod = mod as PythonModule;
System.Diagnostics.Debug.Assert(pmod != null);
}
rp.fullName = fullName;
pmod.InnerModule = rp;
__dict__[SymbolTable.StringToId(name)] = pmod;
return rp;
}
private static object ImportFromPath(SystemState state, string name, string fullName, List path)
{
PythonModule ret = null;
foreach (object dirname in path) {
string str;
if (Converter.TryConvertToString(dirname, out str)) { // ignore non-string
string pathname = Path.Combine(str, name);
if (TryLoadPackage(state, fullName, pathname, out ret))
break;
if (TryLoadModule(state, fullName, pathname, out ret))
break;
}
}
return ret;
}
private static bool TryLoadModule(SystemState state, string fullName, string pathName, out PythonModule ret)
{
string binary = pathName + ".exe";
string source = pathName + ".py";
PythonModule pmod = null;
if (ShouldLoadPreCompiledModule(binary, source)) {
pmod = LoadPreCompiled(state, fullName, binary);
pmod.Filename = binary;
} else if (File.Exists(source)) {
pmod = LoadFromSource(state, fullName, source);
pmod.Filename = source;
} else {
ret = null;
return false;
}
pmod.ModuleName = fullName;
ret = InitializeModule(fullName, pmod);
return true;
}
public PythonModule(string name, IDictionary <object, object> dict, SystemState state, InitializeModule init)
: this(name, dict, state, init, CallerContextFlags.None)
{
}
internal static PythonModule MakePythonModule(SystemState state, string name, ReflectedType type)
{
type.Initialize();
FieldIdDict dict = new FieldIdDict();
foreach (string attrName in type.GetAttrNames(DefaultContext.Default)) {
dict[SymbolTable.StringToId(attrName)] = type.GetAttr(DefaultContext.Default, null, SymbolTable.StringToId(attrName));
}
PythonModule ret = new PythonModule(name, dict, state);
state.modules[name] = ret;
return ret;
}
internal PythonModule(string name, IAttributesDictionary dict, SystemState state)
: this(name, dict, state, null, CallerContextAttributes.None)
{
}
internal PythonModule(string name, IAttributesDictionary dict, SystemState state, InitializeModule init, CallerContextAttributes callerContextFlags)
{
Debug.Assert(state != null);
__dict__ = dict;
ModuleName = name;
__dict__[SymbolTable.Builtins] = TypeCache.Builtin;
initialize = init;
contextFlags = callerContextFlags;
systemState = state;
}
internal static PythonModule Load(string moduleName, Type compiledModuleType, SystemState state)
{
CompiledModule compiledModule = (CompiledModule)compiledModuleType.GetConstructor(Type.EmptyTypes).Invoke(Ops.EMPTY);
return compiledModule.Load(moduleName, new InitializeModule(compiledModule.Initialize), state);
}
internal ReflectedPackage GetOrMakePackage(SystemState state, string fullName, string name, bool isolated)
{
object ret;
if (__dict__.TryGetValue(SymbolTable.StringToId(name), out ret)) {
// if it's not a module we'll wipe it out below, eg def System(): pass then
// import System will result in the namespace being visible.
PythonModule pm = ret as PythonModule;
ReflectedPackage res;
do {
res = pm.InnerModule as ReflectedPackage;
if (res != null) return res;
pm = pm.InnerModule as PythonModule;
} while (pm != null);
}
return MakePackage(state, fullName, name, isolated);
}
public Tokenizer(char[] data, bool verbatim, SystemState state, CompilerContext context)
{
this.data = data;
this.length = data.Length;
this.verbatim = verbatim;
this.current.Line = 1;
this.startLoc.Line = 1;
this.endLoc.Line = 1;
this.context = context;
this.systemState = state;
if (Options.WarningOnIndentationInconsistency || Options.ErrorOnIndentationInconsistency) {
indentFormat = new StringBuilder[MAX_INDENT];
}
}
public Tokenizer(SystemState state, CompilerContext context, char[] data)
: this(data, false, state, context)
{
}
private static string RawEncode(SystemState state, string s, object encodingType, string errors)
{
string encoding = null;
if (encodingType == Missing.Value) {
encoding = state.getdefaultencoding();
} else {
encoding = encodingType as string;
if (encoding == null) {
if (encodingType != null)
throw Ops.TypeError("encode() expected string, got {0}", Ops.StringRepr(Ops.GetDynamicType(encodingType).Name));
encoding = state.getdefaultencoding();
}
}
if (encoding != null) {
string normalizedName = NormalizeEncodingName(encoding);
if ("raw_unicode_escape" == normalizedName) {
return RawUnicodeEscapeEncode(s);
} else if ("string_escape" == normalizedName) {
bool dummy = false;
return ReprEncode(s, '\'', ref dummy);
}
}
Encoding e = state.DefaultEncoding;
if (encoding == null || TryGetEncoding(state, encoding, out e)) {
// CLR's encoder exceptions have a 1-1 mapping w/ Python's encoder exceptions
// so we just clone the encoding & set the fallback to throw in strict mode
e = (Encoding)e.Clone();
switch (errors) {
case "strict": e.EncoderFallback = EncoderFallback.ExceptionFallback; break;
case "replace": e.EncoderFallback = EncoderFallback.ReplacementFallback; break;
default:
e.EncoderFallback = new PythonEncoderFallback(encoding,
s,
Modules.PythonCodecs.LookupError(errors));
break;
}
return FromByteArray(e.GetBytes(s));
}
// look for user-registered codecs
Tuple codecTuple = Modules.PythonCodecs.Lookup(encoding);
if (codecTuple != null) {
return UserDecodeOrEncode(codecTuple[Modules.PythonCodecs.EncoderIndex], s);
}
throw Ops.LookupError("unknown encoding: {0}", encoding);
}
internal PythonModule(ICallerContext context)
{
contextFlags = context.ContextFlags;
systemState = context.SystemState;
}
public void Initialize(SystemState state)
{
if (initialized != 0) return;
if (System.Threading.Interlocked.Exchange(ref initialized, 1) == 0) {
// add mscorlib
state.ClrModule.AddReferenceByName(typeof(string).Assembly.FullName);
// add system.dll
state.ClrModule.AddReferenceByName(typeof(System.Diagnostics.Debug).Assembly.FullName);
InitializeBuiltins(state);
}
}
internal PythonModule(string name, CompiledModule compiledModule, SystemState state, InitializeModule init)
: this(name, compiledModule, state, init, CallerContextAttributes.None)
{
}
/// <summary>
/// returns the package associated with the specified namespace and
/// updates the associated module to mark the package as imported.
/// </summary>
public PythonModule TryGetPackage(SystemState state, string name)
{
PythonModule pm = TryGetPackageAny(state, name) as PythonModule;
if (pm != null) {
pm.PackageImported = true;
return pm;
}
return null;
}
public int Exec(ref Guid pguidCmdGroup, uint nCmdID, uint nCmdexecopt, IntPtr pvaIn, IntPtr pvaOut)
{
// Handle VS commands to support code snippets
if (pguidCmdGroup == VSConstants.VSStd2K)
{
if (nCmdID == (uint)VSConstants.VSStd2KCmdID.INSERTSNIPPET || nCmdID == (uint)VSConstants.VSStd2KCmdID.SURROUNDWITH)
{
IVsTextManager2 textManager = (IVsTextManager2)this.serviceProvider.GetService(typeof(SVsTextManager));
IVsExpansionManager expansionManager;
if (VSConstants.S_OK == textManager.GetExpansionManager(out expansionManager))
{
expansionManager.InvokeInsertionUI(
vsTextView,
this,
Constants.IronPythonLanguageServiceGuid,
null,
0,
1,
null,
0,
1,
"Insert Snippet",
string.Empty);
}
return VSConstants.S_OK;
}
if (this.expansionSession != null)
{
// Handle VS Expansion (Code Snippets) keys
if ((nCmdID == (uint)VSConstants.VSStd2KCmdID.TAB))
{
if (expansionSession.GoToNextExpansionField(0) == VSConstants.S_OK)
return VSConstants.S_OK;
}
else if ((nCmdID == (uint)VSConstants.VSStd2KCmdID.BACKTAB))
{
if (expansionSession.GoToPreviousExpansionField() == VSConstants.S_OK)
return VSConstants.S_OK;
}
else if ((nCmdID == (uint)VSConstants.VSStd2KCmdID.RETURN || nCmdID == (uint)VSConstants.VSStd2KCmdID.CANCEL))
{
if (expansionSession.EndCurrentExpansion(0) == VSConstants.S_OK)
{
expansionSession = null;
return VSConstants.S_OK;
}
}
}
// Handle Edit.ListMembers or Edit.CompleteWord commands
if ((nCmdID == (uint)VSConstants.VSStd2KCmdID.SHOWMEMBERLIST || nCmdID == (uint)VSConstants.VSStd2KCmdID.COMPLETEWORD))
{
if (activeSession != null)
{
activeSession.Dismiss();
}
ShowCompletion();
return VSConstants.S_OK;
}
// Handle Enter/Tab commit keys
if (activeSession != null && (nCmdID == (uint)VSConstants.VSStd2KCmdID.RETURN || nCmdID == (uint)VSConstants.VSStd2KCmdID.TAB))
{
if (activeSession.SelectedCompletionSet.SelectionStatus.IsSelected)
activeSession.Commit();
else
activeSession.Dismiss();
return VSConstants.S_OK;
}
// Handle Code Snippets after pressing the Tab key without completion
if (activeSession == null && (nCmdID == (uint)VSConstants.VSStd2KCmdID.TAB))
{
using (var systemState = new SystemState())
{
// Get the current line text until the cursor
var line = this.textView.GetTextViewLineContainingBufferPosition(this.textView.Caret.Position.BufferPosition);
var text = this.textView.TextSnapshot.GetText(line.Start.Position, this.textView.Caret.Position.BufferPosition - line.Start.Position);
// Create a tokenizer for the text
var tokenizer = new Tokenizer(text.ToCharArray(), true, systemState, new CompilerContext(string.Empty, new QuietCompilerSink()));
// Get the last token in the text
Token currentToken, lastToken = null;
while ((currentToken = tokenizer.Next()).Kind != TokenKind.NewLine)
{
lastToken = currentToken;
}
if (lastToken != null && lastToken.Kind != TokenKind.Constant)
{
var expansionManager = (IVsTextManager2)this.serviceProvider.GetService(typeof(SVsTextManager));
var snippetsEnumerator = new SnippetsEnumerator(expansionManager, Constants.IronPythonLanguageServiceGuid);
// Search a snippet that matched the token text
var expansion = snippetsEnumerator.FirstOrDefault(e => e.title == lastToken.Value.ToString());
if (expansion.title != null)
{
// Set the location where the snippet will be inserted
int startLine, startColumn, endLine, endColumn;
this.vsTextView.GetCaretPos(out startLine, out endColumn);
startColumn = endColumn - expansion.title.Length;
endLine = startLine;
// Insert the snippet
InsertCodeExpansion(expansion, startLine, startColumn, endLine, endColumn);
return VSConstants.S_OK;
}
}
}
}
// Hanlde other keys
if ((nCmdID == (uint)VSConstants.VSStd2KCmdID.TYPECHAR))
{
char typedChar = (char)(ushort)Marshal.GetObjectForNativeVariant(pvaIn);
if (activeSession == null)
{
// Handle trigger keys
// Check if the typed char is a trigger
if (IsTriggerKey(typedChar))
{
var result = this.nextCommandTarget.Exec(pguidCmdGroup, nCmdID, nCmdexecopt, pvaIn, pvaOut);
ShowCompletion();
return result;
}
}
else
{
// Handle commit keys
// Check if the typed char is a commit key
if (IsCommitKey(typedChar))
{
if (activeSession.SelectedCompletionSet.SelectionStatus.IsSelected)
activeSession.Commit();
else
activeSession.Dismiss();
var result = this.nextCommandTarget.Exec(pguidCmdGroup, nCmdID, nCmdexecopt, pvaIn, pvaOut);
// Check we should trigger completion after comitting the previous session (for example, after typing dot '.')
if (IsTriggerKey(typedChar))
ShowCompletion();
return result;
}
}
}
}
// we haven't handled this command so pass it onto the next target
return this.nextCommandTarget.Exec(pguidCmdGroup, nCmdID, nCmdexecopt, pvaIn, pvaOut);
}
public object TryGetPackageAny(SystemState state, string name)
{
Initialize(state);
object ret;
if (__dict__.TryGetValue(SymbolTable.StringToId(name), out ret)) {
return ret;
}
return null;
}
internal static PythonModule LoadPreCompiled(SystemState state, string fullName, string fileName)
{
Assembly asm = Assembly.LoadFile(Path.GetFullPath(fileName));
Type[] types = asm.GetTypes();
// TODO: temporary way to make sure this is pre-compiled module
if (types.Length != 1) {
throw Ops.SystemError("{0} is not pre-compiled module; try again after deleting it.", fileName);
}
Type type = types[0];
CompiledModule cm = Activator.CreateInstance(type) as CompiledModule;
return cm.Load(fullName, new InitializeModule(cm.Initialize), state);
}
internal bool LoadAssembly(SystemState state, Assembly assem)
{
return LoadAssembly(state, assem, false);
}
internal static bool TryGetExistingModule(SystemState state, string fullName, out object ret)
{
// Python uses None/null as a key here to indicate a missing module
if (state.modules.TryGetValue(fullName, out ret)) {
return ret != null;
}
return false;
}
internal bool LoadAssembly(SystemState state, Assembly assem, bool isInteropAssembly)
{
bool loaded;
if (loadedAssemblies.TryGetValue(assem, out loaded)) {
return false;
}
if (!loaded) {
foreach (PythonModuleAttribute pma in assem.GetCustomAttributes(typeof(PythonModuleAttribute), false)) {
builtins.Add(pma.name, pma.type);
builtinModuleNames[pma.type] = pma.name;
}
}
// GetExportedTypes does not work on dynamic assemblies, and this could be an Interop assembly
// generated by Marshal.GetTypeForITypeInfo using Reflection.Emit,
// isInteropAssembly flag can now be removed and replaced w/ a call to LoadTypesFromAssembly.
Type[] types = isInteropAssembly ? assem.GetTypes() : LoadTypesFromAssembly(assem);
foreach (Type type in types) {
// Skip nested types. They get loaded during parent type initalization.
//
if (type.IsNested || (!type.IsPublic && !Compiler.Options.PrivateBinding)) {
continue;
}
// save all the namespaces, types will be lazily initialized
// on demand in GetAttr
ReflectedPackage pkg = GetOrMakeTopPackage(state, assem, type.Namespace);
if (!loaded) {
// We dont save all types since it requires us to hold on to the Type object unnecessarily.
// We do load all the types, so its not clear if this optimizations is really useful.
// Publish all COM types immediately so that ComObject can access it when needed
// for generic Runtime-callable-wrappers
ComObject.AddType(type.GUID, type);
// We need to save top-level types immediately
if (String.IsNullOrEmpty(type.Namespace)) {
pkg.SaveType(type);
}
} else {
// doing a non-lazy reload... force all types to get loaded
pkg.LoadAllTypes();
}
}
// Assembly was loaded
loadedAssemblies[assem] = true;
return true;
}
private static PythonModule LoadFromSource(SystemState state, string fullName, string fileName)
{
CompilerContext context = new CompilerContext(fileName);
Parser parser = Parser.FromFile(state, context);
Statement s = parser.ParseFileInput();
return OutputGenerator.GenerateModule(state, context, s, fullName);
}
private ReflectedPackage GetOrMakeTopPackage(SystemState state, Assembly assm, string ns)
{
ReflectedPackage ret = this;
if (ns != null) {
string[] pieces = ns.Split(Type.Delimiter);
for (int i = 0; i < pieces.Length; i++) {
if (!ret.packageAssemblies.Contains(assm)) ret.packageAssemblies.Add(assm);
ret = ret.GetOrMakePackage(state, String.Join(".", pieces, 0, i + 1), pieces[i], isolated);
}
}
if (!ret.packageAssemblies.Contains(assm)) ret.packageAssemblies.Add(assm);
return ret;
}
// Returning True: such module/file found, and TryLoadXXX should finish.
private static bool TryLoadPackage(SystemState state, string fullName, string pathName, out PythonModule ret)
{
if (!Directory.Exists(pathName)) {
ret = null;
return false;
}
string binary = Path.Combine(pathName, "__init__.exe");
string source = Path.Combine(pathName, "__init__.py");
bool loadBinary = ShouldLoadPreCompiledModule(binary, source);
bool loadSource = File.Exists(source);
if (loadBinary || loadSource) {
PythonModule pmod;
if (loadBinary) {
pmod = LoadPreCompiled(state, fullName, binary);
pmod.Filename = binary;
} else {
pmod = LoadFromSource(state, fullName, source);
pmod.Filename = source;
}
pmod.ModuleName = fullName;
pmod.SetImportedAttr(DefaultContext.Default, SymbolTable.Path, Ops.MakeList(Path.GetFullPath(pathName)));
ret = InitializeModule(fullName, pmod);
return true;
} else {
ret = null;
return false;
}
}
private void InitializeBuiltins(SystemState state)
{
LoadAssembly(state, typeof(Builtin).Assembly);
if (Environment.OSVersion.Platform == PlatformID.Unix) {
// we make our nt package show up as a posix package
// on unix platforms. Because we build on top of the
// CLI for all file operations we should be good from
// there, but modules that check for the presence of
// names (e.g. os) will do the right thing.
builtins.Add("posix", typeof(PythonNT));
builtins.Remove("nt");
}
state.builtin_module_names = Tuple.Make(builtins.Keys);
}
internal PythonModule Load(string moduleName, InitializeModule init, SystemState state)
{
InitializeBuiltins();
PythonModule pmod = new PythonModule(moduleName, this, state, init);
this.Module = pmod;
return pmod;
}
private static PythonModule LoadModuleFromSource(SystemState state, string fullName, string fileName)
{
PythonModule mod = LoadFromSource(state, fullName, fileName);
return(InitializeModule(fullName, mod));
}