public void OnPythonRuntimeInstanceLoaded()
{
PythonRuntimeInfo pyrtInfo = _process.GetPythonRuntimeInfo();
PythonDllBreakpointHandlers handlers = new PythonDllBreakpointHandlers(this);
_exceptionBreakpoints.AddRange(LocalComponent.CreateRuntimeDllFunctionExitBreakpoints(
pyrtInfo.DLLs.Python, "PyErr_SetObject", handlers.PyErr_SetObject, enable: _monitorExceptions));
if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27)
{
_exceptionBreakpoints.AddRange(LocalComponent.CreateRuntimeDllFunctionExitBreakpoints(
pyrtInfo.DLLs.Python, "do_raise", handlers.do_raise, enable: _monitorExceptions));
}
}
public Types(DkmProcess process, PythonRuntimeInfo pyrtInfo)
{
PyBytes_Type = PyObject.GetPyType <PyBytesObject>(process).Address;
if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27)
{
PyUnicode_Type = PyObject.GetPyType <PyUnicodeObject27>(process).Address;
}
else
{
PyUnicode_Type = PyObject.GetPyType <PyUnicodeObject33>(process).Address;
}
}
private static bool Py_IsInitialized(PythonRuntimeInfo pyrtInfo)
{
var ver = pyrtInfo.LanguageVersion;
var objName = ver < PythonLanguageVersion.V35 ? "pythonrun.obj" :
ver < PythonLanguageVersion.V37 ? "pylifecycle.obj" :
null;
var varName = ver < PythonLanguageVersion.V37 ? "initialized" : "_Py_Initialized";
try {
return(pyrtInfo.DLLs.Python.GetStaticVariable <Int32Proxy>(varName, objName).Read() != 0);
} catch (ArgumentException) {
return(false);
}
}
private static void InjectHelperDll(DkmProcess process)
{
var injectionData = process.GetDataItem <HelperDllInjectionDataHolder>();
if (injectionData != null)
{
// Injection is already in progress.
return;
}
injectionData = new HelperDllInjectionDataHolder();
process.SetDataItem(DkmDataCreationDisposition.CreateNew, injectionData);
PythonRuntimeInfo pyrtInfo = process.GetPythonRuntimeInfo();
// Loading the helper is done via CreateRemoteThread(LoadLibrary), which is inherently asynchronous.
// On the other hand, we will not handle breakpoints until it is loaded - they won't even be bound.
// If any Python code is running in the meantime, this may cause us to skip breakpoints, which is
// very surprising in the run (F5) scenario, as the user expects all preset breakpoints to be hit.
// To fix that, we need block the Python interpreter loop until the helper is fully loaded.
//
// Pausing all threads is not a good way to do this, because one of the threads may be holding the
// loader lock, which will prevent the helper from loading and result in a deadlock. So instead,
// block at a known location at the beginning of PyInitialize_Ex, and only freeze the thread that
// calls it - this is sufficient to prevent execution of Python code in run scenario before helper
// is loaded.
//
// For attach-to-running-process scenario, we do nothing because the attach itself is inherently
// asynchronous, and so there's no user expectation that breakpoints light up instantly.
// If Python is already initialized, this is attach-to-running-process - don't block.
if (!Py_IsInitialized(pyrtInfo))
{
// When Py_InitializeEx is hit, suspend the thread.
DkmRuntimeBreakpoint makePendingCallsBP = null;
makePendingCallsBP = CreateRuntimeDllExportedFunctionBreakpoint(pyrtInfo.DLLs.Python, "Py_InitializeEx", (thread, frameBase, vFrame, retAddr) =>
{
makePendingCallsBP.Close();
if (process.GetPythonRuntimeInstance() == null)
{
thread.Suspend(true);
injectionData.SuspendedThread = thread;
}
});
makePendingCallsBP.Enable();
}
// Inject the helper DLL; OnHelperDllInitialized will resume the thread once the DLL is loaded and initialized.
DebugAttach.AttachDkm(process.LivePart.Id);
}
private Int32Proxy GetTracingPossible(PythonRuntimeInfo pyrtInfo, DkmProcess process)
{
if (pyrtInfo.LanguageVersion == PythonLanguageVersion.V37)
{
return(_pyrtInfo.GetRuntimeState().ceval.tracing_possible);
}
else if (pyrtInfo.LanguageVersion > PythonLanguageVersion.V37)
{
foreach (var interp in PyInterpreterState.GetInterpreterStates(process))
{
return(interp.ceval.tracing_possible);
}
}
return(pyrtInfo.DLLs.Python.GetStaticVariable <Int32Proxy>("_Py_TracingPossible"));
}
void IDkmModuleSymbolsLoadedNotification.OnModuleSymbolsLoaded(DkmModuleInstance moduleInstance, DkmModule module, bool isReload, DkmWorkList workList, DkmEventDescriptor eventDescriptor)
{
var process = moduleInstance.Process;
var engines = process.DebugLaunchSettings.EngineFilter;
if (engines == null || !engines.Contains(Guids.PythonDebugEngineGuid))
{
return;
}
PythonRuntimeInfo pyrtInfo = process.GetPythonRuntimeInfo();
var nativeModuleInstance = moduleInstance as DkmNativeModuleInstance;
if (nativeModuleInstance != null)
{
if (PythonDLLs.CTypesNames.Contains(moduleInstance.Name))
{
pyrtInfo.DLLs.CTypes = nativeModuleInstance;
var traceHelper = process.GetDataItem <TraceManagerLocalHelper>();
if (traceHelper != null)
{
traceHelper.OnCTypesLoaded(nativeModuleInstance);
}
}
}
if (process.GetPythonRuntimeInstance() != null)
{
return;
}
if (pyrtInfo.DLLs.Python != null && pyrtInfo.DLLs.Python.HasSymbols())
{
if (process.LivePart == null || pyrtInfo.DLLs.DebuggerHelper != null)
{
CreatePythonRuntimeInstance(process);
}
else
{
InjectHelperDll(process);
}
}
}
private static void CreatePythonRuntimeInstance(DkmProcess process)
{
PythonRuntimeInfo pyrtInfo = process.GetPythonRuntimeInfo();
var pythonDllId = pyrtInfo.DLLs.Python.UniqueId;
var debuggerHelperDllId = pyrtInfo.DLLs.DebuggerHelper != null ? pyrtInfo.DLLs.DebuggerHelper.UniqueId : Guid.Empty;
new LocalStackWalkingComponent.BeforeCreatePythonRuntimeNotification
{
PythonDllModuleInstanceId = pythonDllId,
DebuggerHelperDllModuleInstanceId = debuggerHelperDllId
}.SendHigher(process);
new RemoteComponent.CreatePythonRuntimeRequest
{
PythonDllModuleInstanceId = pythonDllId,
DebuggerHelperDllModuleInstanceId = debuggerHelperDllId
}.SendLower(process);
}
public FieldOffsets(DkmProcess process, PythonRuntimeInfo pyrtInfo)
{
PyObject = new PyObject_FieldOffsets(process);
PyVarObject = new PyVarObject_FieldOffsets(process);
PyFrameObject = new PyFrameObject_FieldOffsets(process);
PyCodeObject = new PyCodeObject_FieldOffsets(process);
PyBytesObject = new PyBytesObject_FieldOffsets(process);
if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27)
{
PyUnicodeObject27 = new PyUnicodeObject27_FieldOffsets(process);
PyUnicodeObject33 = new PyUnicodeObject33_FieldOffsets();
}
else
{
PyUnicodeObject27 = new PyUnicodeObject27_FieldOffsets();
PyUnicodeObject33 = new PyUnicodeObject33_FieldOffsets(process);
}
}
private static void OnHelperDllInitialized(DkmNativeModuleInstance moduleInstance)
{
var process = moduleInstance.Process;
PythonRuntimeInfo pyrtInfo = process.GetPythonRuntimeInfo();
pyrtInfo.DLLs.DebuggerHelper = moduleInstance;
if (pyrtInfo.DLLs.Python != null && pyrtInfo.DLLs.Python.HasSymbols())
{
CreatePythonRuntimeInstance(process);
}
// If there was a suspended thread, resume it.
var injectionData = process.GetDataItem <HelperDllInjectionDataHolder>();
if (injectionData != null && injectionData.SuspendedThread != null)
{
injectionData.SuspendedThread.Resume(true);
}
}
public PythonDLLs(PythonRuntimeInfo pyrtInfo)
{
_pyrtInfo = pyrtInfo;
}
public unsafe TraceManagerLocalHelper(DkmProcess process, Kind kind)
{
_process = process;
_pyrtInfo = process.GetPythonRuntimeInfo();
_traceFunc = _pyrtInfo.DLLs.DebuggerHelper.GetExportedFunctionAddress("TraceFunc");
_evalFrameFunc = _pyrtInfo.LanguageVersion >= PythonLanguageVersion.V39 ?
_pyrtInfo.DLLs.DebuggerHelper.GetExportedFunctionAddress("EvalFrameFunc_39"):
_pyrtInfo.DLLs.DebuggerHelper.GetExportedFunctionAddress("EvalFrameFunc");
_defaultEvalFrameFunc = _pyrtInfo.LanguageVersion >= PythonLanguageVersion.V39 ?
_pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <PointerProxy>("DefaultEvalFrameFunc_39") :
_pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <PointerProxy>("DefaultEvalFrameFunc");
_isTracing = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <ByteProxy>("isTracing");
if (kind == Kind.StepIn)
{
var fieldOffsets = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <CliStructProxy <FieldOffsets> >("fieldOffsets");
fieldOffsets.Write(new FieldOffsets(process, _pyrtInfo));
var types = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <CliStructProxy <Types> >("types");
types.Write(new Types(process, _pyrtInfo));
var functionPointers = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <CliStructProxy <FunctionPointers> >("functionPointers");
functionPointers.Write(new FunctionPointers(process, _pyrtInfo));
var stringEquals = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <PointerProxy>("stringEquals");
if (_pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27)
{
stringEquals.Write(_pyrtInfo.DLLs.DebuggerHelper.GetExportedFunctionAddress("StringEquals27").GetPointer());
}
else
{
stringEquals.Write(_pyrtInfo.DLLs.DebuggerHelper.GetExportedFunctionAddress("StringEquals33").GetPointer());
}
foreach (var interp in PyInterpreterState.GetInterpreterStates(process))
{
if (_pyrtInfo.LanguageVersion >= PythonLanguageVersion.V36)
{
RegisterJITTracing(interp);
}
foreach (var tstate in interp.GetThreadStates())
{
RegisterTracing(tstate);
}
}
_handlers = new PythonDllBreakpointHandlers(this);
LocalComponent.CreateRuntimeDllFunctionExitBreakpoints(_pyrtInfo.DLLs.Python, "new_threadstate", _handlers.new_threadstate, enable: true);
LocalComponent.CreateRuntimeDllFunctionExitBreakpoints(_pyrtInfo.DLLs.Python, "PyInterpreterState_New", _handlers.PyInterpreterState_New, enable: true);
foreach (var methodInfo in _handlers.GetType().GetMethods())
{
var stepInAttr = (StepInGateAttribute)Attribute.GetCustomAttribute(methodInfo, typeof(StepInGateAttribute));
if (stepInAttr != null &&
(stepInAttr.MinVersion == PythonLanguageVersion.None || _pyrtInfo.LanguageVersion >= stepInAttr.MinVersion) &&
(stepInAttr.MaxVersion == PythonLanguageVersion.None || _pyrtInfo.LanguageVersion <= stepInAttr.MaxVersion))
{
var handler = (StepInGateHandler)Delegate.CreateDelegate(typeof(StepInGateHandler), _handlers, methodInfo);
AddStepInGate(handler, _pyrtInfo.DLLs.Python, methodInfo.Name, stepInAttr.HasMultipleExitPoints);
}
}
if (_pyrtInfo.DLLs.CTypes != null)
{
OnCTypesLoaded(_pyrtInfo.DLLs.CTypes);
}
}
}
public CallStackFilter(DkmProcess process)
{
_process = process;
_pyrtInfo = process.GetPythonRuntimeInfo();
}
public override void Handle(DkmProcess process)
{
if (process.LivePart == null)
{
// When debugging dumps, there's no stepping or live expression evaluation. Hence, we don't
// need the helper DLL nor _ctypes.pyd for anything, and even if they are loaded in the dump,
// we don't care about them at all.
return;
}
PythonRuntimeInfo pyrtInfo = process.GetPythonRuntimeInfo();
var moduleInstance = process.GetNativeRuntimeInstance().GetNativeModuleInstances().Single(mi => mi.UniqueId == ModuleInstanceId);
if (pyrtInfo.DLLs.CTypes == null && PythonDLLs.CTypesNames.Contains(moduleInstance.Name))
{
if (!moduleInstance.HasSymbols())
{
moduleInstance.TryLoadSymbols();
}
if (moduleInstance.HasSymbols())
{
pyrtInfo.DLLs.CTypes = moduleInstance;
var traceHelper = process.GetDataItem <TraceManagerLocalHelper>();
if (traceHelper != null)
{
traceHelper.OnCTypesLoaded(moduleInstance);
}
}
}
if (process.GetPythonRuntimeInstance() != null)
{
return;
}
if (PythonDLLs.GetPythonLanguageVersion(moduleInstance) != PythonLanguageVersion.None)
{
pyrtInfo.DLLs.Python = moduleInstance;
for (int i = 0; i < 2; ++i)
{
if (moduleInstance.HasSymbols())
{
if (process.LivePart == null)
{
// If debugging crash dumps, runtime can be created as soon as Python symbols are resolved.
CreatePythonRuntimeInstance(process);
}
else
{
// If not, we need to check for debugger helper DLL as well, and inject it if it isn't there yet.
if (pyrtInfo.DLLs.DebuggerHelper != null)
{
CreatePythonRuntimeInstance(process);
}
else
{
InjectHelperDll(process);
}
}
return;
}
if (!moduleInstance.HasSymbols())
{
moduleInstance.TryLoadSymbols();
}
}
var symWarnMsg = DkmCustomMessage.Create(process.Connection, process, Guid.Empty, (int)VsPackageMessage.WarnAboutPythonSymbols, moduleInstance.Name, null);
symWarnMsg.SendToVsService(Guids.CustomDebuggerEventHandlerGuid, IsBlocking: true);
}
else if (PythonDLLs.DebuggerHelperNames.Contains(moduleInstance.Name))
{
if (!moduleInstance.HasSymbols())
{
moduleInstance.TryLoadSymbols();
}
// When the module is reported is loaded, it is not necessarily fully initialized yet - it is possible to get into a state
// where its import table is not processed yet. If we register TraceFunc and it gets called by Python when in that state,
// we'll get a crash as soon as any imported WinAPI function is called. So check whether DllMain has already run - if it
// is, we're good to go, and if not, set a breakpoint on a hook that will be called once it is run, and defer runtime
// creation until that breakpoint is hit.
bool isInitialized = moduleInstance.GetExportedStaticVariable <ByteProxy>("isInitialized").Read() != 0;
if (isInitialized)
{
OnHelperDllInitialized(moduleInstance);
}
else
{
DkmRuntimeBreakpoint initBP = null;
initBP = CreateRuntimeDllExportedFunctionBreakpoint(moduleInstance, "OnInitialized", (thread, frameBase, vFrame, retAddr) =>
{
initBP.Close();
OnHelperDllInitialized(moduleInstance);
});
initBP.Enable();
}
}
}