public TraceManager(DkmProcess process)
{
_process = process;
_pyrtInfo = process.GetPythonRuntimeInfo();
_breakpointData = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <ArrayProxy <CliStructProxy <BreakpointData> > >("breakpointData");
_currentBreakpointData = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <ByteProxy>("currentBreakpointData");
_breakpointDataInUseByTraceFunc = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <ByteProxy>("breakpointDataInUseByTraceFunc");
_currentSourceLocation = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <CliStructProxy <CurrentSourceLocation> >("currentSourceLocation");
_stepKind = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <Int32Proxy>("stepKind");
_stepThreadId = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <UInt64Proxy>("stepThreadId");
_steppingStackDepth = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <Int32Proxy>("steppingStackDepth");
var onBreakpointHit = _pyrtInfo.DLLs.DebuggerHelper.FindExportName("OnBreakpointHit", true);
_onBreakpointHitBP = DkmRuntimeInstructionBreakpoint.Create(Guids.PythonTraceManagerSourceGuid, null, onBreakpointHit, false, null);
_onBreakpointHitBP.Enable();
var onStepComplete = _pyrtInfo.DLLs.DebuggerHelper.FindExportName("OnStepComplete", true);
_onStepCompleteBP = DkmRuntimeInstructionBreakpoint.Create(Guids.PythonTraceManagerSourceGuid, null, onStepComplete, false, null);
_onStepCompleteBP.Enable();
var onStepFallThrough = _pyrtInfo.DLLs.DebuggerHelper.FindExportName("OnStepFallThrough", true);
_onStepFallThroughBP = DkmRuntimeInstructionBreakpoint.Create(Guids.PythonTraceManagerSourceGuid, null, onStepFallThrough, false, null);
_onStepFallThroughBP.Enable();
WriteBreakpoints();
}
public ValuesHolder(DkmProcess process)
{
var pythonDll = process.GetPythonRuntimeInfo().DLLs.Python;
False = pythonDll.GetStaticVariable <PyBoolObject33>("_Py_FalseStruct");
True = pythonDll.GetStaticVariable <PyBoolObject33>("_Py_TrueStruct");
}
public PyObjectAllocator(DkmProcess process)
{
_process = process;
var pyrtInfo = process.GetPythonRuntimeInfo();
_objectsToRelease = pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <UInt64Proxy>("objectsToRelease");
}
public SourceLocation(ReadOnlyCollection <byte> encodedLocation, DkmProcess process = null)
{
var buffer = encodedLocation.ToArray();
using (var stream = new MemoryStream(buffer))
using (var reader = new BinaryReader(stream)) {
FileName = reader.ReadString();
bool hasFunctionName = reader.ReadBoolean();
if (hasFunctionName)
{
FunctionName = reader.ReadString();
}
LineNumber = reader.ReadInt32();
bool hasNativeAddress = reader.ReadBoolean();
if (hasNativeAddress && process != null)
{
var ip = reader.ReadUInt64();
var rva = reader.ReadUInt32();
NativeAddress = DkmNativeInstructionAddress.Create(
process.GetNativeRuntimeInstance(),
process.GetPythonRuntimeInfo().DLLs.Python,
rva,
new DkmNativeInstructionAddress.CPUInstruction(ip));
}
else
{
NativeAddress = null;
}
}
}
public InterpHeadHolder(DkmProcess process)
{
var pyrtInfo = process.GetPythonRuntimeInfo();
Proxy = pyrtInfo.GetRuntimeState()?.interpreters.head
?? pyrtInfo.DLLs.Python.GetStaticVariable <PointerProxy <PyInterpreterState> >("interp_head");
}
public unsafe TraceManagerLocalHelper(DkmProcess process, Kind kind)
{
_process = process;
_pyrtInfo = process.GetPythonRuntimeInfo();
_traceFunc = _pyrtInfo.DLLs.DebuggerHelper.GetExportedFunctionAddress("TraceFunc");
_isTracing = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <ByteProxy>("isTracing");
_pyTracingPossible = _pyrtInfo.DLLs.Python.GetStaticVariable <UInt32Proxy>("_Py_TracingPossible");
if (kind == Kind.StepIn)
{
var fieldOffsets = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <CliStructProxy <FieldOffsets> >("fieldOffsets");
fieldOffsets.Write(new FieldOffsets(process, _pyrtInfo));
var types = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <CliStructProxy <Types> >("types");
types.Write(new Types(process, _pyrtInfo));
var functionPointers = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <CliStructProxy <FunctionPointers> >("functionPointers");
functionPointers.Write(new FunctionPointers(process, _pyrtInfo));
var stringEquals = _pyrtInfo.DLLs.DebuggerHelper.GetExportedStaticVariable <PointerProxy>("stringEquals");
if (_pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27)
{
stringEquals.Write(_pyrtInfo.DLLs.DebuggerHelper.GetExportedFunctionAddress("StringEquals27").GetPointer());
}
else
{
stringEquals.Write(_pyrtInfo.DLLs.DebuggerHelper.GetExportedFunctionAddress("StringEquals33").GetPointer());
}
foreach (var interp in PyInterpreterState.GetInterpreterStates(process))
{
foreach (var tstate in interp.GetThreadStates())
{
RegisterTracing(tstate);
}
}
_handlers = new PythonDllBreakpointHandlers(this);
LocalComponent.CreateRuntimeDllFunctionExitBreakpoints(_pyrtInfo.DLLs.Python, "new_threadstate", _handlers.new_threadstate, enable: true);
foreach (var methodInfo in _handlers.GetType().GetMethods())
{
var stepInAttr = (StepInGateAttribute)Attribute.GetCustomAttribute(methodInfo, typeof(StepInGateAttribute));
if (stepInAttr != null &&
(stepInAttr.MinVersion == PythonLanguageVersion.None || _pyrtInfo.LanguageVersion >= stepInAttr.MinVersion) &&
(stepInAttr.MaxVersion == PythonLanguageVersion.None || _pyrtInfo.LanguageVersion <= stepInAttr.MaxVersion))
{
var handler = (StepInGateHandler)Delegate.CreateDelegate(typeof(StepInGateHandler), _handlers, methodInfo);
AddStepInGate(handler, _pyrtInfo.DLLs.Python, methodInfo.Name, stepInAttr.HasMultipleExitPoints);
}
}
if (_pyrtInfo.DLLs.CTypes != null)
{
OnCTypesLoaded(_pyrtInfo.DLLs.CTypes);
}
}
}
public DummyHolder(DkmProcess process)
{
PythonRuntimeInfo pyrtInfo = process.GetPythonRuntimeInfo();
Dummy =
pyrtInfo.LanguageVersion >= PythonLanguageVersion.V34 ?
pyrtInfo.DLLs.Python.GetStaticVariable <PointerProxy <PyObject> >("_PySet_Dummy") :
pyrtInfo.DLLs.Python.GetStaticVariable <PointerProxy <PyObject> >("dummy", "setobject.obj");
}
public ModuleManager(DkmProcess process)
{
_process = process;
_pyrtInfo = process.GetPythonRuntimeInfo();
LoadInitialPythonModules();
LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, "PyCode_New", PythonDllBreakpointHandlers.PyCode_New, enable: true, debugStart: true);
LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, "PyCode_NewEmpty", PythonDllBreakpointHandlers.PyCode_NewEmpty, enable: true, debugStart: true);
}
public ModuleManager(DkmProcess process)
{
_process = process;
_pyrtInfo = process.GetPythonRuntimeInfo();
string pyCodeFunctionName = (_pyrtInfo.LanguageVersion < Parsing.PythonLanguageVersion.V38) ? "PyCode_New" : "PyCode_NewWithPosOnlyArgs";
LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, pyCodeFunctionName, PythonDllBreakpointHandlers.PyCode_New, enable: true, debugStart: true);
LocalComponent.CreateRuntimeDllFunctionBreakpoint(_pyrtInfo.DLLs.Python, "PyCode_NewEmpty", PythonDllBreakpointHandlers.PyCode_NewEmpty, enable: true, debugStart: true);
}
public StructMetadata(DkmProcess process, string name)
{
Process = process;
Name = name;
using (var pythonSymbols = process.GetPythonRuntimeInfo().DLLs.Python.GetSymbols()) {
_symbol = pythonSymbols.Object.GetTypeSymbol(name);
}
Size = (long)Symbol.length;
}
private static void InjectHelperDll(DkmProcess process)
{
var injectionData = process.GetDataItem <HelperDllInjectionDataHolder>();
if (injectionData != null)
{
// Injection is already in progress.
return;
}
injectionData = new HelperDllInjectionDataHolder();
process.SetDataItem(DkmDataCreationDisposition.CreateNew, injectionData);
var pyrtInfo = process.GetPythonRuntimeInfo();
// Loading the helper is done via CreateRemoteThread(LoadLibrary), which is inherently asynchronous.
// On the other hand, we will not handle breakpoints until it is loaded - they won't even be bound.
// If any Python code is running in the meantime, this may cause us to skip breakpoints, which is
// very surprising in the run (F5) scenario, as the user expects all preset breakpoints to be hit.
// To fix that, we need block the Python interpreter loop until the helper is fully loaded.
//
// Pausing all threads is not a good way to do this, because one of the threads may be holding the
// loader lock, which will prevent the helper from loading and result in a deadlock. So instead,
// block at a known location at the beginning of PyInitialize_Ex, and only freeze the thread that
// calls it - this is sufficient to prevent execution of Python code in run scenario before helper
// is loaded.
//
// For attach-to-running-process scenario, we do nothing because the attach itself is inherently
// asynchronous, and so there's no user expectation that breakpoints light up instantly.
// If Python is already initialized, this is attach-to-running-process - don't block.
var initialized = pyrtInfo.DLLs.Python.GetStaticVariable <Int32Proxy>(
"initialized",
GetPyInitializeObjectFile(pyrtInfo.LanguageVersion)
);
if (initialized.Read() == 0)
{
// When Py_InitializeEx is hit, suspend the thread.
DkmRuntimeBreakpoint makePendingCallsBP = null;
makePendingCallsBP = CreateRuntimeDllExportedFunctionBreakpoint(pyrtInfo.DLLs.Python, "Py_InitializeEx", (thread, frameBase, vFrame) => {
makePendingCallsBP.Close();
if (process.GetPythonRuntimeInstance() == null)
{
thread.Suspend(true);
injectionData.SuspendedThread = thread;
}
});
makePendingCallsBP.Enable();
}
// Inject the helper DLL; OnHelperDllInitialized will resume the thread once the DLL is loaded and initialized.
DebugAttach.AttachDkm(process.LivePart.Id);
}
public override void Handle(DkmProcess process)
{
var pyrtInfo = process.GetPythonRuntimeInfo();
var nativeModules = process.GetNativeRuntimeInstance().GetNativeModuleInstances();
pyrtInfo.DLLs.Python = nativeModules.Single(mi => mi.UniqueId == PythonDllModuleInstanceId);
if (DebuggerHelperDllModuleInstanceId != Guid.Empty)
{
pyrtInfo.DLLs.DebuggerHelper = nativeModules.Single(mi => mi.UniqueId == DebuggerHelperDllModuleInstanceId);
}
}
public void OnPythonRuntimeInstanceLoaded()
{
PythonRuntimeInfo pyrtInfo = _process.GetPythonRuntimeInfo();
PythonDllBreakpointHandlers handlers = new PythonDllBreakpointHandlers(this);
_exceptionBreakpoints.AddRange(LocalComponent.CreateRuntimeDllFunctionExitBreakpoints(
pyrtInfo.DLLs.Python, "PyErr_SetObject", handlers.PyErr_SetObject, enable: _monitorExceptions));
if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27)
{
_exceptionBreakpoints.AddRange(LocalComponent.CreateRuntimeDllFunctionExitBreakpoints(
pyrtInfo.DLLs.Python, "do_raise", handlers.do_raise, enable: _monitorExceptions));
}
}
public SourceLocation(ReadOnlyCollection <byte> encodedLocation, DkmProcess process = null)
{
var buffer = encodedLocation.ToArray();
using (var stream = new MemoryStream(buffer))
using (var reader = new BinaryReader(stream))
{
FileName = reader.ReadString();
bool hasFunctionName = reader.ReadBoolean();
if (hasFunctionName)
{
FunctionName = reader.ReadString();
}
LineNumber = reader.ReadInt32();
bool hasNativeAddress = reader.ReadBoolean();
if (hasNativeAddress && process != null)
{
var ip = reader.ReadUInt64();
var rva = reader.ReadUInt32();
PythonDLLs dlls = process.GetPythonRuntimeInfo().DLLs;
DkmNativeModuleInstance dll = null;
switch (reader.ReadInt32())
{
case 0:
dll = dlls.Python;
break;
case 1:
dll = dlls.DebuggerHelper;
break;
}
if (dll != null)
{
NativeAddress = DkmNativeInstructionAddress.Create(
process.GetNativeRuntimeInstance(),
dll,
rva,
new DkmNativeInstructionAddress.CPUInstruction(ip));
}
}
else
{
NativeAddress = null;
}
}
}
private static void CreatePythonRuntimeInstance(DkmProcess process)
{
var pyrtInfo = process.GetPythonRuntimeInfo();
var pythonDllId = pyrtInfo.DLLs.Python.UniqueId;
var debuggerHelperDllId = pyrtInfo.DLLs.DebuggerHelper != null ? pyrtInfo.DLLs.DebuggerHelper.UniqueId : Guid.Empty;
new LocalStackWalkingComponent.BeforeCreatePythonRuntimeNotification {
PythonDllModuleInstanceId = pythonDllId,
DebuggerHelperDllModuleInstanceId = debuggerHelperDllId
}.SendHigher(process);
new RemoteComponent.CreatePythonRuntimeRequest {
PythonDllModuleInstanceId = pythonDllId,
DebuggerHelperDllModuleInstanceId = debuggerHelperDllId
}.SendLower(process);
}
public PyFrameObject(DkmProcess process, ulong address)
: base(process, address)
{
PythonRuntimeInfo pythonInfo = process.GetPythonRuntimeInfo();
if (pythonInfo.LanguageVersion <= PythonLanguageVersion.V35)
{
InitializeStruct(this, out Fields_27_35 fields);
_fields = fields;
}
else
{
InitializeStruct(this, out Fields_36 fields);
_fields = fields;
}
CheckPyType <PyFrameObject>();
}
public ProxyTypes(DkmProcess process)
{
var langVer = process.GetPythonRuntimeInfo().LanguageVersion;
var proxyTypes = typeof(PyObject).Assembly.GetTypes().Where(t => typeof(PyObject).IsAssignableFrom(t) && !t.IsAbstract);
foreach (var proxyType in proxyTypes)
{
string typeVarName = null;
var pyTypeAttrs = (PyTypeAttribute[])Attribute.GetCustomAttributes(proxyType, typeof(PyTypeAttribute), inherit: false);
if (pyTypeAttrs.Length == 0)
{
typeVarName = ComputeVariableName(proxyType);
}
else
{
foreach (var pyTypeAttr in pyTypeAttrs)
{
if (pyTypeAttr.MinVersion != PythonLanguageVersion.None && langVer < pyTypeAttr.MinVersion)
{
continue;
}
if (pyTypeAttr.MaxVersion != PythonLanguageVersion.None && langVer > pyTypeAttr.MaxVersion)
{
continue;
}
typeVarName = pyTypeAttr.VariableName ?? ComputeVariableName(proxyType);
break;
}
if (typeVarName == null)
{
continue;
}
}
var pyType = PyTypeObject.FromNativeGlobalVariable(process, typeVarName);
var proxyInfo = new ProxyInfo(proxyType);
ProxyInfoFromPyTypePtr.Add(pyType.Address, proxyInfo);
PyTypeFromType.Add(proxyType, pyType);
}
}
public PyFrameObject_FieldOffsets(DkmProcess process)
{
if (process.GetPythonRuntimeInfo().LanguageVersion <= PythonLanguageVersion.V35)
{
var fields = StructProxy.GetStructFields <PyFrameObject, PyFrameObject.Fields_27_35>(process);
f_back = -1;
f_code = fields.f_code.Offset;
f_globals = fields.f_globals.Offset;
f_locals = fields.f_locals.Offset;
f_lineno = fields.f_lineno.Offset;
}
else
{
var fields = StructProxy.GetStructFields <PyFrameObject, PyFrameObject.Fields_36>(process);
f_back = fields.f_back.Offset;
f_code = fields.f_code.Offset;
f_globals = fields.f_globals.Offset;
f_locals = fields.f_locals.Offset;
f_lineno = fields.f_lineno.Offset;
}
}
public override void Handle(DkmProcess process)
{
var pyrtInfo = process.GetPythonRuntimeInfo();
var nativeModules = process.GetNativeRuntimeInstance().GetNativeModuleInstances();
pyrtInfo.DLLs.Python = nativeModules.Single(mi => mi.UniqueId == PythonDllModuleInstanceId);
pyrtInfo.DLLs.Python.FlagAsTransitionModule();
if (DebuggerHelperDllModuleInstanceId != Guid.Empty)
{
pyrtInfo.DLLs.DebuggerHelper = nativeModules.Single(mi => mi.UniqueId == DebuggerHelperDllModuleInstanceId);
pyrtInfo.DLLs.DebuggerHelper.FlagAsTransitionModule();
process.SetDataItem(DkmDataCreationDisposition.CreateNew, new TraceManager(process));
}
var runtimeId = new DkmRuntimeInstanceId(Guids.PythonRuntimeTypeGuid, 0);
var runtimeInstance = DkmCustomRuntimeInstance.Create(process, runtimeId, null);
new CreateModuleRequest {
ModuleId = Guids.UnknownPythonModuleGuid
}.Handle(process);
}
public DummyHolder(DkmProcess process)
{
Dummy = process.GetPythonRuntimeInfo().DLLs.Python.GetStaticVariable <PointerProxy <PyObject> >("dummy", "dictobject.obj");
}
public override void Handle(DkmProcess process)
{
if (process.LivePart == null)
{
// When debugging dumps, there's no stepping or live expression evaluation. Hence, we don't
// need the helper DLL nor _ctypes.pyd for anything, and even if they are loaded in the dump,
// we don't care about them at all.
return;
}
var pyrtInfo = process.GetPythonRuntimeInfo();
var moduleInstance = process.GetNativeRuntimeInstance().GetNativeModuleInstances().Single(mi => mi.UniqueId == ModuleInstanceId);
if (pyrtInfo.DLLs.CTypes == null && PythonDLLs.CTypesNames.Contains(moduleInstance.Name))
{
moduleInstance.TryLoadSymbols();
if (moduleInstance.HasSymbols())
{
pyrtInfo.DLLs.CTypes = moduleInstance;
var traceHelper = process.GetDataItem <TraceManagerLocalHelper>();
if (traceHelper != null)
{
traceHelper.OnCTypesLoaded(moduleInstance);
}
}
}
if (process.GetPythonRuntimeInstance() != null)
{
return;
}
if (PythonDLLs.GetPythonLanguageVersion(moduleInstance) != PythonLanguageVersion.None)
{
pyrtInfo.DLLs.Python = moduleInstance;
for (int i = 0; i < 2; ++i)
{
if (moduleInstance.HasSymbols())
{
if (IsModuleCompiledWithPGO(moduleInstance))
{
pyrtInfo.DLLs.Python = null;
var pgoWarnMsg = DkmCustomMessage.Create(process.Connection, process, Guid.Empty, (int)VsPackageMessage.WarnAboutPGO, moduleInstance.Name, null);
pgoWarnMsg.SendToVsService(Guids.CustomDebuggerEventHandlerGuid, IsBlocking: true);
return;
}
if (process.LivePart == null)
{
// If debugging crash dumps, runtime can be created as soon as Python symbols are resolved.
CreatePythonRuntimeInstance(process);
}
else
{
// If not, we need to check for debugger helper DLL as well, and inject it if it isn't there yet.
if (pyrtInfo.DLLs.DebuggerHelper != null)
{
CreatePythonRuntimeInstance(process);
}
else
{
InjectHelperDll(process);
}
}
return;
}
moduleInstance.TryLoadSymbols();
}
var symWarnMsg = DkmCustomMessage.Create(process.Connection, process, Guid.Empty, (int)VsPackageMessage.WarnAboutPythonSymbols, moduleInstance.Name, null);
symWarnMsg.SendToVsService(Guids.CustomDebuggerEventHandlerGuid, IsBlocking: true);
}
else if (PythonDLLs.DebuggerHelperNames.Contains(moduleInstance.Name))
{
moduleInstance.TryLoadSymbols();
// When the module is reported is loaded, it is not necessarily fully initialized yet - it is possible to get into a state
// where its import table is not processed yet. If we register TraceFunc and it gets called by Python when in that state,
// we'll get a crash as soon as any imported WinAPI function is called. So check whether DllMain has already run - if it
// is, we're good to go, and if not, set a breakpoint on a hook that will be called once it is run, and defer runtime
// creation until that breakpoint is hit.
bool isInitialized = moduleInstance.GetExportedStaticVariable <ByteProxy>("isInitialized").Read() != 0;
if (isInitialized)
{
OnHelperDllInitialized(moduleInstance);
}
else
{
DkmRuntimeBreakpoint initBP = null;
initBP = CreateRuntimeDllExportedFunctionBreakpoint(moduleInstance, "OnInitialized", (thread, frameBase, vFrame) => {
initBP.Close();
OnHelperDllInitialized(moduleInstance);
});
initBP.Enable();
}
}
}
public CallStackFilter(DkmProcess process)
{
_process = process;
_pyrtInfo = process.GetPythonRuntimeInfo();
}
public ReprOptions(DkmProcess process)
{
Is64Bit = process.Is64Bit();
LanguageVersion = process.GetPythonRuntimeInfo().LanguageVersion;
}
public unsafe static PyTypeObject FromNativeGlobalVariable(DkmProcess process, string name)
{
var addr = process.GetPythonRuntimeInfo().DLLs.Python.GetStaticVariableAddress(name);
return(new PyTypeObject(process, addr));
}
public NoneHolder(DkmProcess process)
{
None = new PyObject(process, process.GetPythonRuntimeInfo().DLLs.Python.GetStaticVariableAddress("_Py_NoneStruct"));
}
public InterpHeadHolder(DkmProcess process)
{
Proxy = process.GetPythonRuntimeInfo().DLLs.Python.GetStaticVariable <PointerProxy <PyInterpreterState> >("interp_head");
}