Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[mypyc] Fix __dict__ in CPython 3.11 #13206

Closed
wants to merge 20 commits into from
Closed

[mypyc] Fix __dict__ in CPython 3.11 #13206

Changes from 2 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
ba461cb
Fix __dict__ in CPython 3.11
Fidget-Spinner Jul 20, 2022
d313a11
fix typos
Fidget-Spinner Jul 20, 2022
6714bfe
fix flake8 problems
Fidget-Spinner Jul 20, 2022
b395a87
Use if compiler directives
Fidget-Spinner Jul 20, 2022
c257d13
Remove sys import
Fidget-Spinner Jul 20, 2022
20f85db
please flake8
Fidget-Spinner Jul 20, 2022
ab6cdf8
guard Py_TPFLAGS_MANAGED_DICT
Fidget-Spinner Jul 20, 2022
0e40a79
typo fix
Fidget-Spinner Jul 20, 2022
5da842a
remove unsued code
Fidget-Spinner Jul 20, 2022
df0182b
Include patchlevel.h
Fidget-Spinner Jul 21, 2022
06db6d6
use runtime checks
Fidget-Spinner Jul 21, 2022
71378b4
fix flag
Fidget-Spinner Jul 21, 2022
0ae48e0
fix tp_flag issues
Fidget-Spinner Jul 21, 2022
5a7d194
Fix GC Problems (maybe)
Fidget-Spinner Jul 21, 2022
1e12b36
Support GC for managed dicts in >=3.11b5
Fidget-Spinner Jul 26, 2022
ba91315
Merge remote-tracking branch 'upstream/master' into 3_11_dict_compat
Fidget-Spinner Jul 27, 2022
ecfbe94
Update emitclass.py
Fidget-Spinner Jul 27, 2022
51a9c44
blacken code
Fidget-Spinner Jul 27, 2022
b4f5650
Movev comment
Fidget-Spinner Jul 27, 2022
392ec4f
handle error condition on visit
Fidget-Spinner Jul 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 66 additions & 29 deletions mypyc/codegen/emitclass.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Code generation for native classes and related wrappers."""
import sys

from typing import Optional, List, Tuple, Dict, Callable, Mapping, Set

Expand All @@ -18,6 +19,7 @@
from mypyc.sametype import is_same_type
from mypyc.namegen import NameGenerator

PY_3_11 = sys.version_info >= (3, 11)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this isn't right: the running version of Python doesn't necessarily equal the version used at runtime. Maybe we need to emit #ifdef'ed C code instead?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh right. Yeah my bad.

Copy link
Collaborator

@ichard26 ichard26 Jul 20, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Currently mypyc does not support cross Python version compilation, but yes conditional C code is probably better since cross version compilation is indeed a future goal. The Emitter class does have the capi_version attribute for future reference (it's currently always set to the running version of Python, could be changed down the road).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For some reason the PY_VERSION_HEX checks are not working (I suspect the appropriate headers aren't being included properly). So for now I'm using capi_version and leaving TODOs to use PY_VERSION_HEX in the future.


def native_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str:
return f'{NATIVE_PREFIX}{fn.cname(emitter.names)}'
Expand Down Expand Up @@ -187,6 +189,7 @@ def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None:
"""
name = cl.name
name_prefix = cl.name_prefix(emitter.names)
flags = []

setup_name = f'{name_prefix}_setup'
new_name = f'{name_prefix}_new'
Expand Down Expand Up @@ -257,21 +260,39 @@ def emit_line() -> None:

# XXX: there is no reason for the __weakref__ stuff to be mixed up with __dict__
if cl.has_dict:
# __dict__ lives right after the struct and __weakref__ lives right after that
# TODO: They should get members in the struct instead of doing this nonsense.
weak_offset = f'{base_size} + sizeof(PyObject *)'
emitter.emit_lines(
f'PyMemberDef {members_name}[] = {{',
f'{{"__dict__", T_OBJECT_EX, {base_size}, 0, NULL}},',
f'{{"__weakref__", T_OBJECT_EX, {weak_offset}, 0, NULL}},',
'{0}',
'};',
)
if PY_3_11:
# In CPython 3.11, __dict__ slot is automatically set before the type. So
# we need to let CPython manage it.
flags.append('Py_TPFLAGS_MANAGED_DICT')
# __weakref__ lives right after the struct
# TODO: __weakref__ should get members in the struct instead of doing this nonsense.
weak_offset = f'{base_size}'
emitter.emit_lines(
f'PyMemberDef {members_name}[] = {{',
f'{{"__weakref__", T_OBJECT_EX, {weak_offset}, 0, NULL}},',
'{0}',
'};',
)

fields['tp_members'] = members_name
fields['tp_basicsize'] = f'{base_size} + sizeof(PyObject *)'
fields['tp_weaklistoffset'] = weak_offset
else:
# __dict__ lives right after the struct and __weakref__ lives right after that
# TODO: They should get members in the struct instead of doing this nonsense.
weak_offset = f'{base_size} + sizeof(PyObject *)'
emitter.emit_lines(
f'PyMemberDef {members_name}[] = {{',
f'{{"__dict__", T_OBJECT_EX, {base_size}, 0, NULL}},',
f'{{"__weakref__", T_OBJECT_EX, {weak_offset}, 0, NULL}},',
'{0}',
'};',
)

fields['tp_members'] = members_name
fields['tp_basicsize'] = f'{base_size} + 2*sizeof(PyObject *)'
fields['tp_dictoffset'] = base_size
fields['tp_weaklistoffset'] = weak_offset
fields['tp_members'] = members_name
fields['tp_basicsize'] = f'{base_size} + 2*sizeof(PyObject *)'
fields['tp_dictoffset'] = base_size
fields['tp_weaklistoffset'] = weak_offset
else:
fields['tp_basicsize'] = base_size

Expand Down Expand Up @@ -315,7 +336,7 @@ def emit_line() -> None:
generate_methods_table(cl, methods_name, emitter)
emit_line()

flags = ['Py_TPFLAGS_DEFAULT', 'Py_TPFLAGS_HEAPTYPE', 'Py_TPFLAGS_BASETYPE']
flags += ['Py_TPFLAGS_DEFAULT', 'Py_TPFLAGS_HEAPTYPE', 'Py_TPFLAGS_BASETYPE']
if generate_full:
flags.append('Py_TPFLAGS_HAVE_GC')
if cl.has_method('__call__') and emitter.use_vectorcall():
Expand Down Expand Up @@ -698,13 +719,21 @@ def generate_traverse_for_class(cl: ClassIR,
emitter.emit_gc_visit(f'self->{emitter.attr(attr)}', rtype)
if cl.has_dict:
struct_name = cl.struct_name(emitter.names)
# __dict__ lives right after the struct and __weakref__ lives right after that
emitter.emit_gc_visit('*((PyObject **)((char *)self + sizeof({})))'.format(
struct_name), object_rprimitive)
emitter.emit_gc_visit(
'*((PyObject **)((char *)self + sizeof(PyObject *) + sizeof({})))'.format(
struct_name),
object_rprimitive)
if PY_3_11:
# __dict__ lives right before the struct and __weakref__ lives right after the struct
emitter.emit_gc_visit('*((PyObject **)((char *)self - sizeof(PyObject *)))', object_rprimitive)
emitter.emit_gc_visit(
'*((PyObject **)((char *)self + sizeof({})))'.format(
struct_name),
object_rprimitive)
else:
# __dict__ lives right after the struct and __weakref__ lives right after that
emitter.emit_gc_visit('*((PyObject **)((char *)self + sizeof({})))'.format(
struct_name), object_rprimitive)
emitter.emit_gc_visit(
'*((PyObject **)((char *)self + sizeof(PyObject *) + sizeof({})))'.format(
struct_name),
object_rprimitive)
emitter.emit_line('return 0;')
emitter.emit_line('}')

Expand All @@ -720,13 +749,21 @@ def generate_clear_for_class(cl: ClassIR,
emitter.emit_gc_clear(f'self->{emitter.attr(attr)}', rtype)
if cl.has_dict:
struct_name = cl.struct_name(emitter.names)
# __dict__ lives right after the struct and __weakref__ lives right after that
emitter.emit_gc_clear('*((PyObject **)((char *)self + sizeof({})))'.format(
struct_name), object_rprimitive)
emitter.emit_gc_clear(
'*((PyObject **)((char *)self + sizeof(PyObject *) + sizeof({})))'.format(
struct_name),
object_rprimitive)
if PY_3_11:
# __dict__ lives right before the struct and __weakref__ lives right after the struct
emitter.emit_gc_clear('*((PyObject **)((char *)self - sizeof(PyObject *)))', object_rprimitive)
emitter.emit_gc_clear(
'*((PyObject **)((char *)self + sizeof({})))'.format(
struct_name),
object_rprimitive)
else:
# __dict__ lives right after the struct and __weakref__ lives right after that
emitter.emit_gc_clear('*((PyObject **)((char *)self + sizeof({})))'.format(
struct_name), object_rprimitive)
emitter.emit_gc_clear(
'*((PyObject **)((char *)self + sizeof(PyObject *) + sizeof({})))'.format(
struct_name),
object_rprimitive)
emitter.emit_line('return 0;')
emitter.emit_line('}')

Expand Down