Skip to content

bpo-36540: PEP 570 -- Implementation #12701

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 19 commits into from
Apr 29, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 44 additions & 2 deletions Grammar/Grammar
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,55 @@ async_funcdef: ASYNC funcdef
funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] func_body_suite

parameters: '(' [typedargslist] ')'
typedargslist: (tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] [

# The following definition for typedarglist is equivalent to this set of rules:
#
# arguments = argument (',' [TYPE_COMMENT] argument)*
# argument = tfpdef ['=' test]
# kwargs = '**' tfpdef [','] [TYPE_COMMENT]
# args = '*' [tfpdef]
# kwonly_kwargs = (',' [TYPE_COMMENT] argument)* (TYPE_COMMENT | [',' [TYPE_COMMENT] [kwargs]])
# args_kwonly_kwargs = args kwonly_kwargs | kwargs
# poskeyword_args_kwonly_kwargs = arguments ( TYPE_COMMENT | [',' [TYPE_COMMENT] [args_kwonly_kwargs]])
# typedargslist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs
# typedarglist = (arguments ',' [TYPE_COMMENT] '/' [',' [[TYPE_COMMENT] typedargslist_no_posonly]])|(typedargslist_no_posonly)"
#
# It needs to be fully expanded to allow our LL(1) parser to work on it.

typedargslist: (
(tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])* ',' [TYPE_COMMENT] '/' [',' [ [TYPE_COMMENT] tfpdef ['=' test] (
',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] [
'*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]])
| '**' tfpdef [','] [TYPE_COMMENT]]])
| '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]])
| '**' tfpdef [','] [TYPE_COMMENT]]] )
| (tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] [
'*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]])
| '**' tfpdef [','] [TYPE_COMMENT]]])
| '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]])
| '**' tfpdef [','] [TYPE_COMMENT])
)
tfpdef: NAME [':' test]
varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [

# The following definition for varargslist is equivalent to this set of rules:
#
# arguments = argument (',' argument )*
# argument = vfpdef ['=' test]
# kwargs = '**' vfpdef [',']
# args = '*' [vfpdef]
# kwonly_kwargs = (',' argument )* [',' [kwargs]]
# args_kwonly_kwargs = args kwonly_kwargs | kwargs
# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]]
# vararglist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs
# varargslist = arguments ',' '/' [','[(vararglist_no_posonly)]] | (vararglist_no_posonly)
#
# It needs to be fully expanded to allow our LL(1) parser to work on it.

varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
'*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
| '**' vfpdef [',']]]
| '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
| '**' vfpdef [',']) ]] | (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
'*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
| '**' vfpdef [',']]]
| '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
Expand Down
10 changes: 6 additions & 4 deletions Include/Python-ast.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion Include/code.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ typedef uint16_t _Py_CODEUNIT;
typedef struct {
PyObject_HEAD
int co_argcount; /* #arguments, except *args */
int co_posonlyargcount; /* #positional only arguments */
int co_kwonlyargcount; /* #keyword only arguments */
int co_nlocals; /* #local variables */
int co_stacksize; /* #entries needed for evaluation stack */
Expand Down Expand Up @@ -102,7 +103,7 @@ PyAPI_DATA(PyTypeObject) PyCode_Type;

/* Public interface */
PyAPI_FUNC(PyCodeObject *) PyCode_New(
int, int, int, int, int, PyObject *, PyObject *,
int, int, int, int, int, int, PyObject *, PyObject *,
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Adding a required argument to the code type constructor will breaking every library that creates types.CodeType instances. Are you sure this is a good idea?

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You missed the end of the story:

commit 4a2edc34a405150d0b23ecfdcb401e7cf59f4650
Author: Pablo Galindo <[email protected]>
Date:   Mon Jul 1 11:35:05 2019 +0100

    bpo-37221: Add PyCode_NewWithPosOnlyArgs to be used internally and set PyCode_New as a compatibility wrapper (GH-13959)
    
    Add PyCode_NewEx to be used internally and set PyCode_New as a compatibility wrapper

Copy link
Member Author

@pablogsal pablogsal Feb 18, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, types.CodeType's constructor is considered private. We now even have this warning in the docs (https://docs.python.org/3.8/library/types.html#standard-interpreter-types):

If you instantiate any of these types, note that signatures may vary between Python versions.

Additionally, you have now available types.CodeType.replace in case you can to create a copy with some fields changed.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for the explanation!
I hope in future we'll have a more stable API for dynamic function generation.

Copy link

@ManifoldFR ManifoldFR Oct 1, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This breaks pickle.loads for functions constructed as

def constant_fn(val: float) -> Callable:
    def func(_):
        return val
    
    return func
fun = constant_fn(1.)

and pickled using cloudpickle under Python 3.7; these can't be loaded back up under Python 3.8.
Should this be addressed downstream in cloudpickle ?

Copy link
Member Author

@pablogsal pablogsal Oct 1, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be addressed downstream in cloudpickle ?

Yes, they need to update whatever code they are using to create code objects as code objects in Python3.8 require an extra field. Please, check these sections in the docs:

https://docs.python.org/3/library/types.html#standard-interpreter-types

If you instantiate any of these types, note that signatures may vary between Python versions.

Also, check this new C-API function:

https://docs.python.org/3/c-api/code.html#c.PyCode_NewWithPosOnlyArgs

and this new convenience function:

https://docs.python.org/3/library/types.html#types.CodeType.replace

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Alright, I'll file an issue there. Thanks.

PyObject *, PyObject *, PyObject *, PyObject *,
PyObject *, PyObject *, int, PyObject *);
/* same as struct above */
Expand Down
6 changes: 3 additions & 3 deletions Lib/ctypes/test/test_values.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,9 @@ class struct_frozen(Structure):
continue
items.append((entry.name.decode("ascii"), entry.size))

expected = [("__hello__", 139),
("__phello__", -139),
("__phello__.spam", 139),
expected = [("__hello__", 141),
("__phello__", -141),
("__phello__.spam", 141),
]
self.assertEqual(items, expected, "PyImport_FrozenModules example "
"in Doc/library/ctypes.rst may be out of date")
Expand Down
1 change: 1 addition & 0 deletions Lib/dis.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ def _format_code_info(co):
lines.append("Name: %s" % co.co_name)
lines.append("Filename: %s" % co.co_filename)
lines.append("Argument count: %s" % co.co_argcount)
lines.append("Positional-only arguments: %s" % co.co_posonlyargcount)
lines.append("Kw-only arguments: %s" % co.co_kwonlyargcount)
lines.append("Number of locals: %s" % co.co_nlocals)
lines.append("Stack size: %s" % co.co_stacksize)
Expand Down
3 changes: 2 additions & 1 deletion Lib/importlib/_bootstrap_external.py
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,7 @@ def _write_atomic(path, data, mode=0o666):
# this might affected the first line number #32911)
# Python 3.8a1 3400 (move frame block handling to compiler #17611)
# Python 3.8a1 3401 (add END_ASYNC_FOR #33041)
# Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540)
#
# MAGIC must change whenever the bytecode emitted by the compiler may no
# longer be understood by older implementations of the eval loop (usually
Expand All @@ -273,7 +274,7 @@ def _write_atomic(path, data, mode=0o666):
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
# in PC/launcher.c must also be updated.

MAGIC_NUMBER = (3401).to_bytes(2, 'little') + b'\r\n'
MAGIC_NUMBER = (3410).to_bytes(2, 'little') + b'\r\n'
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c

_PYCACHE = '__pycache__'
Expand Down
96 changes: 59 additions & 37 deletions Lib/inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ def iscode(object):
| 16=nested | 32=generator | 64=nofree | 128=coroutine
| 256=iterable_coroutine | 512=async_generator
co_freevars tuple of names of free variables
co_posonlyargcount number of positional only arguments
co_kwonlyargcount number of keyword only arguments (not including ** arg)
co_lnotab encoded mapping of line numbers to bytecode indices
co_name name with which this code object was defined
Expand Down Expand Up @@ -1031,26 +1032,20 @@ def getargs(co):
'args' is the list of argument names. Keyword-only arguments are
appended. 'varargs' and 'varkw' are the names of the * and **
arguments or None."""
args, varargs, kwonlyargs, varkw = _getfullargs(co)
return Arguments(args + kwonlyargs, varargs, varkw)

def _getfullargs(co):
"""Get information about the arguments accepted by a code object.

Four things are returned: (args, varargs, kwonlyargs, varkw), where
'args' and 'kwonlyargs' are lists of argument names, and 'varargs'
and 'varkw' are the names of the * and ** arguments or None."""

if not iscode(co):
raise TypeError('{!r} is not a code object'.format(co))

nargs = co.co_argcount
names = co.co_varnames
nargs = co.co_argcount
nposonlyargs = co.co_posonlyargcount
nkwargs = co.co_kwonlyargcount
args = list(names[:nargs])
kwonlyargs = list(names[nargs:nargs+nkwargs])
nposargs = nargs + nposonlyargs
posonlyargs = list(names[:nposonlyargs])
args = list(names[nposonlyargs:nposonlyargs+nargs])
kwonlyargs = list(names[nposargs:nposargs+nkwargs])
step = 0

nargs += nposonlyargs
nargs += nkwargs
varargs = None
if co.co_flags & CO_VARARGS:
Expand All @@ -1059,8 +1054,7 @@ def _getfullargs(co):
varkw = None
if co.co_flags & CO_VARKEYWORDS:
varkw = co.co_varnames[nargs]
return args, varargs, kwonlyargs, varkw

return Arguments(posonlyargs + args + kwonlyargs, varargs, varkw)

ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults')

Expand All @@ -1087,15 +1081,16 @@ def getargspec(func):
warnings.warn("inspect.getargspec() is deprecated since Python 3.0, "
"use inspect.signature() or inspect.getfullargspec()",
DeprecationWarning, stacklevel=2)
args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \
getfullargspec(func)
if kwonlyargs or ann:
raise ValueError("Function has keyword-only parameters or annotations"
", use getfullargspec() API which can support them")
args, varargs, varkw, defaults, posonlyargs, kwonlyargs, \
kwonlydefaults, ann = getfullargspec(func)
if posonlyargs or kwonlyargs or ann:
raise ValueError("Function has positional-only, keyword-only parameters"
" or annotations, use getfullargspec() API which can"
" support them")
return ArgSpec(args, varargs, varkw, defaults)

FullArgSpec = namedtuple('FullArgSpec',
'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations')
'args, varargs, varkw, defaults, posonlyargs, kwonlyargs, kwonlydefaults, annotations')

def getfullargspec(func):
"""Get the names and default values of a callable object's parameters.
Expand Down Expand Up @@ -1145,6 +1140,7 @@ def getfullargspec(func):
args = []
varargs = None
varkw = None
posonlyargs = []
kwonlyargs = []
defaults = ()
annotations = {}
Expand All @@ -1159,7 +1155,9 @@ def getfullargspec(func):
name = param.name

if kind is _POSITIONAL_ONLY:
args.append(name)
posonlyargs.append(name)
if param.default is not param.empty:
defaults += (param.default,)
elif kind is _POSITIONAL_OR_KEYWORD:
args.append(name)
if param.default is not param.empty:
Expand All @@ -1185,7 +1183,7 @@ def getfullargspec(func):
defaults = None

return FullArgSpec(args, varargs, varkw, defaults,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is breaking change. getfullargspec() was added to avoid breaking old getargspec(). You need to add a new function (getfullerargspec()?) and deprecate getfullargspec().

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suspect that there may be a longer discussion for these cases so I opened https://bugs.python.org/issue36751 to track it and decide there.

kwonlyargs, kwdefaults, annotations)
posonlyargs, kwonlyargs, kwdefaults, annotations)


ArgInfo = namedtuple('ArgInfo', 'args varargs keywords locals')
Expand Down Expand Up @@ -1216,7 +1214,8 @@ def _formatannotation(annotation):
return _formatannotation

def formatargspec(args, varargs=None, varkw=None, defaults=None,
kwonlyargs=(), kwonlydefaults={}, annotations={},
posonlyargs=(), kwonlyargs=(), kwonlydefaults={},
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since all these parameters are positional-or-keyword, inserting a new parameter in the middle is breaking change. You can add it only to the end. Or add a new function.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suspect that there may be a longer discussion for these cases so I opened https://bugs.python.org/issue36751 to track it and decide there.

annotations={},
formatarg=str,
formatvarargs=lambda name: '*' + name,
formatvarkw=lambda name: '**' + name,
Expand Down Expand Up @@ -1249,12 +1248,17 @@ def formatargandannotation(arg):
return result
specs = []
if defaults:
firstdefault = len(args) - len(defaults)
for i, arg in enumerate(args):
firstdefault = len(posonlyargs) + len(args) - len(defaults)
posonly_left = len(posonlyargs)
for i, arg in enumerate([*posonlyargs, *args]):
spec = formatargandannotation(arg)
if defaults and i >= firstdefault:
spec = spec + formatvalue(defaults[i - firstdefault])
specs.append(spec)
posonly_left -= 1
if posonlyargs and posonly_left == 0:
specs.append('/')

if varargs is not None:
specs.append(formatvarargs(formatargandannotation(varargs)))
else:
Expand Down Expand Up @@ -1342,7 +1346,8 @@ def getcallargs(*func_and_positional, **named):
func = func_and_positional[0]
positional = func_and_positional[1:]
spec = getfullargspec(func)
args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = spec
(args, varargs, varkw, defaults, posonlyargs,
kwonlyargs, kwonlydefaults, ann) = spec
f_name = func.__name__
arg2value = {}

Expand All @@ -1351,12 +1356,16 @@ def getcallargs(*func_and_positional, **named):
# implicit 'self' (or 'cls' for classmethods) argument
positional = (func.__self__,) + positional
num_pos = len(positional)
num_posonlyargs = len(posonlyargs)
num_args = len(args)
num_defaults = len(defaults) if defaults else 0

n = min(num_pos, num_posonlyargs)
for i in range(num_posonlyargs):
arg2value[posonlyargs[i]] = positional[i]
n = min(num_pos, num_args)
for i in range(n):
arg2value[args[i]] = positional[i]
arg2value[args[i]] = positional[num_posonlyargs+i]
if varargs:
arg2value[varargs] = tuple(positional[n:])
possible_kwargs = set(args + kwonlyargs)
Expand Down Expand Up @@ -2137,9 +2146,12 @@ def _signature_from_function(cls, func):
func_code = func.__code__
pos_count = func_code.co_argcount
arg_names = func_code.co_varnames
positional = tuple(arg_names[:pos_count])
posonly_count = func_code.co_posonlyargcount
positional_count = posonly_count + pos_count
positional_only = tuple(arg_names[:posonly_count])
positional = tuple(arg_names[posonly_count:positional_count])
keyword_only_count = func_code.co_kwonlyargcount
keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)]
keyword_only = arg_names[positional_count:(positional_count + keyword_only_count)]
annotations = func.__annotations__
defaults = func.__defaults__
kwdefaults = func.__kwdefaults__
Expand All @@ -2151,23 +2163,33 @@ def _signature_from_function(cls, func):

parameters = []

non_default_count = positional_count - pos_default_count
all_positional = positional_only + positional

posonly_left = posonly_count

# Non-keyword-only parameters w/o defaults.
non_default_count = pos_count - pos_default_count
for name in positional[:non_default_count]:
for name in all_positional[:non_default_count]:
kind = _POSITIONAL_ONLY if posonly_left else _POSITIONAL_OR_KEYWORD
annotation = annotations.get(name, _empty)
parameters.append(Parameter(name, annotation=annotation,
kind=_POSITIONAL_OR_KEYWORD))
kind=kind))
if posonly_left:
posonly_left -= 1

# ... w/ defaults.
for offset, name in enumerate(positional[non_default_count:]):
for offset, name in enumerate(all_positional[non_default_count:]):
kind = _POSITIONAL_ONLY if posonly_left else _POSITIONAL_OR_KEYWORD
annotation = annotations.get(name, _empty)
parameters.append(Parameter(name, annotation=annotation,
kind=_POSITIONAL_OR_KEYWORD,
kind=kind,
default=defaults[offset]))
if posonly_left:
posonly_left -= 1

# *args
if func_code.co_flags & CO_VARARGS:
name = arg_names[pos_count + keyword_only_count]
name = arg_names[positional_count + keyword_only_count]
annotation = annotations.get(name, _empty)
parameters.append(Parameter(name, annotation=annotation,
kind=_VAR_POSITIONAL))
Expand All @@ -2184,7 +2206,7 @@ def _signature_from_function(cls, func):
default=default))
# **kwargs
if func_code.co_flags & CO_VARKEYWORDS:
index = pos_count + keyword_only_count
index = positional_count + keyword_only_count
if func_code.co_flags & CO_VARARGS:
index += 1

Expand Down
5 changes: 3 additions & 2 deletions Lib/modulefinder.py
Original file line number Diff line number Diff line change
Expand Up @@ -619,8 +619,9 @@ def replace_paths_in_code(self, co):
if isinstance(consts[i], type(co)):
consts[i] = self.replace_paths_in_code(consts[i])

return types.CodeType(co.co_argcount, co.co_kwonlyargcount,
co.co_nlocals, co.co_stacksize, co.co_flags,
return types.CodeType(co.co_argcount, co.co_posonlyargcount,
co.co_kwonlyargcount, co.co_nlocals,
co.co_stacksize, co.co_flags,
co.co_code, tuple(consts), co.co_names,
co.co_varnames, new_filename, co.co_name,
co.co_firstlineno, co.co_lnotab, co.co_freevars,
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/inspect_fodder.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# line 5

# line 7
def spam(a, b, c, d=3, e=4, f=5, *g, **h):
def spam(a, /, b, c, d=3, e=4, f=5, *g, **h):
eggs(b + d, c + f)

# line 11
Expand Down
Loading