From bdb1c537fa74c5358a6a8d60dbc807a98ea91633 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 5 May 2022 10:29:13 +0300 Subject: [PATCH 1/5] gh-91162: Fix substitution of unpacked tuples in generic aliases --- Include/internal/pycore_global_strings.h | 3 +- Include/internal/pycore_runtime_init.h | 3 +- Lib/test/test_typing.py | 68 ++++++------- Lib/typing.py | 68 +++++++++---- Objects/genericaliasobject.c | 118 ++++++++++++++++++++--- 5 files changed, 193 insertions(+), 67 deletions(-) diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 28fffa95071a04..890f98d286a059 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -201,8 +201,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(__subclasshook__) STRUCT_FOR_ID(__truediv__) STRUCT_FOR_ID(__trunc__) + STRUCT_FOR_ID(__typing_is_unpacked_typevartuple__) STRUCT_FOR_ID(__typing_subst__) - STRUCT_FOR_ID(__typing_unpacked__) + STRUCT_FOR_ID(__typing_unpacked_tuple_args__) STRUCT_FOR_ID(__warningregistry__) STRUCT_FOR_ID(__weakref__) STRUCT_FOR_ID(__xor__) diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 941badfc8cb6a8..46817c48c7816c 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -824,8 +824,9 @@ extern "C" { INIT_ID(__subclasshook__), \ INIT_ID(__truediv__), \ INIT_ID(__trunc__), \ + INIT_ID(__typing_is_unpacked_typevartuple__), \ INIT_ID(__typing_subst__), \ - INIT_ID(__typing_unpacked__), \ + INIT_ID(__typing_unpacked_tuple_args__), \ INIT_ID(__warningregistry__), \ INIT_ID(__weakref__), \ INIT_ID(__xor__), \ diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 8399465f6052da..4240fbe97f6bbc 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -603,22 +603,16 @@ class C(Generic[T]): pass ('generic[T]', '[int]', 'generic[int]'), ('generic[T]', '[int, str]', 'TypeError'), ('generic[T]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'), + ('generic[T]', '[*tuple_type[int]]', 'generic[int]'), # Should raise TypeError: a) according to the tentative spec, # unpacked types cannot be used as arguments to aliases that expect # a fixed number of arguments; b) it's equivalent to generic[()]. - ('generic[T]', '[*tuple[()]]', 'generic[*tuple[()]]'), - ('generic[T]', '[*Tuple[()]]', 'TypeError'), + ('generic[T]', '[*tuple_type[()]]', 'TypeError'), + ('generic[T]', '[*tuple_type[int, str]]', 'TypeError'), # Should raise TypeError according to the tentative spec: unpacked # types cannot be used as arguments to aliases that expect a fixed # number of arguments. - ('generic[T]', '[*tuple[int]]', 'generic[*tuple[int]]'), - ('generic[T]', '[*Tuple[int]]', 'TypeError'), - # Ditto. - ('generic[T]', '[*tuple[int, str]]', 'generic[*tuple[int, str]]'), - ('generic[T]', '[*Tuple[int, str]]', 'TypeError'), - # Ditto. - ('generic[T]', '[*tuple[int, ...]]', 'generic[*tuple[int, ...]]'), - ('generic[T]', '[*Tuple[int, ...]]', 'TypeError'), + ('generic[T]', '[*tuple_type[int, ...]]', 'TypeError'), ('generic[T]', '[*Ts]', 'TypeError'), ('generic[T]', '[T, *Ts]', 'TypeError'), ('generic[T]', '[*Ts, T]', 'TypeError'), @@ -664,23 +658,29 @@ class C(Generic[T1, T2]): pass ('generic[T1, T2]', '[int, str]', 'generic[int, str]'), ('generic[T1, T2]', '[int, str, bool]', 'TypeError'), ('generic[T1, T2]', '[*tuple_type[int]]', 'TypeError'), - ('generic[T1, T2]', '[*tuple_type[int, str]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, str]]', 'generic[int, str]'), ('generic[T1, T2]', '[*tuple_type[int, str, bool]]', 'TypeError'), - # Should raise TypeError according to the tentative spec: unpacked - # types cannot be used as arguments to aliases that expect a fixed - # number of arguments. - ('generic[T1, T2]', '[*tuple[int, str], *tuple[float, bool]]', 'generic[*tuple[int, str], *tuple[float, bool]]'), - ('generic[T1, T2]', '[*Tuple[int, str], *Tuple[float, bool]]', 'TypeError'), + ('generic[T1, T2]', '[int, *tuple_type[str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int], str]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int], *tuple_type[str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int, str], *tuple_type[()]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[()], *tuple_type[int, str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int], *tuple_type[()]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[()], *tuple_type[int]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, str], *tuple_type[float]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int], *tuple_type[str, float]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, str], *tuple_type[float, bool]]', 'TypeError'), ('generic[T1, T2]', '[tuple_type[int, ...]]', 'TypeError'), ('generic[T1, T2]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'), + # Should raise TypeError according to the tentative spec: unpacked + # types cannot be used as arguments to aliases that expect a fixed + # number of arguments. ('generic[T1, T2]', '[*tuple_type[int, ...]]', 'TypeError'), - - # Ditto. - ('generic[T1, T2]', '[*tuple[int, ...], *tuple[str, ...]]', 'generic[*tuple[int, ...], *tuple[str, ...]]'), - ('generic[T1, T2]', '[*Tuple[int, ...], *Tuple[str, ...]]', 'TypeError'), - + ('generic[T1, T2]', '[int, *tuple_type[str, ...]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, ...], str]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'), ('generic[T1, T2]', '[*Ts]', 'TypeError'), ('generic[T1, T2]', '[T, *Ts]', 'TypeError'), ('generic[T1, T2]', '[*Ts, T]', 'TypeError'), @@ -720,7 +720,7 @@ class C(Generic[T1, T2, T3]): pass tests = [ # Alias # Args # Expected result ('generic[T1, bool, T2]', '[int, str]', 'generic[int, bool, str]'), - ('generic[T1, bool, T2]', '[*tuple_type[int, str]]', 'TypeError'), + ('generic[T1, bool, T2]', '[*tuple_type[int, str]]', 'generic[int, bool, str]'), ] for alias_template, args_template, expected_template in tests: @@ -765,17 +765,17 @@ class C(Generic[*Ts]): pass ('tuple[*Ts]', '[int, str]', 'tuple[int, str]'), ('Tuple[*Ts]', '[int, str]', 'Tuple[int, str]'), - ('C[*Ts]', '[*tuple_type[int]]', 'C[*tuple_type[int]]'), # Should be C[int] - ('tuple[*Ts]', '[*tuple_type[int]]', 'tuple[*tuple_type[int]]'), # Should be tuple[int] - ('Tuple[*Ts]', '[*tuple_type[int]]', 'Tuple[*tuple_type[int]]'), # Should be Tuple[int] + ('C[*Ts]', '[*tuple_type[int]]', 'C[int]'), + ('tuple[*Ts]', '[*tuple_type[int]]', 'tuple[int]'), + ('Tuple[*Ts]', '[*tuple_type[int]]', 'Tuple[int]'), - ('C[*Ts]', '[*tuple_type[*Ts]]', 'C[*tuple_type[*Ts]]'), # Should be C[*Ts] - ('tuple[*Ts]', '[*tuple_type[*Ts]]', 'tuple[*tuple_type[*Ts]]'), # Should be tuple[*Ts] - ('Tuple[*Ts]', '[*tuple_type[*Ts]]', 'Tuple[*tuple_type[*Ts]]'), # Should be Tuple[*Ts] + ('C[*Ts]', '[*tuple_type[*Ts]]', 'C[*Ts]'), + ('tuple[*Ts]', '[*tuple_type[*Ts]]', 'tuple[*Ts]'), + ('Tuple[*Ts]', '[*tuple_type[*Ts]]', 'Tuple[*Ts]'), - ('C[*Ts]', '[*tuple_type[int, str]]', 'C[*tuple_type[int, str]]'), # Should be C[int, str] - ('tuple[*Ts]', '[*tuple_type[int, str]]', 'tuple[*tuple_type[int, str]]'), # Should be tuple[int, str] - ('Tuple[*Ts]', '[*tuple_type[int, str]]', 'Tuple[*tuple_type[int, str]]'), # Should be Tuple[int, str] + ('C[*Ts]', '[*tuple_type[int, str]]', 'C[int, str]'), + ('tuple[*Ts]', '[*tuple_type[int, str]]', 'tuple[int, str]'), + ('Tuple[*Ts]', '[*tuple_type[int, str]]', 'Tuple[int, str]'), ('C[*Ts]', '[tuple_type[int, ...]]', 'C[tuple_type[int, ...]]'), ('tuple[*Ts]', '[tuple_type[int, ...]]', 'tuple[tuple_type[int, ...]]'), @@ -820,11 +820,11 @@ class C(Generic[*Ts]): pass ('tuple[T, *Ts]', '[int, str, bool]', 'tuple[int, str, bool]'), ('Tuple[T, *Ts]', '[int, str, bool]', 'Tuple[int, str, bool]'), - ('C[T, *Ts]', '[*tuple[int, ...]]', 'C[*tuple[int, ...]]'), # Should be C[int, *tuple[int, ...]] + ('C[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be C[int, *tuple[int, ...]] ('C[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Ditto - ('tuple[T, *Ts]', '[*tuple[int, ...]]', 'tuple[*tuple[int, ...]]'), # Should be tuple[int, *tuple[int, ...]] + ('tuple[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be tuple[int, *tuple[int, ...]] ('tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Should be tuple[int, *Tuple[int, ...]] - ('Tuple[T, *Ts]', '[*tuple[int, ...]]', 'Tuple[*tuple[int, ...]]'), # Should be Tuple[int, *tuple[int, ...]] + ('Tuple[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be Tuple[int, *tuple[int, ...]] ('Tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Should be Tuple[int, *Tuple[int, ...]] ('C[*Ts, T]', '[int]', 'C[int]'), diff --git a/Lib/typing.py b/Lib/typing.py index bdc14e39033dcf..13e3186b223804 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -271,6 +271,16 @@ def _check_generic(cls, parameters, elen): raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};" f" actual {alen}, expected {elen}") +def _unpack_args(args): + newargs = [] + for arg in args: + subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) + if subargs is not None and not (subargs and subargs[-1] is ...): + newargs.extend(subargs) + else: + newargs.append(arg) + return newargs + def _prepare_paramspec_params(cls, params): """Prepares the parameters for a Generic containing ParamSpec variables (internal helper). @@ -995,7 +1005,8 @@ def __init__(self, name, *constraints, bound=None, def __typing_subst__(self, arg): msg = "Parameters to generic types must be types." arg = _type_check(arg, msg, is_argument=True) - if (isinstance(arg, _GenericAlias) and arg.__origin__ is Unpack): + if ((isinstance(arg, _GenericAlias) and arg.__origin__ is Unpack) or + (isinstance(arg, GenericAlias) and getattr(arg, '__unpacked__', False))): raise TypeError(f"{arg} is not valid as type argument") return arg @@ -1354,19 +1365,17 @@ def __getitem__(self, args): if self.__origin__ in (Generic, Protocol): # Can't subscript Generic[...] or Protocol[...]. raise TypeError(f"Cannot subscript already-subscripted {self}") + if not self.__parameters__: + raise TypeError(f"{self} is not a generic class") # Preprocess `args`. if not isinstance(args, tuple): args = (args,) args = tuple(_type_convert(p) for p in args) + args = _unpack_args(args) if (self._paramspec_tvars and any(isinstance(t, ParamSpec) for t in self.__parameters__)): args = _prepare_paramspec_params(self, args) - elif not any(isinstance(p, TypeVarTuple) for p in self.__parameters__): - # We only run this if there are no TypeVarTuples, because we - # don't check variadic generic arity at runtime (to reduce - # complexity of typing.py). - _check_generic(self, args, len(self.__parameters__)) new_args = self._determine_new_args(args) r = self.copy_with(new_args) @@ -1390,16 +1399,28 @@ def _determine_new_args(self, args): params = self.__parameters__ # In the example above, this would be {T3: str} new_arg_by_param = {} + typevartuple_index = None for i, param in enumerate(params): if isinstance(param, TypeVarTuple): - j = len(args) - (len(params) - i - 1) - if j < i: - raise TypeError(f"Too few arguments for {self}") - new_arg_by_param.update(zip(params[:i], args[:i])) - new_arg_by_param[param] = args[i: j] - new_arg_by_param.update(zip(params[i + 1:], args[j:])) - break + if typevartuple_index is not None: + raise TypeError(f"More than one TypeVarTuple parameter in {self}") + typevartuple_index = i + + alen = len(args) + plen = len(params) + if typevartuple_index is not None: + i = typevartuple_index + j = alen - (plen - i - 1) + if j < i: + raise TypeError(f"Too few arguments for {self};" + f" actual {alen}, expected at least {plen-1}") + new_arg_by_param.update(zip(params[:i], args[:i])) + new_arg_by_param[params[i]] = tuple(args[i: j]) + new_arg_by_param.update(zip(params[i + 1:], args[j:])) else: + if alen != plen: + raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};" + f" actual {alen}, expected {plen}") new_arg_by_param.update(zip(params, args)) new_args = [] @@ -1707,14 +1728,25 @@ def __repr__(self): return '*' + repr(self.__args__[0]) def __getitem__(self, args): - if self.__typing_unpacked__(): + if self.__typing_is_unpacked_typevartuple__: return args return super().__getitem__(args) - def __typing_unpacked__(self): - # If x is Unpack[tuple[...]], __parameters__ will be empty. - return bool(self.__parameters__ and - isinstance(self.__parameters__[0], TypeVarTuple)) + @property + def __typing_unpacked_tuple_args__(self): + assert self.__origin__ is Unpack + assert len(self.__args__) == 1 + arg, = self.__args__ + if isinstance(arg, _GenericAlias): + assert arg.__origin__ is tuple + return arg.__args__ + return None + + @property + def __typing_is_unpacked_typevartuple__(self): + assert self.__origin__ is Unpack + assert len(self.__args__) == 1 + return isinstance(self.__args__[0], TypeVarTuple) class Generic: diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index c6ed1611bd29ec..c8a86c957a5953 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -320,20 +320,80 @@ subs_tvars(PyObject *obj, PyObject *params, static int _is_unpacked_typevartuple(PyObject *arg) { - PyObject *meth; - int res = _PyObject_LookupAttr(arg, &_Py_ID(__typing_unpacked__), &meth); + PyObject *tmp; + if (PyType_Check(arg)) { // TODO: Add test + return 0; + } + int res = _PyObject_LookupAttr(arg, &_Py_ID(__typing_is_unpacked_typevartuple__), &tmp); if (res > 0) { - PyObject *tmp = PyObject_CallNoArgs(meth); - Py_DECREF(meth); - if (tmp == NULL) { - return -1; - } res = PyObject_IsTrue(tmp); Py_DECREF(tmp); } return res; } +static PyObject * +_unpacked_tuple_args(PyObject *arg) +{ + PyObject *result; + assert(!PyType_Check(arg)); + // Fast path + if (_PyGenericAlias_Check(arg) && ((gaobject *)arg)->starred && ((gaobject *)arg)->origin == (PyObject *)&PyTuple_Type) { + result = ((gaobject *)arg)->args; + Py_INCREF(result); + return result; + } + if (_PyObject_LookupAttr(arg, &_Py_ID(__typing_unpacked_tuple_args__), &result) > 0) { + if (result == Py_None) { + Py_DECREF(result); + return NULL; + } + return result; + } + return NULL; +} + +static PyObject * +_unpack_args(PyObject *item) +{ + PyObject *newargs = PyList_New(0); + if (newargs == NULL) { + return NULL; + } + int is_tuple = PyTuple_Check(item); + Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1; + PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item; + for (Py_ssize_t i = 0; i < nitems; i++) { + item = argitems[i]; + if (!PyType_Check(item)) { + PyObject *subargs = _unpacked_tuple_args(item); + if (subargs != NULL && + PyTuple_Check(subargs) && + !(PyTuple_GET_SIZE(subargs) && + PyTuple_GET_ITEM(subargs, PyTuple_GET_SIZE(subargs)-1) == Py_Ellipsis)) + { + if (PyList_SetSlice(newargs, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, subargs) < 0) { + Py_DECREF(subargs); + Py_DECREF(newargs); + return NULL; + } + Py_DECREF(subargs); + continue; + } + if (PyErr_Occurred()) { + Py_DECREF(newargs); + return NULL; + } + } + if (PyList_Append(newargs, item) < 0) { + Py_DECREF(newargs); + return NULL; + } + } + Py_SETREF(newargs, PySequence_Tuple(newargs)); + return newargs; +} + PyObject * _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObject *item) { @@ -343,18 +403,26 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje "%R is not a generic class", self); } + item = _unpack_args(item); int is_tuple = PyTuple_Check(item); Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1; PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item; - Py_ssize_t varparam = 0; - for (; varparam < nparams; varparam++) { - PyObject *param = PyTuple_GET_ITEM(parameters, varparam); + Py_ssize_t varparam = nparams; + for (Py_ssize_t i = 0; i < nparams; i++) { + PyObject *param = PyTuple_GET_ITEM(parameters, i); if (Py_TYPE(param)->tp_iter) { // TypeVarTuple - break; + if (varparam < nparams) { + Py_DECREF(item); + return PyErr_Format(PyExc_TypeError, + "More than one TypeVarTuple parameter in %S", + self); + } + varparam = i; } } if (varparam < nparams) { if (nitems < nparams - 1) { + Py_DECREF(item); return PyErr_Format(PyExc_TypeError, "Too few arguments for %R", self); @@ -362,10 +430,11 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } else { if (nitems != nparams) { + Py_DECREF(item); return PyErr_Format(PyExc_TypeError, - "Too %s arguments for %R", + "Too %s arguments for %R; actual %zd, expected %zd", nitems > nparams ? "many" : "few", - self); + self, nitems, nparams); } } /* Replace all type variables (specified by parameters) @@ -377,6 +446,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje Py_ssize_t nargs = PyTuple_GET_SIZE(args); PyObject *newargs = PyTuple_New(nargs); if (newargs == NULL) { + Py_DECREF(item); return NULL; } for (Py_ssize_t iarg = 0, jarg = 0; iarg < nargs; iarg++) { @@ -384,11 +454,13 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje int unpack = _is_unpacked_typevartuple(arg); if (unpack < 0) { Py_DECREF(newargs); + Py_DECREF(item); return NULL; } PyObject *subst; if (_PyObject_LookupAttr(arg, &_Py_ID(__typing_subst__), &subst) < 0) { Py_DECREF(newargs); + Py_DECREF(item); return NULL; } if (subst) { @@ -397,6 +469,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje if (iparam == varparam) { Py_DECREF(subst); Py_DECREF(newargs); + Py_DECREF(item); PyErr_SetString(PyExc_TypeError, "Substitution of bare TypeVarTuple is not supported"); return NULL; @@ -412,6 +485,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } if (arg == NULL) { Py_DECREF(newargs); + Py_DECREF(item); return NULL; } if (unpack) { @@ -419,6 +493,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje &PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg)); Py_DECREF(arg); if (jarg < 0) { + Py_DECREF(item); return NULL; } } @@ -428,6 +503,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } } + Py_DECREF(item); return newargs; } @@ -518,6 +594,7 @@ static const char* const attr_exceptions[] = { "__args__", "__unpacked__", "__parameters__", + "__typing_unpacked_tuple_args__", "__mro_entries__", "__reduce_ex__", // needed so we don't look up object.__reduce_ex__ "__reduce__", @@ -567,6 +644,9 @@ ga_richcompare(PyObject *a, PyObject *b, int op) gaobject *aa = (gaobject *)a; gaobject *bb = (gaobject *)b; + if (aa->starred != bb->starred) { + Py_RETURN_FALSE; + } int eq = PyObject_RichCompareBool(aa->origin, bb->origin, Py_EQ); if (eq < 0) { return NULL; @@ -676,8 +756,20 @@ ga_parameters(PyObject *self, void *unused) return alias->parameters; } +static PyObject * +ga_unpacked_tuple_args(PyObject *self, void *unused) +{ + gaobject *alias = (gaobject *)self; + if (alias->starred && alias->origin == (PyObject *)&PyTuple_Type) { + Py_INCREF(alias->args); + return alias->args; + } + Py_RETURN_NONE; +} + static PyGetSetDef ga_properties[] = { {"__parameters__", ga_parameters, (setter)NULL, "Type variables in the GenericAlias.", NULL}, + {"__typing_unpacked_tuple_args__", ga_unpacked_tuple_args, (setter)NULL, NULL}, {0} }; From 1397f0a27a49bc4c087c2f559a14a98ca52a890b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 6 May 2022 09:37:17 +0300 Subject: [PATCH 2/5] Fix more old bugs. --- Lib/test/test_typing.py | 5 +++++ Lib/typing.py | 8 ++------ Objects/genericaliasobject.c | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 4240fbe97f6bbc..b963d5b4ec67e4 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -843,6 +843,11 @@ class C(Generic[*Ts]): pass ('generic[T1, T2, *tuple_type[int, ...]]', '[str, bool]', 'generic[str, bool, *tuple_type[int, ...]]'), ('generic[T1, *tuple_type[int, ...], T2]', '[str, bool]', 'generic[str, *tuple_type[int, ...], bool]'), ('generic[T1, *tuple_type[int, ...], T2]', '[str, bool, float]', 'TypeError'), + + ('generic[T1, *tuple_type[T2, ...]]', '[int, str]', 'generic[int, *tuple_type[str, ...]]'), + ('generic[*tuple_type[T1, ...], T2]', '[int, str]', 'generic[*tuple_type[int, ...], str]'), + ('generic[T1, *tuple_type[generic[*Ts], ...]]', '[int, str, bool]', 'generic[int, *tuple_type[generic[str, bool], ...]]'), + ('generic[*tuple_type[generic[*Ts], ...], T1]', '[int, str, bool]', 'generic[*tuple_type[generic[int, str], ...], bool]'), ] for alias_template, args_template, expected_template in tests: diff --git a/Lib/typing.py b/Lib/typing.py index 13e3186b223804..00014dc658d76d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -887,12 +887,8 @@ def __repr__(self): def _is_unpacked_typevartuple(x: Any) -> bool: - return ( - isinstance(x, _UnpackGenericAlias) - # If x is Unpack[tuple[...]], __parameters__ will be empty. - and x.__parameters__ - and isinstance(x.__parameters__[0], TypeVarTuple) - ) + return ((not isinstance(x, type)) and + getattr(x, '__typing_is_unpacked_typevartuple__', False)) def _is_typevar_like(x: Any) -> bool: diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 0ee716336735db..8f873e3565bee6 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -530,6 +530,7 @@ ga_getitem(PyObject *self, PyObject *item) } PyObject *res = Py_GenericAlias(alias->origin, newargs); + ((gaobject *)res)->starred = alias->starred; Py_DECREF(newargs); return res; From 0067dc8abf40051f7eaf33b5e7d50191995da758 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 6 May 2022 09:53:43 +0300 Subject: [PATCH 3/5] Improve formatting --- Objects/genericaliasobject.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 8f873e3565bee6..b21cd5c44be343 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -338,11 +338,15 @@ _unpacked_tuple_args(PyObject *arg) PyObject *result; assert(!PyType_Check(arg)); // Fast path - if (_PyGenericAlias_Check(arg) && ((gaobject *)arg)->starred && ((gaobject *)arg)->origin == (PyObject *)&PyTuple_Type) { + if (_PyGenericAlias_Check(arg) && + ((gaobject *)arg)->starred && + ((gaobject *)arg)->origin == (PyObject *)&PyTuple_Type) + { result = ((gaobject *)arg)->args; Py_INCREF(result); return result; } + if (_PyObject_LookupAttr(arg, &_Py_ID(__typing_unpacked_tuple_args__), &result) > 0) { if (result == Py_None) { Py_DECREF(result); @@ -368,9 +372,9 @@ _unpack_args(PyObject *item) if (!PyType_Check(item)) { PyObject *subargs = _unpacked_tuple_args(item); if (subargs != NULL && - PyTuple_Check(subargs) && - !(PyTuple_GET_SIZE(subargs) && - PyTuple_GET_ITEM(subargs, PyTuple_GET_SIZE(subargs)-1) == Py_Ellipsis)) + PyTuple_Check(subargs) && + !(PyTuple_GET_SIZE(subargs) && + PyTuple_GET_ITEM(subargs, PyTuple_GET_SIZE(subargs)-1) == Py_Ellipsis)) { if (PyList_SetSlice(newargs, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, subargs) < 0) { Py_DECREF(subargs); From f0ba8cde437c2218d96068b03368d14ee7e0574f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 6 May 2022 11:48:10 +0300 Subject: [PATCH 4/5] Fix a leak. --- Objects/genericaliasobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index b21cd5c44be343..39fd70999ecbe5 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -384,6 +384,7 @@ _unpack_args(PyObject *item) Py_DECREF(subargs); continue; } + Py_XDECREF(subargs); if (PyErr_Occurred()) { Py_DECREF(newargs); return NULL; From 8e4e43d74a566521f63bfcb1ad69c1f945025908 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 7 May 2022 18:46:21 +0300 Subject: [PATCH 5/5] Simplify tests. --- Lib/test/test_typing.py | 110 +++++++++------------------------------- 1 file changed, 23 insertions(+), 87 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b963d5b4ec67e4..87d758fd61147d 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -604,14 +604,8 @@ class C(Generic[T]): pass ('generic[T]', '[int, str]', 'TypeError'), ('generic[T]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'), ('generic[T]', '[*tuple_type[int]]', 'generic[int]'), - # Should raise TypeError: a) according to the tentative spec, - # unpacked types cannot be used as arguments to aliases that expect - # a fixed number of arguments; b) it's equivalent to generic[()]. ('generic[T]', '[*tuple_type[()]]', 'TypeError'), ('generic[T]', '[*tuple_type[int, str]]', 'TypeError'), - # Should raise TypeError according to the tentative spec: unpacked - # types cannot be used as arguments to aliases that expect a fixed - # number of arguments. ('generic[T]', '[*tuple_type[int, ...]]', 'TypeError'), ('generic[T]', '[*Ts]', 'TypeError'), ('generic[T]', '[T, *Ts]', 'TypeError'), @@ -753,91 +747,33 @@ class C(Generic[*Ts]): pass # Tuple because tuple currently behaves differently. tests = [ # Alias # Args # Expected result - ('C[*Ts]', '[()]', 'C[()]'), - ('tuple[*Ts]', '[()]', 'tuple[()]'), - ('Tuple[*Ts]', '[()]', 'Tuple[()]'), - - ('C[*Ts]', '[int]', 'C[int]'), - ('tuple[*Ts]', '[int]', 'tuple[int]'), - ('Tuple[*Ts]', '[int]', 'Tuple[int]'), - - ('C[*Ts]', '[int, str]', 'C[int, str]'), - ('tuple[*Ts]', '[int, str]', 'tuple[int, str]'), - ('Tuple[*Ts]', '[int, str]', 'Tuple[int, str]'), - - ('C[*Ts]', '[*tuple_type[int]]', 'C[int]'), - ('tuple[*Ts]', '[*tuple_type[int]]', 'tuple[int]'), - ('Tuple[*Ts]', '[*tuple_type[int]]', 'Tuple[int]'), - - ('C[*Ts]', '[*tuple_type[*Ts]]', 'C[*Ts]'), - ('tuple[*Ts]', '[*tuple_type[*Ts]]', 'tuple[*Ts]'), - ('Tuple[*Ts]', '[*tuple_type[*Ts]]', 'Tuple[*Ts]'), - - ('C[*Ts]', '[*tuple_type[int, str]]', 'C[int, str]'), - ('tuple[*Ts]', '[*tuple_type[int, str]]', 'tuple[int, str]'), - ('Tuple[*Ts]', '[*tuple_type[int, str]]', 'Tuple[int, str]'), - - ('C[*Ts]', '[tuple_type[int, ...]]', 'C[tuple_type[int, ...]]'), - ('tuple[*Ts]', '[tuple_type[int, ...]]', 'tuple[tuple_type[int, ...]]'), - ('Tuple[*Ts]', '[tuple_type[int, ...]]', 'Tuple[tuple_type[int, ...]]'), - - ('C[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'C[tuple_type[int, ...], tuple_type[str, ...]]'), - ('tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'tuple[tuple_type[int, ...], tuple_type[str, ...]]'), - ('Tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'Tuple[tuple_type[int, ...], tuple_type[str, ...]]'), - - ('C[*Ts]', '[*tuple_type[int, ...]]', 'C[*tuple_type[int, ...]]'), - ('tuple[*Ts]', '[*tuple_type[int, ...]]', 'tuple[*tuple_type[int, ...]]'), - ('Tuple[*Ts]', '[*tuple_type[int, ...]]', 'Tuple[*tuple_type[int, ...]]'), + ('generic[*Ts]', '[()]', 'generic[()]'), + ('generic[*Ts]', '[int]', 'generic[int]'), + ('generic[*Ts]', '[int, str]', 'generic[int, str]'), + ('generic[*Ts]', '[*tuple_type[int]]', 'generic[int]'), + ('generic[*Ts]', '[*tuple_type[*Ts]]', 'generic[*Ts]'), + ('generic[*Ts]', '[*tuple_type[int, str]]', 'generic[int, str]'), + ('generic[*Ts]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'), + ('generic[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'), + ('generic[*Ts]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...]]'), # Technically, multiple unpackings are forbidden by PEP 646, but we # choose to be less restrictive at runtime, to allow folks room # to experiment. So all three of these should be valid. - ('C[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'C[*tuple_type[int, ...], *tuple_type[str, ...]]'), - ('tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'tuple[*tuple_type[int, ...], *tuple_type[str, ...]]'), - ('Tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'Tuple[*tuple_type[int, ...], *tuple_type[str, ...]]'), - - ('C[*Ts]', '[*Ts]', 'C[*Ts]'), - ('tuple[*Ts]', '[*Ts]', 'tuple[*Ts]'), - ('Tuple[*Ts]', '[*Ts]', 'Tuple[*Ts]'), - - ('C[*Ts]', '[T, *Ts]', 'C[T, *Ts]'), - ('tuple[*Ts]', '[T, *Ts]', 'tuple[T, *Ts]'), - ('Tuple[*Ts]', '[T, *Ts]', 'Tuple[T, *Ts]'), - - ('C[*Ts]', '[*Ts, T]', 'C[*Ts, T]'), - ('tuple[*Ts]', '[*Ts, T]', 'tuple[*Ts, T]'), - ('Tuple[*Ts]', '[*Ts, T]', 'Tuple[*Ts, T]'), - - ('C[T, *Ts]', '[int]', 'C[int]'), - ('tuple[T, *Ts]', '[int]', 'tuple[int]'), - ('Tuple[T, *Ts]', '[int]', 'Tuple[int]'), - - ('C[T, *Ts]', '[int, str]', 'C[int, str]'), - ('tuple[T, *Ts]', '[int, str]', 'tuple[int, str]'), - ('Tuple[T, *Ts]', '[int, str]', 'Tuple[int, str]'), - - ('C[T, *Ts]', '[int, str, bool]', 'C[int, str, bool]'), - ('tuple[T, *Ts]', '[int, str, bool]', 'tuple[int, str, bool]'), - ('Tuple[T, *Ts]', '[int, str, bool]', 'Tuple[int, str, bool]'), - - ('C[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be C[int, *tuple[int, ...]] - ('C[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Ditto - ('tuple[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be tuple[int, *tuple[int, ...]] - ('tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Should be tuple[int, *Tuple[int, ...]] - ('Tuple[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be Tuple[int, *tuple[int, ...]] - ('Tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Should be Tuple[int, *Tuple[int, ...]] - - ('C[*Ts, T]', '[int]', 'C[int]'), - ('tuple[*Ts, T]', '[int]', 'tuple[int]'), - ('Tuple[*Ts, T]', '[int]', 'Tuple[int]'), - - ('C[*Ts, T]', '[int, str]', 'C[int, str]'), - ('tuple[*Ts, T]', '[int, str]', 'tuple[int, str]'), - ('Tuple[*Ts, T]', '[int, str]', 'Tuple[int, str]'), - - ('C[*Ts, T]', '[int, str, bool]', 'C[int, str, bool]'), - ('tuple[*Ts, T]', '[int, str, bool]', 'tuple[int, str, bool]'), - ('Tuple[*Ts, T]', '[int, str, bool]', 'Tuple[int, str, bool]'), + ('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'generic[*tuple_type[int, ...], *tuple_type[str, ...]]'), + + ('generic[*Ts]', '[*Ts]', 'generic[*Ts]'), + ('generic[*Ts]', '[T, *Ts]', 'generic[T, *Ts]'), + ('generic[*Ts]', '[*Ts, T]', 'generic[*Ts, T]'), + ('generic[T, *Ts]', '[int]', 'generic[int]'), + ('generic[T, *Ts]', '[int, str]', 'generic[int, str]'), + ('generic[T, *Ts]', '[int, str, bool]', 'generic[int, str, bool]'), + + ('generic[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be generic[int, *tuple[int, ...]] + + ('generic[*Ts, T]', '[int]', 'generic[int]'), + ('generic[*Ts, T]', '[int, str]', 'generic[int, str]'), + ('generic[*Ts, T]', '[int, str, bool]', 'generic[int, str, bool]'), ('generic[T, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'), ('generic[T1, T2, *tuple_type[int, ...]]', '[str, bool]', 'generic[str, bool, *tuple_type[int, ...]]'),