Skip to content

Commit cef452d

Browse files
authored
Fail gracefully on diverging recursive type aliases (#13352)
This is another follow up on #13297. We can't support aliases like `Nested = Union[T, Nested[List[T]]]` (and it looks like no-one can, without hacks like fixed type recursion limit). I would propose to just ban them for now. We can reconsider if people will ask for this.
1 parent cb50e63 commit cef452d

File tree

6 files changed

+131
-16
lines changed

6 files changed

+131
-16
lines changed

mypy/nodes.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1238,6 +1238,7 @@ class AssignmentStmt(Statement):
12381238
"new_syntax",
12391239
"is_alias_def",
12401240
"is_final_def",
1241+
"invalid_recursive_alias",
12411242
)
12421243

12431244
lvalues: List[Lvalue]
@@ -1258,6 +1259,9 @@ class AssignmentStmt(Statement):
12581259
# a final declaration overrides another final declaration (this is checked
12591260
# during type checking when MROs are known).
12601261
is_final_def: bool
1262+
# Stop further processing of this assignment, to prevent flipping back and forth
1263+
# during semantic analysis passes.
1264+
invalid_recursive_alias: bool
12611265

12621266
def __init__(
12631267
self,
@@ -1274,6 +1278,7 @@ def __init__(
12741278
self.new_syntax = new_syntax
12751279
self.is_alias_def = False
12761280
self.is_final_def = False
1281+
self.invalid_recursive_alias = False
12771282

12781283
def accept(self, visitor: StatementVisitor[T]) -> T:
12791284
return visitor.visit_assignment_stmt(self)

mypy/semanal.py

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,7 @@
232232
TypeVarLikeQuery,
233233
analyze_type_alias,
234234
check_for_explicit_any,
235+
detect_diverging_alias,
235236
fix_instance_types,
236237
has_any_from_unimported_type,
237238
no_subscript_builtin_alias,
@@ -263,11 +264,11 @@
263264
PlaceholderType,
264265
ProperType,
265266
StarType,
267+
TrivialSyntheticTypeTranslator,
266268
TupleType,
267269
Type,
268270
TypeAliasType,
269271
TypeOfAny,
270-
TypeTranslator,
271272
TypeType,
272273
TypeVarLikeType,
273274
TypeVarType,
@@ -3014,6 +3015,8 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool:
30143015
Note: the resulting types for subscripted (including generic) aliases
30153016
are also stored in rvalue.analyzed.
30163017
"""
3018+
if s.invalid_recursive_alias:
3019+
return True
30173020
lvalue = s.lvalues[0]
30183021
if len(s.lvalues) > 1 or not isinstance(lvalue, NameExpr):
30193022
# First rule: Only simple assignments like Alias = ... create aliases.
@@ -3107,8 +3110,7 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool:
31073110
check_for_explicit_any(res, self.options, self.is_typeshed_stub_file, self.msg, context=s)
31083111
# When this type alias gets "inlined", the Any is not explicit anymore,
31093112
# so we need to replace it with non-explicit Anys.
3110-
if not has_placeholder(res):
3111-
res = make_any_non_explicit(res)
3113+
res = make_any_non_explicit(res)
31123114
# Note: with the new (lazy) type alias representation we only need to set no_args to True
31133115
# if the expected number of arguments is non-zero, so that aliases like A = List work.
31143116
# However, eagerly expanding aliases like Text = str is a nice performance optimization.
@@ -3127,8 +3129,6 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool:
31273129
no_args=no_args,
31283130
eager=eager,
31293131
)
3130-
if invalid_recursive_alias({alias_node}, alias_node.target):
3131-
self.fail("Invalid recursive alias: a union item of itself", rvalue)
31323132
if isinstance(s.rvalue, (IndexExpr, CallExpr)): # CallExpr is for `void = type(None)`
31333133
s.rvalue.analyzed = TypeAliasExpr(alias_node)
31343134
s.rvalue.analyzed.line = s.line
@@ -3164,8 +3164,28 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool:
31643164
self.add_symbol(lvalue.name, alias_node, s)
31653165
if isinstance(rvalue, RefExpr) and isinstance(rvalue.node, TypeAlias):
31663166
alias_node.normalized = rvalue.node.normalized
3167+
current_node = existing.node if existing else alias_node
3168+
assert isinstance(current_node, TypeAlias)
3169+
self.disable_invalid_recursive_aliases(s, current_node)
31673170
return True
31683171

3172+
def disable_invalid_recursive_aliases(
3173+
self, s: AssignmentStmt, current_node: TypeAlias
3174+
) -> None:
3175+
"""Prohibit and fix recursive type aliases that are invalid/unsupported."""
3176+
messages = []
3177+
if invalid_recursive_alias({current_node}, current_node.target):
3178+
messages.append("Invalid recursive alias: a union item of itself")
3179+
if detect_diverging_alias(
3180+
current_node, current_node.target, self.lookup_qualified, self.tvar_scope
3181+
):
3182+
messages.append("Invalid recursive alias: type variable nesting on right hand side")
3183+
if messages:
3184+
current_node.target = AnyType(TypeOfAny.from_error)
3185+
s.invalid_recursive_alias = True
3186+
for msg in messages:
3187+
self.fail(msg, s.rvalue)
3188+
31693189
def analyze_lvalue(
31703190
self,
31713191
lval: Lvalue,
@@ -6056,7 +6076,7 @@ def make_any_non_explicit(t: Type) -> Type:
60566076
return t.accept(MakeAnyNonExplicit())
60576077

60586078

6059-
class MakeAnyNonExplicit(TypeTranslator):
6079+
class MakeAnyNonExplicit(TrivialSyntheticTypeTranslator):
60606080
def visit_any(self, t: AnyType) -> Type:
60616081
if t.type_of_any == TypeOfAny.explicit:
60626082
return t.copy_modified(TypeOfAny.special_form)

mypy/semanal_typeargs.py

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
UnpackType,
3131
get_proper_type,
3232
get_proper_types,
33-
invalid_recursive_alias,
3433
)
3534

3635

@@ -73,12 +72,11 @@ def visit_type_alias_type(self, t: TypeAliasType) -> None:
7372
# types, since errors there have already been reported.
7473
return
7574
self.seen_aliases.add(t)
76-
assert t.alias is not None, f"Unfixed type alias {t.type_ref}"
77-
if invalid_recursive_alias({t.alias}, t.alias.target):
78-
# Fix type arguments for invalid aliases (error is already reported).
79-
t.args = []
80-
t.alias.target = AnyType(TypeOfAny.from_error)
81-
return
75+
# Some recursive aliases may produce spurious args. In principle this is not very
76+
# important, as we would simply ignore them when expanding, but it is better to keep
77+
# correct aliases.
78+
if t.alias and len(t.args) != len(t.alias.alias_tvars):
79+
t.args = [AnyType(TypeOfAny.from_error) for _ in t.alias.alias_tvars]
8280
get_proper_type(t).accept(self)
8381

8482
def visit_instance(self, t: Instance) -> None:

mypy/type_visitor.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,10 @@ def __init__(self, strategy: Callable[[Iterable[T]], T]) -> None:
317317
# Keep track of the type aliases already visited. This is needed to avoid
318318
# infinite recursion on types like A = Union[int, List[A]].
319319
self.seen_aliases: Set[TypeAliasType] = set()
320+
# By default, we eagerly expand type aliases, and query also types in the
321+
# alias target. In most cases this is a desired behavior, but we may want
322+
# to skip targets in some cases (e.g. when collecting type variables).
323+
self.skip_alias_target = False
320324

321325
def visit_unbound_type(self, t: UnboundType) -> T:
322326
return self.query_types(t.args)
@@ -398,6 +402,8 @@ def visit_placeholder_type(self, t: PlaceholderType) -> T:
398402
return self.query_types(t.args)
399403

400404
def visit_type_alias_type(self, t: TypeAliasType) -> T:
405+
if self.skip_alias_target:
406+
return self.query_types(t.args)
401407
return get_proper_type(t).accept(self)
402408

403409
def query_types(self, types: Iterable[Type]) -> T:

mypy/typeanal.py

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
RequiredType,
6666
StarType,
6767
SyntheticTypeVisitor,
68+
TrivialSyntheticTypeTranslator,
6869
TupleType,
6970
Type,
7071
TypeAliasType,
@@ -1611,6 +1612,10 @@ def __init__(
16111612
self.scope = scope
16121613
self.include_bound_tvars = include_bound_tvars
16131614
super().__init__(flatten_tvars)
1615+
# Only include type variables in type aliases args. This would be anyway
1616+
# that case if we expand (as target variables would be overridden with args)
1617+
# and it may cause infinite recursion on invalid (diverging) recursive aliases.
1618+
self.skip_alias_target = True
16141619

16151620
def _seems_like_callable(self, type: UnboundType) -> bool:
16161621
if not type.args:
@@ -1656,6 +1661,75 @@ def visit_callable_type(self, t: CallableType) -> TypeVarLikeList:
16561661
return []
16571662

16581663

1664+
class DivergingAliasDetector(TrivialSyntheticTypeTranslator):
1665+
"""See docstring of detect_diverging_alias() for details."""
1666+
1667+
# TODO: this doesn't really need to be a translator, but we don't have a trivial visitor.
1668+
def __init__(
1669+
self,
1670+
seen_nodes: Set[TypeAlias],
1671+
lookup: Callable[[str, Context], Optional[SymbolTableNode]],
1672+
scope: "TypeVarLikeScope",
1673+
) -> None:
1674+
self.seen_nodes = seen_nodes
1675+
self.lookup = lookup
1676+
self.scope = scope
1677+
self.diverging = False
1678+
1679+
def is_alias_tvar(self, t: Type) -> bool:
1680+
# Generic type aliases use unbound type variables.
1681+
if not isinstance(t, UnboundType) or t.args:
1682+
return False
1683+
node = self.lookup(t.name, t)
1684+
if (
1685+
node
1686+
and isinstance(node.node, TypeVarLikeExpr)
1687+
and self.scope.get_binding(node) is None
1688+
):
1689+
return True
1690+
return False
1691+
1692+
def visit_type_alias_type(self, t: TypeAliasType) -> Type:
1693+
assert t.alias is not None, f"Unfixed type alias {t.type_ref}"
1694+
if t.alias in self.seen_nodes:
1695+
for arg in t.args:
1696+
if not self.is_alias_tvar(arg) and bool(
1697+
arg.accept(TypeVarLikeQuery(self.lookup, self.scope))
1698+
):
1699+
self.diverging = True
1700+
return t
1701+
# All clear for this expansion chain.
1702+
return t
1703+
new_nodes = self.seen_nodes | {t.alias}
1704+
visitor = DivergingAliasDetector(new_nodes, self.lookup, self.scope)
1705+
_ = get_proper_type(t).accept(visitor)
1706+
if visitor.diverging:
1707+
self.diverging = True
1708+
return t
1709+
1710+
1711+
def detect_diverging_alias(
1712+
node: TypeAlias,
1713+
target: Type,
1714+
lookup: Callable[[str, Context], Optional[SymbolTableNode]],
1715+
scope: "TypeVarLikeScope",
1716+
) -> bool:
1717+
"""This detects type aliases that will diverge during type checking.
1718+
1719+
For example F = Something[..., F[List[T]]]. At each expansion step this will produce
1720+
*new* type aliases: e.g. F[List[int]], F[List[List[int]]], etc. So we can't detect
1721+
recursion. It is a known problem in the literature, recursive aliases and generic types
1722+
don't always go well together. It looks like there is no known systematic solution yet.
1723+
1724+
# TODO: should we handle such aliases using type_recursion counter and some large limit?
1725+
They may be handy in rare cases, e.g. to express a union of non-mixed nested lists:
1726+
Nested = Union[T, Nested[List[T]]] ~> Union[T, List[T], List[List[T]], ...]
1727+
"""
1728+
visitor = DivergingAliasDetector({node}, lookup, scope)
1729+
_ = target.accept(visitor)
1730+
return visitor.diverging
1731+
1732+
16591733
def check_for_explicit_any(
16601734
typ: Optional[Type],
16611735
options: Options,

test-data/unit/check-recursive-types.test

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -405,13 +405,19 @@ NR = List[int]
405405
NR2 = Union[NR, NR]
406406
NR3 = Union[NR, Union[NR2, NR2]]
407407

408+
T = TypeVar("T")
409+
NRG = Union[int, T]
410+
NR4 = NRG[str]
411+
NR5 = Union[NRG[int], NR4]
412+
408413
A = Union[B, int] # E: Invalid recursive alias: a union item of itself
409-
B = Union[int, A] # E: Invalid recursive alias: a union item of itself
414+
B = Union[int, A] # Error reported above
410415
def f() -> A: ...
411-
reveal_type(f()) # N: Revealed type is "Union[Any, builtins.int]"
416+
reveal_type(f()) # N: Revealed type is "Any"
412417

413-
T = TypeVar("T")
414418
G = Union[T, G[T]] # E: Invalid recursive alias: a union item of itself
419+
GL = Union[T, GL[List[T]]] # E: Invalid recursive alias: a union item of itself \
420+
# E: Invalid recursive alias: type variable nesting on right hand side
415421
def g() -> G[int]: ...
416422
reveal_type(g()) # N: Revealed type is "Any"
417423

@@ -425,4 +431,10 @@ S = Type[S] # E: Type[...] cannot contain another Type[...]
425431
U = Type[Union[int, U]] # E: Type[...] cannot contain another Type[...]
426432
x: U
427433
reveal_type(x) # N: Revealed type is "Type[Any]"
434+
435+
D = List[F[List[T]]] # E: Invalid recursive alias: type variable nesting on right hand side
436+
F = D[T] # Error reported above
437+
E = List[E[E[T]]] # E: Invalid recursive alias: type variable nesting on right hand side
438+
d: D
439+
reveal_type(d) # N: Revealed type is "Any"
428440
[builtins fixtures/isinstancelist.pyi]

0 commit comments

Comments
 (0)