subtypes: fast path for Union/Union subtype check (#14277)

Enums are exploded into Union of Literal when narrowed.

Conditional branches on enum values can result in multiple distinct
narrowing of the same enum which are later subject to subtype checks 
(most notably via `is_same_type`, when exiting frame context in the binder). 
Such checks would have quadratic complexity: `O(N*M)` where `N` and `M` 
are the number of entries in each narrowed enum variable, and led to drastic 
slowdown if any of the enums involved has a large number of values.

Implement a linear-time fast path where literals are quickly filtered, with
a fallback to the slow path for more complex values.

In our codebase there is one method with a chain of a dozen `if`
statements operating on instances of an enum with a hundreds of values. 
Prior to the regression it was typechecked in less than 1s. After the regression 
it takes over 13min to typecheck. This patch fully fixes the regression for us.

Fixes #13821.

Author: hugues-aff

mypy/subtypes.py

@@ -57,6 +57,7 @@
     UninhabitedType,
     UnionType,
     UnpackType,
+    _flattened,
     get_proper_type,
     is_named_instance,
 )
@@ -891,6 +892,35 @@ def visit_union_type(self, left: UnionType) -> bool:
                 if not self._is_subtype(item, self.orig_right):
                     return False
             return True
+
+        elif isinstance(self.right, UnionType):
+            # prune literals early to avoid nasty quadratic behavior which would otherwise arise when checking
+            # subtype relationships between slightly different narrowings of an Enum
+            # we achieve O(N+M) instead of O(N*M)
+
+            fast_check: set[ProperType] = set()
+
+            for item in _flattened(self.right.relevant_items()):
+                p_item = get_proper_type(item)
+                if isinstance(p_item, LiteralType):
+                    fast_check.add(p_item)
+                elif isinstance(p_item, Instance):
+                    if p_item.last_known_value is None:
+                        fast_check.add(p_item)
+                    else:
+                        fast_check.add(p_item.last_known_value)
+
+            for item in left.relevant_items():
+                p_item = get_proper_type(item)
+                if p_item in fast_check:
+                    continue
+                lit_type = mypy.typeops.simple_literal_type(p_item)
+                if lit_type in fast_check:
+                    continue
+                if not self._is_subtype(item, self.orig_right):
+                    return False
+            return True
+
         return all(self._is_subtype(item, self.orig_right) for item in left.items)
 
     def visit_partial_type(self, left: PartialType) -> bool:

mypy/types.py

@@ -3346,6 +3346,15 @@ def has_recursive_types(typ: Type) -> bool:
     return typ.accept(_has_recursive_type)
 
 
+def _flattened(types: Iterable[Type]) -> Iterable[Type]:
+    for t in types:
+        tp = get_proper_type(t)
+        if isinstance(tp, UnionType):
+            yield from _flattened(tp.items)
+        else:
+            yield t
+
+
 def flatten_nested_unions(
     types: Iterable[Type], handle_type_alias_type: bool = True
 ) -> list[Type]: