From 178ad41c5a915bc5157e9d4c87af310b65991262 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 14 Sep 2017 13:20:35 +0200 Subject: [PATCH 01/17] Clean-up most of checker; started checkexpr --- mypy/checker.py | 80 ++++++++++++++++++++++++--------------------- mypy/checkexpr.py | 20 ++++++------ mypy_self_check.ini | 6 ---- 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index e9ca186bd566..383e76c4c71e 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -205,7 +205,7 @@ def check_first_pass(self) -> None: self.fail(messages.ALL_MUST_BE_SEQ_STR.format(str_seq_s, all_s), all_node) - def check_second_pass(self, todo: List[DeferredNode] = None) -> bool: + def check_second_pass(self, todo: Optional[List[DeferredNode]] = None) -> bool: """Run second or following pass of type checking. This goes through deferred nodes, returning True if there were any. @@ -279,8 +279,8 @@ def accept(self, stmt: Statement) -> None: except Exception as err: report_internal_error(err, self.errors.file, stmt.line, self.errors, self.options) - def accept_loop(self, body: Statement, else_body: Statement = None, *, - exit_condition: Expression = None) -> None: + def accept_loop(self, body: Statement, else_body: Optional[Statement] = None, *, + exit_condition: Optional[Expression] = None) -> None: """Repeatedly type check a loop body until the frame doesn't change. If exit_condition is set, assume it must be False on exit from the loop. @@ -562,15 +562,15 @@ def visit_func_def(self, defn: FuncDef) -> None: 'original type') def check_func_item(self, defn: FuncItem, - type_override: CallableType = None, - name: str = None) -> None: + type_override: Optional[CallableType] = None, + name: Optional[str] = None) -> None: """Type check a function. If type_override is provided, use it as the function type. """ # We may be checking a function definition or an anonymous function. In # the first case, set up another reference with the precise type. - fdef = None # type: FuncDef + fdef = None # type: Optional[FuncDef] if isinstance(defn, FuncDef): fdef = defn @@ -597,7 +597,7 @@ def enter_attribute_inference_context(self) -> Iterator[None]: yield None self.inferred_attribute_types = old_types - def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: + def check_func_def(self, defn: FuncItem, typ: CallableType, name: Optional[str]) -> None: """Type check a function definition.""" # Expand type variables with value restrictions to ordinary types. for item, typ in self.expand_typevars(defn, typ): @@ -610,7 +610,7 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: # function. In the first case, set up another reference with the # precise type. if isinstance(item, FuncDef): - fdef = item + fdef = item # type: Optional[FuncDef] else: fdef = None @@ -634,12 +634,13 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: str) -> None: self.msg.unimported_type_becomes_any(prefix, arg_type, fdef) check_for_explicit_any(fdef.type, self.options, self.is_typeshed_stub, self.msg, context=fdef) - if name in nodes.reverse_op_method_set: - self.check_reverse_op_method(item, typ, name) - elif name in ('__getattr__', '__getattribute__'): - self.check_getattr_method(typ, defn, name) - elif name == '__setattr__': - self.check_setattr_method(typ, defn) + if name: # Special method names + if name in nodes.reverse_op_method_set: + self.check_reverse_op_method(item, typ, name) + elif name in ('__getattr__', '__getattribute__'): + self.check_getattr_method(typ, defn, name) + elif name == '__setattr__': + self.check_setattr_method(typ, defn) # Refuse contravariant return type variable if isinstance(typ.ret_type, TypeVarType): if typ.ret_type.variance == CONTRAVARIANT: @@ -1454,7 +1455,7 @@ def check_compatibility_super(self, lvalue: NameExpr, lvalue_type: Optional[Type # lvalue had a type defined; this is handled by other # parts, and all we have to worry about in that case is # that lvalue is compatible with the base class. - compare_node = None # type: Node + compare_node = None # type: Optional[Node] if lvalue_type: compare_type = lvalue_type compare_node = lvalue.node @@ -1533,7 +1534,7 @@ def lvalue_type_from_base(self, expr_node: Var, return None, None def check_compatibility_classvar_super(self, node: Var, - base: TypeInfo, base_node: Node) -> bool: + base: TypeInfo, base_node: Optional[Node]) -> bool: if not isinstance(base_node, Var): return True if node.is_classvar and not base_node.is_classvar: @@ -1600,7 +1601,7 @@ def check_multi_assignment(self, lvalues: List[Lvalue], rvalue: Expression, context: Context, infer_lvalue_type: bool = True, - msg: str = None) -> None: + msg: Optional[str] = None) -> None: """Check the assignment of one rvalue to a number of lvalues.""" # Infer the type of an ordinary rvalue expression. @@ -1801,7 +1802,7 @@ def infer_variable_type(self, name: Var, lvalue: Lvalue, self.fail(messages.NEED_ANNOTATION_FOR_VAR, context) self.set_inference_error_fallback_type(name, lvalue, init_type, context) elif (isinstance(lvalue, MemberExpr) and self.inferred_attribute_types is not None - and lvalue.def_var in self.inferred_attribute_types + and lvalue.def_var and lvalue.def_var in self.inferred_attribute_types and not is_same_type(self.inferred_attribute_types[lvalue.def_var], init_type)): # Multiple, inconsistent types inferred for an attribute. self.fail(messages.NEED_ANNOTATION_FOR_VAR, context) @@ -1844,6 +1845,7 @@ def set_inferred_type(self, var: Var, lvalue: Lvalue, type: Type) -> None: var.is_inferred = True if isinstance(lvalue, MemberExpr) and self.inferred_attribute_types is not None: # Store inferred attribute type so that we can check consistency afterwards. + assert lvalue.def_var is not None self.inferred_attribute_types[lvalue.def_var] = type self.store_type(lvalue, type) @@ -2223,22 +2225,25 @@ def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: self.accept(s.body) for i in range(len(s.handlers)): with self.binder.frame_context(can_skip=True, fall_through=4): - if s.types[i]: - t = self.check_except_handler_test(s.types[i]) - if s.vars[i]: + typ = s.types[i] + if typ: + t = self.check_except_handler_test(typ) + var = s.vars[i] + if var: # To support local variables, we make this a definition line, # causing assignment to set the variable's type. - s.vars[i].is_def = True + var.is_def = True # We also temporarily set current_node_deferred to False to # make sure the inference happens. # TODO: Use a better solution, e.g. a # separate Var for each except block. am_deferring = self.current_node_deferred self.current_node_deferred = False - self.check_assignment(s.vars[i], self.temp_node(t, s.vars[i])) + self.check_assignment(var, self.temp_node(t, var)) self.current_node_deferred = am_deferring self.accept(s.handlers[i]) - if s.vars[i]: + var = s.vars[i] + if var: # Exception variables are deleted in python 3 but not python 2. # But, since it's bad form in python 2 and the type checking # wouldn't work very well, we delete it anyway. @@ -2246,14 +2251,13 @@ def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None: # Unfortunately, this doesn't let us detect usage before the # try/except block. if self.options.python_version[0] >= 3: - source = s.vars[i].name + source = var.name else: source = ('(exception variable "{}", which we do not ' 'accept outside except: blocks even in ' - 'python 2)'.format(s.vars[i].name)) - var = cast(Var, s.vars[i].node) - var.type = DeletedType(source=source) - self.binder.cleanse(s.vars[i]) + 'python 2)'.format(var.name)) + cast(Var, var.node).type = DeletedType(source=source) + self.binder.cleanse(var) if s.else_body: self.accept(s.else_body) @@ -2452,7 +2456,7 @@ def check_untyped_after_decorator(self, typ: Type, func: FuncDef) -> None: if mypy.checkexpr.has_any_type(typ): self.msg.untyped_decorated_function(typ, func) - def check_async_with_item(self, expr: Expression, target: Expression, + def check_async_with_item(self, expr: Expression, target: Optional[Expression], infer_lvalue_type: bool) -> None: echk = self.expr_checker ctx = echk.accept(expr) @@ -2468,7 +2472,7 @@ def check_async_with_item(self, expr: Expression, target: Expression, echk.check_awaitable_expr( res, expr, messages.INCOMPATIBLE_TYPES_IN_ASYNC_WITH_AEXIT) - def check_with_item(self, expr: Expression, target: Expression, + def check_with_item(self, expr: Expression, target: Optional[Expression], infer_lvalue_type: bool) -> None: echk = self.expr_checker ctx = echk.accept(expr) @@ -2502,8 +2506,8 @@ def visit_continue_stmt(self, s: ContinueStmt) -> None: def check_subtype(self, subtype: Type, supertype: Type, context: Context, msg: str = messages.INCOMPATIBLE_TYPES, - subtype_label: str = None, - supertype_label: str = None) -> bool: + subtype_label: Optional[str] = None, + supertype_label: Optional[str] = None) -> bool: """Generate an error if the subtype is not compatible with supertype.""" if is_subtype(subtype, supertype): @@ -2621,7 +2625,9 @@ def lookup_qualified(self, name: str) -> SymbolTableNode: parts = name.split('.') n = self.modules[parts[0]] for i in range(1, len(parts) - 1): - n = cast(MypyFile, n.names.get(parts[i], None).node) + sym = n.names.get(parts[i]) + assert sym is not None, "Internal error: attempted lookup of unknown name" + n = cast(MypyFile, sym.node) last = parts[-1] if last in n.names: return n.names[last] @@ -2659,7 +2665,7 @@ def find_partial_types(self, var: Var) -> Optional[Dict[Var, Context]]: return partial_types return None - def temp_node(self, t: Type, context: Context = None) -> TempNode: + def temp_node(self, t: Type, context: Optional[Context] = None) -> TempNode: """Create a temporary node with the given, fixed type.""" temp = TempNode(t) if context: @@ -2904,7 +2910,7 @@ def or_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: def convert_to_typetype(type_map: TypeMap) -> TypeMap: - converted_type_map = {} # type: TypeMap + converted_type_map = {} # type: Dict[Expression, Type] if type_map is None: return None for expr, typ in type_map.items(): @@ -3278,7 +3284,7 @@ def is_valid_inferred_type_component(typ: Type) -> bool: return True -def is_node_static(node: Node) -> Optional[bool]: +def is_node_static(node: Optional[Node]) -> Optional[bool]: """Find out if a node describes a static function method.""" if isinstance(node, FuncDef): diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index eb1f315eb6ba..8cef0e37e130 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -21,7 +21,7 @@ ConditionalExpr, ComparisonExpr, TempNode, SetComprehension, DictionaryComprehension, ComplexExpr, EllipsisExpr, StarExpr, AwaitExpr, YieldExpr, YieldFromExpr, TypedDictExpr, PromoteExpr, NewTypeExpr, NamedTupleExpr, TypeVarExpr, - TypeAliasExpr, BackquoteExpr, EnumCallExpr, + TypeAliasExpr, BackquoteExpr, EnumCallExpr, SymbolTableNode, ARG_POS, ARG_NAMED, ARG_STAR, ARG_STAR2, MODULE_REF, TVAR, LITERAL_TYPE, ) from mypy.literals import literal @@ -201,11 +201,12 @@ def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type: and len(e.args) == 2): for typ in mypy.checker.flatten(e.args[1]): if isinstance(typ, NameExpr): + node = None # type: Optional[SymbolTableNode] try: node = self.chk.lookup_qualified(typ.name) except KeyError: # Undefined names should already be reported in semantic analysis. - node = None + pass if ((isinstance(typ, IndexExpr) and isinstance(typ.analyzed, (TypeApplication, TypeAliasExpr))) # node.kind == TYPE_ALIAS only for aliases like It = Iterable[int]. @@ -217,6 +218,7 @@ def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type: elif typ.node.is_newtype: self.msg.fail(messages.CANNOT_ISINSTANCE_NEWTYPE, e) self.try_infer_partial_type(e) + type_context = None # type: Optional[CallableType] if isinstance(e.callee, LambdaExpr): formal_to_actual = map_actuals_to_formals( e.arg_kinds, e.arg_names, @@ -228,8 +230,6 @@ def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type: type_context = CallableType(arg_types, e.callee.arg_kinds, e.callee.arg_names, ret_type=self.object_type(), fallback=self.named_type('builtins.function')) - else: - type_context = None callee_type = self.accept(e.callee, type_context, always_allow_any=True) if (self.chk.options.disallow_untyped_calls and self.chk.in_checked_function() and @@ -487,9 +487,9 @@ def check_call_expr_with_callee_type(self, def check_call(self, callee: Type, args: List[Expression], arg_kinds: List[int], context: Context, - arg_names: List[str] = None, - callable_node: Expression = None, - arg_messages: MessageBuilder = None, + arg_names: Optional[List[str]] = None, + callable_node: Optional[Expression] = None, + arg_messages: Optional[MessageBuilder] = None, callable_name: Optional[str] = None, object_type: Optional[Type] = None) -> Tuple[Type, Type]: """Type check a call. @@ -976,8 +976,8 @@ def check_argument_types(self, arg_types: List[Type], arg_kinds: List[int], callee: CallableType, formal_to_actual: List[List[int]], context: Context, - messages: MessageBuilder = None, - check_arg: ArgChecker = None) -> None: + messages: Optional[MessageBuilder] = None, + check_arg: Optional[ArgChecker] = None) -> None: """Check argument types against a callable type. Report errors if the argument types are not compatible. @@ -1059,7 +1059,7 @@ def check_arg(self, caller_type: Type, original_caller_type: Type, def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], arg_names: List[str], overload: Overloaded, context: Context, - messages: MessageBuilder = None) -> Type: + messages: Optional[MessageBuilder] = None) -> Type: """Infer the correct overload item to call with given argument types. The return value may be CallableType or AnyType (if an unique item diff --git a/mypy_self_check.ini b/mypy_self_check.ini index c54a3bad3ce9..aa4683befee4 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -12,12 +12,6 @@ warn_unused_ignores = True [mypy-mypy.binder] disallow_any = unimported -[mypy-mypy.checker] -strict_optional = False - -[mypy-mypy.checkexpr] -strict_optional = False - [mypy-mypy.semanal] strict_optional = False From 9d77fbc84390b0e4ed100c55ce218fda574661d4 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 14 Sep 2017 17:27:23 +0200 Subject: [PATCH 02/17] More clean-up --- mypy/applytype.py | 13 ++++--- mypy/checkexpr.py | 95 +++++++++++++++++++++++++++------------------ mypy/constraints.py | 4 +- mypy/fastparse.py | 6 ++- mypy/infer.py | 4 +- mypy/messages.py | 8 ++-- mypy/nodes.py | 6 +-- 7 files changed, 80 insertions(+), 56 deletions(-) diff --git a/mypy/applytype.py b/mypy/applytype.py index 75831116562e..894b03a09627 100644 --- a/mypy/applytype.py +++ b/mypy/applytype.py @@ -1,4 +1,4 @@ -from typing import List, Dict +from typing import List, Dict, Sequence, Optional import mypy.subtypes from mypy.sametypes import is_same_type @@ -8,7 +8,7 @@ from mypy.nodes import Context -def apply_generic_arguments(callable: CallableType, types: List[Type], +def apply_generic_arguments(callable: CallableType, orig_types: Sequence[Optional[Type]], msg: MessageBuilder, context: Context) -> CallableType: """Apply generic type arguments to a callable type. @@ -18,10 +18,10 @@ def apply_generic_arguments(callable: CallableType, types: List[Type], Note that each type can be None; in this case, it will not be applied. """ tvars = callable.variables - assert len(tvars) == len(types) + assert len(tvars) == len(orig_types) # Check that inferred type variable values are compatible with allowed # values and bounds. Also, promote subtype values to allowed values. - types = types[:] + types = list(orig_types) for i, type in enumerate(types): values = callable.variables[i].values if values and type: @@ -47,8 +47,9 @@ def apply_generic_arguments(callable: CallableType, types: List[Type], # Create a map from type variable id to target type. id_to_type = {} # type: Dict[TypeVarId, Type] for i, tv in enumerate(tvars): - if types[i]: - id_to_type[tv.id] = types[i] + typ = types[i] + if typ: + id_to_type[tv.id] = typ # Apply arguments to argument types. arg_types = [expand_type(at, id_to_type) for at in callable.arg_types] diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 8cef0e37e130..f32d4a0cb852 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1,7 +1,7 @@ """Expression type checker. This file is conceptually part of TypeChecker.""" from collections import OrderedDict -from typing import cast, Dict, Set, List, Tuple, Callable, Union, Optional, Iterable +from typing import cast, Dict, Set, List, Tuple, Callable, Union, Optional, Iterable, Sequence, Any from mypy.errors import report_internal_error from mypy.typeanal import has_any_from_unimported_type, check_for_explicit_any, set_any_tvars @@ -281,12 +281,13 @@ def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type: def check_typeddict_call(self, callee: TypedDictType, arg_kinds: List[int], - arg_names: List[str], + arg_names: Sequence[Optional[str]], args: List[Expression], context: Context) -> Type: if len(args) >= 1 and all([ak == ARG_NAMED for ak in arg_kinds]): # ex: Point(x=42, y=1337) - item_names = arg_names + assert all(arg_name is not None for arg_name in arg_names) + item_names = cast(List[str], arg_names) item_args = args return self.check_typeddict_call_with_kwargs( callee, OrderedDict(zip(item_names, item_args)), context) @@ -487,7 +488,7 @@ def check_call_expr_with_callee_type(self, def check_call(self, callee: Type, args: List[Expression], arg_kinds: List[int], context: Context, - arg_names: Optional[List[str]] = None, + arg_names: Optional[Sequence[Optional[str]]] = None, callable_node: Optional[Expression] = None, arg_messages: Optional[MessageBuilder] = None, callable_name: Optional[str] = None, @@ -675,8 +676,7 @@ def infer_arg_types_in_context(self, callee: Optional[CallableType], if callee: fixed = min(fixed, callee.max_fixed_args()) - arg_type = None # type: Type - ctx = None # type: Type + ctx = None # type: Optional[Type] for i, arg in enumerate(args): if i < fixed: if callee and i < len(callee.arg_types): @@ -703,7 +703,8 @@ def infer_arg_types_in_context2( Returns the inferred types of *actual arguments*. """ - res = [None] * len(args) # type: List[Type] + dummy = None # type: Any + res = [dummy] * len(args) # type: List[Type] for i, actuals in enumerate(formal_to_actual): for ai in actuals: @@ -750,7 +751,7 @@ def infer_function_type_arguments_using_context( ret_type = NoneTyp() args = infer_type_arguments(callable.type_var_ids(), ret_type, erased_ctx) # Only substitute non-Uninhabited and non-erased types. - new_args = [] # type: List[Type] + new_args = [] # type: List[Optional[Type]] for arg in args: if isinstance(arg, UninhabitedType) or has_erased_component(arg): new_args.append(None) @@ -793,7 +794,7 @@ def infer_function_type_arguments(self, callee_type: CallableType, inferred_args = infer_function_type_arguments( callee_type, pass1_args, arg_kinds, formal_to_actual, - strict=self.chk.in_checked_function()) # type: List[Type] + strict=self.chk.in_checked_function()) if 2 in arg_pass_nums: # Second pass of type inference. @@ -810,9 +811,10 @@ def infer_function_type_arguments(self, callee_type: CallableType, # if they shuffle type variables around, as we assume that there is a 1-1 # correspondence with dict type variables. This is a marginal issue and # a little tricky to fix so it's left unfixed for now. - if isinstance(inferred_args[0], (NoneTyp, UninhabitedType)): + first_arg = inferred_args[0] + if isinstance(first_arg, (NoneTyp, UninhabitedType)): inferred_args[0] = self.named_type('builtins.str') - elif not is_subtype(self.named_type('builtins.str'), inferred_args[0]): + elif not first_arg or not is_subtype(self.named_type('builtins.str'), first_arg): self.msg.fail(messages.KEYWORD_ARGUMENT_REQUIRES_STR_KEY_TYPE, context) else: @@ -827,8 +829,8 @@ def infer_function_type_arguments_pass2( args: List[Expression], arg_kinds: List[int], formal_to_actual: List[List[int]], - inferred_args: List[Type], - context: Context) -> Tuple[CallableType, List[Type]]: + old_inferred_args: Sequence[Optional[Type]], + context: Context) -> Tuple[CallableType, List[Optional[Type]]]: """Perform second pass of generic function type argument inference. The second pass is needed for arguments with types such as Callable[[T], S], @@ -843,6 +845,7 @@ def infer_function_type_arguments_pass2( # None or erased types in inferred types mean that there was not enough # information to infer the argument. Replace them with None values so # that they are not applied yet below. + inferred_args = list(old_inferred_args) for i, arg in enumerate(inferred_args): if isinstance(arg, (NoneTyp, UninhabitedType)) or has_erased_component(arg): inferred_args[i] = None @@ -875,7 +878,7 @@ def get_arg_infer_passes(self, arg_types: List[Type], return res def apply_inferred_arguments(self, callee_type: CallableType, - inferred_args: List[Type], + inferred_args: Sequence[Optional[Type]], context: Context) -> CallableType: """Apply inferred values of type arguments to a generic function. @@ -896,9 +899,10 @@ def apply_inferred_arguments(self, callee_type: CallableType, return self.apply_generic_arguments(callee_type, inferred_args, context) def check_argument_count(self, callee: CallableType, actual_types: List[Type], - actual_kinds: List[int], actual_names: List[str], + actual_kinds: List[int], + actual_names: Optional[Sequence[Optional[str]]], formal_to_actual: List[List[int]], - context: Context, + context: Optional[Context], messages: Optional[MessageBuilder]) -> bool: """Check that there is a value for all required arguments to a function. @@ -925,11 +929,16 @@ def check_argument_count(self, callee: CallableType, actual_types: List[Type], ok = False if kind != nodes.ARG_NAMED: if messages: + assert context, "Internal error: messages given without context" messages.too_many_arguments(callee, context) else: if messages: + assert context, "Internal error: messages given without context" + assert actual_names, "Internal error: named kinds without names given" + act_name = actual_names[i] + assert act_name is not None messages.unexpected_keyword_argument( - callee, actual_names[i], context) + callee, act_name, context) is_unexpected_arg_error = True elif kind == nodes.ARG_STAR and ( nodes.ARG_STAR not in formal_kinds): @@ -938,6 +947,7 @@ def check_argument_count(self, callee: CallableType, actual_types: List[Type], if all_actuals.count(i) < len(actual_type.items): # Too many tuple items as some did not match. if messages: + assert context, "Internal error: messages given without context" messages.too_many_arguments(callee, context) ok = False # *args can be applied even if the function takes a fixed @@ -948,13 +958,17 @@ def check_argument_count(self, callee: CallableType, actual_types: List[Type], not is_unexpected_arg_error): # No actual for a mandatory positional formal. if messages: + assert context, "Internal error: messages given without context" messages.too_few_arguments(callee, context, actual_names) ok = False elif kind == nodes.ARG_NAMED and (not formal_to_actual[i] and not is_unexpected_arg_error): # No actual for a mandatory named formal if messages: - messages.missing_named_argument(callee, context, callee.arg_names[i]) + argname = callee.arg_names[i] + assert argname is not None + assert context, "Internal error: messages given without context" + messages.missing_named_argument(callee, context, argname) ok = False elif kind in [nodes.ARG_POS, nodes.ARG_OPT, nodes.ARG_NAMED, nodes.ARG_NAMED_OPT] and is_duplicate_mapping( @@ -962,12 +976,14 @@ def check_argument_count(self, callee: CallableType, actual_types: List[Type], if (self.chk.in_checked_function() or isinstance(actual_types[formal_to_actual[i][0]], TupleType)): if messages: + assert context, "Internal error: messages given without context" messages.duplicate_argument_value(callee, i, context) ok = False elif (kind in (nodes.ARG_NAMED, nodes.ARG_NAMED_OPT) and formal_to_actual[i] and actual_kinds[formal_to_actual[i][0]] not in [nodes.ARG_NAMED, nodes.ARG_STAR2]): # Positional argument when expecting a keyword argument. if messages: + assert context, "Internal error: messages given without context" messages.too_many_positional_arguments(callee, context) ok = False return ok @@ -1057,7 +1073,7 @@ def check_arg(self, caller_type: Type, original_caller_type: Type, self.msg.note_call(original_caller_type, call, context) def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], + arg_names: Optional[Sequence[Optional[str]]], overload: Overloaded, context: Context, messages: Optional[MessageBuilder] = None) -> Type: """Infer the correct overload item to call with given argument types. @@ -1120,7 +1136,8 @@ def overload_call_target(self, arg_types: List[Type], arg_kinds: List[int], return match[0] def erased_signature_similarity(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], callee: CallableType, + arg_names: Optional[Sequence[Optional[str]]], + callee: CallableType, context: Context) -> int: """Determine whether arguments could match the signature at runtime. @@ -1160,7 +1177,7 @@ def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, return similarity def match_signature_types(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[str], callee: CallableType, + arg_names: Optional[Sequence[Optional[str]]], callee: CallableType, context: Context) -> bool: """Determine whether arguments types match the signature. @@ -1186,7 +1203,7 @@ def check_arg(caller_type: Type, original_caller_type: Type, caller_kind: int, context=context, check_arg=check_arg) return ok - def apply_generic_arguments(self, callable: CallableType, types: List[Type], + def apply_generic_arguments(self, callable: CallableType, types: Sequence[Optional[Type]], context: Context) -> CallableType: """Simple wrapper around mypy.applytype.apply_generic_arguments.""" return applytype.apply_generic_arguments(callable, types, self.msg, context) @@ -1245,7 +1262,6 @@ def analyze_descriptor_access(self, instance_type: Type, descriptor_type: Type, bound_method = bind_self(function, descriptor_type) typ = map_instance_to_supertype(descriptor_type, dunder_get.info) dunder_get_type = expand_type_by_instance(bound_method, typ) - owner_type = None # type: Type if isinstance(instance_type, FunctionLike) and instance_type.is_type_obj(): owner_type = instance_type.items()[0].ret_type @@ -1351,13 +1367,13 @@ def visit_comparison_expr(self, e: ComparisonExpr) -> Type: Comparison expressions are type checked consecutive-pair-wise That is, 'a < b > c == d' is check as 'a < b and b > c and c == d' """ - result = None # type: mypy.types.Type + result = None # type: Optional[mypy.types.Type] # Check each consecutive operand pair and their operator for left, right, operator in zip(e.operands, e.operands[1:], e.operators): left_type = self.accept(left) - method_type = None # type: mypy.types.Type + method_type = None # type: Optional[mypy.types.Type] if operator == 'in' or operator == 'not in': right_type = self.accept(right) # TODO only evaluate if needed @@ -1407,6 +1423,7 @@ def visit_comparison_expr(self, e: ComparisonExpr) -> Type: else: result = join.join_types(result, sub_result) + assert result is not None return result def get_operator_method(self, op: str) -> str: @@ -1679,9 +1696,9 @@ def visit_index_expr_helper(self, e: IndexExpr) -> Type: return result def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr) -> Type: - begin = None # type: int - end = None # type: int - stride = None # type:int + begin = None # type: Optional[int] + end = None # type: Optional[int] + stride = None # type: Optional[int] if slic.begin_index: begin = self._get_value(slic.begin_index) @@ -1698,6 +1715,7 @@ def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr) -> Typ if stride is None: return self.nonliteral_tuple_index_helper(left_type, slic) + assert begin and stride and end return left_type.slice(begin, stride, end) def nonliteral_tuple_index_helper(self, left_type: TupleType, index: Expression) -> Type: @@ -1872,7 +1890,7 @@ def visit_tuple_expr(self, e: TupleExpr) -> Type: if isinstance(type_context, TupleType): type_context_items = type_context.items - elif is_named_instance(type_context, 'builtins.tuple'): + elif type_context and is_named_instance(type_context, 'builtins.tuple'): assert isinstance(type_context, Instance) if type_context.args: type_context_items = [type_context.args[0]] * len(e.items) @@ -1887,7 +1905,6 @@ def visit_tuple_expr(self, e: TupleExpr) -> Type: j = 0 # Index into type_context_items; irrelevant if type_context_items is none for i in range(len(e.items)): item = e.items[i] - tt = None # type: Type if isinstance(item, StarExpr): # Special handling for star expressions. # TODO: If there's a context, and item.expr is a @@ -1977,9 +1994,10 @@ def visit_dict_expr(self, e: DictExpr) -> Type: else: method = self.analyze_external_member_access('update', rv, arg) self.check_call(method, [arg], [nodes.ARG_POS], arg) + assert rv is not None return rv - def find_typeddict_context(self, context: Type) -> Optional[TypedDictType]: + def find_typeddict_context(self, context: Optional[Type]) -> Optional[TypedDictType]: if isinstance(context, TypedDictType): return context elif isinstance(context, UnionType): @@ -2148,7 +2166,9 @@ def analyze_super(self, e: SuperExpr, is_lvalue: bool) -> Type: if self.chk.scope.active_class() is not None: self.chk.fail('super() outside of a method is not supported', e) return AnyType(TypeOfAny.from_error) - args = self.chk.scope.top_function().arguments + method = self.chk.scope.top_function() + assert method is not None + args = method.arguments # super() in a function with empty args is an error; we # need something in declared_self. if not args: @@ -2157,6 +2177,7 @@ def analyze_super(self, e: SuperExpr, is_lvalue: bool) -> Type: 'enclosing function', e) return AnyType(TypeOfAny.from_error) declared_self = args[0].variable.type + assert declared_self is not None, "Internal error: type of self is None" return analyze_member_access(name=e.name, typ=fill_typevars(e.info), node=e, is_lvalue=False, is_super=True, is_operator=False, builtin_type=self.named_type, @@ -2457,12 +2478,11 @@ def visit_yield_from_expr(self, e: YieldFromExpr, allow_none_return: bool = Fals # the context should be Generator[X, Y, T], where T is the # context of the 'yield from' itself (but it isn't known). subexpr_type = self.accept(e.expr) - iter_type = None # type: Type # Check that the expr is an instance of Iterable and get the type of the iterator produced # by __iter__. if isinstance(subexpr_type, AnyType): - iter_type = AnyType(TypeOfAny.from_another_any, source_any=subexpr_type) + iter_type = AnyType(TypeOfAny.from_another_any, source_any=subexpr_type) # type: Type elif self.chk.type_is_iterable(subexpr_type): if is_async_def(subexpr_type) and not has_coroutine_decorator(return_type): self.chk.msg.yield_from_invalid_operand_type(subexpr_type, e) @@ -2609,7 +2629,7 @@ def is_async_def(t: Type) -> bool: def map_actuals_to_formals(caller_kinds: List[int], - caller_names: List[Optional[str]], + caller_names: Optional[Sequence[Optional[str]]], callee_kinds: List[int], callee_names: List[Optional[str]], caller_arg_type: Callable[[int], @@ -2659,6 +2679,7 @@ def map_actuals_to_formals(caller_kinds: List[int], break j += 1 elif kind in (nodes.ARG_NAMED, nodes.ARG_NAMED_OPT): + assert caller_names is not None, "Internal error: named kinds without names given" name = caller_names[i] if name in callee_names: map[callee_names.index(name)].append(i) @@ -2719,7 +2740,7 @@ def visit_type_var(self, t: TypeVarType) -> bool: return True -def has_erased_component(t: Type) -> bool: +def has_erased_component(t: Optional[Type]) -> bool: return t is not None and t.accept(HasErasedComponentsQuery()) @@ -2832,7 +2853,7 @@ def overload_arg_similarity(actual: Type, formal: Type) -> int: def any_arg_causes_overload_ambiguity(items: List[CallableType], arg_types: List[Type], arg_kinds: List[int], - arg_names: List[Optional[str]]) -> bool: + arg_names: Optional[Sequence[Optional[str]]]) -> bool: """May an Any actual argument cause ambiguous result type on call to overloaded function? Note that this sometimes returns True even if there is no ambiguity, since a correct diff --git a/mypy/constraints.py b/mypy/constraints.py index 7180dd00f25f..4e7c2ecba269 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1,6 +1,6 @@ """Type inference constraints.""" -from typing import Iterable, List, Optional +from typing import Iterable, List, Optional, Sequence from mypy import experiments from mypy.types import ( @@ -42,7 +42,7 @@ def __repr__(self) -> str: def infer_constraints_for_callable( - callee: CallableType, arg_types: List[Optional[Type]], arg_kinds: List[int], + callee: CallableType, arg_types: Sequence[Optional[Type]], arg_kinds: List[int], formal_to_actual: List[List[int]]) -> List[Constraint]: """Infer type variable constraints for a callable and actual arguments. diff --git a/mypy/fastparse.py b/mypy/fastparse.py index d9119c39b361..cb1eeba02c20 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -885,7 +885,8 @@ def visit_JoinedStr(self, n: ast3.JoinedStr) -> Expression: join_method.set_line(empty_string) result_expression = CallExpr(join_method, [strs_to_join], - [ARG_POS]) + [ARG_POS], + [None]) return result_expression # FormattedValue(expr value) @@ -902,7 +903,8 @@ def visit_FormattedValue(self, n: ast3.FormattedValue) -> Expression: format_method.set_line(format_string) result_expression = CallExpr(format_method, [exp], - [ARG_POS]) + [ARG_POS], + [None]) return result_expression # Bytes(bytes s) diff --git a/mypy/infer.py b/mypy/infer.py index 6820a2c05eb4..b7d0dca8b9b4 100644 --- a/mypy/infer.py +++ b/mypy/infer.py @@ -1,6 +1,6 @@ """Utilities for type argument inference.""" -from typing import List, Optional +from typing import List, Optional, Sequence from mypy.constraints import infer_constraints, infer_constraints_for_callable from mypy.types import Type, TypeVarId, CallableType @@ -9,7 +9,7 @@ def infer_function_type_arguments(callee_type: CallableType, - arg_types: List[Optional[Type]], + arg_types: Sequence[Optional[Type]], arg_kinds: List[int], formal_to_actual: List[List[int]], strict: bool = True) -> List[Optional[Type]]: diff --git a/mypy/messages.py b/mypy/messages.py index 51fa0c7d9955..2f3473b3e59d 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -165,7 +165,7 @@ def enable_errors(self) -> None: def is_errors(self) -> bool: return self.errors.is_errors() - def report(self, msg: str, context: Context, severity: str, + def report(self, msg: str, context: Optional[Context], severity: str, file: Optional[str] = None, origin: Optional[Context] = None, offset: int = 0) -> None: """Report an error or note (unless disabled).""" @@ -175,7 +175,7 @@ def report(self, msg: str, context: Context, severity: str, msg.strip(), severity=severity, file=file, offset=offset, origin_line=origin.get_line() if origin else None) - def fail(self, msg: str, context: Context, file: Optional[str] = None, + def fail(self, msg: str, context: Optional[Context], file: Optional[str] = None, origin: Optional[Context] = None) -> None: """Report an error message (unless disabled).""" self.report(msg, context, 'error', file=file, origin=origin) @@ -641,7 +641,7 @@ def invalid_index_type(self, index_type: Type, expected_type: Type, base_str: st self.format(index_type), base_str, self.format(expected_type)), context) def too_few_arguments(self, callee: CallableType, context: Context, - argument_names: List[str]) -> None: + argument_names: Optional[Sequence[Optional[str]]]) -> None: if (argument_names is not None and not all(k is None for k in argument_names) and len(argument_names) >= 1): diff = [k for k in callee.arg_names if k not in argument_names] @@ -695,7 +695,7 @@ def duplicate_argument_value(self, callee: CallableType, index: int, format(capitalize(callable_name(callee)), callee.arg_names[index]), context) - def does_not_return_value(self, callee_type: Type, context: Context) -> None: + def does_not_return_value(self, callee_type: Optional[Type], context: Context) -> None: """Report an error about use of an unusable type.""" name = None # type: Optional[str] if isinstance(callee_type, FunctionLike): diff --git a/mypy/nodes.py b/mypy/nodes.py index 330a03676889..7f7ceb5fb8fe 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1241,7 +1241,7 @@ def __init__(self, callee: Expression, args: List[Expression], arg_kinds: List[int], - arg_names: Optional[List[Optional[str]]] = None, + arg_names: List[Optional[str]], analyzed: Optional[Expression] = None) -> None: if not arg_names: arg_names = [None] * len(args) @@ -1731,13 +1731,13 @@ class TypeAliasExpr(Expression): type = None # type: mypy.types.Type # Simple fallback type for aliases that are invalid in runtime expressions # (for example Union, Tuple, Callable). - fallback = None # type: Optional[mypy.types.Type] + fallback = None # type: mypy.types.Type # This type alias is subscripted in a runtime expression like Alias[int](42) # (not in a type context like type annotation or base class). in_runtime = False # type: bool def __init__(self, type: 'mypy.types.Type', tvars: List[str], - fallback: 'Optional[mypy.types.Type]' = None, in_runtime: bool = False) -> None: + fallback: 'mypy.types.Type', in_runtime: bool = False) -> None: self.type = type self.fallback = fallback self.in_runtime = in_runtime From 1f7310606644b137ae574157bb98b989b8d055e0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 14 Sep 2017 19:44:32 +0200 Subject: [PATCH 03/17] Finish clean-up of checker, checkexpr and myunit --- mypy/binder.py | 2 +- mypy/checker.py | 10 ++++++---- mypy/checkexpr.py | 14 +++++++------- mypy/checkmember.py | 2 +- mypy/myunit/__init__.py | 2 +- mypy/types.py | 3 ++- mypy_self_check.ini | 11 +---------- 7 files changed, 19 insertions(+), 25 deletions(-) diff --git a/mypy/binder.py b/mypy/binder.py index 094d51b912ea..956c950c5394 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -212,7 +212,7 @@ def pop_frame(self, can_skip: bool, fall_through: int) -> Frame: def assign_type(self, expr: Expression, type: Type, - declared_type: Type, + declared_type: Optional[Type], restrict_any: bool = False) -> None: if not isinstance(expr, BindableTypes): return None diff --git a/mypy/checker.py b/mypy/checker.py index 383e76c4c71e..e53964e6345b 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1433,6 +1433,7 @@ def check_compatibility_all_supers(self, lvalue: NameExpr, lvalue_type: Optional base_type, base_node = self.lvalue_type_from_base(lvalue_node, base) if base_type: + assert base_node is not None if not self.check_compatibility_super(lvalue, lvalue_type, rvalue, @@ -1516,6 +1517,7 @@ def lvalue_type_from_base(self, expr_node: Var, if base_type: if not has_no_typevars(base_type): self_type = self.scope.active_self_type() + assert self_type is not None, "Internal error: base lookup outside class" if isinstance(self_type, TupleType): instance = self_type.fallback else: @@ -1864,7 +1866,7 @@ def set_inference_error_fallback_type(self, var: Var, lvalue: Lvalue, type: Type if context.get_line() in self.errors.ignored_lines[self.errors.file]: self.set_inferred_type(var, lvalue, AnyType(TypeOfAny.from_error)) - def check_simple_assignment(self, lvalue_type: Type, rvalue: Expression, + def check_simple_assignment(self, lvalue_type: Optional[Type], rvalue: Expression, context: Context, msg: str = messages.INCOMPATIBLE_TYPES_IN_ASSIGNMENT, lvalue_name: str = 'variable', @@ -1880,7 +1882,7 @@ def check_simple_assignment(self, lvalue_type: Type, rvalue: Expression, self.msg.deleted_as_rvalue(rvalue_type, context) if isinstance(lvalue_type, DeletedType): self.msg.deleted_as_lvalue(lvalue_type, context) - else: + elif lvalue_type: self.check_subtype(rvalue_type, lvalue_type, context, msg, '{} has type'.format(rvalue_name), '{} has type'.format(lvalue_name)) @@ -2770,7 +2772,7 @@ def conditional_type_map(expr: Expression, return {}, {} -def partition_by_callable(type: Optional[Type]) -> Tuple[List[Type], List[Type]]: +def partition_by_callable(type: Type) -> Tuple[List[Type], List[Type]]: """Takes in a type and partitions that type into callable subtypes and uncallable subtypes. @@ -2801,7 +2803,7 @@ def partition_by_callable(type: Optional[Type]) -> Tuple[List[Type], List[Type]] if isinstance(type, Instance): method = type.type.get_method('__call__') - if method: + if method and method.type: callables, uncallables = partition_by_callable(method.type) if len(callables) and not len(uncallables): # Only consider the type callable if its __call__ method is diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index f32d4a0cb852..6849420fd159 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -236,7 +236,7 @@ def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type: isinstance(callee_type, CallableType) and callee_type.implicit): return self.msg.untyped_function_call(callee_type, e) - # Figure out the full name of the callee for plugin loopup. + # Figure out the full name of the callee for plugin lookup. object_type = None if not isinstance(e.callee, RefExpr): fullname = None @@ -260,6 +260,7 @@ def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type: # Apply plugin signature hook that may generate a better signature. signature_hook = self.plugin.get_method_signature_hook(fullname) if signature_hook: + assert object_type is not None callee_type = self.apply_method_signature_hook( e, callee_type, object_type, signature_hook) ret_type = self.check_call_expr_with_callee_type(callee_type, e, fullname, object_type) @@ -1715,7 +1716,6 @@ def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr) -> Typ if stride is None: return self.nonliteral_tuple_index_helper(left_type, slic) - assert begin and stride and end return left_type.slice(begin, stride, end) def nonliteral_tuple_index_helper(self, left_type: TupleType, index: Expression) -> Type: @@ -1960,6 +1960,7 @@ def visit_dict_expr(self, e: DictExpr) -> Type: vtdef = TypeVarDef('VT', -2, [], self.object_type()) kt = TypeVarType(ktdef) vt = TypeVarType(vtdef) + rv = None # type: Optional[Type] # Call dict(*args), unless it's empty and stargs is not. if args or not stargs: # The callable type represents a function like this: @@ -1976,7 +1977,7 @@ def visit_dict_expr(self, e: DictExpr) -> Type: rv = self.check_call(constructor, args, [nodes.ARG_POS] * len(args), e)[0] else: # dict(...) will be called below. - rv = None + pass # Call rv.update(arg) for each arg in **stargs, # except if rv isn't set yet, then set rv = dict(arg). if stargs: @@ -2176,8 +2177,7 @@ def analyze_super(self, e: SuperExpr, is_lvalue: bool) -> Type: 'super() requires one or more positional arguments in ' 'enclosing function', e) return AnyType(TypeOfAny.from_error) - declared_self = args[0].variable.type - assert declared_self is not None, "Internal error: type of self is None" + declared_self = args[0].variable.type or fill_typevars(e.info) return analyze_member_access(name=e.name, typ=fill_typevars(e.info), node=e, is_lvalue=False, is_super=True, is_operator=False, builtin_type=self.named_type, @@ -2836,7 +2836,7 @@ def overload_arg_similarity(actual: Type, formal: Type) -> int: item = actual.item if formal.type.fullname() in {"builtins.object", "builtins.type"}: return 2 - elif isinstance(item, Instance): + elif isinstance(item, Instance) and item.type.metaclass_type: # FIX: this does not handle e.g. Union of instances return overload_arg_similarity(item.type.metaclass_type, formal) else: @@ -2900,7 +2900,7 @@ def all_same_types(types: Iterable[Type]) -> bool: def map_formals_to_actuals(caller_kinds: List[int], - caller_names: List[Optional[str]], + caller_names: Optional[Sequence[Optional[str]]], callee_kinds: List[int], callee_names: List[Optional[str]], caller_arg_type: Callable[[int], diff --git a/mypy/checkmember.py b/mypy/checkmember.py index ceb9a4ff00ce..ea8aff82d209 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -630,7 +630,7 @@ class B(A): pass """ if isinstance(method, Overloaded): - return cast(F, Overloaded([bind_self(c, method) for c in method.items()])) + return cast(F, Overloaded([bind_self(c, original_type) for c in method.items()])) assert isinstance(method, CallableType) func = method if not func.arg_types: diff --git a/mypy/myunit/__init__.py b/mypy/myunit/__init__.py index 013353daed2c..92ba802530eb 100644 --- a/mypy/myunit/__init__.py +++ b/mypy/myunit/__init__.py @@ -112,7 +112,7 @@ def __init__(self, name: str, suite: 'Optional[Suite]' = None, self.name = name self.suite = suite self.old_cwd = None # type: Optional[str] - self.tmpdir = None # type: Optional[tempfile.TemporaryDirectory] + self.tmpdir = None # type: Optional[tempfile.TemporaryDirectory[str]] def run(self) -> None: if self.func: diff --git a/mypy/types.py b/mypy/types.py index 904694c81558..a53085a66a36 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1011,7 +1011,8 @@ def copy_modified(self, *, fallback: Optional[Instance] = None, items = self.items return TupleType(items, fallback, self.line, self.column) - def slice(self, begin: int, stride: int, end: int) -> 'TupleType': + def slice(self, begin: Optional[int], stride: Optional[int], + end: Optional[int]) -> 'TupleType': return TupleType(self.items[begin:end:stride], self.fallback, self.line, self.column, self.implicit) diff --git a/mypy_self_check.ini b/mypy_self_check.ini index aa4683befee4..e97c379d677a 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -8,15 +8,6 @@ disallow_any = generics, unimported warn_redundant_casts = True warn_unused_ignores = True -; historical exceptions -[mypy-mypy.binder] -disallow_any = unimported - +; historical exception [mypy-mypy.semanal] strict_optional = False - -[mypy-mypy.myunit] -disallow_any = unimported - -[mypy-mypy.nodes] -disallow_any = unimported From 218253a9c8cd8f44df0af0957a3d46aa5f515a25 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 17 Sep 2017 17:22:29 +0200 Subject: [PATCH 04/17] Address CR --- mypy/checker.py | 10 ++++------ mypy/checkexpr.py | 20 ++++++++++---------- mypy_self_check.ini | 2 +- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index e53964e6345b..3fcc3c665e12 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -610,11 +610,7 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: Optional[str]) # function. In the first case, set up another reference with the # precise type. if isinstance(item, FuncDef): - fdef = item # type: Optional[FuncDef] - else: - fdef = None - - if fdef: + fdef = item # Check if __init__ has an invalid, non-None return type. if (fdef.info and fdef.name() in ('__init__', '__init_subclass__') and not isinstance(typ.ret_type, NoneTyp) and @@ -634,6 +630,7 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: Optional[str]) self.msg.unimported_type_becomes_any(prefix, arg_type, fdef) check_for_explicit_any(fdef.type, self.options, self.is_typeshed_stub, self.msg, context=fdef) + if name: # Special method names if name in nodes.reverse_op_method_set: self.check_reverse_op_method(item, typ, name) @@ -641,6 +638,7 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: Optional[str]) self.check_getattr_method(typ, defn, name) elif name == '__setattr__': self.check_setattr_method(typ, defn) + # Refuse contravariant return type variable if isinstance(typ.ret_type, TypeVarType): if typ.ret_type.variance == CONTRAVARIANT: @@ -1456,7 +1454,7 @@ def check_compatibility_super(self, lvalue: NameExpr, lvalue_type: Optional[Type # lvalue had a type defined; this is handled by other # parts, and all we have to worry about in that case is # that lvalue is compatible with the base class. - compare_node = None # type: Optional[Node] + compare_node = None if lvalue_type: compare_type = lvalue_type compare_node = lvalue.node diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 6849420fd159..36c89b3ceda4 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -21,7 +21,7 @@ ConditionalExpr, ComparisonExpr, TempNode, SetComprehension, DictionaryComprehension, ComplexExpr, EllipsisExpr, StarExpr, AwaitExpr, YieldExpr, YieldFromExpr, TypedDictExpr, PromoteExpr, NewTypeExpr, NamedTupleExpr, TypeVarExpr, - TypeAliasExpr, BackquoteExpr, EnumCallExpr, SymbolTableNode, + TypeAliasExpr, BackquoteExpr, EnumCallExpr, ARG_POS, ARG_NAMED, ARG_STAR, ARG_STAR2, MODULE_REF, TVAR, LITERAL_TYPE, ) from mypy.literals import literal @@ -201,7 +201,7 @@ def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type: and len(e.args) == 2): for typ in mypy.checker.flatten(e.args[1]): if isinstance(typ, NameExpr): - node = None # type: Optional[SymbolTableNode] + node = None try: node = self.chk.lookup_qualified(typ.name) except KeyError: @@ -218,7 +218,7 @@ def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type: elif typ.node.is_newtype: self.msg.fail(messages.CANNOT_ISINSTANCE_NEWTYPE, e) self.try_infer_partial_type(e) - type_context = None # type: Optional[CallableType] + type_context = None if isinstance(e.callee, LambdaExpr): formal_to_actual = map_actuals_to_formals( e.arg_kinds, e.arg_names, @@ -677,7 +677,7 @@ def infer_arg_types_in_context(self, callee: Optional[CallableType], if callee: fixed = min(fixed, callee.max_fixed_args()) - ctx = None # type: Optional[Type] + ctx = None for i, arg in enumerate(args): if i < fixed: if callee and i < len(callee.arg_types): @@ -908,7 +908,7 @@ def check_argument_count(self, callee: CallableType, actual_types: List[Type], """Check that there is a value for all required arguments to a function. Also check that there are no duplicate values for arguments. Report found errors - using 'messages' if it's not None. + using 'messages' if it's not None. If 'messages' is given, 'context' must also be given. Return False if there were any errors. Otherwise return True """ @@ -1368,7 +1368,7 @@ def visit_comparison_expr(self, e: ComparisonExpr) -> Type: Comparison expressions are type checked consecutive-pair-wise That is, 'a < b > c == d' is check as 'a < b and b > c and c == d' """ - result = None # type: Optional[mypy.types.Type] + result = None # Check each consecutive operand pair and their operator for left, right, operator in zip(e.operands, e.operands[1:], e.operators): @@ -1697,9 +1697,9 @@ def visit_index_expr_helper(self, e: IndexExpr) -> Type: return result def visit_tuple_slice_helper(self, left_type: TupleType, slic: SliceExpr) -> Type: - begin = None # type: Optional[int] - end = None # type: Optional[int] - stride = None # type: Optional[int] + begin = None + end = None + stride = None if slic.begin_index: begin = self._get_value(slic.begin_index) @@ -1960,7 +1960,7 @@ def visit_dict_expr(self, e: DictExpr) -> Type: vtdef = TypeVarDef('VT', -2, [], self.object_type()) kt = TypeVarType(ktdef) vt = TypeVarType(vtdef) - rv = None # type: Optional[Type] + rv = None # Call dict(*args), unless it's empty and stargs is not. if args or not stargs: # The callable type represents a function like this: diff --git a/mypy_self_check.ini b/mypy_self_check.ini index e97c379d677a..6b97ed660d9b 100644 --- a/mypy_self_check.ini +++ b/mypy_self_check.ini @@ -8,6 +8,6 @@ disallow_any = generics, unimported warn_redundant_casts = True warn_unused_ignores = True -; historical exception +# historical exception [mypy-mypy.semanal] strict_optional = False From 233c298592f133e00fb86dc5cb602cb3800cc9bd Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 17 Sep 2017 23:12:04 +0200 Subject: [PATCH 05/17] Fix some more strict optional errors --- mypy/checker.py | 11 +++++++---- mypy/checkexpr.py | 8 ++++---- mypy/nodes.py | 4 ++-- mypy/types.py | 6 +++--- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 3fcc3c665e12..5bd0b570efa1 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -1743,9 +1743,9 @@ def check_multi_assignment_from_iterable(self, lvalues: List[Lvalue], rvalue_typ def check_lvalue(self, lvalue: Lvalue) -> Tuple[Optional[Type], Optional[IndexExpr], Optional[Var]]: - lvalue_type = None # type: Optional[Type] - index_lvalue = None # type: Optional[IndexExpr] - inferred = None # type: Optional[Var] + lvalue_type = None + index_lvalue = None + inferred = None if self.is_definition(lvalue): if isinstance(lvalue, NameExpr): @@ -1765,7 +1765,10 @@ def check_lvalue(self, lvalue: Lvalue) -> Tuple[Optional[Type], lvalue_type = self.expr_checker.analyze_ref_expr(lvalue, lvalue=True) self.store_type(lvalue, lvalue_type) elif isinstance(lvalue, TupleExpr) or isinstance(lvalue, ListExpr): - types = [self.check_lvalue(sub_expr)[0] for sub_expr in lvalue.items] + types = [self.check_lvalue(sub_expr)[0] or + # This type will be used as a context for further inference of rvalue + # we put Any if there is no information available from lvalue. + AnyType(TypeOfAny.special_form) for sub_expr in lvalue.items] lvalue_type = TupleType(types, self.named_type('builtins.tuple')) else: lvalue_type = self.expr_checker.accept(lvalue) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 36c89b3ceda4..16cd5a164f69 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -704,8 +704,7 @@ def infer_arg_types_in_context2( Returns the inferred types of *actual arguments*. """ - dummy = None # type: Any - res = [dummy] * len(args) # type: List[Type] + res = [None] * len(args) # type: List[Optional[Type]] for i, actuals in enumerate(formal_to_actual): for ai in actuals: @@ -716,7 +715,8 @@ def infer_arg_types_in_context2( for i, t in enumerate(res): if not t: res[i] = self.accept(args[i]) - return res + assert all(tp is not None for tp in res) + return cast(List[Type], res) def infer_function_type_arguments_using_context( self, callable: CallableType, error_context: Context) -> CallableType: @@ -2631,7 +2631,7 @@ def is_async_def(t: Type) -> bool: def map_actuals_to_formals(caller_kinds: List[int], caller_names: Optional[Sequence[Optional[str]]], callee_kinds: List[int], - callee_names: List[Optional[str]], + callee_names: Sequence[Optional[str]], caller_arg_type: Callable[[int], Type]) -> List[List[int]]: """Calculate mapping between actual (caller) args and formals. diff --git a/mypy/nodes.py b/mypy/nodes.py index 7f7ceb5fb8fe..a23f5a2f1259 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -4,7 +4,7 @@ from abc import abstractmethod from collections import OrderedDict from typing import ( - Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional, Callable, + Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional, Callable, Sequence, ) import mypy.strconv @@ -2538,7 +2538,7 @@ def check_arg_kinds(arg_kinds: List[int], nodes: List[T], fail: Callable[[str, T is_kw_arg = True -def check_arg_names(names: List[Optional[str]], nodes: List[T], fail: Callable[[str, T], None], +def check_arg_names(names: Sequence[Optional[str]], nodes: List[T], fail: Callable[[str, T], None], description: str = 'function definition') -> None: seen_names = set() # type: Set[Optional[str]] for name, node in zip(names, nodes): diff --git a/mypy/types.py b/mypy/types.py index a53085a66a36..472919d9d10b 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -5,7 +5,7 @@ from collections import OrderedDict from typing import ( Any, TypeVar, Dict, List, Tuple, cast, Generic, Set, Optional, Union, Iterable, NamedTuple, - Callable + Callable, Sequence ) import mypy.nodes @@ -642,7 +642,7 @@ class CallableType(FunctionLike): def __init__(self, arg_types: List[Type], arg_kinds: List[int], - arg_names: List[Optional[str]], + arg_names: Sequence[Optional[str]], ret_type: Type, fallback: Instance, name: Optional[str] = None, @@ -663,7 +663,7 @@ def __init__(self, assert not any(tp is None for tp in arg_types), "No annotation must be Any, not None" self.arg_types = arg_types self.arg_kinds = arg_kinds - self.arg_names = arg_names + self.arg_names = list(arg_names) self.min_args = arg_kinds.count(ARG_POS) self.is_var_arg = ARG_STAR in arg_kinds self.is_kw_arg = ARG_STAR2 in arg_kinds From 6ecd80b240de05ee91496a03ba869599baed708e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 7 Oct 2017 11:55:34 +0200 Subject: [PATCH 06/17] Deal with lookup functions --- mypy/semanal.py | 39 ++++++++++++++++++++++----------------- mypy/typeanal.py | 20 +++++++++++++------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 3c80b3bd37ce..c009e7090521 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -407,6 +407,7 @@ def visit_func_def(self, defn: FuncDef) -> None: # Top-level function if not defn.is_decorated and not defn.is_overload: symbol = self.globals.get(defn.name()) + assert symbol, "Global function not found in globals" if isinstance(symbol.node, FuncDef) and symbol.node != defn: # This is redefinition. Conditional redefinition is okay. if not self.set_original_def(symbol.node, defn): @@ -1250,6 +1251,7 @@ def class_type(self, info: TypeInfo) -> Type: def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: sym = self.lookup_qualified(qualified_name, None) + assert sym, "Internal error: attempted to construct unknown type" node = sym.node assert isinstance(node, TypeInfo) if args: @@ -1521,25 +1523,26 @@ def process_import_over_existing_name(self, return False def normalize_type_alias(self, node: SymbolTableNode, - ctx: Context) -> SymbolTableNode: + ctx: Context) -> Optional[SymbolTableNode]: normalized = False fullname = node.fullname if fullname in type_aliases: # Node refers to an aliased type such as typing.List; normalize. - node = self.lookup_qualified(type_aliases[fullname], ctx) - if node is None: + new_node = self.lookup_qualified(type_aliases[fullname], ctx) + if new_node is None: self.add_fixture_note(fullname, ctx) return None normalized = True if fullname in collections_type_aliases: # Similar, but for types from the collections module like typing.DefaultDict self.add_module_symbol('collections', '__mypy_collections__', False, ctx) - node = self.lookup_qualified(collections_type_aliases[fullname], ctx) + new_node = self.lookup_qualified(collections_type_aliases[fullname], ctx) normalized = True if normalized: - node = SymbolTableNode(node.kind, node.node, - node.mod_id, node.type_override, - normalized=True, alias_tvars=node.alias_tvars) + assert new_node is not None, "Collection node not found" + node = SymbolTableNode(new_node.kind, new_node.node, + new_node.mod_id, new_node.type_override, + normalized=True, alias_tvars=new_node.alias_tvars) return node def add_fixture_note(self, fullname: str, ctx: Context) -> None: @@ -1572,21 +1575,22 @@ def visit_import_all(self, i: ImportAll) -> None: m = self.modules[i_id] self.add_submodules_to_parent_modules(i_id, True) for name, node in m.names.items(): - node = self.normalize_type_alias(node, i) + new_node = self.normalize_type_alias(node, i) # if '__all__' exists, all nodes not included have had module_public set to # False, and we can skip checking '_' because it's been explicitly included. - if node.module_public and (not name.startswith('_') or '__all__' in m.names): + if (new_node and new_node.module_public and + (not name.startswith('_') or '__all__' in m.names)): existing_symbol = self.globals.get(name) if existing_symbol: # Import can redefine a variable. They get special treatment. if self.process_import_over_existing_name( - name, existing_symbol, node, i): + name, existing_symbol, new_node, i): continue - self.add_symbol(name, SymbolTableNode(node.kind, node.node, + self.add_symbol(name, SymbolTableNode(new_node.kind, new_node.node, self.cur_mod_id, - node.type_override, - normalized=node.normalized, - alias_tvars=node.alias_tvars), i) + new_node.type_override, + normalized=new_node.normalized, + alias_tvars=new_node.alias_tvars), i) else: # Don't add any dummy symbols for 'from x import *' if 'x' is unknown. pass @@ -2252,6 +2256,7 @@ def process_namedtuple_definition(self, s: AssignmentStmt) -> None: return # Yes, it's a valid namedtuple definition. Add it to the symbol table. node = self.lookup(name, s) + assert node is not None node.kind = GDEF # TODO locally defined namedtuple node.node = named_tuple @@ -3655,13 +3660,13 @@ def lookup_fully_qualified(self, name: str) -> SymbolTableNode: next_sym = n.names[parts[i]] assert isinstance(next_sym.node, MypyFile) n = next_sym.node - return n.names.get(parts[-1]) + return n.names[parts[-1]] def lookup_fully_qualified_or_none(self, name: str) -> Optional[SymbolTableNode]: """Lookup a fully qualified name. - Assume that the name is defined. This happens in the global namespace -- the local - module namespace is ignored. + Don't assume that the name is defined. This happens in the global namespace -- + the local module namespace is ignored. """ assert '.' in name parts = name.split('.') diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 3119c19a05e3..260f9baa56d5 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -53,7 +53,7 @@ def analyze_type_alias(node: Expression, - lookup_func: Callable[[str, Context], SymbolTableNode], + lookup_func: Callable[[str, Context], Optional[SymbolTableNode]], lookup_fqn_func: Callable[[str], SymbolTableNode], tvar_scope: TypeVarScope, fail_func: Callable[[str, Context], None], @@ -143,7 +143,7 @@ class TypeAnalyser(SyntheticTypeVisitor[Type], AnalyzerPluginInterface): global_scope = True # type: bool def __init__(self, - lookup_func: Callable[[str, Context], SymbolTableNode], + lookup_func: Callable[[str, Context], Optional[SymbolTableNode]], lookup_fqn_func: Callable[[str], SymbolTableNode], tvar_scope: Optional[TypeVarScope], fail_func: Callable[[str, Context], None], @@ -548,7 +548,9 @@ def bind_function_type_variables(self, return [] # We are in third pass, nothing new here if fun_type.variables: for var in fun_type.variables: - var_expr = self.lookup(var.name, var).node + var_node = self.lookup(var.name, var) + assert var_node, "Binding for function type variable not found within function" + var_expr = var_node.node assert isinstance(var_expr, TypeVarExpr) self.tvar_scope.bind(var.name, var_expr) return fun_type.variables @@ -568,8 +570,12 @@ def bind_function_type_variables(self, return defs def is_defined_type_var(self, tvar: str, context: Context) -> bool: - return (self.tvar_scope is not None and - self.tvar_scope.get_binding(self.lookup(tvar, context)) is not None) + if self.tvar_scope is None: + return False + tvar_node = self.lookup(tvar, context) + if not tvar_node: + return False + return self.tvar_scope.get_binding(tvar_node) is not None def anal_array(self, a: List[Type], nested: bool = True) -> List[Type]: res = [] # type: List[Type] @@ -631,7 +637,7 @@ class TypeAnalyserPass3(TypeVisitor[None]): """ def __init__(self, - lookup_func: Callable[[str, Context], SymbolTableNode], + lookup_func: Callable[[str, Context], Optional[SymbolTableNode]], lookup_fqn_func: Callable[[str], SymbolTableNode], fail_func: Callable[[str, Context], None], note_func: Callable[[str, Context], None], @@ -867,7 +873,7 @@ def flatten_tvars(ll: Iterable[List[T]]) -> List[T]: class TypeVariableQuery(TypeQuery[TypeVarList]): def __init__(self, - lookup: Callable[[str, Context], SymbolTableNode], + lookup: Callable[[str, Context], Optional[SymbolTableNode]], scope: 'TypeVarScope', *, include_callables: bool = True, From b105456086c3e12bf73f144811e8ba471b1ef38c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 7 Oct 2017 12:56:44 +0200 Subject: [PATCH 07/17] Fix most signatures and other fixes --- mypy/checkexpr.py | 3 +-- mypy/semanal.py | 53 +++++++++++++++++++++++------------------------ 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index f382052bc5df..68d119fd1448 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -719,8 +719,7 @@ def infer_arg_types_in_context2( Returns the inferred types of *actual arguments*. """ - dummy = None # type: Any - res = [dummy] * len(args) # type: List[Type] + res = [None] * len(args) # type: List[Optional[Type]] for i, actuals in enumerate(formal_to_actual): for ai in actuals: diff --git a/mypy/semanal.py b/mypy/semanal.py index c009e7090521..b12a2b09df5d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -906,7 +906,7 @@ def analyze_typevar_declaration(self, t: Type) -> Optional[TypeVarList]: return tvars return None - def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: + def analyze_unbound_tvar(self, t: Type) -> Optional[Tuple[str, TypeVarExpr]]: if not isinstance(t, UnboundType): return None unbound = t @@ -1249,7 +1249,7 @@ def class_type(self, info: TypeInfo) -> Type: else: return leading_type - def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: + def named_type(self, qualified_name: str, args: Optional[List[Type]] = None) -> Instance: sym = self.lookup_qualified(qualified_name, None) assert sym, "Internal error: attempted to construct unknown type" node = sym.node @@ -1259,7 +1259,8 @@ def named_type(self, qualified_name: str, args: List[Type] = None) -> Instance: return Instance(node, args) return Instance(node, [AnyType(TypeOfAny.special_form)] * len(node.defn.type_vars)) - def named_type_or_none(self, qualified_name: str, args: List[Type] = None) -> Instance: + def named_type_or_none(self, qualified_name: str, + args: Optional[List[Type]] = None) -> Optional[Instance]: sym = self.lookup_fully_qualified_or_none(qualified_name) if not sym: return None @@ -1337,9 +1338,9 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> bool: return False def check_typeddict_classdef(self, defn: ClassDef, - oldfields: List[str] = None) -> Tuple[List[str], - List[Type], - Set[str]]: + oldfields: Optional[List[str]] = None) -> Tuple[List[str], + List[Type], + Set[str]]: TPDICT_CLASS_ERROR = ('Invalid statement in TypedDict definition; ' 'expected "field_name: field_type"') if self.options.python_version < (3, 6): @@ -1622,7 +1623,7 @@ def visit_block(self, b: Block) -> None: self.accept(s) self.block_depth[-1] -= 1 - def visit_block_maybe(self, b: Block) -> None: + def visit_block_maybe(self, b: Optional[Block]) -> None: if b: self.visit_block(b) @@ -1654,16 +1655,11 @@ def anal_type(self, t: Type, *, allow_tuple_literal: bool = False, aliasing: bool = False, third_pass: bool = False) -> Type: - if t: - a = self.type_analyzer( - tvar_scope=tvar_scope, - aliasing=aliasing, - allow_tuple_literal=allow_tuple_literal, - third_pass=third_pass) - return t.accept(a) - - else: - return None + a = self.type_analyzer(tvar_scope=tvar_scope, + aliasing=aliasing, + allow_tuple_literal=allow_tuple_literal, + third_pass=third_pass) + return t.accept(a) def visit_assignment_stmt(self, s: AssignmentStmt) -> None: for lval in s.lvalues: @@ -2260,7 +2256,8 @@ def process_namedtuple_definition(self, s: AssignmentStmt) -> None: node.kind = GDEF # TODO locally defined namedtuple node.node = named_tuple - def check_namedtuple(self, node: Expression, var_name: str = None) -> Optional[TypeInfo]: + def check_namedtuple(self, node: Expression, + var_name: Optional[str] = None) -> Optional[TypeInfo]: """Check if a call defines a namedtuple. The optional var_name argument is the name of the variable to @@ -2280,6 +2277,7 @@ def check_namedtuple(self, node: Expression, var_name: str = None) -> Optional[T fullname = callee.fullname if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): return None + assert fullname items, types, ok = self.parse_namedtuple_args(call, fullname) if not ok: # Error. Construct dummy return value. @@ -2440,7 +2438,7 @@ def add_field(var: Var, is_initialized_in_class: bool = False, def add_method(funcname: str, ret: Type, args: List[Argument], - name: str = None, + name: Optional[str] = None, is_classmethod: bool = False, ) -> None: if is_classmethod: @@ -2513,7 +2511,8 @@ def process_typeddict_definition(self, s: AssignmentStmt) -> None: node.kind = GDEF # TODO locally defined TypedDict node.node = typed_dict - def check_typeddict(self, node: Expression, var_name: str = None) -> Optional[TypeInfo]: + def check_typeddict(self, node: Expression, + var_name: Optional[str] = None) -> Optional[TypeInfo]: """Check if a call defines a TypedDict. The optional var_name argument is the name of the variable to @@ -2533,7 +2532,7 @@ def check_typeddict(self, node: Expression, var_name: str = None) -> Optional[Ty fullname = callee.fullname if fullname != 'mypy_extensions.TypedDict': return None - items, types, total, ok = self.parse_typeddict_args(call, fullname) + items, types, total, ok = self.parse_typeddict_args(call) if not ok: # Error. Construct dummy return value. info = self.build_typeddict_typeinfo('TypedDict', [], [], set()) @@ -2559,8 +2558,7 @@ def check_typeddict(self, node: Expression, var_name: str = None) -> Optional[Ty call.analyzed.set_line(call.line, call.column) return info - def parse_typeddict_args(self, call: CallExpr, - fullname: str) -> Tuple[List[str], List[Type], bool, bool]: + def parse_typeddict_args(self, call: CallExpr) -> Tuple[List[str], List[Type], bool, bool]: # TODO: Share code with check_argument_count in checkexpr.py? args = call.args if len(args) < 2: @@ -2663,7 +2661,7 @@ def check_classvar(self, s: AssignmentStmt) -> None: # Other kinds of member assignments should be already reported self.fail_invalid_classvar(lvalue) - def is_classvar(self, typ: Type) -> bool: + def is_classvar(self, typ: Optional[Type]) -> bool: if not isinstance(typ, UnboundType): return False sym = self.lookup_qualified(typ.name, typ) @@ -2749,7 +2747,8 @@ def process_enum_call(self, s: AssignmentStmt) -> None: node.kind = GDEF # TODO locally defined Enum node.node = enum_call - def check_enum_call(self, node: Expression, var_name: str = None) -> Optional[TypeInfo]: + def check_enum_call(self, node: Expression, + var_name: Optional[str] = None) -> Optional[TypeInfo]: """Check if a call defines an Enum. Example: @@ -4407,7 +4406,7 @@ def fail(self, msg: str, ctx: Context, *, blocker: bool = False) -> None: def fail_blocker(self, msg: str, ctx: Context) -> None: self.fail(msg, ctx, blocker=True) - def builtin_type(self, name: str, args: List[Type] = None) -> Instance: + def builtin_type(self, name: str, args: Optional[List[Type]] = None) -> Instance: names = self.modules['builtins'] sym = names.names[name] node = sym.node @@ -4472,7 +4471,7 @@ def calculate_class_mro(defn: ClassDef, fail: Callable[[str, Context], None]) -> defn.info.fallback_to_any = any(baseinfo.fallback_to_any for baseinfo in defn.info.mro) -def find_duplicate(list: List[T]) -> T: +def find_duplicate(list: List[T]) -> Optional[T]: """If the list has duplicates, return one of the duplicates. Otherwise, return None. From 363407c6115fd0365da8d6e27f91d44a900c3d78 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 7 Oct 2017 16:10:50 +0200 Subject: [PATCH 08/17] Allow optional removal after container 'in' checks --- mypy/checker.py | 21 +++++++++++++++++---- mypy/semanal.py | 36 +++++++++++++++++++++++------------- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 4afb187a2401..8a267387d016 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2702,12 +2702,10 @@ def iterable_item_type(self, instance: Instance) -> Type: def function_type(self, func: FuncBase) -> FunctionLike: return function_type(func, self.named_type('builtins.function')) - # TODO: These next two functions should refer to TypeMap below - def find_isinstance_check(self, n: Expression) -> Tuple[Optional[Dict[Expression, Type]], - Optional[Dict[Expression, Type]]]: + def find_isinstance_check(self, n: Expression) -> 'Tuple[TypeMap, TypeMap]': return find_isinstance_check(n, self.type_map) - def push_type_map(self, type_map: Optional[Dict[Expression, Type]]) -> None: + def push_type_map(self, type_map: 'TypeMap') -> None: if type_map is None: self.binder.unreachable() else: @@ -2874,6 +2872,16 @@ def remove_optional(typ: Type) -> Type: return typ +def is_non_optional_container(tp: Type) -> bool: + # This is only safe for built-in containers, where we know the behavior of __contains__. + if not isinstance(tp, Instance): + return False + if tp.type.fullname() not in ['builtins.list', 'builtins.dict', + 'builtins.set', 'builtins.frozenfet']: + return False + return not is_optional(tp.args[0]) + + def and_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: """Calculate what information we can learn from the truth of (e1 and e2) in terms of the information that we can learn from the truth of e1 and @@ -3020,6 +3028,11 @@ def find_isinstance_check(node: Expression, optional_expr = node.operands[1] if is_overlapping_types(optional_type, comp_type): return {optional_expr: remove_optional(optional_type)}, {} + elif node.operators == ['in']: + item_type = type_map[node.operands[0]] + cont_type = type_map[node.operands[1]] + if is_optional(item_type) and is_non_optional_container(cont_type): + return {node.operands[0]: remove_optional(item_type)}, {} elif isinstance(node, RefExpr): # Restrict the type of the variable to True-ish/False-ish in the if and else branches # respectively diff --git a/mypy/semanal.py b/mypy/semanal.py index b12a2b09df5d..06f2bf68808f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -450,7 +450,7 @@ def prepare_method_signature(self, func: FuncDef) -> None: leading_type = fill_typevars(self.type) func.type = replace_implicit_first_type(functype, leading_type) - def set_original_def(self, previous: Node, new: FuncDef) -> bool: + def set_original_def(self, previous: Optional[Node], new: FuncDef) -> bool: """If 'new' conditionally redefine 'previous', set 'previous' as original We reject straight redefinitions of functions, as they are usually @@ -2650,7 +2650,7 @@ def check_classvar(self, s: AssignmentStmt) -> None: lvalue = s.lvalues[0] if len(s.lvalues) != 1 or not isinstance(lvalue, RefExpr): return - if not self.is_classvar(s.type): + if not s.type or not self.is_classvar(s.type): return if self.is_class_scope() and isinstance(lvalue, NameExpr): node = lvalue.node @@ -2661,7 +2661,7 @@ def check_classvar(self, s: AssignmentStmt) -> None: # Other kinds of member assignments should be already reported self.fail_invalid_classvar(lvalue) - def is_classvar(self, typ: Optional[Type]) -> bool: + def is_classvar(self, typ: Type) -> bool: if not isinstance(typ, UnboundType): return False sym = self.lookup_qualified(typ.name, typ) @@ -3710,12 +3710,13 @@ def is_module_scope(self) -> bool: def add_symbol(self, name: str, node: SymbolTableNode, context: Context) -> None: if self.is_func_scope(): - if name in self.locals[-1]: + localns = self.locals[-1] + assert localns is not None + if name in localns: # Flag redefinition unless this is a reimport of a module. - if not (node.kind == MODULE_REF and - self.locals[-1][name].node == node.node): + if not (node.kind == MODULE_REF and localns[name].node == node.node): self.name_already_defined(name, context) - self.locals[-1][name] = node + localns[name] = node elif self.type: self.type.names[name] = node else: @@ -3733,11 +3734,13 @@ def add_symbol(self, name: str, node: SymbolTableNode, self.globals[name] = node def add_local(self, node: Union[Var, FuncDef, OverloadedFuncDef], ctx: Context) -> None: + localns = self.locals[-1] + assert localns is not None, "Should not add locals outside a function" name = node.name() - if name in self.locals[-1]: + if name in localns: self.name_already_defined(name, ctx) node._fullname = name - self.locals[-1][name] = SymbolTableNode(LDEF, node) + localns[name] = SymbolTableNode(LDEF, node) def add_exports(self, *exps: Expression) -> None: for exp in exps: @@ -4203,7 +4206,7 @@ def visit_decorator(self, dec: Decorator) -> None: for expr in dec.decorators: preserve_type = False if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): - if is_identity_signature(expr.node.type): + if expr.node.type and is_identity_signature(expr.node.type): preserve_type = True if not preserve_type: decorator_preserves_type = False @@ -4297,14 +4300,17 @@ def perform_transform(self, node: Union[Node, SymbolTableNode], transform: Callable[[Type], Type]) -> None: """Apply transform to all types associated with node.""" if isinstance(node, ForStmt): - node.index_type = transform(node.index_type) + if node.index_type: + node.index_type = transform(node.index_type) self.transform_types_in_lvalue(node.index, transform) if isinstance(node, WithStmt): - node.target_type = transform(node.target_type) + if node.target_type: + node.target_type = transform(node.target_type) for n in node.target: if isinstance(n, NameExpr) and isinstance(n.node, Var) and n.node.type: n.node.type = transform(n.node.type) if isinstance(node, (FuncDef, CastExpr, AssignmentStmt, TypeAliasExpr, Var)): + assert node.type, "Scheduled patch for non-existent type" node.type = transform(node.type) if isinstance(node, NewTypeExpr): node.old_type = transform(node.old_type) @@ -4314,14 +4320,17 @@ def perform_transform(self, node: Union[Node, SymbolTableNode], if node.values: node.values = [transform(v) for v in node.values] if isinstance(node, TypedDictExpr): + assert node.info.typeddict_type, "Scheduled patch for non-existent type" node.info.typeddict_type = cast(TypedDictType, transform(node.info.typeddict_type)) if isinstance(node, NamedTupleExpr): + assert node.info.tuple_type, "Scheduled patch for non-existent type" node.info.tuple_type = cast(TupleType, transform(node.info.tuple_type)) if isinstance(node, TypeApplication): node.types = [transform(t) for t in node.types] if isinstance(node, SymbolTableNode): + assert node.type_override, "Scheduled patch for non-existent type" node.type_override = transform(node.type_override) if isinstance(node, TypeInfo): for tvar in node.defn.type_vars: @@ -4347,7 +4356,8 @@ def transform_types_in_lvalue(self, lvalue: Lvalue, if isinstance(lvalue, RefExpr): if isinstance(lvalue.node, Var): var = lvalue.node - var.type = transform(var.type) + if var.type: + var.type = transform(var.type) elif isinstance(lvalue, TupleExpr): for item in lvalue.items: self.transform_types_in_lvalue(item, transform) From eaa6edb0f1ef7c414a8d5b2b84a702ab6f690aeb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 7 Oct 2017 17:16:28 +0200 Subject: [PATCH 09/17] More fixes --- mypy/checker.py | 24 ++++++++++++++++++------ mypy/nodes.py | 4 ++-- mypy/semanal.py | 19 ++++++++++++++----- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 8a267387d016..012d1851b991 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2874,12 +2874,19 @@ def remove_optional(typ: Type) -> Type: def is_non_optional_container(tp: Type) -> bool: # This is only safe for built-in containers, where we know the behavior of __contains__. - if not isinstance(tp, Instance): - return False - if tp.type.fullname() not in ['builtins.list', 'builtins.dict', - 'builtins.set', 'builtins.frozenfet']: - return False - return not is_optional(tp.args[0]) + if isinstance(tp, Instance) and tp.type.fullname() in ['builtins.list', 'builtins.tuple', + 'builtins.dict', 'builtins.set', + 'builtins.frozenfet']: + if not tp.args: + # TODO: make lib-stub contain generic tuple. + return False + return not is_optional(tp.args[0]) and not isinstance(tp.args[0], AnyType) + if isinstance(tp, TupleType): + return all(not is_optional(it) and not isinstance(it, AnyType) for it in tp.items) + if isinstance(tp, TypedDictType): + # TypedDict always has non-optional string keys. + return True + return False def and_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: @@ -3033,6 +3040,11 @@ def find_isinstance_check(node: Expression, cont_type = type_map[node.operands[1]] if is_optional(item_type) and is_non_optional_container(cont_type): return {node.operands[0]: remove_optional(item_type)}, {} + elif node.operators == ['not in']: + item_type = type_map[node.operands[0]] + cont_type = type_map[node.operands[1]] + if is_optional(item_type) and is_non_optional_container(cont_type): + return {}, {node.operands[0]: remove_optional(item_type)} elif isinstance(node, RefExpr): # Restrict the type of the variable to True-ish/False-ish in the if and else branches # respectively diff --git a/mypy/nodes.py b/mypy/nodes.py index 7d38a1a229f3..4dd16613ae0e 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1808,11 +1808,11 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T: class NewTypeExpr(Expression): """NewType expression NewType(...).""" name = None # type: str - old_type = None # type: mypy.types.Type + old_type = None # type: Optional[mypy.types.Type] info = None # type: Optional[TypeInfo] - def __init__(self, name: str, old_type: 'mypy.types.Type', line: int) -> None: + def __init__(self, name: str, old_type: 'Optional[mypy.types.Type]', line: int) -> None: self.name = name self.old_type = old_type diff --git a/mypy/semanal.py b/mypy/semanal.py index 06f2bf68808f..1fbf8defc70c 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1945,7 +1945,8 @@ def is_self_member_ref(self, memberexpr: MemberExpr) -> bool: node = memberexpr.expr.node return isinstance(node, Var) and node.is_self - def check_lvalue_validity(self, node: Union[Expression, SymbolNode], ctx: Context) -> None: + def check_lvalue_validity(self, node: Union[Expression, SymbolNode, None], + ctx: Context) -> None: if isinstance(node, TypeVarExpr): self.fail('Invalid assignment target', ctx) elif isinstance(node, TypeInfo): @@ -2139,6 +2140,7 @@ def process_typevar_declaration(self, s: AssignmentStmt) -> None: # Yes, it's a valid type variable definition! Add it to the symbol table. node = self.lookup(name, s) assert node is not None + assert node.fullname is not None node.kind = TVAR TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) TypeVar.line = call.line @@ -2405,7 +2407,8 @@ def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Typ info.tuple_type = TupleType(types, fallback) def patch() -> None: - # Calculate the correct value type for the fallback Mapping. + # Calculate the correct value type for the fallback tuple. + assert info.tuple_type, "TupleType type deleted before calling the patch" fallback.args[0] = join.join_type_list(list(info.tuple_type.items)) # We can't calculate the complete fallback type until after semantic @@ -2450,7 +2453,9 @@ def add_method(funcname: str, types = [arg.type_annotation for arg in args] items = [arg.variable.name() for arg in args] arg_kinds = [arg.kind for arg in args] - signature = CallableType(types, arg_kinds, items, ret, function_type, + assert None not in types + signature = CallableType(cast(List[Type], types), arg_kinds, items, ret, + function_type, name=name or info.name() + '.' + funcname) signature.variables = [tvd] func = FuncDef(funcname, args, Block([]), typ=signature) @@ -2577,7 +2582,7 @@ def parse_typeddict_args(self, call: CallExpr) -> Tuple[List[str], List[Type], b if not isinstance(args[1], DictExpr): return self.fail_typeddict_arg( "TypedDict() expects a dictionary literal as the second argument", call) - total = True + total = True # type: Optional[bool] if len(args) == 3: total = self.parse_bool(call.args[2]) if total is None: @@ -2593,6 +2598,7 @@ def parse_typeddict_args(self, call: CallExpr) -> Tuple[List[str], List[Type], b for t in types: if has_any_from_unimported_type(t): self.msg.unimported_type_becomes_any("Type of a TypedDict key", t, dictexpr) + assert total is not None return items, types, total, ok def parse_bool(self, expr: Expression) -> Optional[bool]: @@ -2638,6 +2644,7 @@ def build_typeddict_typeinfo(self, name: str, items: List[str], def patch() -> None: # Calculate the correct value type for the fallback Mapping. + assert info.typeddict_type, "TypedDict type deleted before calling the patch" fallback.args[1] = join.join_type_list(list(info.typeddict_type.items.values())) # We can't calculate the complete fallback type until after semantic @@ -3253,7 +3260,7 @@ def translate_dict_call(self, call: CallExpr) -> Optional[DictExpr]: for a in call.args: a.accept(self) return None - expr = DictExpr([(StrExpr(key), value) + expr = DictExpr([(StrExpr(cast(str, key)), value) # since they are all ARG_NAMED for key, value in zip(call.arg_names, call.args)]) expr.set_line(call) expr.accept(self) @@ -3385,6 +3392,7 @@ def visit_index_expr(self, expr: IndexExpr) -> None: # Special form -- subscripting a generic type alias. # Perform the type substitution and create a new alias. res, alias_tvars = self.analyze_alias(expr, allow_unnormalized=self.is_stub_file) + assert res is not None, "Failed analyzing already defined alias" expr.analyzed = TypeAliasExpr(res, alias_tvars, fallback=self.alias_fallback(res), in_runtime=True) expr.analyzed.line = expr.line @@ -4313,6 +4321,7 @@ def perform_transform(self, node: Union[Node, SymbolTableNode], assert node.type, "Scheduled patch for non-existent type" node.type = transform(node.type) if isinstance(node, NewTypeExpr): + assert node.old_type, "Scheduled patch for non-existent type" node.old_type = transform(node.old_type) if isinstance(node, TypeVarExpr): if node.upper_bound: From 03028119f47f264c3809e167e8f599b9253df779 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 7 Oct 2017 23:43:41 +0200 Subject: [PATCH 10/17] More fixes --- mypy/semanal.py | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 1fbf8defc70c..36c29a20449d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -395,6 +395,7 @@ def visit_func_def(self, defn: FuncDef) -> None: self.prepare_method_signature(defn) elif self.is_func_scope(): # Nested function + assert self.locals[-1] is not None, "No locals at function scope" if not defn.is_decorated and not defn.is_overload: if defn.name() in self.locals[-1]: # Redefinition. Conditional redefinition is okay. @@ -430,7 +431,9 @@ def visit_func_def(self, defn: FuncDef) -> None: # has external return type `Awaitable[T]`. defn.type = defn.type.copy_modified( ret_type = self.named_type_or_none('typing.Awaitable', - [defn.type.ret_type])) + [defn.type.ret_type]) or + # We are running tests + self.named_type('typing_full.Awaitable', [defn.type.ret_type])) self.errors.pop_function() def prepare_method_signature(self, func: FuncDef) -> None: @@ -649,9 +652,8 @@ def analyze_function(self, defn: FuncItem) -> None: self.function_stack.pop() def check_classvar_in_signature(self, typ: Type) -> None: - t = None # type: Type if isinstance(typ, Overloaded): - for t in typ.items(): + for t in typ.items(): # type: Type self.check_classvar_in_signature(t) return if not isinstance(typ, CallableType): @@ -710,8 +712,10 @@ def analyze_class_body(self, defn: ClassDef) -> Iterator[bool]: if prohibited in named_tuple_info.names: if nt_names.get(prohibited) is named_tuple_info.names[prohibited]: continue + ctx = named_tuple_info.names[prohibited].node + assert ctx is not None self.fail('Cannot overwrite NamedTuple attribute "{}"'.format(prohibited), - named_tuple_info.names[prohibited].node) + ctx) # Restore the names in the original symbol table. This ensures that the symbol # table contains the field objects created by build_namedtuple_typeinfo. Exclude @@ -779,7 +783,7 @@ def calculate_abstract_status(self, typ: TypeInfo) -> None: # check arbitrarily the first overload item. If the # different items have a different abstract status, there # should be an error reported elsewhere. - func = node.items[0] # type: Node + func = node.items[0] # type: Optional[Node] else: func = node if isinstance(func, Decorator): @@ -799,7 +803,7 @@ def setup_type_promotion(self, defn: ClassDef) -> None: This includes things like 'int' being compatible with 'float'. """ - promote_target = None # type: Type + promote_target = None # type: Optional[Type] for decorator in defn.decorators: if isinstance(decorator, CallExpr): analyzed = decorator.analyzed @@ -913,7 +917,7 @@ def analyze_unbound_tvar(self, t: Type) -> Optional[Tuple[str, TypeVarExpr]]: sym = self.lookup_qualified(unbound.name, unbound) if sym is None or sym.kind != TVAR: return None - elif not self.tvar_scope.allow_binding(sym.fullname): + elif sym.fullname and not self.tvar_scope.allow_binding(sym.fullname): # It's bound by our type variable scope return None else: @@ -1158,6 +1162,7 @@ def expr_to_analyzed_type(self, expr: Expression) -> Type: # Some form of namedtuple is the only valid type that looks like a call # expression. This isn't a valid type. raise TypeTranslationError() + assert info.tuple_type, "NamedTuple without tuple type" fallback = Instance(info, []) return TupleType(info.tuple_type.items, fallback=fallback) typ = expr_to_unanalyzed_type(expr) @@ -1198,11 +1203,12 @@ def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool: def analyze_metaclass(self, defn: ClassDef) -> None: if defn.metaclass: + metaclass_name = None if isinstance(defn.metaclass, NameExpr): metaclass_name = defn.metaclass.name elif isinstance(defn.metaclass, MemberExpr): metaclass_name = get_member_expr_fullname(defn.metaclass) - else: + if metaclass_name is None: self.fail("Dynamic metaclass not supported for '%s'" % defn.name, defn.metaclass) return sym = self.lookup_qualified(metaclass_name, defn.metaclass) @@ -1250,7 +1256,7 @@ def class_type(self, info: TypeInfo) -> Type: return leading_type def named_type(self, qualified_name: str, args: Optional[List[Type]] = None) -> Instance: - sym = self.lookup_qualified(qualified_name, None) + sym = self.lookup_qualified(qualified_name, Context()) assert sym, "Internal error: attempted to construct unknown type" node = sym.node assert isinstance(node, TypeInfo) @@ -1378,7 +1384,7 @@ def check_typeddict_classdef(self, defn: ClassDef, elif not isinstance(stmt.rvalue, TempNode): # x: int assigns rvalue to TempNode(AnyType()) self.fail('Right hand side values are not supported in TypedDict', stmt) - total = True + total = True # type: Optional[bool] if 'total' in defn.keywords: total = self.parse_bool(defn.keywords['total']) if total is None: @@ -1845,7 +1851,7 @@ def analyze_lvalue(self, lval: Lvalue, nested: bool = False, # Since the is_def flag is set, this must have been analyzed # already in the first pass and added to the symbol table. assert lval.node.name() in self.globals - elif (self.is_func_scope() and lval.name not in self.locals[-1] and + elif (self.locals[-1] is not None and lval.name not in self.locals[-1] and lval.name not in self.global_decls[-1] and lval.name not in self.nonlocal_decls[-1]): # Define new local name. @@ -2279,7 +2285,6 @@ def check_namedtuple(self, node: Expression, fullname = callee.fullname if fullname not in ('collections.namedtuple', 'typing.NamedTuple'): return None - assert fullname items, types, ok = self.parse_namedtuple_args(call, fullname) if not ok: # Error. Construct dummy return value. @@ -3718,13 +3723,12 @@ def is_module_scope(self) -> bool: def add_symbol(self, name: str, node: SymbolTableNode, context: Context) -> None: if self.is_func_scope(): - localns = self.locals[-1] - assert localns is not None - if name in localns: + assert self.locals[-1] is not None + if name in self.locals[-1]: # Flag redefinition unless this is a reimport of a module. - if not (node.kind == MODULE_REF and localns[name].node == node.node): + if not (node.kind == MODULE_REF and self.locals[-1][name].node == node.node): self.name_already_defined(name, context) - localns[name] = node + self.locals[-1][name] = node elif self.type: self.type.names[name] = node else: @@ -3742,13 +3746,12 @@ def add_symbol(self, name: str, node: SymbolTableNode, self.globals[name] = node def add_local(self, node: Union[Var, FuncDef, OverloadedFuncDef], ctx: Context) -> None: - localns = self.locals[-1] - assert localns is not None, "Should not add locals outside a function" + assert self.locals[-1] is not None, "Should not add locals outside a function" name = node.name() - if name in localns: + if name in self.locals[-1]: self.name_already_defined(name, ctx) node._fullname = name - localns[name] = SymbolTableNode(LDEF, node) + self.locals[-1][name] = SymbolTableNode(LDEF, node) def add_exports(self, *exps: Expression) -> None: for exp in exps: From 638735975506ff529eb9e19b97636b8a515eb0c3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 8 Oct 2017 02:46:28 +0200 Subject: [PATCH 11/17] Generalize/simplify container check --- mypy/checker.py | 50 ++++++++++++++++++++++++------------------------- mypy/semanal.py | 3 ++- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 012d1851b991..e404020cf5d6 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -53,7 +53,7 @@ from mypy.erasetype import erase_typevars from mypy.expandtype import expand_type, expand_type_by_instance from mypy.visitor import NodeVisitor -from mypy.join import join_types +from mypy.join import join_types, join_type_list from mypy.treetransform import TransformVisitor from mypy.binder import ConditionalTypeBinder, get_declaration from mypy.meet import is_overlapping_types @@ -2872,21 +2872,22 @@ def remove_optional(typ: Type) -> Type: return typ -def is_non_optional_container(tp: Type) -> bool: - # This is only safe for built-in containers, where we know the behavior of __contains__. - if isinstance(tp, Instance) and tp.type.fullname() in ['builtins.list', 'builtins.tuple', - 'builtins.dict', 'builtins.set', - 'builtins.frozenfet']: - if not tp.args: - # TODO: make lib-stub contain generic tuple. - return False - return not is_optional(tp.args[0]) and not isinstance(tp.args[0], AnyType) - if isinstance(tp, TupleType): - return all(not is_optional(it) and not isinstance(it, AnyType) for it in tp.items) - if isinstance(tp, TypedDictType): +def builtin_item_type(tp: Type) -> Optional[Type]: + # This is only OK for built-in containers, where we know the behavior of __contains__. + if isinstance(tp, Instance): + if tp.type.fullname() in ['builtins.list', 'builtins.tuple', 'builtins.dict', + 'builtins.set', 'builtins.frozenfet']: + if not tp.args: + # TODO: make lib-stub/builtins.pyi define generic tuple. + return None + if not isinstance(tp.args[0], AnyType): + return tp.args[0] + elif isinstance(tp, TupleType) and all(not isinstance(it, AnyType) for it in tp.items): + return join_type_list(tp.items) + elif isinstance(tp, TypedDictType) and tp.fallback.type.fullname() == 'typing.Mapping': # TypedDict always has non-optional string keys. - return True - return False + return tp.fallback.args[0] + return None def and_conditional_maps(m1: TypeMap, m2: TypeMap) -> TypeMap: @@ -3001,6 +3002,15 @@ def find_isinstance_check(node: Expression, if literal(expr) == LITERAL_TYPE: vartype = type_map[expr] return conditional_callable_type_map(expr, vartype) + elif isinstance(node, ComparisonExpr) and node.operators in [['in'], ['not in']]: + expr = node.operands[0] + cont_type = type_map[node.operands[1]] + item_type = builtin_item_type(cont_type) + if item_type and literal(expr) == LITERAL_TYPE and not is_literal_none(expr): + if node.operators == ['in']: + return {expr: item_type}, {} + if node.operators == ['not in']: + return {}, {expr: item_type} elif isinstance(node, ComparisonExpr) and experiments.STRICT_OPTIONAL: # Check for `x is None` and `x is not None`. is_not = node.operators == ['is not'] @@ -3035,16 +3045,6 @@ def find_isinstance_check(node: Expression, optional_expr = node.operands[1] if is_overlapping_types(optional_type, comp_type): return {optional_expr: remove_optional(optional_type)}, {} - elif node.operators == ['in']: - item_type = type_map[node.operands[0]] - cont_type = type_map[node.operands[1]] - if is_optional(item_type) and is_non_optional_container(cont_type): - return {node.operands[0]: remove_optional(item_type)}, {} - elif node.operators == ['not in']: - item_type = type_map[node.operands[0]] - cont_type = type_map[node.operands[1]] - if is_optional(item_type) and is_non_optional_container(cont_type): - return {}, {node.operands[0]: remove_optional(item_type)} elif isinstance(node, RefExpr): # Restrict the type of the variable to True-ish/False-ish in the if and else branches # respectively diff --git a/mypy/semanal.py b/mypy/semanal.py index 36c29a20449d..dffc6db116aa 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3726,7 +3726,8 @@ def add_symbol(self, name: str, node: SymbolTableNode, assert self.locals[-1] is not None if name in self.locals[-1]: # Flag redefinition unless this is a reimport of a module. - if not (node.kind == MODULE_REF and self.locals[-1][name].node == node.node): + if not (node.kind == MODULE_REF and + self.locals[-1][name].node == node.node): self.name_already_defined(name, context) self.locals[-1][name] = node elif self.type: From 424334ae52fa952cecc4e158437e4c08fd8b062d Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 8 Oct 2017 10:33:12 +0200 Subject: [PATCH 12/17] Address CR --- mypy/checker.py | 2 +- mypy/semanal.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index e404020cf5d6..64e90d000e24 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2876,7 +2876,7 @@ def builtin_item_type(tp: Type) -> Optional[Type]: # This is only OK for built-in containers, where we know the behavior of __contains__. if isinstance(tp, Instance): if tp.type.fullname() in ['builtins.list', 'builtins.tuple', 'builtins.dict', - 'builtins.set', 'builtins.frozenfet']: + 'builtins.set', 'builtins.frozenset']: if not tp.args: # TODO: make lib-stub/builtins.pyi define generic tuple. return None diff --git a/mypy/semanal.py b/mypy/semanal.py index dffc6db116aa..b443dd8748de 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -429,11 +429,11 @@ def visit_func_def(self, defn: FuncDef) -> None: else: # A coroutine defined as `async def foo(...) -> T: ...` # has external return type `Awaitable[T]`. - defn.type = defn.type.copy_modified( - ret_type = self.named_type_or_none('typing.Awaitable', - [defn.type.ret_type]) or - # We are running tests - self.named_type('typing_full.Awaitable', [defn.type.ret_type])) + ret_type = self.named_type_or_none('typing.Awaitable', [defn.type.ret_type]) + if ret_type is None: + # We are running tests. + ret_type = self.named_type('typing_full.Awaitable', [defn.type.ret_type]) + defn.type = defn.type.copy_modified(ret_type=ret_type) self.errors.pop_function() def prepare_method_signature(self, func: FuncDef) -> None: From 80c1b38fd6d28c0ac76f1cad3cc01f640a8dfa6b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 8 Oct 2017 13:29:04 +0200 Subject: [PATCH 13/17] Sync with #4072 --- mypy/checker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 64e90d000e24..6cb9311ab8f5 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2886,7 +2886,10 @@ def builtin_item_type(tp: Type) -> Optional[Type]: return join_type_list(tp.items) elif isinstance(tp, TypedDictType) and tp.fallback.type.fullname() == 'typing.Mapping': # TypedDict always has non-optional string keys. - return tp.fallback.args[0] + if tp.fallback.type.fullname() == 'typing.Mapping': + return tp.fallback.args[0] + elif tp.fallback.type.bases[0].type.fullname() == 'typing.Mapping': + return tp.fallback.type.bases[0].args[0] return None From 2a96e50c8cf324ac5c96e9a677f8376bd8740e56 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 8 Oct 2017 13:37:03 +0200 Subject: [PATCH 14/17] Sync with #4072 for realz --- mypy/checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index 6cb9311ab8f5..06b9f8d4e9cb 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -2884,7 +2884,7 @@ def builtin_item_type(tp: Type) -> Optional[Type]: return tp.args[0] elif isinstance(tp, TupleType) and all(not isinstance(it, AnyType) for it in tp.items): return join_type_list(tp.items) - elif isinstance(tp, TypedDictType) and tp.fallback.type.fullname() == 'typing.Mapping': + elif isinstance(tp, TypedDictType): # TypedDict always has non-optional string keys. if tp.fallback.type.fullname() == 'typing.Mapping': return tp.fallback.args[0] From a877957d2aee8a2f2fe99cab7c2cd5f10e7d5a25 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 13 Oct 2017 16:40:37 +0200 Subject: [PATCH 15/17] Fix merge and sync with #4072 --- mypy/checker.py | 29 +++++++++++++++++------------ mypy/semanal_pass3.py | 18 +++++++++++++----- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index d745c10cacca..36343c29f845 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -53,7 +53,7 @@ from mypy.erasetype import erase_typevars from mypy.expandtype import expand_type, expand_type_by_instance from mypy.visitor import NodeVisitor -from mypy.join import join_types, join_type_list +from mypy.join import join_types from mypy.treetransform import TransformVisitor from mypy.binder import ConditionalTypeBinder, get_declaration from mypy.meet import is_overlapping_types @@ -2957,12 +2957,12 @@ def builtin_item_type(tp: Type) -> Optional[Type]: if tp.type.fullname() in ['builtins.list', 'builtins.tuple', 'builtins.dict', 'builtins.set', 'builtins.frozenset']: if not tp.args: - # TODO: make lib-stub/builtins.pyi define generic tuple. + # TODO: fix tuple in lib-stub/builtins.pyi (it should be generic). return None if not isinstance(tp.args[0], AnyType): return tp.args[0] elif isinstance(tp, TupleType) and all(not isinstance(it, AnyType) for it in tp.items): - return join_type_list(tp.items) + return UnionType.make_simplified_union(tp.items) # this type is not externally visible elif isinstance(tp, TypedDictType): # TypedDict always has non-optional string keys. if tp.fallback.type.fullname() == 'typing.Mapping': @@ -3084,15 +3084,6 @@ def find_isinstance_check(node: Expression, if literal(expr) == LITERAL_TYPE: vartype = type_map[expr] return conditional_callable_type_map(expr, vartype) - elif isinstance(node, ComparisonExpr) and node.operators in [['in'], ['not in']]: - expr = node.operands[0] - cont_type = type_map[node.operands[1]] - item_type = builtin_item_type(cont_type) - if item_type and literal(expr) == LITERAL_TYPE and not is_literal_none(expr): - if node.operators == ['in']: - return {expr: item_type}, {} - if node.operators == ['not in']: - return {}, {expr: item_type} elif isinstance(node, ComparisonExpr) and experiments.STRICT_OPTIONAL: # Check for `x is None` and `x is not None`. is_not = node.operators == ['is not'] @@ -3127,6 +3118,20 @@ def find_isinstance_check(node: Expression, optional_expr = node.operands[1] if is_overlapping_types(optional_type, comp_type): return {optional_expr: remove_optional(optional_type)}, {} + elif node.operators in [['in'], ['not in']]: + expr = node.operands[0] + left_type = type_map[expr] + right_type = builtin_item_type(type_map[node.operands[1]]) + right_ok = right_type and (not is_optional(right_type) and + (not isinstance(right_type, Instance) or + right_type.type.fullname() != 'builtins.object')) + if (right_type and right_ok and is_optional(left_type) and + literal(expr) == LITERAL_TYPE and not is_literal_none(expr) and + is_overlapping_types(left_type, right_type)): + if node.operators == ['in']: + return {expr: remove_optional(left_type)}, {} + if node.operators == ['not in']: + return {}, {expr: remove_optional(left_type)} elif isinstance(node, RefExpr): # Restrict the type of the variable to True-ish/False-ish in the if and else branches # respectively diff --git a/mypy/semanal_pass3.py b/mypy/semanal_pass3.py index 9e9ba0d0deff..859a6f033dcc 100644 --- a/mypy/semanal_pass3.py +++ b/mypy/semanal_pass3.py @@ -152,7 +152,7 @@ def visit_decorator(self, dec: Decorator) -> None: for expr in dec.decorators: preserve_type = False if isinstance(expr, RefExpr) and isinstance(expr.node, FuncDef): - if is_identity_signature(expr.node.type): + if expr.node.type and is_identity_signature(expr.node.type): preserve_type = True if not preserve_type: decorator_preserves_type = False @@ -246,16 +246,20 @@ def perform_transform(self, node: Union[Node, SymbolTableNode], transform: Callable[[Type], Type]) -> None: """Apply transform to all types associated with node.""" if isinstance(node, ForStmt): - node.index_type = transform(node.index_type) + if node.index_type: + node.index_type = transform(node.index_type) self.transform_types_in_lvalue(node.index, transform) if isinstance(node, WithStmt): - node.target_type = transform(node.target_type) + if node.target_type: + node.target_type = transform(node.target_type) for n in node.target: if isinstance(n, NameExpr) and isinstance(n.node, Var) and n.node.type: n.node.type = transform(n.node.type) if isinstance(node, (FuncDef, CastExpr, AssignmentStmt, TypeAliasExpr, Var)): + assert node.type, "Scheduled patch for non-existent type" node.type = transform(node.type) if isinstance(node, NewTypeExpr): + assert node.old_type, "Scheduled patch for non-existent type" node.old_type = transform(node.old_type) if isinstance(node, TypeVarExpr): if node.upper_bound: @@ -263,14 +267,17 @@ def perform_transform(self, node: Union[Node, SymbolTableNode], if node.values: node.values = [transform(v) for v in node.values] if isinstance(node, TypedDictExpr): + assert node.info.typeddict_type, "Scheduled patch for non-existent type" node.info.typeddict_type = cast(TypedDictType, transform(node.info.typeddict_type)) if isinstance(node, NamedTupleExpr): + assert node.info.tuple_type, "Scheduled patch for non-existent type" node.info.tuple_type = cast(TupleType, transform(node.info.tuple_type)) if isinstance(node, TypeApplication): node.types = [transform(t) for t in node.types] if isinstance(node, SymbolTableNode): + assert node.type_override, "Scheduled patch for non-existent type" node.type_override = transform(node.type_override) if isinstance(node, TypeInfo): for tvar in node.defn.type_vars: @@ -296,7 +303,8 @@ def transform_types_in_lvalue(self, lvalue: Lvalue, if isinstance(lvalue, RefExpr): if isinstance(lvalue.node, Var): var = lvalue.node - var.type = transform(var.type) + if var.type: + var.type = transform(var.type) elif isinstance(lvalue, TupleExpr): for item in lvalue.items: self.transform_types_in_lvalue(item, transform) @@ -355,7 +363,7 @@ def fail(self, msg: str, ctx: Context, *, blocker: bool = False) -> None: def fail_blocker(self, msg: str, ctx: Context) -> None: self.fail(msg, ctx, blocker=True) - def builtin_type(self, name: str, args: List[Type] = None) -> Instance: + def builtin_type(self, name: str, args: Optional[List[Type]] = None) -> Instance: names = self.modules['builtins'] sym = names.names[name] node = sym.node From 9969ecb46bd3c6c2f27c4a25eb2f8ebaa2de73d8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 13 Oct 2017 16:48:44 +0200 Subject: [PATCH 16/17] Fix typeshed commit --- typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typeshed b/typeshed index 78587dc89597..c10dc67ad256 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 78587dc89597fa2019e037700f0007607933ce73 +Subproject commit c10dc67ad2569fa0c8de4124461b298e23e2f4d7 From 25493869c6f9ab754f930382e08167c1239b21ab Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 21 Oct 2017 00:27:41 +0200 Subject: [PATCH 17/17] Address CR --- mypy/semanal.py | 12 +++++------- test-data/unit/check-classes.test | 5 ++++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 3450c5b8a673..2d4b3c5b923c 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -380,7 +380,7 @@ def visit_func_def(self, defn: FuncDef) -> None: if not self.set_original_def(n, defn): self.name_already_defined(defn.name(), defn) self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) - self.prepare_method_signature(defn) + self.prepare_method_signature(defn, self.type) elif self.is_func_scope(): # Nested function assert self.locals[-1] is not None, "No locals at function scope" @@ -395,8 +395,7 @@ def visit_func_def(self, defn: FuncDef) -> None: else: # Top-level function if not defn.is_decorated and not defn.is_overload: - symbol = self.globals.get(defn.name()) - assert symbol, "Global function not found in globals" + symbol = self.globals[defn.name()] if isinstance(symbol.node, FuncDef) and symbol.node != defn: # This is redefinition. Conditional redefinition is okay. if not self.set_original_def(symbol.node, defn): @@ -424,10 +423,9 @@ def visit_func_def(self, defn: FuncDef) -> None: defn.type = defn.type.copy_modified(ret_type=ret_type) self.errors.pop_function() - def prepare_method_signature(self, func: FuncDef) -> None: + def prepare_method_signature(self, func: FuncDef, info: TypeInfo) -> None: """Check basic signature validity and tweak annotation of self/cls argument.""" # Only non-static methods are special. - assert self.type is not None, "This method should be only called inside a class" functype = func.type if not func.is_static: if not func.arguments: @@ -436,9 +434,9 @@ def prepare_method_signature(self, func: FuncDef) -> None: self_type = functype.arg_types[0] if isinstance(self_type, AnyType): if func.is_class or func.name() in ('__new__', '__init_subclass__'): - leading_type = self.class_type(self.type) + leading_type = self.class_type(info) else: - leading_type = fill_typevars(self.type) + leading_type = fill_typevars(info) func.type = replace_implicit_first_type(functype, leading_type) def set_original_def(self, previous: Optional[Node], new: FuncDef) -> bool: diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index f5ea5d24a071..2379c71d9983 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -2904,10 +2904,13 @@ class B(object, A): # E: Cannot determine consistent method resolution order (MR __iter__ = readlines [case testDynamicMetaclass] - class C(metaclass=int()): # E: Dynamic metaclass not supported for 'C' pass +[case testDynamicMetaclassCrash] +class C(metaclass=int().x): # E: Dynamic metaclass not supported for 'C' + pass + [case testVariableSubclass] class A: a = 1 # type: int