From 249d70fc858c982087a59f1ca29f03f845ade9c4 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 11 Nov 2016 14:56:24 -0800 Subject: [PATCH 01/28] Implement Callable[[Arg('name', Type)], ret] syntax --- extensions/mypy_extensions.py | 25 +++++ mypy/erasetype.py | 4 +- mypy/expandtype.py | 4 +- mypy/exprtotype.py | 45 +++++++- mypy/fastparse.py | 8 +- mypy/fixup.py | 4 +- mypy/indirection.py | 2 +- mypy/join.py | 4 +- mypy/meet.py | 4 +- mypy/parsetype.py | 112 ++++++++++++++++++-- mypy/sametypes.py | 4 +- mypy/semanal.py | 6 +- mypy/subtypes.py | 4 +- mypy/typeanal.py | 14 +-- mypy/types.py | 48 ++++++--- test-data/unit/check-functions.test | 61 +++++++++++ test-data/unit/fixtures/dict.pyi | 3 + test-data/unit/lib-stub/mypy_extensions.pyi | 22 +++- test-data/unit/lib-stub/typing.pyi | 5 +- test-data/unit/parse.test | 8 +- test-data/unit/semanal-errors.test | 2 +- 21 files changed, 329 insertions(+), 60 deletions(-) diff --git a/extensions/mypy_extensions.py b/extensions/mypy_extensions.py index 248da3de4aae..33f048b5f10f 100644 --- a/extensions/mypy_extensions.py +++ b/extensions/mypy_extensions.py @@ -5,6 +5,8 @@ from mypy_extensions import TypedDict """ +from typing import Any + # NOTE: This module must support Python 2.7 in addition to Python 3.x import sys @@ -88,3 +90,26 @@ class Point2D(TypedDict): The latter syntax is only supported in Python 3.6+, while two other syntax forms work for Python 2.7 and 3.2+ """ + new_dict.__name__ = typename + new_dict.__supertype__ = dict + return new_dict + +class Arg(object): + def __init__(name=None, typ=Any, keyword_only=False): + self.name = name + self.typ = typ + self.named_only = named_only + +class DefaultArg(object): + def __init__(name=None, typ=Any, keyword_only=False): + self.name = name + self.typ = typ + self.named_only = named_only + +class StarArg(object): + def __init__(typ=Any) -> None: + self.typ = typ + +class KwArg(object): + def __init__(typ=Any) -> None: + self.typ = typ diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 47a0d7241bb2..559b37ee4d02 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -3,7 +3,7 @@ from mypy.types import ( Type, TypeVisitor, UnboundType, ErrorType, AnyType, Void, NoneTyp, TypeVarId, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, - ErasedType, PartialType, DeletedType, TypeTranslator, TypeList, UninhabitedType, TypeType + ErasedType, PartialType, DeletedType, TypeTranslator, ArgumentList, UninhabitedType, TypeType ) from mypy import experiments @@ -32,7 +32,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: def visit_error_type(self, t: ErrorType) -> Type: return t - def visit_type_list(self, t: TypeList) -> Type: + def visit_type_list(self, t: ArgumentList) -> Type: assert False, 'Not supported' def visit_any(self, t: AnyType) -> Type: diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 785c77d005e9..9576820757b9 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -3,7 +3,7 @@ from mypy.types import ( Type, Instance, CallableType, TypeVisitor, UnboundType, ErrorType, AnyType, Void, NoneTyp, TypeVarType, Overloaded, TupleType, TypedDictType, UnionType, - ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId + ErasedType, ArgumentList, PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId ) @@ -42,7 +42,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: def visit_error_type(self, t: ErrorType) -> Type: return t - def visit_type_list(self, t: TypeList) -> Type: + def visit_type_list(self, t: ArgumentList) -> Type: assert False, 'Not supported' def visit_any(self, t: AnyType) -> Type: diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index abc091a5ddc7..c196cdc5f925 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -2,10 +2,11 @@ from mypy.nodes import ( Expression, NameExpr, MemberExpr, IndexExpr, TupleExpr, - ListExpr, StrExpr, BytesExpr, UnicodeExpr, EllipsisExpr + ListExpr, StrExpr, BytesExpr, UnicodeExpr, EllipsisExpr, CallExpr, + ARG_POS, ARG_NAMED, ) from mypy.parsetype import parse_str_as_type, TypeParseError -from mypy.types import Type, UnboundType, TypeList, EllipsisType +from mypy.types import Type, UnboundType, ArgumentList, EllipsisType, AnyType class TypeTranslationError(Exception): @@ -15,7 +16,7 @@ class TypeTranslationError(Exception): def expr_to_unanalyzed_type(expr: Expression) -> Type: """Translate an expression to the corresponding type. - The result is not semantically analyzed. It can be UnboundType or TypeList. + The result is not semantically analyzed. It can be UnboundType or ArgumentList. Raise TypeTranslationError if the expression cannot represent a type. """ if isinstance(expr, NameExpr): @@ -43,9 +44,47 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: else: raise TypeTranslationError() elif isinstance(expr, ListExpr): +<<<<<<< HEAD return TypeList([expr_to_unanalyzed_type(t) for t in expr.items], line=expr.line, column=expr.column) elif isinstance(expr, (StrExpr, BytesExpr, UnicodeExpr)): +======= + types = [] # type: List[Type] + names = [] # type: List[Optional[str]] + kinds = [] # type: List[int] + for it in expr.items: + if isinstance(expr_to_unanalyzed_type(it), CallExpr): + if not isinstance(it.callee, NameExpr): + raise TypeTranslationError() + arg_const = it.callee.name + if arg_const == 'Arg': + if len(it.args) > 0: + name = it.args[0] + if not isinstance(name, StrLit): + raise TypeTranslationError() + names.append(name.parsed()) + else: + names.append(None) + + if len(it.args) > 1: + typ = it.args[1] + types.append(expr_to_unanalyzed_type(typ)) + else: + types.append(AnyType) + + if len(it.args) > 2: + kinds.append(ARG_NAMED) + else: + kinds.append(ARG_POS) + + else: + types.append(expr_to_unanalyzed_type(it)) + names.append(None) + kinds.append(ARG_POS) + return ArgumentList(types, names, kinds, + line=expr.line) + elif isinstance(expr, (StrExpr, BytesExpr)): +>>>>>>> c45f8c6... Implement Callable[[Arg('name', Type)], ret] syntax # Parse string literal type. try: result = parse_str_as_type(expr.value, expr.line) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 97e916c4b028..7d0d58d4fd2a 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -19,7 +19,7 @@ ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR2 ) from mypy.types import ( - Type, CallableType, AnyType, UnboundType, TupleType, TypeList, EllipsisType, + Type, CallableType, AnyType, UnboundType, TupleType, ArgumentList, EllipsisType, ) from mypy import defaults from mypy import experiments @@ -911,7 +911,11 @@ def visit_Ellipsis(self, n: ast35.Ellipsis) -> Type: # List(expr* elts, expr_context ctx) def visit_List(self, n: ast35.List) -> Type: - return TypeList(self.translate_expr_list(n.elts), line=self.line) + return ArgumentList( + self.translate_expr_list(n.elts), + [None]*len(n.elts), + [0]*len(n.elts), + line=self.line) class TypeCommentParseError(Exception): diff --git a/mypy/fixup.py b/mypy/fixup.py index 8375b9fa58ae..e95f606a2fb4 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -10,7 +10,7 @@ ) from mypy.types import ( CallableType, EllipsisType, Instance, Overloaded, TupleType, TypedDictType, - TypeList, TypeVarType, UnboundType, UnionType, TypeVisitor, + ArgumentList, TypeVarType, UnboundType, UnionType, TypeVisitor, TypeType ) from mypy.visitor import NodeVisitor @@ -203,7 +203,7 @@ def visit_typeddict_type(self, tdt: TypedDictType) -> None: if tdt.fallback is not None: tdt.fallback.accept(self) - def visit_type_list(self, tl: TypeList) -> None: + def visit_type_list(self, tl: ArgumentList) -> None: for t in tl.items: t.accept(self) diff --git a/mypy/indirection.py b/mypy/indirection.py index b36d999fd19f..7d8f15194fca 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -42,7 +42,7 @@ def _visit(self, *typs: types.Type) -> Set[str]: def visit_unbound_type(self, t: types.UnboundType) -> Set[str]: return self._visit(*t.args) - def visit_type_list(self, t: types.TypeList) -> Set[str]: + def visit_type_list(self, t: types.ArgumentList) -> Set[str]: return self._visit(*t.items) def visit_error_type(self, t: types.ErrorType) -> Set[str]: diff --git a/mypy/join.py b/mypy/join.py index d14b83ded9fb..6f403fbbd362 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -5,7 +5,7 @@ from mypy.types import ( Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType, - ErrorType, TypeVarType, CallableType, TupleType, TypedDictType, ErasedType, TypeList, + ErrorType, TypeVarType, CallableType, TupleType, TypedDictType, ErasedType, ArgumentList, UnionType, FunctionLike, Overloaded, PartialType, DeletedType, UninhabitedType, TypeType, true_or_false ) @@ -115,7 +115,7 @@ def visit_union_type(self, t: UnionType) -> Type: def visit_error_type(self, t: ErrorType) -> Type: return t - def visit_type_list(self, t: TypeList) -> Type: + def visit_type_list(self, t: ArgumentList) -> Type: assert False, 'Not supported' def visit_any(self, t: AnyType) -> Type: diff --git a/mypy/meet.py b/mypy/meet.py index 7aa479c0eefc..e6124805a84e 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -4,7 +4,7 @@ from mypy.join import is_similar_callables, combine_similar_callables, join_type_list from mypy.types import ( Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, - Instance, CallableType, TupleType, TypedDictType, ErasedType, TypeList, UnionType, PartialType, + Instance, CallableType, TupleType, TypedDictType, ErasedType, ArgumentList, UnionType, PartialType, DeletedType, UninhabitedType, TypeType ) from mypy.subtypes import is_equivalent, is_subtype @@ -134,7 +134,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: def visit_error_type(self, t: ErrorType) -> Type: return t - def visit_type_list(self, t: TypeList) -> Type: + def visit_type_list(self, t: ArgumentList) -> Type: assert False, 'Not supported' def visit_any(self, t: AnyType) -> Type: diff --git a/mypy/parsetype.py b/mypy/parsetype.py index 1af035221abe..75f6d55a3c71 100644 --- a/mypy/parsetype.py +++ b/mypy/parsetype.py @@ -3,8 +3,8 @@ from typing import List, Tuple, Union, Optional from mypy.types import ( - Type, UnboundType, TupleType, TypeList, CallableType, StarType, - EllipsisType + Type, UnboundType, TupleType, ArgumentList, CallableType, StarType, + EllipsisType, AnyType ) from mypy.lex import Token, Name, StrLit, lex from mypy import nodes @@ -57,7 +57,7 @@ def parse_type(self) -> Type: if isinstance(t, Name): return self.parse_named_type() elif t.string == '[': - return self.parse_type_list() + return self.parse_argument_list() elif t.string == '*': return self.parse_star_type() elif t.string == '...': @@ -102,19 +102,114 @@ def parse_types(self) -> Type: type = TupleType(items, None, type.line, implicit=True) return type - def parse_type_list(self) -> TypeList: + def parse_argument_spec(self) -> Tuple[Type, Optional[str], int]: + current = self.current_token() + nxt = self.next_token() + # This is a small recreation of a subset of parsing a CallExpr; just + # enough to parse what happens in an arugment list. + # TODO: Doesn't handle an explicit name of None yet. + if isinstance(current, Name) and nxt is not None and nxt.string == '(': + arg_const = self.expect_type(Name).string + name = None # type: Optional[str] + typ = AnyType # type: Type + if arg_const == 'Arg': + kind = nodes.ARG_POS + name, typ, keyword_only = self.parse_arguments( + read_name = True, + read_typ = True, + read_keyword_only = True) + if keyword_only: + kind = nodes.ARG_NAMED + elif arg_const == 'DefaultArg': + kind = nodes.ARG_OPT + name, typ, keyword_only = self.parse_arguments( + read_name = True, + read_typ = True, + read_keyword_only = True) + if keyword_only: + kind = nodes.ARG_NAMED_OPT + elif arg_const == 'StarArg': + # Takes one type + kind = nodes.ARG_STAR + _, typ, _ = self.parse_arguments( + read_name = False, + read_typ = True, + read_keyword_only = False) + elif arg_const == 'KwArg': + # Takes one type + kind = nodes.ARG_STAR2 + _, typ, _ = self.parse_arguments( + read_name = False, + read_typ = True, + read_keyword_only = False) + else: + self.parse_error() + return typ, name, kind + else: + return self.parse_type(), None, nodes.ARG_POS + + def parse_arguments(self, + *, + read_name: bool, + read_typ: bool, + read_keyword_only: bool) -> Tuple[ + Optional[str], + Optional[Type], + bool]: + self.expect('(') + name = None + typ = AnyType + keyword_only = False + try: + if self.current_token_str() == ')': + return name, typ, keyword_only + if read_name: + if self.current_token_str() != 'None': + name = self.expect_type(StrLit).parsed() + else: + self.skip() + if self.current_token_str() == ')': + return name, typ, keyword_only + else: + self.expect(',') + if read_typ: + typ = self.parse_type() + if self.current_token_str() == ')': + return name, typ, keyword_only + else: + self.expect(',') + if read_keyword_only: + if self.current_token_str() == 'True': + keyword_only = True + self.skip() + else: + self.expect('False') + if self.current_token_str() == ')': + return name, typ, keyword_only + else: + self.expect(',') + finally: + self.expect(')') + + + def parse_argument_list(self) -> ArgumentList: """Parse type list [t, ...].""" lbracket = self.expect('[') commas = [] # type: List[Token] items = [] # type: List[Type] + names = [] # type: List[Optional[str]] + kinds = [] # type: List[int] while self.current_token_str() != ']': - t = self.parse_type() + t, name, kind = self.parse_argument_spec() items.append(t) + names.append(name) + kinds.append(kind) + if self.current_token_str() != ',': break commas.append(self.skip()) self.expect(']') - return TypeList(items, line=lbracket.line) + return ArgumentList(items, names, kinds, line=lbracket.line) def parse_named_type(self) -> Type: line = self.current_token().line @@ -178,6 +273,11 @@ def expect_type(self, typ: type) -> Token: def current_token(self) -> Token: return self.tok[self.ind] + def next_token(self) -> Optional[Token]: + if self.ind + 1>= len(self.tok): + return None + return self.tok[self.ind + 1] + def current_token_str(self) -> str: return self.current_token().string diff --git a/mypy/sametypes.py b/mypy/sametypes.py index 48782738962a..ce38938ddbf2 100644 --- a/mypy/sametypes.py +++ b/mypy/sametypes.py @@ -3,7 +3,7 @@ from mypy.types import ( Type, UnboundType, ErrorType, AnyType, NoneTyp, Void, TupleType, TypedDictType, UnionType, CallableType, TypeVarType, Instance, TypeVisitor, ErasedType, - TypeList, Overloaded, PartialType, DeletedType, UninhabitedType, TypeType + ArgumentList, Overloaded, PartialType, DeletedType, UninhabitedType, TypeType ) @@ -58,7 +58,7 @@ def visit_unbound_type(self, left: UnboundType) -> bool: def visit_error_type(self, left: ErrorType) -> bool: return False - def visit_type_list(self, t: TypeList) -> bool: + def visit_type_list(self, t: ArgumentList) -> bool: assert False, 'Not supported' def visit_any(self, left: AnyType) -> bool: diff --git a/mypy/semanal.py b/mypy/semanal.py index 8771fb768d80..767e999c8066 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -72,7 +72,7 @@ from mypy.errors import Errors, report_internal_error from mypy.types import ( NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, TypeList, TypeVarDef, TypeType, + FunctionLike, UnboundType, ArgumentList, TypeVarDef, TypeType, TupleType, UnionType, StarType, EllipsisType, function_type, TypedDictType, ) from mypy.nodes import implicit_module_attrs @@ -402,8 +402,8 @@ def find_type_variables_in_type(self, type: Type) -> List[Tuple[str, TypeVarExpr result.append((name, node.node)) for arg in type.args: result.extend(self.find_type_variables_in_type(arg)) - elif isinstance(type, TypeList): - for item in type.items: + elif isinstance(type, ArgumentList): + for item in type.types: result.extend(self.find_type_variables_in_type(item)) elif isinstance(type, UnionType): for item in type.items: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 43f28c18d029..c094a5594727 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -3,7 +3,7 @@ from mypy.types import ( Type, AnyType, UnboundType, TypeVisitor, ErrorType, FormalArgument, Void, NoneTyp, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, - ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance + ErasedType, ArgumentList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance ) import mypy.applytype import mypy.constraints @@ -102,7 +102,7 @@ def visit_unbound_type(self, left: UnboundType) -> bool: def visit_error_type(self, left: ErrorType) -> bool: return False - def visit_type_list(self, t: TypeList) -> bool: + def visit_type_list(self, t: ArgumentList) -> bool: assert False, 'Not supported' def visit_any(self, left: AnyType) -> bool: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 4d9f1709a9ce..fe8f49406d6f 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -5,7 +5,7 @@ from mypy.types import ( Type, UnboundType, TypeVarType, TupleType, TypedDictType, UnionType, Instance, - AnyType, CallableType, Void, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, + AnyType, CallableType, Void, NoneTyp, DeletedType, ArgumentList, TypeVarDef, TypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, ) from mypy.nodes import ( @@ -277,7 +277,7 @@ def visit_uninhabited_type(self, t: UninhabitedType) -> Type: def visit_deleted_type(self, t: DeletedType) -> Type: return t - def visit_type_list(self, t: TypeList) -> Type: + def visit_type_list(self, t: ArgumentList) -> Type: self.fail('Invalid type', t) return AnyType() @@ -339,12 +339,12 @@ def analyze_callable_type(self, t: UnboundType) -> Type: is_ellipsis_args=True) elif len(t.args) == 2: ret_type = t.args[1].accept(self) - if isinstance(t.args[0], TypeList): + if isinstance(t.args[0], ArgumentList): # Callable[[ARG, ...], RET] (ordinary callable type) - args = t.args[0].items + args = t.args[0].types return CallableType(self.anal_array(args), - [nodes.ARG_POS] * len(args), - [None] * len(args), + t.args[0].kinds, + t.args[0].names, ret_type=ret_type, fallback=fallback) elif isinstance(t.args[0], EllipsisType): @@ -508,7 +508,7 @@ def visit_uninhabited_type(self, t: UninhabitedType) -> None: def visit_deleted_type(self, t: DeletedType) -> None: pass - def visit_type_list(self, t: TypeList) -> None: + def visit_type_list(self, t: ArgumentList) -> None: self.fail('Invalid type', t) def visit_type_var(self, t: TypeVarType) -> None: diff --git a/mypy/types.py b/mypy/types.py index e852f96e5730..359d104a6396 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -231,8 +231,8 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_error_type(self) -class TypeList(Type): - """A list of types [...]. +class ArgumentList(Type): + """Information about argument types and names [...]. This is only used for the arguments of a Callable type, i.e. for [arg, ...] in Callable[[arg, ...], ret]. This is not a real type @@ -240,23 +240,41 @@ class TypeList(Type): """ items = None # type: List[Type] + names = None # type: List[Optional[str]] + kinds = None # type: List[int] - def __init__(self, items: List[Type], line: int = -1, column: int = -1) -> None: + def __init__(self, + types: List[Type], + names: List[Optional[str]], + kinds: List[int], + line: int = -1, + column: int = -1) -> None: super().__init__(line, column) - self.items = items + self.types = types + self.names = names + self.kinds = kinds def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_type_list(self) def serialize(self) -> JsonDict: - return {'.class': 'TypeList', - 'items': [t.serialize() for t in self.items], - } + return {'.class': 'ArgumentList', + 'items': [t.serialize() for t in self.types], + 'names': self.names, + 'kinds': self.kinds, + } @classmethod - def deserialize(cls, data: JsonDict) -> 'TypeList': - assert data['.class'] == 'TypeList' - return TypeList([Type.deserialize(t) for t in data['items']]) + def deserialize(cls, data: JsonDict) -> 'ArgumentList': + assert data['.class'] == 'ArgumentList' or data['.class'] == 'TypeList' + types = [Type.deserialize(t) for t in data['items']] + names = cast(List[Optional[str]], data.get('names', [None]*len(types))) + kinds = cast(List[int], data.get('kinds', [ARG_POS]*len(types))) + return ArgumentList( + types=[Type.deserialize(t) for t in data['items']], + names=names, + kinds=kinds, + ) class AnyType(Type): @@ -1201,7 +1219,7 @@ def _notimplemented_helper(self, name: str) -> NotImplementedError: def visit_unbound_type(self, t: UnboundType) -> T: pass - def visit_type_list(self, t: TypeList) -> T: + def visit_type_list(self, t: ArgumentList) -> T: raise self._notimplemented_helper('type_list') def visit_error_type(self, t: ErrorType) -> T: @@ -1282,7 +1300,7 @@ class TypeTranslator(TypeVisitor[Type]): def visit_unbound_type(self, t: UnboundType) -> Type: return t - def visit_type_list(self, t: TypeList) -> Type: + def visit_type_list(self, t: ArgumentList) -> Type: return t def visit_error_type(self, t: ErrorType) -> Type: @@ -1384,8 +1402,8 @@ def visit_unbound_type(self, t: UnboundType)-> str: s += '[{}]'.format(self.list_str(t.args)) return s - def visit_type_list(self, t: TypeList) -> str: - return ''.format(self.list_str(t.items)) + def visit_type_list(self, t: ArgumentList) -> str: + return ''.format(self.list_str(t.items)) def visit_error_type(self, t: ErrorType) -> str: return '' @@ -1556,7 +1574,7 @@ def __init__(self, default: bool, strategy: int) -> None: def visit_unbound_type(self, t: UnboundType) -> bool: return self.default - def visit_type_list(self, t: TypeList) -> bool: + def visit_type_list(self, t: ArgumentList) -> bool: return self.default def visit_error_type(self, t: ErrorType) -> bool: diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 31a61688eeee..4b133b9bd11a 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1355,6 +1355,67 @@ f('x') # E: Argument 1 to "f" has incompatible type "str"; expected "int" g('x') g(1) # E: Argument 1 to "g" has incompatible type "int"; expected "str" +-- Callable with specific arg list +-- ------------------------------- + +[case testCallableWithNamedArg] +from typing import Callable +from mypy_extensions import Arg + +def a(f: Callable[[Arg('x', int)], int]): + f(x=4) + f(5) + f(y=3) # E: Unexpected keyword argument "y" + +[builtins fixtures/dict.pyi] + +[out] +main: note: In function "a": + +[case testCallableWithOptionalArg] +from typing import Callable +from mypy_extensions import DefaultArg + +def a(f: Callable[[DefaultArg('x', int)], int]): + f(x=4) + f(2) + f() + f(y=3) # E: Unexpected keyword argument "y" + f("foo") # E: Argument 1 has incompatible type "str"; expected "int" + +[builtins fixtures/dict.pyi] + +[out] +main: note: In function "a": + +[case testCallableArgKindSubtyping] +from typing import Callable +from mypy_extensions import Arg, DefaultArg + +int_str_fun = None # type: Callable[[int, str], str] +int_opt_str_fun = None # type: Callable[[int, DefaultArg(None, str)], str] +int_named_str_fun = None # type: Callable[[int, Arg('s', str)], str] + +def isf(ii: int, ss: str) -> str: + return ss + +def iosf(i: int, s: str = "bar") -> str: + return s + +def isf_unnamed(__i: int, __s: str) -> str: + return __s + +int_str_fun = isf +int_str_fun = isf_unnamed +int_named_str_fun = isf_unnamed # E: Incompatible types in assignment (expression has type Callable[[int, str], str], variable has type Callable[[int, Arg('s', str, False)], str]) +int_opt_str_fun = iosf +int_str_fun = iosf +int_opt_str_fun = isf # E: Incompatible types in assignment (expression has type Callable[[Arg('ii', int, False), Arg('ss', str, False)], str], variable has type Callable[[int, DefaultArg('None', str, False)], str]) + +int_named_str_fun = isf # E: Incompatible types in assignment (expression has type Callable[[Arg('ii', int, False), Arg('ss', str, False)], str], variable has type Callable[[int, Arg('s', str, False)], str]) +int_named_str_fun = iosf + +[builtins fixtures/dict.pyi] -- Callable[..., T] -- ---------------- diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index 5a7886439692..c79a21f7e6bc 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -32,3 +32,6 @@ class list(Iterable[T], Generic[T]): # needed by some test cases class tuple: pass class function: pass class float: pass +class bool: pass + +class ellipsis: pass \ No newline at end of file diff --git a/test-data/unit/lib-stub/mypy_extensions.pyi b/test-data/unit/lib-stub/mypy_extensions.pyi index 2bfc072b940e..83a452b73c29 100644 --- a/test-data/unit/lib-stub/mypy_extensions.pyi +++ b/test-data/unit/lib-stub/mypy_extensions.pyi @@ -1,6 +1,24 @@ -from typing import Dict, Type, TypeVar +from typing import Dict, Type, TypeVar, Optional, Any T = TypeVar('T') -def TypedDict(typename: str, fields: Dict[str, Type[T]]) -> Type[dict]: pass +def TypedDict(typename: str, fields: Dict[str, Type[T]]) -> Type[dict]: ... + +class Arg(object): + def __init__(name: Optional[str]=..., + typ: Type[T]=..., + keyword_only: Optional[bool]=...) -> None: + ... + +class DefaultArg(object): + def __init__(name: Optional[str]=..., + typ: Type[T]=..., + keyword_only: Optional[bool]=...) -> None: + ... + +class StarArg(object): + def __init__(typ: Type[T]=...) -> None: ... + +class KwArg(object): + def __init__(typ: Type[T]=...) -> None: ... diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 77a7b349e4cd..57b20f740b6c 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -75,15 +75,16 @@ class AsyncIterator(AsyncIterable[T], Generic[T]): def __anext__(self) -> Awaitable[T]: pass class Sequence(Iterable[T], Generic[T]): + # Use int because slice isn't defined in the default test builtins @abstractmethod - def __getitem__(self, n: Any) -> T: pass + def __getitem__(self, n: int) -> T: pass class Mapping(Generic[T, U]): pass class MutableMapping(Generic[T, U]): pass def NewType(name: str, tp: Type[T]) -> Callable[[T], T]: - def new_type(x): + def new_type(x: T) -> T: return x return new_type diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index 0dec0f397281..af217494a9d9 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -2407,7 +2407,7 @@ MypyFile:1( AssignmentStmt:1( NameExpr(f) NameExpr(None) - Callable?[, None?])) + Callable?[, None?])) [case testFunctionTypeWithArgument] f = None # type: Callable[[str], int] @@ -2416,7 +2416,7 @@ MypyFile:1( AssignmentStmt:1( NameExpr(f) NameExpr(None) - Callable?[, int?])) + Callable?[, int?])) [case testFunctionTypeWithTwoArguments] f = None # type: Callable[[a[b], x.y], List[int]] @@ -2425,7 +2425,7 @@ MypyFile:1( AssignmentStmt:1( NameExpr(f) NameExpr(None) - Callable?[, List?[int?]])) + Callable?[, List?[int?]])) [case testFunctionTypeWithExtraComma] def f(x: Callable[[str,], int]): pass @@ -2435,7 +2435,7 @@ MypyFile:1( f Args( Var(x)) - def (x: Callable?[, int?]) -> Any + def (x: Callable?[, int?]) -> Any Block:1( PassStmt:1()))) diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 06fd3d0dd226..81db36479862 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -803,7 +803,7 @@ Any(str, None) # E: 'Any' expects 1 argument Any(arg=str) # E: 'Any' must be called with 1 positional argument [out] -[case testTypeListAsType] +[case testArgumentListAsType] def f(x:[int, str]) -> None: # E: Invalid type pass [out] From 7ecdcc1767a7fd5eb107a6d215b8bfb4d8ccb560 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 23 Dec 2016 21:55:02 -0800 Subject: [PATCH 02/28] General cleanup --- extensions/mypy_extensions.py | 44 ++++++------ mypy/exprtotype.py | 10 +-- mypy/messages.py | 4 +- mypy/parsetype.py | 78 ++++++--------------- test-data/unit/check-functions.test | 33 ++++++--- test-data/unit/lib-stub/mypy_extensions.pyi | 24 +++---- 6 files changed, 79 insertions(+), 114 deletions(-) diff --git a/extensions/mypy_extensions.py b/extensions/mypy_extensions.py index 33f048b5f10f..7d4cdd74f943 100644 --- a/extensions/mypy_extensions.py +++ b/extensions/mypy_extensions.py @@ -5,7 +5,7 @@ from mypy_extensions import TypedDict """ -from typing import Any +from typing import Any, TypeVar # NOTE: This module must support Python 2.7 in addition to Python 3.x @@ -14,6 +14,7 @@ # the (convenient) behavior of types provided by typing module. from typing import _type_check # type: ignore +T = TypeVar('T') def _check_fails(cls, other): try: @@ -90,26 +91,21 @@ class Point2D(TypedDict): The latter syntax is only supported in Python 3.6+, while two other syntax forms work for Python 2.7 and 3.2+ """ - new_dict.__name__ = typename - new_dict.__supertype__ = dict - return new_dict - -class Arg(object): - def __init__(name=None, typ=Any, keyword_only=False): - self.name = name - self.typ = typ - self.named_only = named_only - -class DefaultArg(object): - def __init__(name=None, typ=Any, keyword_only=False): - self.name = name - self.typ = typ - self.named_only = named_only - -class StarArg(object): - def __init__(typ=Any) -> None: - self.typ = typ - -class KwArg(object): - def __init__(typ=Any) -> None: - self.typ = typ + +def Arg(name=None, typ: T = Any) -> T: + return typ + +def DefaultArg(name=None, typ: T = Any) -> T: + return typ + +def NamedArg(name=None, typ: T = Any) -> T: + return typ + +def DefaultNamedArg(name=None, typ: T = Any) -> T: + return typ + +def StarArg(typ: T = Any) -> T: + return typ + +def KwArg(typ: T =Any) -> T: + return typ diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index c196cdc5f925..0f51dcde6806 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -44,11 +44,6 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: else: raise TypeTranslationError() elif isinstance(expr, ListExpr): -<<<<<<< HEAD - return TypeList([expr_to_unanalyzed_type(t) for t in expr.items], - line=expr.line, column=expr.column) - elif isinstance(expr, (StrExpr, BytesExpr, UnicodeExpr)): -======= types = [] # type: List[Type] names = [] # type: List[Optional[str]] kinds = [] # type: List[int] @@ -82,9 +77,8 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: names.append(None) kinds.append(ARG_POS) return ArgumentList(types, names, kinds, - line=expr.line) - elif isinstance(expr, (StrExpr, BytesExpr)): ->>>>>>> c45f8c6... Implement Callable[[Arg('name', Type)], ret] syntax + line=expr.line, column=expr.column) + elif isinstance(expr, (StrExpr, BytesExpr, UnicodeExpr)): # Parse string literal type. try: result = parse_str_as_type(expr.value, expr.line) diff --git a/mypy/messages.py b/mypy/messages.py index fb5923dc52c4..cceb179f2bb8 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -203,9 +203,9 @@ def format(self, typ: Type, verbosity: int = 0) -> str: constructor, strip_quotes(self.format(arg_type)))) else: - arg_strings.append("{}('{}', {})".format( + arg_strings.append("{}({}, {})".format( constructor, - arg_name, + repr(arg_name), strip_quotes(self.format(arg_type)))) return 'Callable[[{}], {}]'.format(", ".join(arg_strings), return_type) diff --git a/mypy/parsetype.py b/mypy/parsetype.py index 75f6d55a3c71..a726806c453c 100644 --- a/mypy/parsetype.py +++ b/mypy/parsetype.py @@ -112,82 +112,46 @@ def parse_argument_spec(self) -> Tuple[Type, Optional[str], int]: arg_const = self.expect_type(Name).string name = None # type: Optional[str] typ = AnyType # type: Type - if arg_const == 'Arg': - kind = nodes.ARG_POS - name, typ, keyword_only = self.parse_arguments( - read_name = True, - read_typ = True, - read_keyword_only = True) - if keyword_only: - kind = nodes.ARG_NAMED - elif arg_const == 'DefaultArg': - kind = nodes.ARG_OPT - name, typ, keyword_only = self.parse_arguments( - read_name = True, - read_typ = True, - read_keyword_only = True) - if keyword_only: - kind = nodes.ARG_NAMED_OPT - elif arg_const == 'StarArg': - # Takes one type - kind = nodes.ARG_STAR - _, typ, _ = self.parse_arguments( - read_name = False, - read_typ = True, - read_keyword_only = False) - elif arg_const == 'KwArg': - # Takes one type - kind = nodes.ARG_STAR2 - _, typ, _ = self.parse_arguments( - read_name = False, - read_typ = True, - read_keyword_only = False) + kind = { + 'Arg': nodes.ARG_POS, + 'DefaultArg': nodes.ARG_OPT, + 'NamedArg': nodes.ARG_NAMED, + 'DefaultNamedArg': nodes.ARG_NAMED_OPT, + 'StarArg': nodes.ARG_STAR, + 'KwArg': nodes.ARG_STAR2, + }[arg_const] + if arg_const in {'Arg', 'DefaultArg', 'NamedArg', 'DefaultNamedArg'}: + name, typ = self.parse_arg_args(read_name = True) + elif arg_const in {'StarArg', 'KwArg'}: + _, typ = self.parse_arg_args(read_name = False) else: self.parse_error() return typ, name, kind else: return self.parse_type(), None, nodes.ARG_POS - def parse_arguments(self, - *, - read_name: bool, - read_typ: bool, - read_keyword_only: bool) -> Tuple[ - Optional[str], - Optional[Type], - bool]: + def parse_arg_args(self, *, read_name: bool) -> Tuple[Optional[str], Optional[Type]]: self.expect('(') name = None typ = AnyType - keyword_only = False try: if self.current_token_str() == ')': - return name, typ, keyword_only + return name, typ if read_name: if self.current_token_str() != 'None': name = self.expect_type(StrLit).parsed() else: self.skip() if self.current_token_str() == ')': - return name, typ, keyword_only - else: - self.expect(',') - if read_typ: - typ = self.parse_type() - if self.current_token_str() == ')': - return name, typ, keyword_only - else: - self.expect(',') - if read_keyword_only: - if self.current_token_str() == 'True': - keyword_only = True - self.skip() - else: - self.expect('False') - if self.current_token_str() == ')': - return name, typ, keyword_only + return name, typ else: self.expect(',') + + typ = self.parse_type() + if self.current_token_str() == ')': + return name, typ + else: + self.expect(',') finally: self.expect(')') diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 4b133b9bd11a..d273b427f07d 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1369,9 +1369,6 @@ def a(f: Callable[[Arg('x', int)], int]): [builtins fixtures/dict.pyi] -[out] -main: note: In function "a": - [case testCallableWithOptionalArg] from typing import Callable from mypy_extensions import DefaultArg @@ -1382,11 +1379,31 @@ def a(f: Callable[[DefaultArg('x', int)], int]): f() f(y=3) # E: Unexpected keyword argument "y" f("foo") # E: Argument 1 has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] +[case testCallableWithKeywordOnlyArg] +from typing import Callable +from mypy_extensions import NamedArg + +def a(f: Callable[[NamedArg('x', int)], int]): + f(x=4) + f(2) # E: Too many positional arguments + f() # E: Missing named argument "x" + f(y=3) # E: Unexpected keyword argument "y" + f(x="foo") # E: Argument "x" has incompatible type "str"; expected "int" [builtins fixtures/dict.pyi] -[out] -main: note: In function "a": +[case testCallableWithKeywordOnlyOptionalArg] +from typing import Callable +from mypy_extensions import DefaultNamedArg + +def a(f: Callable[[DefaultNamedArg('x', int)], int]): + f(x=4) + f(2) # E: Too many positional arguments + f() + f(y=3) # E: Unexpected keyword argument "y" + f(x="foo") # E: Argument "x" has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] [case testCallableArgKindSubtyping] from typing import Callable @@ -1407,12 +1424,12 @@ def isf_unnamed(__i: int, __s: str) -> str: int_str_fun = isf int_str_fun = isf_unnamed -int_named_str_fun = isf_unnamed # E: Incompatible types in assignment (expression has type Callable[[int, str], str], variable has type Callable[[int, Arg('s', str, False)], str]) +int_named_str_fun = isf_unnamed # E: Incompatible types in assignment (expression has type Callable[[int, str], str], variable has type Callable[[int, Arg('s', str)], str]) int_opt_str_fun = iosf int_str_fun = iosf -int_opt_str_fun = isf # E: Incompatible types in assignment (expression has type Callable[[Arg('ii', int, False), Arg('ss', str, False)], str], variable has type Callable[[int, DefaultArg('None', str, False)], str]) +int_opt_str_fun = isf # E: Incompatible types in assignment (expression has type Callable[[Arg('ii', int), Arg('ss', str)], str], variable has type Callable[[int, DefaultArg(None, str)], str]) -int_named_str_fun = isf # E: Incompatible types in assignment (expression has type Callable[[Arg('ii', int, False), Arg('ss', str, False)], str], variable has type Callable[[int, Arg('s', str, False)], str]) +int_named_str_fun = isf # E: Incompatible types in assignment (expression has type Callable[[Arg('ii', int), Arg('ss', str)], str], variable has type Callable[[int, Arg('s', str)], str]) int_named_str_fun = iosf [builtins fixtures/dict.pyi] diff --git a/test-data/unit/lib-stub/mypy_extensions.pyi b/test-data/unit/lib-stub/mypy_extensions.pyi index 83a452b73c29..3b1a9d099655 100644 --- a/test-data/unit/lib-stub/mypy_extensions.pyi +++ b/test-data/unit/lib-stub/mypy_extensions.pyi @@ -3,22 +3,16 @@ from typing import Dict, Type, TypeVar, Optional, Any T = TypeVar('T') -def TypedDict(typename: str, fields: Dict[str, Type[T]]) -> Type[dict]: ... +def TypedDict(typename: str, fields: Dict[str, Type[T]]) -> Type[dict]: pass -class Arg(object): - def __init__(name: Optional[str]=..., - typ: Type[T]=..., - keyword_only: Optional[bool]=...) -> None: - ... +def Arg(name=None, typ: T = ...) -> T: pass -class DefaultArg(object): - def __init__(name: Optional[str]=..., - typ: Type[T]=..., - keyword_only: Optional[bool]=...) -> None: - ... +def DefaultArg(name=None, typ: T = ...) -> T: pass -class StarArg(object): - def __init__(typ: Type[T]=...) -> None: ... +def NamedArg(name=None, typ: T = ...) -> T: pass -class KwArg(object): - def __init__(typ: Type[T]=...) -> None: ... +def DefaultNamedArg(name=None, typ: T = ...) -> T: pass + +def StarArg(typ: T = ...) -> T: pass + +def KwArg(typ: T = ...) -> T: pass From 066bd5efc7e78a8efdbdf31949112876e055d51e Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 23 Dec 2016 21:56:09 -0800 Subject: [PATCH 03/28] Change tests back to match old behavior --- test-data/unit/check-functions.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index d273b427f07d..39e98538054e 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1390,7 +1390,7 @@ def a(f: Callable[[NamedArg('x', int)], int]): f(2) # E: Too many positional arguments f() # E: Missing named argument "x" f(y=3) # E: Unexpected keyword argument "y" - f(x="foo") # E: Argument "x" has incompatible type "str"; expected "int" + f(x="foo") # E: Argument 1 has incompatible type "str"; expected "int" [builtins fixtures/dict.pyi] [case testCallableWithKeywordOnlyOptionalArg] @@ -1402,7 +1402,7 @@ def a(f: Callable[[DefaultNamedArg('x', int)], int]): f(2) # E: Too many positional arguments f() f(y=3) # E: Unexpected keyword argument "y" - f(x="foo") # E: Argument "x" has incompatible type "str"; expected "int" + f(x="foo") # E: Argument 1 has incompatible type "str"; expected "int" [builtins fixtures/dict.pyi] [case testCallableArgKindSubtyping] From 213944be35c8ea9aecf1fa4143a3327854915e42 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 23 Dec 2016 23:23:18 -0800 Subject: [PATCH 04/28] lots of lint --- mypy/exprtotype.py | 18 +++++++++--------- mypy/fastparse.py | 4 ++-- mypy/fixup.py | 2 +- mypy/indirection.py | 2 +- mypy/meet.py | 4 ++-- mypy/parsetype.py | 28 ++++++++++++++++------------ mypy/subtypes.py | 3 ++- mypy/types.py | 11 +++++------ 8 files changed, 38 insertions(+), 34 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 0f51dcde6806..30eb967f7b78 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -6,7 +6,7 @@ ARG_POS, ARG_NAMED, ) from mypy.parsetype import parse_str_as_type, TypeParseError -from mypy.types import Type, UnboundType, ArgumentList, EllipsisType, AnyType +from mypy.types import Type, UnboundType, ArgumentList, EllipsisType, AnyType, Optional class TypeTranslationError(Exception): @@ -44,20 +44,20 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: else: raise TypeTranslationError() elif isinstance(expr, ListExpr): - types = [] # type: List[Type] - names = [] # type: List[Optional[str]] - kinds = [] # type: List[int] + types = [] # type: List[Type] + names = [] # type: List[Optional[str]] + kinds = [] # type: List[int] for it in expr.items: - if isinstance(expr_to_unanalyzed_type(it), CallExpr): + if isinstance(it, CallExpr): if not isinstance(it.callee, NameExpr): raise TypeTranslationError() arg_const = it.callee.name if arg_const == 'Arg': if len(it.args) > 0: - name = it.args[0] - if not isinstance(name, StrLit): + arg_name = it.args[0] + if not isinstance(arg_name, StrExpr): raise TypeTranslationError() - names.append(name.parsed()) + names.append(arg_name.value) else: names.append(None) @@ -65,7 +65,7 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: typ = it.args[1] types.append(expr_to_unanalyzed_type(typ)) else: - types.append(AnyType) + types.append(AnyType()) if len(it.args) > 2: kinds.append(ARG_NAMED) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 7d0d58d4fd2a..c783ad39a8f8 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -913,8 +913,8 @@ def visit_Ellipsis(self, n: ast35.Ellipsis) -> Type: def visit_List(self, n: ast35.List) -> Type: return ArgumentList( self.translate_expr_list(n.elts), - [None]*len(n.elts), - [0]*len(n.elts), + [None] * len(n.elts), + [0] * len(n.elts), line=self.line) diff --git a/mypy/fixup.py b/mypy/fixup.py index e95f606a2fb4..16303bd7e3f3 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -204,7 +204,7 @@ def visit_typeddict_type(self, tdt: TypedDictType) -> None: tdt.fallback.accept(self) def visit_type_list(self, tl: ArgumentList) -> None: - for t in tl.items: + for t in tl.types: t.accept(self) def visit_type_var(self, tvt: TypeVarType) -> None: diff --git a/mypy/indirection.py b/mypy/indirection.py index 7d8f15194fca..28aa72b17cd2 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -43,7 +43,7 @@ def visit_unbound_type(self, t: types.UnboundType) -> Set[str]: return self._visit(*t.args) def visit_type_list(self, t: types.ArgumentList) -> Set[str]: - return self._visit(*t.items) + return self._visit(*t.types) def visit_error_type(self, t: types.ErrorType) -> Set[str]: return set() diff --git a/mypy/meet.py b/mypy/meet.py index e6124805a84e..99c5b6d8f9eb 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -4,8 +4,8 @@ from mypy.join import is_similar_callables, combine_similar_callables, join_type_list from mypy.types import ( Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType, - Instance, CallableType, TupleType, TypedDictType, ErasedType, ArgumentList, UnionType, PartialType, - DeletedType, UninhabitedType, TypeType + Instance, CallableType, TupleType, TypedDictType, ErasedType, ArgumentList, UnionType, + PartialType, DeletedType, UninhabitedType, TypeType ) from mypy.subtypes import is_equivalent, is_subtype diff --git a/mypy/parsetype.py b/mypy/parsetype.py index a726806c453c..6d19a220c225 100644 --- a/mypy/parsetype.py +++ b/mypy/parsetype.py @@ -1,14 +1,18 @@ """Type parser""" -from typing import List, Tuple, Union, Optional +from typing import List, Tuple, Union, Optional, TypeVar, cast + +import typing from mypy.types import ( Type, UnboundType, TupleType, ArgumentList, CallableType, StarType, EllipsisType, AnyType ) + from mypy.lex import Token, Name, StrLit, lex from mypy import nodes +T = TypeVar('T', bound=Token) none = Token('') # Empty token @@ -110,8 +114,8 @@ def parse_argument_spec(self) -> Tuple[Type, Optional[str], int]: # TODO: Doesn't handle an explicit name of None yet. if isinstance(current, Name) and nxt is not None and nxt.string == '(': arg_const = self.expect_type(Name).string - name = None # type: Optional[str] - typ = AnyType # type: Type + name = None # type: Optional[str] + typ = AnyType(implicit=True) # type: Type kind = { 'Arg': nodes.ARG_POS, 'DefaultArg': nodes.ARG_OPT, @@ -119,7 +123,7 @@ def parse_argument_spec(self) -> Tuple[Type, Optional[str], int]: 'DefaultNamedArg': nodes.ARG_NAMED_OPT, 'StarArg': nodes.ARG_STAR, 'KwArg': nodes.ARG_STAR2, - }[arg_const] + }[arg_const] if arg_const in {'Arg', 'DefaultArg', 'NamedArg', 'DefaultNamedArg'}: name, typ = self.parse_arg_args(read_name = True) elif arg_const in {'StarArg', 'KwArg'}: @@ -132,8 +136,8 @@ def parse_argument_spec(self) -> Tuple[Type, Optional[str], int]: def parse_arg_args(self, *, read_name: bool) -> Tuple[Optional[str], Optional[Type]]: self.expect('(') - name = None - typ = AnyType + name = None # type: Optional[str] + typ = AnyType(implicit=True) # type: Type try: if self.current_token_str() == ')': return name, typ @@ -154,15 +158,15 @@ def parse_arg_args(self, *, read_name: bool) -> Tuple[Optional[str], Optional[Ty self.expect(',') finally: self.expect(')') - + return name, typ def parse_argument_list(self) -> ArgumentList: """Parse type list [t, ...].""" lbracket = self.expect('[') commas = [] # type: List[Token] items = [] # type: List[Type] - names = [] # type: List[Optional[str]] - kinds = [] # type: List[int] + names = [] # type: List[Optional[str]] + kinds = [] # type: List[int] while self.current_token_str() != ']': t, name, kind = self.parse_argument_spec() items.append(t) @@ -227,10 +231,10 @@ def expect(self, string: str) -> Token: else: raise self.parse_error() - def expect_type(self, typ: type) -> Token: + def expect_type(self, typ: typing.Type[T]) -> T: if isinstance(self.current_token(), typ): self.ind += 1 - return self.tok[self.ind - 1] + return cast(T, self.tok[self.ind - 1]) else: raise self.parse_error() @@ -238,7 +242,7 @@ def current_token(self) -> Token: return self.tok[self.ind] def next_token(self) -> Optional[Token]: - if self.ind + 1>= len(self.tok): + if self.ind + 1 >= len(self.tok): return None return self.tok[self.ind + 1] diff --git a/mypy/subtypes.py b/mypy/subtypes.py index c094a5594727..6fc09159f70f 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -3,7 +3,8 @@ from mypy.types import ( Type, AnyType, UnboundType, TypeVisitor, ErrorType, FormalArgument, Void, NoneTyp, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, - ErasedType, ArgumentList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance + ErasedType, ArgumentList, PartialType, DeletedType, UninhabitedType, TypeType, + is_named_instance ) import mypy.applytype import mypy.constraints diff --git a/mypy/types.py b/mypy/types.py index 359d104a6396..76f950c31313 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -239,7 +239,7 @@ class ArgumentList(Type): but a syntactic AST construct. """ - items = None # type: List[Type] + types = None # type: List[Type] names = None # type: List[Optional[str]] kinds = None # type: List[int] @@ -261,15 +261,14 @@ def serialize(self) -> JsonDict: return {'.class': 'ArgumentList', 'items': [t.serialize() for t in self.types], 'names': self.names, - 'kinds': self.kinds, - } + 'kinds': self.kinds} @classmethod def deserialize(cls, data: JsonDict) -> 'ArgumentList': assert data['.class'] == 'ArgumentList' or data['.class'] == 'TypeList' types = [Type.deserialize(t) for t in data['items']] - names = cast(List[Optional[str]], data.get('names', [None]*len(types))) - kinds = cast(List[int], data.get('kinds', [ARG_POS]*len(types))) + names = cast(List[Optional[str]], data.get('names', [None] * len(types))) + kinds = cast(List[int], data.get('kinds', [ARG_POS] * len(types))) return ArgumentList( types=[Type.deserialize(t) for t in data['items']], names=names, @@ -1403,7 +1402,7 @@ def visit_unbound_type(self, t: UnboundType)-> str: return s def visit_type_list(self, t: ArgumentList) -> str: - return ''.format(self.list_str(t.items)) + return ''.format(self.list_str(t.types)) def visit_error_type(self, t: ErrorType) -> str: return '' From 0e19070805579714ec63477d1d5bd0f66c591c56 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sat, 24 Dec 2016 20:43:24 -0800 Subject: [PATCH 05/28] Oh god I am tired of writing parsers --- extensions/mypy_extensions.py | 16 ++-- mypy/exprtotype.py | 51 +++++++----- mypy/fastparse.py | 77 ++++++++++++++++-- mypy/parsetype.py | 70 ++++++++-------- mypy/sharedparse.py | 13 +++ mypy/typeanal.py | 15 ++-- mypy/types.py | 49 ++++++++++- test-data/unit/check-functions.test | 121 ++++++++++++++++++++++++++++ test-data/unit/lib-stub/typing.pyi | 6 +- 9 files changed, 344 insertions(+), 74 deletions(-) diff --git a/extensions/mypy_extensions.py b/extensions/mypy_extensions.py index 7d4cdd74f943..cd71e6734861 100644 --- a/extensions/mypy_extensions.py +++ b/extensions/mypy_extensions.py @@ -5,7 +5,7 @@ from mypy_extensions import TypedDict """ -from typing import Any, TypeVar +from typing import Any # NOTE: This module must support Python 2.7 in addition to Python 3.x @@ -14,8 +14,6 @@ # the (convenient) behavior of types provided by typing module. from typing import _type_check # type: ignore -T = TypeVar('T') - def _check_fails(cls, other): try: if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: @@ -92,20 +90,20 @@ class Point2D(TypedDict): syntax forms work for Python 2.7 and 3.2+ """ -def Arg(name=None, typ: T = Any) -> T: +def Arg(name=None, typ=Any): return typ -def DefaultArg(name=None, typ: T = Any) -> T: +def DefaultArg(name=None, typ=Any): return typ -def NamedArg(name=None, typ: T = Any) -> T: +def NamedArg(name=None, typ=Any): return typ -def DefaultNamedArg(name=None, typ: T = Any) -> T: +def DefaultNamedArg(name=None, typ=Any): return typ -def StarArg(typ: T = Any) -> T: +def StarArg(typ=Any): return typ -def KwArg(typ: T =Any) -> T: +def KwArg(typ=Any): return typ diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 30eb967f7b78..5650947e7d91 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -5,6 +5,7 @@ ListExpr, StrExpr, BytesExpr, UnicodeExpr, EllipsisExpr, CallExpr, ARG_POS, ARG_NAMED, ) +from mypy.sharedparse import ARG_KINDS_BY_CONSTRUCTOR, STAR_ARG_CONSTRUCTORS from mypy.parsetype import parse_str_as_type, TypeParseError from mypy.types import Type, UnboundType, ArgumentList, EllipsisType, AnyType, Optional @@ -12,6 +13,14 @@ class TypeTranslationError(Exception): """Exception raised when an expression is not valid as a type.""" +def _extract_str(expr: Expression) -> Optional[str]: + if isinstance(expr, NameExpr) and expr.name == 'None': + return None + elif isinstance(expr, StrExpr): + return expr.value + else: + raise TypeTranslationError() + def expr_to_unanalyzed_type(expr: Expression) -> Type: """Translate an expression to the corresponding type. @@ -52,26 +61,32 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: if not isinstance(it.callee, NameExpr): raise TypeTranslationError() arg_const = it.callee.name - if arg_const == 'Arg': - if len(it.args) > 0: - arg_name = it.args[0] - if not isinstance(arg_name, StrExpr): + try: + kind = ARG_KINDS_BY_CONSTRUCTOR[arg_const] + except KeyError: + raise TypeTranslationError() + name = None + typ = AnyType(implicit=True) + star = arg_const in STAR_ARG_CONSTRUCTORS + for i, arg in enumerate(it.args): + if it.arg_names[i] is not None: + if it.arg_names[i] == "name": + name = _extract_str(arg) + continue + elif it.arg_names[i] == "typ": + typ = expr_to_unanalyzed_type(arg) + continue + else: raise TypeTranslationError() - names.append(arg_name.value) + elif i == 0 and not star: + name = _extract_str(arg) + elif i == 1 and not star or i == 0 and star: + typ = expr_to_unanalyzed_type(arg) else: - names.append(None) - - if len(it.args) > 1: - typ = it.args[1] - types.append(expr_to_unanalyzed_type(typ)) - else: - types.append(AnyType()) - - if len(it.args) > 2: - kinds.append(ARG_NAMED) - else: - kinds.append(ARG_POS) - + raise TypeTranslationError() + names.append(name) + types.append(typ) + kinds.append(kind) else: types.append(expr_to_unanalyzed_type(it)) names.append(None) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index c783ad39a8f8..02ebc9bfb320 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -2,7 +2,10 @@ import sys from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, cast, List -from mypy.sharedparse import special_function_elide_names, argument_elide_name +from mypy.sharedparse import ( + special_function_elide_names, argument_elide_name, + ARG_KINDS_BY_CONSTRUCTOR, STAR_ARG_CONSTRUCTORS, +) from mypy.nodes import ( MypyFile, Node, ImportBase, Import, ImportAll, ImportFrom, FuncDef, OverloadedFuncDef, ClassDef, Decorator, Block, Var, OperatorAssignmentStmt, @@ -862,6 +865,72 @@ def visit_NoneType(self, n: Any) -> Type: def translate_expr_list(self, l: Sequence[ast35.AST]) -> List[Type]: return [self.visit(e) for e in l] + def translate_argument_list(self, l: Sequence[ast35.AST]) -> ArgumentList: + types = [] # type: List[Type] + names = [] # type: List[Optional[str]] + kinds = [] # type: List[int] + for e in l: + if isinstance(e, ast35.Call): + # Parse the arg constructor + f = e.func + if not isinstance(f, ast35.Name): + raise FastParserError("Expected arg constructor name", + e.lineno, e.col_offset) + try: + kind = ARG_KINDS_BY_CONSTRUCTOR[f.id] + star = f.id in STAR_ARG_CONSTRUCTORS + except KeyError: + raise FastParserError("Unknown argument constructor {}".format(f.id), + f.lineno, f.col_offset) + + name = None # type: Optional[str] + typ = AnyType(implicit=True) # type: Type + for i, arg in enumerate(e.args): + if i == 0 and not star: + if isinstance(arg, ast35.Name) and arg.id == 'None': + name = None + elif isinstance(arg, ast35.Str): + name = arg.s + else: + raise FastParserError("Bad type for name of argument", + value.lineno, value.col_offset) + elif i == 1 and not star or i == 0 and star: + try: + typ = self.visit(arg) + except TypeCommentParseError: + raise FastParserError("Bad type for callable argument", + arg.lineno, arg.col_offset) + else: + raise FastParserError("Too many arguments for argument constructor", + f.lineno, f.col_offset) + for k in e.keywords: + value = k.value + if k.arg == "name" and not star: + if isinstance(value, ast35.Name) and value.id == 'None': + name = None + elif isinstance(value, ast35.Str): + name = value.s + elif k.arg == "typ": + try: + typ = self.visit(value) + except TypeCommentParseError: + raise FastParserError("Bad type for callable argument", + value.lineno, value.col_offset) + else: + raise FastParserError( + 'Unexpected argument "{}" for argument constructor'.format(k.arg), + value.lineno, value.col_offset) + + types.append(typ) + names.append(name) + kinds.append(kind) + else: + types.append(self.visit(e)) + names.append(None) + kinds.append(ARG_POS) + + return ArgumentList(types, names, kinds, line=self.line) + def visit_Name(self, n: ast35.Name) -> Type: return UnboundType(n.id, line=self.line) @@ -911,11 +980,7 @@ def visit_Ellipsis(self, n: ast35.Ellipsis) -> Type: # List(expr* elts, expr_context ctx) def visit_List(self, n: ast35.List) -> Type: - return ArgumentList( - self.translate_expr_list(n.elts), - [None] * len(n.elts), - [0] * len(n.elts), - line=self.line) + return self.translate_argument_list(n.elts) class TypeCommentParseError(Exception): diff --git a/mypy/parsetype.py b/mypy/parsetype.py index 6d19a220c225..5e0d0644388e 100644 --- a/mypy/parsetype.py +++ b/mypy/parsetype.py @@ -9,6 +9,8 @@ EllipsisType, AnyType ) +from mypy.sharedparse import ARG_KINDS_BY_CONSTRUCTOR, STAR_ARG_CONSTRUCTORS + from mypy.lex import Token, Name, StrLit, lex from mypy import nodes @@ -116,20 +118,11 @@ def parse_argument_spec(self) -> Tuple[Type, Optional[str], int]: arg_const = self.expect_type(Name).string name = None # type: Optional[str] typ = AnyType(implicit=True) # type: Type - kind = { - 'Arg': nodes.ARG_POS, - 'DefaultArg': nodes.ARG_OPT, - 'NamedArg': nodes.ARG_NAMED, - 'DefaultNamedArg': nodes.ARG_NAMED_OPT, - 'StarArg': nodes.ARG_STAR, - 'KwArg': nodes.ARG_STAR2, - }[arg_const] - if arg_const in {'Arg', 'DefaultArg', 'NamedArg', 'DefaultNamedArg'}: - name, typ = self.parse_arg_args(read_name = True) - elif arg_const in {'StarArg', 'KwArg'}: - _, typ = self.parse_arg_args(read_name = False) - else: - self.parse_error() + try: + kind = ARG_KINDS_BY_CONSTRUCTOR[arg_const] + except KeyError: + raise self.parse_error("Unknown argument constructor {}".format(arg_const)) + name, typ = self.parse_arg_args(read_name = arg_const not in STAR_ARG_CONSTRUCTORS) return typ, name, kind else: return self.parse_type(), None, nodes.ARG_POS @@ -138,26 +131,37 @@ def parse_arg_args(self, *, read_name: bool) -> Tuple[Optional[str], Optional[Ty self.expect('(') name = None # type: Optional[str] typ = AnyType(implicit=True) # type: Type - try: - if self.current_token_str() == ')': - return name, typ - if read_name: - if self.current_token_str() != 'None': - name = self.expect_type(StrLit).parsed() + i = 0 + while self.current_token_str() != ')': + if i > 0: + self.expect(',') + if self.next_token() and self.next_token().string == '=': + arg_arg_name = self.current_token_str() + if arg_arg_name == 'name' and read_name: + self.expect('name') + self.expect('=') + if self.current_token_str() == 'None': + self.expect('None') + else: + name = self.expect_type(StrLit).parsed() + elif arg_arg_name == 'typ': + self.expect('typ') + self.expect('=') + typ = self.parse_type() else: - self.skip() - if self.current_token_str() == ')': - return name, typ + raise self.parse_error( + 'Unexpected argument "{}" for argument constructor'.format(arg_arg_name)) + elif i == 0 and read_name: + if self.current_token_str() == 'None': + self.expect('None') else: - self.expect(',') - - typ = self.parse_type() - if self.current_token_str() == ')': - return name, typ + name = self.expect_type(StrLit).parsed() + elif i == 0 and not read_name or i == 1 and read_name: + typ = self.parse_type() else: - self.expect(',') - finally: - self.expect(')') + raise self.parse_error("Unexpected argument for argument constructor") + i += 1 + self.expect(')') return name, typ def parse_argument_list(self) -> ArgumentList: @@ -249,8 +253,8 @@ def next_token(self) -> Optional[Token]: def current_token_str(self) -> str: return self.current_token().string - def parse_error(self) -> TypeParseError: - return TypeParseError(self.tok[self.ind], self.ind) + def parse_error(self, message: Optional[str] = None) -> TypeParseError: + return TypeParseError(self.tok[self.ind], self.ind, message=message) def parse_str_as_type(typestr: str, line: int) -> Type: diff --git a/mypy/sharedparse.py b/mypy/sharedparse.py index ba1da1eb17db..a673968f7e77 100644 --- a/mypy/sharedparse.py +++ b/mypy/sharedparse.py @@ -1,5 +1,7 @@ from typing import Union, Tuple +from mypy.nodes import (ARG_POS, ARG_NAMED, ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2) + """Shared logic between our three mypy parser files.""" @@ -90,6 +92,17 @@ MAGIC_METHODS_POS_ARGS_ONLY = MAGIC_METHODS - MAGIC_METHODS_ALLOWING_KWARGS +ARG_KINDS_BY_CONSTRUCTOR = { + 'Arg': ARG_POS, + 'DefaultArg': ARG_OPT, + 'NamedArg': ARG_NAMED, + 'DefaultNamedArg': ARG_NAMED_OPT, + 'StarArg': ARG_STAR, + 'KwArg': ARG_STAR2, +} + +STAR_ARG_CONSTRUCTORS = {'StarArg', 'KwArg'} + def special_function_elide_names(name: str) -> bool: return name in MAGIC_METHODS_POS_ARGS_ONLY diff --git a/mypy/typeanal.py b/mypy/typeanal.py index fe8f49406d6f..cfca422f8a07 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -7,6 +7,7 @@ Type, UnboundType, TypeVarType, TupleType, TypedDictType, UnionType, Instance, AnyType, CallableType, Void, NoneTyp, DeletedType, ArgumentList, TypeVarDef, TypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, + ArgKindException, ArgNameException ) from mypy.nodes import ( BOUND_TVAR, UNBOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, @@ -342,11 +343,15 @@ def analyze_callable_type(self, t: UnboundType) -> Type: if isinstance(t.args[0], ArgumentList): # Callable[[ARG, ...], RET] (ordinary callable type) args = t.args[0].types - return CallableType(self.anal_array(args), - t.args[0].kinds, - t.args[0].names, - ret_type=ret_type, - fallback=fallback) + try: + return CallableType(self.anal_array(args), + t.args[0].kinds, + t.args[0].names, + ret_type=ret_type, + fallback=fallback) + except (ArgKindException, ArgNameException) as e: + self.fail(e.message, t) + return AnyType() elif isinstance(t.args[0], EllipsisType): # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) return CallableType([AnyType(), AnyType()], diff --git a/mypy/types.py b/mypy/types.py index 76f950c31313..32866aa0c456 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -567,6 +567,16 @@ def deserialize(cls, data: JsonDict) -> 'FunctionLike': ('typ', Type), ('required', bool)]) +class ArgKindException(Exception): + message = None # type: str + def __init__(self, message: str): + self.message = message + +class ArgNameException(Exception): + message = None # type: str + def __init__(self, message: str): + self.message = message + class CallableType(FunctionLike): """Type of a non-overloaded callable object (function).""" @@ -576,6 +586,7 @@ class CallableType(FunctionLike): arg_names = None # type: List[str] # None if not a keyword argument min_args = 0 # Minimum number of arguments; derived from arg_kinds is_var_arg = False # Is it a varargs function? derived from arg_kinds + is_kw_arg = False ret_type = None # type: Type # Return value type name = '' # Name (may be None; for error messages) definition = None # type: SymbolNode # For error messages. May be None. @@ -608,6 +619,8 @@ def __init__(self, is_classmethod_class: bool = False, special_sig: Optional[str] = None, ) -> None: + self._process_kinds_on_init(arg_kinds) + self._process_names_on_init(arg_names) if variables is None: variables = [] assert len(arg_types) == len(arg_kinds) @@ -615,8 +628,6 @@ def __init__(self, self.arg_kinds = arg_kinds self.arg_names = arg_names self.min_args = arg_kinds.count(ARG_POS) - self.is_var_arg = ARG_STAR in arg_kinds - self.is_kw_arg = ARG_STAR2 in arg_kinds self.ret_type = ret_type self.fallback = fallback assert not name or ' Date: Sat, 24 Dec 2016 20:57:46 -0800 Subject: [PATCH 06/28] Tighten fastparse a little --- mypy/fastparse.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 02ebc9bfb320..c193f1059589 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -887,13 +887,7 @@ def translate_argument_list(self, l: Sequence[ast35.AST]) -> ArgumentList: typ = AnyType(implicit=True) # type: Type for i, arg in enumerate(e.args): if i == 0 and not star: - if isinstance(arg, ast35.Name) and arg.id == 'None': - name = None - elif isinstance(arg, ast35.Str): - name = arg.s - else: - raise FastParserError("Bad type for name of argument", - value.lineno, value.col_offset) + name = _extract_str(arg) elif i == 1 and not star or i == 0 and star: try: typ = self.visit(arg) @@ -906,10 +900,7 @@ def translate_argument_list(self, l: Sequence[ast35.AST]) -> ArgumentList: for k in e.keywords: value = k.value if k.arg == "name" and not star: - if isinstance(value, ast35.Name) and value.id == 'None': - name = None - elif isinstance(value, ast35.Str): - name = value.s + name = _extract_str(value) elif k.arg == "typ": try: typ = self.visit(value) @@ -992,3 +983,14 @@ def __init__(self, msg: str, lineno: int, offset: int) -> None: class FastParserError(TypeCommentParseError): pass + +def _extract_str(arg: ast35.AST) -> Optional[str]: + if isinstance(arg, ast35.Name) and arg.id == 'None': + return None + elif isinstance(arg, ast35.NameConstant) and arg.value is None: + return None + elif isinstance(arg, ast35.Str): + return arg.s + else: + raise FastParserError("Bad type for name of argument", + arg.lineno, arg.col_offset) From 0b69630eeed92202aab23994e71e2bb5b210ff56 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sat, 24 Dec 2016 22:31:21 -0800 Subject: [PATCH 07/28] make all tests pass now --- mypy/exprtotype.py | 3 ++- mypy/fastparse.py | 3 ++- mypy/parse.py | 1 + mypy/parsetype.py | 17 +++++++++++------ mypy/sharedparse.py | 1 + mypy/types.py | 13 ++++++++++--- test-data/unit/check-functions.test | 11 +++++++---- test-data/unit/lib-stub/typing.pyi | 6 +----- test-data/unit/parse-errors.test | 11 ----------- typeshed | 2 +- 10 files changed, 36 insertions(+), 32 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 5650947e7d91..efee8be56d12 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -13,6 +13,7 @@ class TypeTranslationError(Exception): """Exception raised when an expression is not valid as a type.""" + def _extract_str(expr: Expression) -> Optional[str]: if isinstance(expr, NameExpr) and expr.name == 'None': return None @@ -66,7 +67,7 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: except KeyError: raise TypeTranslationError() name = None - typ = AnyType(implicit=True) + typ = AnyType(implicit=True) # type: Type star = arg_const in STAR_ARG_CONSTRUCTORS for i, arg in enumerate(it.args): if it.arg_names[i] is not None: diff --git a/mypy/fastparse.py b/mypy/fastparse.py index c193f1059589..29d7a2aca685 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -984,7 +984,8 @@ def __init__(self, msg: str, lineno: int, offset: int) -> None: class FastParserError(TypeCommentParseError): pass -def _extract_str(arg: ast35.AST) -> Optional[str]: + +def _extract_str(arg: ast35.expr) -> Optional[str]: if isinstance(arg, ast35.Name) and arg.id == 'None': return None elif isinstance(arg, ast35.NameConstant) and arg.value is None: diff --git a/mypy/parse.py b/mypy/parse.py index 4e83b77e9001..550d7ec80264 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -433,6 +433,7 @@ def parse_function(self, no_type_checks: bool = False) -> FuncDef: self.errors.report( def_tok.line, def_tok.column, 'Function has duplicate type signatures') sig = cast(CallableType, comment_type) + if sig.is_ellipsis_args: # When we encounter an ellipsis, fill in the arg_types with # a bunch of AnyTypes, emulating Callable[..., T] diff --git a/mypy/parsetype.py b/mypy/parsetype.py index 5e0d0644388e..fbf6376226f5 100644 --- a/mypy/parsetype.py +++ b/mypy/parsetype.py @@ -6,7 +6,7 @@ from mypy.types import ( Type, UnboundType, TupleType, ArgumentList, CallableType, StarType, - EllipsisType, AnyType + EllipsisType, AnyType, ArgNameException, ArgKindException ) from mypy.sharedparse import ARG_KINDS_BY_CONSTRUCTOR, STAR_ARG_CONSTRUCTORS @@ -279,6 +279,8 @@ def parse_signature(tokens: List[Token]) -> Tuple[CallableType, int]: i = 0 if tokens[i].string != '(': raise TypeParseError(tokens[i], i) + begin = tokens[i] + begin_idx = i i += 1 arg_types = [] # type: List[Type] arg_kinds = [] # type: List[int] @@ -314,8 +316,11 @@ def parse_signature(tokens: List[Token]) -> Tuple[CallableType, int]: raise TypeParseError(tokens[i], i) i += 1 ret_type, i = parse_type(tokens, i) - return CallableType(arg_types, - arg_kinds, - [None] * len(arg_types), - ret_type, None, - is_ellipsis_args=encountered_ellipsis), i + try: + return CallableType(arg_types, + arg_kinds, + [None] * len(arg_types), + ret_type, None, + is_ellipsis_args=encountered_ellipsis), i + except (ArgKindException, ArgNameException) as e: + raise TypeParseError(begin, begin_idx, e.message) diff --git a/mypy/sharedparse.py b/mypy/sharedparse.py index a673968f7e77..9b4e9946ba0a 100644 --- a/mypy/sharedparse.py +++ b/mypy/sharedparse.py @@ -103,6 +103,7 @@ STAR_ARG_CONSTRUCTORS = {'StarArg', 'KwArg'} + def special_function_elide_names(name: str) -> bool: return name in MAGIC_METHODS_POS_ARGS_ONLY diff --git a/mypy/types.py b/mypy/types.py index 32866aa0c456..298ef8361f6c 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -567,14 +567,20 @@ def deserialize(cls, data: JsonDict) -> 'FunctionLike': ('typ', Type), ('required', bool)]) + class ArgKindException(Exception): + """Raised when the argument kinds are not in the right order""" message = None # type: str - def __init__(self, message: str): + + def __init__(self, message: str) -> None: self.message = message + class ArgNameException(Exception): + """Raised when there are duplicate arg names""" message = None # type: str - def __init__(self, message: str): + + def __init__(self, message: str) -> None: self.message = message @@ -660,7 +666,8 @@ def _process_kinds_on_init(self, arg_kinds): "after default, named or star args") elif kind == ARG_OPT: if self.is_var_arg or self.is_kw_arg or seen_named: - raise ArgKindException("Positional default args may not appear after named or star args") + raise ArgKindException("Positional default args may not appear " + "after named or star args") seen_opt = True elif kind == ARG_STAR: if self.is_var_arg or self.is_kw_arg or seen_named: diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 4899de965e50..a6933aea0a03 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1412,10 +1412,13 @@ from typing import Callable from mypy_extensions import Arg, StarArg, KwArg def WrongArg(x, y): return y - -F = Callable[[WrongArg('x', int)], int] # E: Invalid type alias -G = Callable[[Arg('x', 1)], int] # E: Invalid type alias -H = Callable[[StarArg('x', int)], int] # E: Invalid type alias # E: Too many arguments for "StarArg" +# Note that for this test, the 'Value of type "int" is not indexable' errors are silly, +# and a consequence of Callable being set to an int in the test stub. We can't set it to +# something else sensible, because other tests require the stub not have anything +# that looks like a function call. +F = Callable[[WrongArg('x', int)], int] # E: Invalid type alias # E: Value of type "int" is not indexable +G = Callable[[Arg('x', 1)], int] # E: Invalid type alias # E: Value of type "int" is not indexable +H = Callable[[StarArg('x', int)], int] # E: Invalid type alias # E: Value of type "int" is not indexable # E: Too many arguments for "StarArg" I = Callable[[StarArg(int)], int] # ok J = Callable[[StarArg(), KwArg()], int] # ok K = Callable[[StarArg(), int], int] # E: Required positional args may not appear after default, named or star args diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index ca7bbdce6eea..dfab7c2f99b7 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -16,6 +16,7 @@ _promote = 0 NamedTuple = 0 Type = 0 no_type_check = 0 +Callable = 0 # Type aliases. List = 0 @@ -27,11 +28,6 @@ U = TypeVar('U') V = TypeVar('V') S = TypeVar('S') -class _Callable(object): - def __getattr__(self, o): pass - -Callable = _Callable() - class Container(Generic[T]): @abstractmethod # Use int because bool isn't in the default test builtins diff --git a/test-data/unit/parse-errors.test b/test-data/unit/parse-errors.test index f2251a9e6e2a..1587e87812c3 100644 --- a/test-data/unit/parse-errors.test +++ b/test-data/unit/parse-errors.test @@ -287,17 +287,6 @@ def g(*x): # type: (X) -> Y file:1: error: Inconsistent use of '*' in function signature file:3: error: Inconsistent use of '*' in function signature -[case testCommentFunctionAnnotationVarArgMispatch2] -def f(*x, **y): # type: (**X, *Y) -> Z - pass -def g(*x, **y): # type: (*X, *Y) -> Z - pass -[out] -file:1: error: Inconsistent use of '*' in function signature -file:1: error: Inconsistent use of '**' in function signature -file:3: error: Inconsistent use of '*' in function signature -file:3: error: Inconsistent use of '**' in function signature - [case testPrintStatementInPython3] print 1 [out] diff --git a/typeshed b/typeshed index 231f00d7daae..fadd7d884741 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 231f00d7daaea17b352aaedc52fafa602a68c78c +Subproject commit fadd7d8847412ac1f9d3439e8aa14f944daf9db5 From f4ccf92b17663a68b4e0fe2c49f71d5678d04142 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sat, 24 Dec 2016 22:34:01 -0800 Subject: [PATCH 08/28] go back to master version of typeshed I guess --- typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typeshed b/typeshed index fadd7d884741..231f00d7daae 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit fadd7d8847412ac1f9d3439e8aa14f944daf9db5 +Subproject commit 231f00d7daaea17b352aaedc52fafa602a68c78c From bb5134efda9f588c3217b6eab70c0870f00852e7 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Mon, 17 Apr 2017 19:29:24 -0700 Subject: [PATCH 09/28] Tests all pass again after merge --- mypy/exprtotype.py | 2 ++ mypy/fastparse.py | 29 +++++++++++--------- mypy/nodes.py | 42 ++++++++++++++++++++++++++++- mypy/server/astdiff.py | 4 +-- mypy/typeanal.py | 5 +++- mypy/types.py | 39 ++------------------------- test-data/unit/check-functions.test | 16 +++++------ 7 files changed, 74 insertions(+), 63 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index f359710ba620..3b98e61cedf8 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -92,6 +92,8 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: types.append(expr_to_unanalyzed_type(it)) names.append(None) kinds.append(ARG_POS) + def fail(message, Expression): + raise TypeTranslationError(message) return ArgumentList(types, names, kinds, line=expr.line, column=expr.column) elif isinstance(expr, (StrExpr, BytesExpr, UnicodeExpr)): diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 01953cd51a43..473a86d2801d 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -20,7 +20,8 @@ StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension, SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, Argument, AwaitExpr, TempNode, Expression, Statement, - ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR2 + ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR2, + check_arg_names, ) from mypy.types import ( Type, CallableType, AnyType, UnboundType, TupleType, ArgumentList, EllipsisType, @@ -152,6 +153,9 @@ def __init__(self, def fail(self, msg: str, line: int, column: int) -> None: self.errors.report(line, column, msg) + def fail_ast(self, msg: str, n: ast3.AST) -> None: + self.fail(msg, n.lineno, n.col_offset) + def generic_visit(self, node: ast3.AST) -> None: raise RuntimeError('AST node not implemented: ' + str(type(node))) @@ -445,13 +449,7 @@ def make_argument(arg: ast3.arg, default: Optional[ast3.expr], kind: int) -> Arg new_args.append(make_argument(args.kwarg, None, ARG_STAR2)) names.append(args.kwarg) - seen_names = set() # type: Set[str] - for name in names: - if name.arg in seen_names: - self.fail("duplicate argument '{}' in function definition".format(name.arg), - name.lineno, name.col_offset) - break - seen_names.add(name.arg) + check_arg_names([name.arg for name in names], names, self.fail_ast) return new_args @@ -1010,7 +1008,7 @@ def translate_argument_list(self, l: Sequence[ast3.AST]) -> ArgumentList: typ = AnyType(implicit=True) # type: Type for i, arg in enumerate(e.args): if i == 0 and not star: - name = _extract_str(arg) + name = self._extract_str(arg) elif i == 1 and not star or i == 0 and star: typ = self.visit(arg) else: @@ -1019,7 +1017,7 @@ def translate_argument_list(self, l: Sequence[ast3.AST]) -> ArgumentList: for k in e.keywords: value = k.value if k.arg == "name" and not star: - name = _extract_str(value) + name = self._extract_str(value) elif k.arg == "typ": typ = self.visit(value) else: @@ -1037,6 +1035,14 @@ def translate_argument_list(self, l: Sequence[ast3.AST]) -> ArgumentList: return ArgumentList(types, names, kinds, line=self.line) + def _extract_str(self, n: ast3.AST) -> str: + if isinstance(n, ast3.Str): + return n.s.strip() + elif isinstance(n, ast3.NameConstant) and str(n.value) == 'None': + return None + self.fail('Expected string literal for argument name, got "{}"'.format(n), n.lineno, n.col_offset) + return None + def visit_Name(self, n: ast3.Name) -> Type: return UnboundType(n.id, line=self.line) @@ -1089,5 +1095,4 @@ def visit_Ellipsis(self, n: ast3.Ellipsis) -> Type: # List(expr* elts, expr_context ctx) def visit_List(self, n: ast3.List) -> Type: l = len(n.elts) - return ArgumentList( - self.translate_expr_list(n.elts), [None]*l, [ARG_POS]*l, line=self.line) + return self.translate_argument_list(n.elts) diff --git a/mypy/nodes.py b/mypy/nodes.py index c1e3fc8e5f1c..65beb2986763 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -4,7 +4,7 @@ from abc import abstractmethod from typing import ( - Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional + Any, TypeVar, List, Tuple, cast, Set, Dict, Union, Optional, Callable, ) import mypy.strconv @@ -2447,3 +2447,43 @@ def get_member_expr_fullname(expr: MemberExpr) -> str: for key, obj in globals().items() if isinstance(obj, type) and issubclass(obj, SymbolNode) and obj is not SymbolNode } + +def check_arg_kinds(arg_kinds: List[int], nodes: List[T], fail: Callable[[str, T], None]) -> None: + is_var_arg = False + is_kw_arg = False + seen_named = False + seen_opt = False + for kind, node in zip(arg_kinds, nodes): + if kind == ARG_POS: + if is_var_arg or is_kw_arg or seen_named or seen_opt: + fail("Required positional args may not appear " + "after default, named or star args", + node) + break + elif kind == ARG_OPT: + if is_var_arg or is_kw_arg or seen_named: + fail("Positional default args may not appear after named or star args", node) + break + seen_opt = True + elif kind == ARG_STAR: + if is_var_arg or is_kw_arg or seen_named: + fail("Star args may not appear after named or star args", node) + break + is_var_arg = True + elif kind == ARG_NAMED or kind == ARG_NAMED_OPT: + seen_named = True + elif kind == ARG_STAR2: + if is_kw_arg: + fail("You may only have one **kwargs argument", node) + break + is_kw_arg = True + + +def check_arg_names(names: List[str], nodes: List[T], fail: Callable[[str, T], None], + description: str = 'function definition') -> None: + seen_names = set() # type: Set[str] + for name, node in zip(names, nodes): + if name is not None and name in seen_names: + fail("duplicate argument '{}' in {}".format(name, description), node) + break + seen_names.add(name) diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 9b9659d2f4ea..695f9ce1253e 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -11,7 +11,7 @@ from mypy.nodes import SymbolTable, SymbolTableNode, FuncBase, TypeInfo, Var from mypy.types import ( - Type, TypeVisitor, UnboundType, TypeList, AnyType, NoneTyp, UninhabitedType, + Type, TypeVisitor, UnboundType, ArgumentList, AnyType, NoneTyp, UninhabitedType, ErasedType, DeletedType, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, PartialType, TypeType ) @@ -137,7 +137,7 @@ def __init__(self, right: Type) -> None: def visit_unbound_type(self, left: UnboundType) -> bool: return False - def visit_type_list(self, t: TypeList) -> bool: + def visit_type_list(self, t: ArgumentList) -> bool: assert False, 'Not supported' def visit_any(self, left: AnyType) -> bool: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 6b1a36e7c549..65e1dcfebcfe 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -9,10 +9,11 @@ StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, ArgKindException, ArgNameException, get_type_vars, union_items ) + from mypy.nodes import ( BOUND_TVAR, UNBOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, Var, Expression, - IndexExpr, RefExpr, nongen_builtins, + IndexExpr, RefExpr, nongen_builtins, check_arg_names, check_arg_kinds, ) from mypy.sametypes import is_same_type from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError @@ -382,6 +383,8 @@ def analyze_callable_type(self, t: UnboundType) -> Type: # Callable[[ARG, ...], RET] (ordinary callable type) args = t.args[0].types try: + check_arg_names(t.args[0].names, [t]*len(args), self.fail, "Callable") + check_arg_kinds(t.args[0].kinds, [t]*len(args), self.fail) return CallableType(self.anal_array(args), t.args[0].kinds, t.args[0].names, diff --git a/mypy/types.py b/mypy/types.py index fabf5e19bc12..1055a09dfe5b 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -585,13 +585,13 @@ def __init__(self, special_sig: Optional[str] = None, from_type_type: bool = False, ) -> None: - self._process_kinds_on_init(arg_kinds) - self._process_names_on_init(arg_names) if variables is None: variables = [] assert len(arg_types) == len(arg_kinds) self.arg_types = arg_types self.arg_kinds = arg_kinds + self.is_var_arg = ARG_STAR in arg_kinds + self.is_kw_arg = ARG_STAR2 in arg_kinds self.arg_names = arg_names self.min_args = arg_kinds.count(ARG_POS) self.ret_type = ret_type @@ -607,41 +607,6 @@ def __init__(self, self.from_type_type = from_type_type super().__init__(line, column) - def _process_names_on_init(self, arg_names): - seen = set() # type: Set[str] - for name in arg_names: - if name is None: - continue - if name in seen: - raise ArgNameException('Duplicate argument name "{}"'.format(name)) - seen.add(name) - - def _process_kinds_on_init(self, arg_kinds): - self.is_var_arg = False - self.is_kw_arg = False - seen_named = False - seen_opt = False - for kind in arg_kinds: - if kind == ARG_POS: - if self.is_var_arg or self.is_kw_arg or seen_named or seen_opt: - raise ArgKindException("Required positional args may not appear " - "after default, named or star args") - elif kind == ARG_OPT: - if self.is_var_arg or self.is_kw_arg or seen_named: - raise ArgKindException("Positional default args may not appear " - "after named or star args") - seen_opt = True - elif kind == ARG_STAR: - if self.is_var_arg or self.is_kw_arg or seen_named: - raise ArgKindException("Star args may not appear after named or star args") - self.is_var_arg = True - elif kind == ARG_NAMED or kind == ARG_NAMED_OPT: - seen_named = True - elif kind == ARG_STAR2: - if self.is_kw_arg: - raise ArgKindException("You may only have one **kwargs argument") - self.is_kw_arg = True - def copy_modified(self, arg_types: List[Type] = _dummy, arg_kinds: List[int] = _dummy, diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 057c65269b3e..4802fda3fe48 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1474,13 +1474,13 @@ from mypy_extensions import Arg, StarArg, KwArg def WrongArg(x, y): return y -def a(f: Callable[[WrongArg('x', int)], int]): pass # E: Parse error before (: Unknown argument constructor WrongArg # E: Parse error before end of line -def b(f: Callable[[Arg('x', 1)], int]): pass # E: Parse error before numeric literal # E: Parse error before end of line -def c(f: Callable[[StarArg('x', int)], int]): pass # E: Parse error before "int": Unexpected argument for argument constructor # E: Parse error before end of line +def a(f: Callable[[WrongArg('x', int)], int]): pass # E: Unknown argument constructor WrongArg +def b(f: Callable[[Arg('x', 1)], int]): pass # E: invalid type comment or annotation +def c(f: Callable[[StarArg('x', int)], int]): pass # E: Too many arguments for argument constructor def d(f: Callable[[StarArg(int)], int]): pass # ok def e(f: Callable[[StarArg(), KwArg()], int]): pass # ok def g(f: Callable[[Arg(name='x', typ=int)], int]): pass # ok -def h(f: Callable[[Arg(gnome='x', typ=int)], int]): pass # E: Parse error before "gnome": Unexpected argument "gnome" for argument constructor # E: Parse error before end of line +def h(f: Callable[[Arg(gnome='x', typ=int)], int]): pass # E: Unexpected argument "gnome" for argument constructor def i(f: Callable[[Arg(name=None, typ=int)], int]): pass # ok [builtins fixtures/dict.pyi] @@ -1493,21 +1493,18 @@ def a(f: Callable[[WrongArg('x', int)], int]): pass # E: Unknown argument constr [builtins fixtures/dict.pyi] [case testCallableFastParseWrongTypeType] -# flags: --fast-parser from typing import Callable from mypy_extensions import Arg -def b(f: Callable[[Arg('x', 1)], int]): pass # E: Bad type for callable argument +def b(f: Callable[[Arg('x', 1)], int]): pass # E: invalid type comment or annotation [builtins fixtures/dict.pyi] [case testCallableFastParseTooManyStarArg] -# flags: --fast-parser from typing import Callable from mypy_extensions import StarArg def c(f: Callable[[StarArg('x', int)], int]): pass # E: Too many arguments for argument constructor [builtins fixtures/dict.pyi] [case testCallableFastParseGood] -# flags: --fast-parser from typing import Callable from mypy_extensions import StarArg, Arg def d(f: Callable[[StarArg(int)], int]): pass # ok @@ -1517,7 +1514,6 @@ def i(f: Callable[[Arg(name=None, typ=int)], int]): pass # ok [builtins fixtures/dict.pyi] [case testCallableFastParseBadArgArgName] -# flags: --fast-parser from typing import Callable from mypy_extensions import Arg def h(f: Callable[[Arg(gnome='x', typ=int)], int]): pass # E: Unexpected argument "gnome" for argument constructor @@ -1539,7 +1535,7 @@ def j(f: Callable[[NamedArg('x'), DefaultArg('y', int)], int]): pass # E: Positi from typing import Callable from mypy_extensions import Arg, StarArg, KwArg, DefaultArg -def f(f: Callable[[Arg('x', int), int, Arg('x', int)], int]): pass # E: Duplicate argument name "x" +def f(f: Callable[[Arg('x', int), int, Arg('x', int)], int]): pass # E: duplicate argument 'x' in Callable [builtins fixtures/dict.pyi] From 54a5da91eb363bef01d086ec53b93e1077fd7ced Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Wed, 19 Apr 2017 22:50:56 -0700 Subject: [PATCH 10/28] Big refactor. Wait until semanal to get arg kinds, switch order again, visitor stuff --- extensions/mypy_extensions.py | 10 +- mypy/erasetype.py | 8 +- mypy/expandtype.py | 3 - mypy/exprtotype.py | 70 ++++------ mypy/fastparse.py | 128 ++++++++++--------- mypy/fixup.py | 15 ++- mypy/indirection.py | 9 +- mypy/join.py | 3 - mypy/meet.py | 3 - mypy/messages.py | 8 +- mypy/sametypes.py | 3 - mypy/server/astdiff.py | 3 - mypy/sharedparse.py | 14 -- mypy/subtypes.py | 6 - mypy/typeanal.py | 49 ++++++- mypy/types.py | 134 ++++++++++++++------ test-data/unit/check-expressions.test | 6 +- test-data/unit/check-functions.test | 116 +++++++++-------- test-data/unit/check-inference.test | 2 +- test-data/unit/check-varargs.test | 2 +- test-data/unit/lib-stub/mypy_extensions.pyi | 17 +-- test-data/unit/parse-python2.test | 4 +- test-data/unit/parse.test | 2 +- test-data/unit/pythoneval.test | 4 +- typeshed | 2 +- 25 files changed, 342 insertions(+), 279 deletions(-) diff --git a/extensions/mypy_extensions.py b/extensions/mypy_extensions.py index eee68baae6bb..1fcdfc9b3b9e 100644 --- a/extensions/mypy_extensions.py +++ b/extensions/mypy_extensions.py @@ -93,19 +93,19 @@ class Point2D(TypedDict): syntax forms work for Python 2.7 and 3.2+ """ -def Arg(name=None, typ=Any): +def Arg(typ=Any, name=None): return typ -def DefaultArg(name=None, typ=Any): +def DefaultArg(typ=Any, name=None): return typ -def NamedArg(name=None, typ=Any): +def NamedArg(typ=Any, name=None): return typ -def DefaultNamedArg(name=None, typ=Any): +def DefaultNamedArg(typ=Any, name=None): return typ -def StarArg(typ=Any): +def VarArg(typ=Any): return typ def KwArg(typ=Any): diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 23508dc21736..7168e4092754 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -26,11 +26,6 @@ def erase_type(typ: Type) -> Type: class EraseTypeVisitor(TypeVisitor[Type]): - def visit_unbound_type(self, t: UnboundType) -> Type: - assert False, 'Not supported' - - def visit_type_list(self, t: ArgumentList) -> Type: - assert False, 'Not supported' def visit_any(self, t: AnyType) -> Type: return t @@ -49,6 +44,9 @@ def visit_partial_type(self, t: PartialType) -> Type: # Should not get here. raise RuntimeError() + def visit_unbound_type(self, t: UnboundType) -> Type: + raise RuntimeError() + def visit_deleted_type(self, t: DeletedType) -> Type: return t diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 5a04b5c05ba4..ca581fb21fa6 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -63,9 +63,6 @@ def __init__(self, variables: Mapping[TypeVarId, Type]) -> None: def visit_unbound_type(self, t: UnboundType) -> Type: return t - def visit_type_list(self, t: ArgumentList) -> Type: - assert False, 'Not supported' - def visit_any(self, t: AnyType) -> Type: return t diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 3b98e61cedf8..f95bdd803853 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -5,9 +5,10 @@ ListExpr, StrExpr, BytesExpr, UnicodeExpr, EllipsisExpr, CallExpr, ARG_POS, ARG_NAMED, get_member_expr_fullname ) -from mypy.sharedparse import ARG_KINDS_BY_CONSTRUCTOR, STAR_ARG_CONSTRUCTORS from mypy.fastparse import parse_type_comment -from mypy.types import Type, UnboundType, ArgumentList, EllipsisType, AnyType, Optional +from mypy.types import ( + Type, UnboundType, ArgumentList, EllipsisType, AnyType, Optional, CallableArgument, +) class TypeTranslationError(Exception): @@ -53,49 +54,32 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: return base else: raise TypeTranslationError() - elif isinstance(expr, ListExpr): - types = [] # type: List[Type] - names = [] # type: List[Optional[str]] - kinds = [] # type: List[int] - for it in expr.items: - if isinstance(it, CallExpr): - if not isinstance(it.callee, NameExpr): - raise TypeTranslationError() - arg_const = it.callee.name - try: - kind = ARG_KINDS_BY_CONSTRUCTOR[arg_const] - except KeyError: + elif isinstance(expr, CallExpr): + if not isinstance(expr.callee, NameExpr): + raise TypeTranslationError() + arg_const = expr.callee.name + name = None + typ = AnyType(implicit=True) # type: Type + for i, arg in enumerate(expr.args): + if expr.arg_names[i] is not None: + if expr.arg_names[i] == "name": + name = _extract_str(arg) + continue + elif expr.arg_names[i] == "typ": + typ = expr_to_unanalyzed_type(arg) + continue + else: raise TypeTranslationError() - name = None - typ = AnyType(implicit=True) # type: Type - star = arg_const in STAR_ARG_CONSTRUCTORS - for i, arg in enumerate(it.args): - if it.arg_names[i] is not None: - if it.arg_names[i] == "name": - name = _extract_str(arg) - continue - elif it.arg_names[i] == "typ": - typ = expr_to_unanalyzed_type(arg) - continue - else: - raise TypeTranslationError() - elif i == 0 and not star: - name = _extract_str(arg) - elif i == 1 and not star or i == 0 and star: - typ = expr_to_unanalyzed_type(arg) - else: - raise TypeTranslationError() - names.append(name) - types.append(typ) - kinds.append(kind) + elif i == 0: + typ = expr_to_unanalyzed_type(arg) + elif i == 1: + name = _extract_str(arg) else: - types.append(expr_to_unanalyzed_type(it)) - names.append(None) - kinds.append(ARG_POS) - def fail(message, Expression): - raise TypeTranslationError(message) - return ArgumentList(types, names, kinds, - line=expr.line, column=expr.column) + raise TypeTranslationError() + return CallableArgument(typ, name, arg_const, expr.line, expr.column) + elif isinstance(expr, ListExpr): + return ArgumentList([expr_to_unanalyzed_type(it) for it in expr.items], + line=expr.line, column=expr.column) elif isinstance(expr, (StrExpr, BytesExpr, UnicodeExpr)): # Parse string literal type. try: diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 473a86d2801d..eccbc7a65a3f 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -4,7 +4,6 @@ from typing import Tuple, Union, TypeVar, Callable, Sequence, Optional, Any, cast, List, Set from mypy.sharedparse import ( special_function_elide_names, argument_elide_name, - ARG_KINDS_BY_CONSTRUCTOR, STAR_ARG_CONSTRUCTORS, ) from mypy.nodes import ( MypyFile, Node, ImportBase, Import, ImportAll, ImportFrom, FuncDef, @@ -25,6 +24,7 @@ ) from mypy.types import ( Type, CallableType, AnyType, UnboundType, TupleType, ArgumentList, EllipsisType, + CallableArgument, ) from mypy import defaults from mypy import experiments @@ -153,9 +153,6 @@ def __init__(self, def fail(self, msg: str, line: int, column: int) -> None: self.errors.report(line, column, msg) - def fail_ast(self, msg: str, n: ast3.AST) -> None: - self.fail(msg, n.lineno, n.col_offset) - def generic_visit(self, node: ast3.AST) -> None: raise RuntimeError('AST node not implemented: ' + str(type(node))) @@ -449,18 +446,12 @@ def make_argument(arg: ast3.arg, default: Optional[ast3.expr], kind: int) -> Arg new_args.append(make_argument(args.kwarg, None, ARG_STAR2)) names.append(args.kwarg) - check_arg_names([name.arg for name in names], names, self.fail_ast) + def fail_arg(msg: str, arg: ast3.arg): + self.fail(msg, arg.lineno, arg.col_offset) - return new_args + check_arg_names([name.arg for name in names], names, fail_arg) - def stringify_name(self, n: ast3.AST) -> str: - if isinstance(n, ast3.Name): - return n.id - elif isinstance(n, ast3.Attribute): - sv = self.stringify_name(n.value) - if sv is not None: - return "{}.{}".format(sv, n.attr) - return None # Can't do it. + return new_args # ClassDef(identifier name, # expr* bases, @@ -473,7 +464,7 @@ def visit_ClassDef(self, n: ast3.ClassDef) -> ClassDef: metaclass_arg = find(lambda x: x.arg == 'metaclass', n.keywords) metaclass = None if metaclass_arg: - metaclass = self.stringify_name(metaclass_arg.value) + metaclass = stringify_name(metaclass_arg.value) if metaclass is None: metaclass = '' # To be reported later @@ -964,6 +955,21 @@ class TypeConverter(ast3.NodeTransformer): # type: ignore # typeshed PR #931 def __init__(self, errors: Errors, line: int = -1) -> None: self.errors = errors self.line = line + self.node_stack = [] # type: List[ast3.AST] + + def visit(self, node): + """Modified visit -- keep track of the stack of nodes""" + self.node_stack.append(node) + try: + return super().visit(node) + finally: + self.node_stack.pop() + + def parent(self) -> ast3.AST: + """Return the AST node above the one we are processing""" + if len(self.node_stack) < 2: + return None + return self.node_stack[-2] def fail(self, msg: str, line: int, column: int) -> None: self.errors.report(line, column, msg) @@ -984,63 +990,50 @@ def visit_NoneType(self, n: Any) -> Type: def translate_expr_list(self, l: Sequence[ast3.AST]) -> List[Type]: return [self.visit(e) for e in l] + def visit_Call(self, e: ast3.Call) -> Type: + # Parse the arg constructor + if not isinstance(self.parent(), ast3.List): + return self.generic_visit(e) + f = e.func + constructor = stringify_name(f) + if not constructor: + self.fail("Expected arg constructor name", e.lineno, e.col_offset) + constructor = "BadArgConstructor" + name = None # type: Optional[str] + typ = AnyType(implicit=True) # type: Type + for i, arg in enumerate(e.args): + if i == 0: + typ = self.visit(arg) + elif i == 1: + name = self._extract_str(arg) + else: + self.fail("Too many arguments for argument constructor", + f.lineno, f.col_offset) + for k in e.keywords: + value = k.value + if k.arg == "name": + name = self._extract_str(value) + elif k.arg == "typ": + typ = self.visit(value) + else: + self.fail( + 'Unexpected argument "{}" for argument constructor'.format(k.arg), + value.lineno, value.col_offset) + return CallableArgument(typ, name, constructor, e.lineno, e.col_offset) + + def translate_argument_list(self, l: Sequence[ast3.AST]) -> ArgumentList: types = [] # type: List[Type] names = [] # type: List[Optional[str]] kinds = [] # type: List[int] - for e in l: - if isinstance(e, ast3.Call): - # Parse the arg constructor - f = e.func - if not isinstance(f, ast3.Name): - raise FastParserError("Expected arg constructor name", - e.lineno, e.col_offset) - try: - kind = ARG_KINDS_BY_CONSTRUCTOR[f.id] - star = f.id in STAR_ARG_CONSTRUCTORS - except KeyError: - self.fail("Unknown argument constructor {}".format(f.id), - f.lineno, f.col_offset) - kind = ARG_POS - star = False - - name = None # type: Optional[str] - typ = AnyType(implicit=True) # type: Type - for i, arg in enumerate(e.args): - if i == 0 and not star: - name = self._extract_str(arg) - elif i == 1 and not star or i == 0 and star: - typ = self.visit(arg) - else: - self.fail("Too many arguments for argument constructor", - f.lineno, f.col_offset) - for k in e.keywords: - value = k.value - if k.arg == "name" and not star: - name = self._extract_str(value) - elif k.arg == "typ": - typ = self.visit(value) - else: - self.fail( - 'Unexpected argument "{}" for argument constructor'.format(k.arg), - value.lineno, value.col_offset) - - types.append(typ) - names.append(name) - kinds.append(kind) - else: - types.append(self.visit(e)) - names.append(None) - kinds.append(ARG_POS) - - return ArgumentList(types, names, kinds, line=self.line) + return ArgumentList([self.visit(e) for e in l], line=self.line) - def _extract_str(self, n: ast3.AST) -> str: + def _extract_str(self, n: ast3.expr) -> str: if isinstance(n, ast3.Str): return n.s.strip() elif isinstance(n, ast3.NameConstant) and str(n.value) == 'None': return None - self.fail('Expected string literal for argument name, got "{}"'.format(n), n.lineno, n.col_offset) + self.fail('Expected string literal for argument name, got {}'.format(type(n).__name__), self.line, 0) return None def visit_Name(self, n: ast3.Name) -> Type: @@ -1096,3 +1089,12 @@ def visit_Ellipsis(self, n: ast3.Ellipsis) -> Type: def visit_List(self, n: ast3.List) -> Type: l = len(n.elts) return self.translate_argument_list(n.elts) + +def stringify_name(n: ast3.AST) -> Optional[str]: + if isinstance(n, ast3.Name): + return n.id + elif isinstance(n, ast3.Attribute): + sv = stringify_name(n.value) + if sv is not None: + return "{}.{}".format(sv, n.attr) + return None # Can't do it. diff --git a/mypy/fixup.py b/mypy/fixup.py index 2be2ec4ec327..f4bc3ae50757 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -11,7 +11,7 @@ from mypy.types import ( CallableType, EllipsisType, Instance, Overloaded, TupleType, TypedDictType, ArgumentList, TypeVarType, UnboundType, UnionType, TypeVisitor, - TypeType + TypeType, CallableArgument, ) from mypy.visitor import NodeVisitor @@ -177,8 +177,8 @@ def visit_callable_type(self, ct: CallableType) -> None: val.accept(self) v.upper_bound.accept(self) - def visit_ellipsis_type(self, e: EllipsisType) -> None: - pass # Nothing to descend into. +# def visit_ellipsis_type(self, e: EllipsisType) -> None: +# pass # Nothing to descend into. def visit_overloaded(self, t: Overloaded) -> None: for ct in t.items(): @@ -210,9 +210,12 @@ def visit_typeddict_type(self, tdt: TypedDictType) -> None: if tdt.fallback is not None: tdt.fallback.accept(self) - def visit_type_list(self, tl: ArgumentList) -> None: - for t in tl.types: - t.accept(self) + # def visit_type_list(self, tl: ArgumentList) -> None: + # for t in tl.items: + # t.accept(self) + + # def visit_callable_argument(self, t: CallableArgument) -> None: + # t.typ.accept(self) def visit_type_var(self, tvt: TypeVarType) -> None: if tvt.values: diff --git a/mypy/indirection.py b/mypy/indirection.py index b50721650c3a..ba45a8cef41b 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -2,7 +2,7 @@ from abc import abstractmethod from mypy.visitor import NodeVisitor -from mypy.types import TypeVisitor +from mypy.types import SyntheticTypeVisitor from mypy.nodes import MODULE_REF import mypy.nodes as nodes import mypy.types as types @@ -19,7 +19,7 @@ def extract_module_names(type_name: Optional[str]) -> List[str]: return [] -class TypeIndirectionVisitor(TypeVisitor[Set[str]]): +class TypeIndirectionVisitor(SyntheticTypeVisitor[Set[str]]): """Returns all module references within a particular type.""" def __init__(self) -> None: @@ -43,7 +43,10 @@ def visit_unbound_type(self, t: types.UnboundType) -> Set[str]: return self._visit(*t.args) def visit_type_list(self, t: types.ArgumentList) -> Set[str]: - return self._visit(*t.types) + return self._visit(*t.items) + + def visit_callable_argument(self, t: types.CallableArgument) -> Set[str]: + return self._visit(t.typ) def visit_any(self, t: types.AnyType) -> Set[str]: return set() diff --git a/mypy/join.py b/mypy/join.py index f999ef191b1a..49a056a5cc44 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -107,9 +107,6 @@ def visit_union_type(self, t: UnionType) -> Type: else: return UnionType.make_simplified_union([self.s, t]) - def visit_type_list(self, t: ArgumentList) -> Type: - assert False, 'Not supported' - def visit_any(self, t: AnyType) -> Type: return t diff --git a/mypy/meet.py b/mypy/meet.py index 16bd0285aa02..c9221bfe0669 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -139,9 +139,6 @@ def visit_unbound_type(self, t: UnboundType) -> Type: else: return AnyType() - def visit_type_list(self, t: ArgumentList) -> Type: - assert False, 'Not supported' - def visit_any(self, t: AnyType) -> Type: return self.s diff --git a/mypy/messages.py b/mypy/messages.py index cc886b4c8fe6..6d52765115d2 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -93,7 +93,7 @@ ARG_OPT: "DefaultArg", ARG_NAMED: "NamedArg", ARG_NAMED_OPT: "DefaultNamedArg", - ARG_STAR: "StarArg", + ARG_STAR: "VarArg", ARG_STAR2: "KwArg", } @@ -214,15 +214,15 @@ def format(self, typ: Type, verbosity: int = 0) -> str: verbosity = max(verbosity - 1, 0)))) else: constructor = ARG_CONSTRUCTOR_NAMES[arg_kind] - if arg_kind in (ARG_STAR, ARG_STAR2): + if arg_kind in (ARG_STAR, ARG_STAR2) or arg_name is None: arg_strings.append("{}({})".format( constructor, strip_quotes(self.format(arg_type)))) else: arg_strings.append("{}({}, {})".format( constructor, - repr(arg_name), - strip_quotes(self.format(arg_type)))) + strip_quotes(self.format(arg_type)), + repr(arg_name))) return 'Callable[[{}], {}]'.format(", ".join(arg_strings), return_type) else: diff --git a/mypy/sametypes.py b/mypy/sametypes.py index beafe852599f..779032f7ade3 100644 --- a/mypy/sametypes.py +++ b/mypy/sametypes.py @@ -55,9 +55,6 @@ def __init__(self, right: Type) -> None: def visit_unbound_type(self, left: UnboundType) -> bool: return True - def visit_type_list(self, t: ArgumentList) -> bool: - assert False, 'Not supported' - def visit_any(self, left: AnyType) -> bool: return isinstance(self.right, AnyType) diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 20ec561b2dee..584fcb5cb4b2 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -136,9 +136,6 @@ def __init__(self, right: Type) -> None: def visit_unbound_type(self, left: UnboundType) -> bool: return False - def visit_type_list(self, t: ArgumentList) -> bool: - assert False, 'Not supported' - def visit_any(self, left: AnyType) -> bool: return isinstance(self.right, AnyType) diff --git a/mypy/sharedparse.py b/mypy/sharedparse.py index 148b151f1260..1643aaccf321 100644 --- a/mypy/sharedparse.py +++ b/mypy/sharedparse.py @@ -1,7 +1,5 @@ from typing import Union, Tuple -from mypy.nodes import (ARG_POS, ARG_NAMED, ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2) - """Shared logic between our three mypy parser files.""" @@ -94,18 +92,6 @@ MAGIC_METHODS_POS_ARGS_ONLY = MAGIC_METHODS - MAGIC_METHODS_ALLOWING_KWARGS -ARG_KINDS_BY_CONSTRUCTOR = { - 'Arg': ARG_POS, - 'DefaultArg': ARG_OPT, - 'NamedArg': ARG_NAMED, - 'DefaultNamedArg': ARG_NAMED_OPT, - 'StarArg': ARG_STAR, - 'KwArg': ARG_STAR2, -} - -STAR_ARG_CONSTRUCTORS = {'StarArg', 'KwArg'} - - def special_function_elide_names(name: str) -> bool: return name in MAGIC_METHODS_POS_ARGS_ONLY diff --git a/mypy/subtypes.py b/mypy/subtypes.py index bab0a34cc7eb..26b14702ca57 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -110,9 +110,6 @@ def __init__(self, right: Type, def visit_unbound_type(self, left: UnboundType) -> bool: return True - def visit_type_list(self, t: ArgumentList) -> bool: - assert False, 'Not supported' - def visit_any(self, left: AnyType) -> bool: return True @@ -564,9 +561,6 @@ def visit_unbound_type(self, left: UnboundType) -> bool: # from unions, which could filter out some bogus messages. return True - def visit_type_list(self, left: ArgumentList) -> bool: - assert False, 'Should not happen' - def visit_any(self, left: AnyType) -> bool: return isinstance(self.right, AnyType) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 65e1dcfebcfe..ff5c1d0514af 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -7,13 +7,14 @@ Type, UnboundType, TypeVarType, TupleType, TypedDictType, UnionType, Instance, AnyType, CallableType, NoneTyp, DeletedType, ArgumentList, TypeVarDef, TypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, - ArgKindException, ArgNameException, get_type_vars, union_items + ArgKindException, ArgNameException, CallableArgument, get_type_vars, union_items ) from mypy.nodes import ( BOUND_TVAR, UNBOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, Var, Expression, IndexExpr, RefExpr, nongen_builtins, check_arg_names, check_arg_kinds, + ARG_POS, ARG_NAMED, ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, ) from mypy.sametypes import is_same_type from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError @@ -30,6 +31,15 @@ 'typing.Union', } +ARG_KINDS_BY_CONSTRUCTOR = { + 'mypy_extensions.Arg': ARG_POS, + 'mypy_extensions.DefaultArg': ARG_OPT, + 'mypy_extensions.NamedArg': ARG_NAMED, + 'mypy_extensions.DefaultNamedArg': ARG_NAMED_OPT, + 'mypy_extensions.VarArg': ARG_STAR, + 'mypy_extensions.KwArg': ARG_STAR2, +} + def analyze_type_alias(node: Expression, lookup_func: Callable[[str, Context], SymbolTableNode], @@ -381,13 +391,40 @@ def analyze_callable_type(self, t: UnboundType) -> Type: ret_type = self.anal_type(t.args[1]) if isinstance(t.args[0], ArgumentList): # Callable[[ARG, ...], RET] (ordinary callable type) - args = t.args[0].types + args = [] # type: List[Type] + names = [] # type: List[str] + kinds = [] # type: List[int] + for arg in t.args[0].items: + if isinstance(arg, CallableArgument): + args.append(arg.typ) + names.append(arg.name) + if arg.constructor is None: + kinds.append(ARG_POS) + continue + found = self.lookup(arg.constructor, arg) + if found is None: + # Looking it up already put an error message in + kinds.append(ARG_POS) + elif found.fullname not in ARG_KINDS_BY_CONSTRUCTOR: + self.fail('Invalid argument constructor "{}"'.format( + found.fullname), arg) + kinds.append(ARG_POS) + else: + kind = ARG_KINDS_BY_CONSTRUCTOR[found.fullname] + kinds.append(kind) + if arg.name is not None and kind in {ARG_STAR, ARG_STAR2}: + self.fail("{} arguments should not have names".format( + arg.constructor), arg) + else: + args.append(arg) + names.append(None) + kinds.append(ARG_POS) try: - check_arg_names(t.args[0].names, [t]*len(args), self.fail, "Callable") - check_arg_kinds(t.args[0].kinds, [t]*len(args), self.fail) + check_arg_names(names, [t]*len(args), self.fail, "Callable") + check_arg_kinds(kinds, [t]*len(args), self.fail) return CallableType(self.anal_array(args), - t.args[0].kinds, - t.args[0].names, + kinds, + names, ret_type=ret_type, fallback=fallback) except (ArgKindException, ArgNameException) as e: diff --git a/mypy/types.py b/mypy/types.py index 407cf12fd845..616e1773d953 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -222,6 +222,37 @@ def deserialize(cls, data: JsonDict) -> 'UnboundType': return UnboundType(data['name'], [deserialize_type(a) for a in data['args']]) +class CallableArgument(Type): + typ = None # type: Type + name = None # type: Optional[str] + constructor = None # type: str + + def __init__(self, typ: Type, name: str, constructor: str, + line: int = -1, column: int = -1) -> None: + super().__init__(line, column) + self.typ = typ + self.name = name + self.constructor = constructor + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + assert isinstance(visitor, SyntheticTypeVisitor) + return visitor.visit_callable_argument(self) + + def serialize(self) -> JsonDict: + return { + '.class': 'CallableArgument', + 'typ': self.typ.serialize(), + 'name': self.name, + 'constructor': self.constructor, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'CallableArgument': + assert data['.class'] == 'CallableArgument' + return CallableArgument( + typ=deserialize_type(data['typ']), + name=data['name'], + constructor=data['constructor']) class ArgumentList(Type): """Information about argument types and names [...]. @@ -231,42 +262,39 @@ class ArgumentList(Type): but a syntactic AST construct. """ - types = None # type: List[Type] - names = None # type: List[Optional[str]] - kinds = None # type: List[int] + items = None # type: List[Type] - def __init__(self, - types: List[Type], - names: List[Optional[str]], - kinds: List[int], - line: int = -1, - column: int = -1) -> None: + def __init__(self, items: List[Type], line: int = -1, column: int = -1) -> None: super().__init__(line, column) - self.types = types - self.names = names - self.kinds = kinds + self.items = items + + @property + def types(self) -> List[Type]: + return [a.typ if isinstance(a, CallableArgument) else a for a in self.items] + + @property + def names(self) -> List[Optional[str]]: + return [a.name if isinstance(a, CallableArgument) else None for a in self.items] + + @property + def constructors(self) -> List[Optional[str]]: + return [a.constructor if isinstance(a, CallableArgument) else None for a in self.items] def accept(self, visitor: 'TypeVisitor[T]') -> T: + assert isinstance(visitor, SyntheticTypeVisitor) return visitor.visit_type_list(self) def serialize(self) -> JsonDict: - return {'.class': 'ArgumentList', - 'items': [t.serialize() for t in self.types], - 'names': self.names, - 'kinds': self.kinds} + return { + '.class': 'ArgumentList', + 'items': [t.serialize() for t in self.items], + } @classmethod def deserialize(cls, data: JsonDict) -> 'ArgumentList': assert data['.class'] == 'ArgumentList' or data['.class'] == 'TypeList' - types = [deserialize_type(t) for t in data['items']] - names = cast(List[Optional[str]], data.get('names', [None] * len(types))) - kinds = cast(List[int], data.get('kinds', [ARG_POS] * len(types))) - return ArgumentList( - types=types, - names=names, - kinds=kinds, - ) - + items = [Type.deserialize(t) for t in data['items']] + return ArgumentList(items=items) class AnyType(Type): """The type 'Any'.""" @@ -1009,6 +1037,7 @@ def __init__(self, type: Type, line: int = -1, column: int = -1) -> None: super().__init__(line, column) def accept(self, visitor: 'TypeVisitor[T]') -> T: + assert isinstance(visitor, SyntheticTypeVisitor) return visitor.visit_star_type(self) @@ -1150,6 +1179,7 @@ class EllipsisType(Type): """ def accept(self, visitor: 'TypeVisitor[T]') -> T: + assert isinstance(visitor, SyntheticTypeVisitor) return visitor.visit_ellipsis_type(self) def serialize(self) -> JsonDict: @@ -1233,9 +1263,6 @@ def _notimplemented_helper(self, name: str) -> NotImplementedError: def visit_unbound_type(self, t: UnboundType) -> T: pass - def visit_type_list(self, t: ArgumentList) -> T: - raise self._notimplemented_helper('type_list') - @abstractmethod def visit_any(self, t: AnyType) -> T: pass @@ -1278,9 +1305,6 @@ def visit_tuple_type(self, t: TupleType) -> T: def visit_typeddict_type(self, t: TypedDictType) -> T: pass - def visit_star_type(self, t: StarType) -> T: - raise self._notimplemented_helper('star_type') - @abstractmethod def visit_union_type(self, t: UnionType) -> T: pass @@ -1289,15 +1313,34 @@ def visit_union_type(self, t: UnionType) -> T: def visit_partial_type(self, t: PartialType) -> T: pass - def visit_ellipsis_type(self, t: EllipsisType) -> T: - raise self._notimplemented_helper('ellipsis_type') - @abstractmethod def visit_type_type(self, t: TypeType) -> T: pass -class TypeTranslator(TypeVisitor[Type]): +class SyntheticTypeVisitor(TypeVisitor[T]): + """A TypeVisitor that also knows how to visit synthetic AST constructs. + + Not just real types.""" + + @abstractmethod + def visit_star_type(self, t: StarType) -> T: + pass + + @abstractmethod + def visit_type_list(self, t: ArgumentList) -> T: + pass + + @abstractmethod + def visit_callable_argument(self, t: CallableArgument) -> T: + pass + + @abstractmethod + def visit_ellipsis_type(self, t: EllipsisType) -> T: + pass + + +class TypeTranslator(SyntheticTypeVisitor[Type]): """Identity type transformation. Subclass this and override some methods to implement a non-trivial @@ -1310,6 +1353,9 @@ def visit_unbound_type(self, t: UnboundType) -> Type: def visit_type_list(self, t: ArgumentList) -> Type: return t + def visit_callable_argument(self, t: CallableArgument) -> Type: + return t + def visit_any(self, t: AnyType) -> Type: return t @@ -1385,7 +1431,7 @@ def visit_type_type(self, t: TypeType) -> Type: return TypeType(t.item.accept(self), line=t.line, column=t.column) -class TypeStrVisitor(TypeVisitor[str]): +class TypeStrVisitor(SyntheticTypeVisitor[str]): """Visitor for pretty-printing types into strings. This is mostly for debugging/testing. @@ -1407,7 +1453,14 @@ def visit_unbound_type(self, t: UnboundType)-> str: return s def visit_type_list(self, t: ArgumentList) -> str: - return ''.format(self.list_str(t.types)) + return ''.format(self.list_str(t.items)) + + def visit_callable_argument(self, t: CallableArgument) -> str: + typ = t.typ.accept(self) + if t.name is None: + return "{}({})".format(t.constructor, t.typ) + else: + return "{}({}, {})".format(t.constructor, t.typ, t.name) def visit_any(self, t: AnyType) -> str: return 'Any' @@ -1544,7 +1597,7 @@ def keywords_str(self, a: Iterable[Tuple[str, Type]]) -> str: ]) -class TypeQuery(Generic[T], TypeVisitor[T]): +class TypeQuery(SyntheticTypeVisitor[T]): """Visitor for performing queries of types. strategy is used to combine results for a series of types @@ -1559,7 +1612,10 @@ def visit_unbound_type(self, t: UnboundType) -> T: return self.query_types(t.args) def visit_type_list(self, t: ArgumentList) -> T: - return self.query_types(t.types) + return self.query_types(t.items) + + def visit_callable_argument(self, t: CallableArgument) -> T: + return t.typ.accept(self) def visit_any(self, t: AnyType) -> T: return self.strategy([]) diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index bc3f00bfe2df..66248c209859 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1169,7 +1169,7 @@ def foo(a: bytes, b: bytes): b'%s:%s' % (a, b) foo(b'a', b'b') == b'a:b' -[case testStringInterpolationStarArgs] +[case testStringInterpolationVarArgs] x = (1, 2) "%d%d" % (*x,) @@ -1566,7 +1566,7 @@ d = dict([], x=1) d() # E: Dict[str, int] not callable [builtins fixtures/dict.pyi] -[case testDictFromIterableAndStarStarArgs] +[case testDictFromIterableAndStarVarArgs] from typing import Dict it = [('x', 1)] @@ -1581,7 +1581,7 @@ d2() # E: Dict[Any, Any] not callable d3 = dict(it, **kw2) # type: Dict[str, int] # E: Argument 2 to "dict" has incompatible type **Dict[str, str]; expected "int" [builtins fixtures/dict.pyi] -[case testDictFromIterableAndStarStarArgs2] +[case testDictFromIterableAndStarVarArgs2] it = [(1, 'x')] kw = {'x': 'y'} d = dict(it, **kw) # E: Keyword argument only valid with "str" key type in call to "dict" diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 4802fda3fe48..45b978dfde84 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -79,7 +79,7 @@ h = h def l(x) -> None: ... def r(__, *, x) -> None: ... -r = l # E: Incompatible types in assignment (expression has type Callable[[Any], None], variable has type Callable[[Any, NamedArg('x', Any)], None]) +r = l # E: Incompatible types in assignment (expression has type Callable[[Any], None], variable has type Callable[[Any, NamedArg(Any, 'x')], None]) [case testSubtypingFunctionsRequiredLeftArgNotPresent] @@ -114,10 +114,10 @@ hh = h ff = gg ff_nonames = ff ff_nonames = f_nonames # reset -ff = ff_nonames # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[Arg('a', int), Arg('b', str)], None]) +ff = ff_nonames # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[Arg(int, 'a'), Arg(str, 'b')], None]) ff = f # reset -gg = ff # E: Incompatible types in assignment (expression has type Callable[[Arg('a', int), Arg('b', str)], None], variable has type Callable[[Arg('a', int), DefaultArg('b', str)], None]) -gg = hh # E: Incompatible types in assignment (expression has type Callable[[Arg('aa', int), DefaultArg('b', str)], None], variable has type Callable[[Arg('a', int), DefaultArg('b', str)], None]) +gg = ff # E: Incompatible types in assignment (expression has type Callable[[Arg(int, 'a'), Arg(str, 'b')], None], variable has type Callable[[Arg(int, 'a'), DefaultArg(str, 'b')], None]) +gg = hh # E: Incompatible types in assignment (expression has type Callable[[Arg(int, 'aa'), DefaultArg(str, 'b')], None], variable has type Callable[[Arg(int, 'a'), DefaultArg(str, 'b')], None]) [case testSubtypingFunctionsArgsKwargs] from typing import Any, Callable @@ -143,7 +143,7 @@ ee_var = everything ee_var = everywhere ee_var = specific_1 # The difference between Callable[..., blah] and one with a *args: Any, **kwargs: Any is that the ... goes loosely both ways. -ee_def = specific_1 # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[StarArg(Any), KwArg(Any)], None]) +ee_def = specific_1 # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[VarArg(Any), KwArg(Any)], None]) [builtins fixtures/dict.pyi] @@ -174,7 +174,7 @@ ff = f gg = g ff = g -gg = f # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[Arg('a', int), Arg('b', str)], None]) +gg = f # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[Arg(int, 'a'), Arg(str, 'b')], None]) [case testLackOfNamesFastparse] @@ -186,7 +186,7 @@ ff = f gg = g ff = g -gg = f # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[Arg('a', int), Arg('b', str)], None]) +gg = f # E: Incompatible types in assignment (expression has type Callable[[int, str], None], variable has type Callable[[Arg(int, 'a'), Arg(str, 'b')], None]) [case testFunctionTypeCompatibilityWithOtherTypes] from typing import Callable @@ -1401,7 +1401,7 @@ g(1) # E: Argument 1 to "g" has incompatible type "int"; expected "str" from typing import Callable from mypy_extensions import Arg -def a(f: Callable[[Arg('x', int)], int]): +def a(f: Callable[[Arg(int, 'x')], int]): f(x=4) f(5) f(y=3) # E: Unexpected keyword argument "y" @@ -1412,7 +1412,7 @@ def a(f: Callable[[Arg('x', int)], int]): from typing import Callable from mypy_extensions import DefaultArg -def a(f: Callable[[DefaultArg('x', int)], int]): +def a(f: Callable[[DefaultArg(int, 'x')], int]): f(x=4) f(2) f() @@ -1424,7 +1424,7 @@ def a(f: Callable[[DefaultArg('x', int)], int]): from typing import Callable from mypy_extensions import Arg -F = Callable[[Arg('x', int)], int] +F = Callable[[Arg(int, 'x')], int] def a(f: F): f(x=4) @@ -1437,7 +1437,7 @@ def a(f: F): from typing import Callable from mypy_extensions import DefaultArg -F = Callable[[DefaultArg('x', int)], int] +F = Callable[[DefaultArg(int, 'x')], int] def a(f: F): f(x=4) f(2) @@ -1448,19 +1448,19 @@ def a(f: F): [case testCallableParsingFromExpr] from typing import Callable -from mypy_extensions import Arg, StarArg, KwArg +from mypy_extensions import Arg, VarArg, KwArg def WrongArg(x, y): return y # Note that for this test, the 'Value of type "int" is not indexable' errors are silly, # and a consequence of Callable being set to an int in the test stub. We can't set it to # something else sensible, because other tests require the stub not have anything # that looks like a function call. -F = Callable[[WrongArg('x', int)], int] # E: Invalid type alias # E: Value of type "int" is not indexable -G = Callable[[Arg('x', 1)], int] # E: Invalid type alias # E: Value of type "int" is not indexable -H = Callable[[StarArg('x', int)], int] # E: Invalid type alias # E: Value of type "int" is not indexable # E: Too many arguments for "StarArg" -I = Callable[[StarArg(int)], int] # ok -J = Callable[[StarArg(), KwArg()], int] # ok -K = Callable[[StarArg(), int], int] # E: Required positional args may not appear after default, named or star args +F = Callable[[WrongArg(int, 'x')], int] # E: Invalid argument constructor "__main__.WrongArg" +G = Callable[[Arg(1, 'x')], int] # E: Invalid type alias # E: Value of type "int" is not indexable +H = Callable[[VarArg(int, 'x')], int] # E: VarArg arguments should not have names +I = Callable[[VarArg(int)], int] # ok +J = Callable[[VarArg(), KwArg()], int] # ok +K = Callable[[VarArg(), int], int] # E: Required positional args may not appear after default, named or star args L = Callable[[Arg(name='x', typ=int)], int] # ok # I have commented out the following test because I don't know how to expect the "defined here" note part of the error. # M = Callable[[Arg(gnome='x', typ=int)], int] # Invalid type alias, Unexpected keyword argument "gnome" for "Arg" @@ -1470,45 +1470,59 @@ N = Callable[[Arg(name=None, typ=int)], int] # ok [case testCallableParsing] from typing import Callable -from mypy_extensions import Arg, StarArg, KwArg +from mypy_extensions import Arg, VarArg, KwArg def WrongArg(x, y): return y -def a(f: Callable[[WrongArg('x', int)], int]): pass # E: Unknown argument constructor WrongArg -def b(f: Callable[[Arg('x', 1)], int]): pass # E: invalid type comment or annotation -def c(f: Callable[[StarArg('x', int)], int]): pass # E: Too many arguments for argument constructor -def d(f: Callable[[StarArg(int)], int]): pass # ok -def e(f: Callable[[StarArg(), KwArg()], int]): pass # ok +def b(f: Callable[[Arg(1, 'x')], int]): pass # E: invalid type comment or annotation +def d(f: Callable[[VarArg(int)], int]): pass # ok +def e(f: Callable[[VarArg(), KwArg()], int]): pass # ok def g(f: Callable[[Arg(name='x', typ=int)], int]): pass # ok def h(f: Callable[[Arg(gnome='x', typ=int)], int]): pass # E: Unexpected argument "gnome" for argument constructor def i(f: Callable[[Arg(name=None, typ=int)], int]): pass # ok [builtins fixtures/dict.pyi] -[case testCallableFastParseWrongArg] -# flags: --fast-parser +[case testCallableTypeAnalysis] from typing import Callable +from mypy_extensions import Arg, VarArg as VARG, KwArg +import mypy_extensions as ext + def WrongArg(x, y): return y -def a(f: Callable[[WrongArg('x', int)], int]): pass # E: Unknown argument constructor WrongArg +def a(f: Callable[[WrongArg(int, 'x')], int]): pass # E: Invalid argument constructor "__main__.WrongArg" +def b(f: Callable[[BadArg(int, 'x')], int]): pass # E: Name 'BadArg' is not defined +def d(f: Callable[[ext.VarArg(int)], int]): pass # ok +def e(f: Callable[[VARG(), ext.KwArg()], int]): pass # ok +def g(f: Callable[[ext.Arg(name='x', typ=int)], int]): pass # ok +def i(f: Callable[[Arg(name=None, typ=int)], int]): pass # ok + +def f1(*args) -> int: pass +def f2(*args, **kwargs) -> int: pass + +d(f1) +e(f2) +d(f2) +e(f1) # E: Argument 1 to "e" has incompatible type Callable[[VarArg(Any)], int]; expected Callable[[VarArg(Any), KwArg(Any)], int] + [builtins fixtures/dict.pyi] -[case testCallableFastParseWrongTypeType] +[case testCallableWrongTypeType] from typing import Callable from mypy_extensions import Arg -def b(f: Callable[[Arg('x', 1)], int]): pass # E: invalid type comment or annotation +def b(f: Callable[[Arg(1, 'x')], int]): pass # E: invalid type comment or annotation [builtins fixtures/dict.pyi] -[case testCallableFastParseTooManyStarArg] +[case testCallableTooManyVarArg] from typing import Callable -from mypy_extensions import StarArg -def c(f: Callable[[StarArg('x', int)], int]): pass # E: Too many arguments for argument constructor +from mypy_extensions import VarArg +def c(f: Callable[[VarArg(int, 'x')], int]): pass # E: VarArg arguments should not have names [builtins fixtures/dict.pyi] [case testCallableFastParseGood] from typing import Callable -from mypy_extensions import StarArg, Arg -def d(f: Callable[[StarArg(int)], int]): pass # ok -def e(f: Callable[[StarArg(), KwArg()], int]): pass # ok +from mypy_extensions import VarArg, Arg, KwArg +def d(f: Callable[[VarArg(int)], int]): pass # ok +def e(f: Callable[[VarArg(), KwArg()], int]): pass # ok def g(f: Callable[[Arg(name='x', typ=int)], int]): pass # ok def i(f: Callable[[Arg(name=None, typ=int)], int]): pass # ok [builtins fixtures/dict.pyi] @@ -1520,22 +1534,22 @@ def h(f: Callable[[Arg(gnome='x', typ=int)], int]): pass # E: Unexpected argumen [builtins fixtures/dict.pyi] [case testCallableKindsOrdering] -from typing import Callable -from mypy_extensions import Arg, StarArg, KwArg, DefaultArg +from typing import Callable, Any +from mypy_extensions import Arg, VarArg, KwArg, DefaultArg, NamedArg -def f(f: Callable[[StarArg(), int], int]): pass # E: Required positional args may not appear after default, named or star args -def g(f: Callable[[StarArg(), StarArg()], int]): pass # E: Star args may not appear after named or star args +def f(f: Callable[[VarArg(), int], int]): pass # E: Required positional args may not appear after default, named or star args +def g(f: Callable[[VarArg(), VarArg()], int]): pass # E: Star args may not appear after named or star args def h(f: Callable[[KwArg(), KwArg()], int]): pass # E: You may only have one **kwargs argument def i(f: Callable[[DefaultArg(), int], int]): pass # E: Required positional args may not appear after default, named or star args -def j(f: Callable[[NamedArg('x'), DefaultArg('y', int)], int]): pass # E: Positional default args may not appear after named or star args +def j(f: Callable[[NamedArg(Any, 'x'), DefaultArg(int, 'y')], int]): pass # E: Positional default args may not appear after named or star args [builtins fixtures/dict.pyi] [case testCallableDuplicateNames] from typing import Callable -from mypy_extensions import Arg, StarArg, KwArg, DefaultArg +from mypy_extensions import Arg, VarArg, KwArg, DefaultArg -def f(f: Callable[[Arg('x', int), int, Arg('x', int)], int]): pass # E: duplicate argument 'x' in Callable +def f(f: Callable[[Arg(int, 'x'), int, Arg(int, 'x')], int]): pass # E: duplicate argument 'x' in Callable [builtins fixtures/dict.pyi] @@ -1544,7 +1558,7 @@ def f(f: Callable[[Arg('x', int), int, Arg('x', int)], int]): pass # E: duplicat from typing import Callable from mypy_extensions import NamedArg -def a(f: Callable[[NamedArg('x', int)], int]): +def a(f: Callable[[NamedArg(int, 'x')], int]): f(x=4) f(2) # E: Too many positional arguments f() # E: Missing named argument "x" @@ -1556,7 +1570,7 @@ def a(f: Callable[[NamedArg('x', int)], int]): from typing import Callable from mypy_extensions import DefaultNamedArg -def a(f: Callable[[DefaultNamedArg('x', int)], int]): +def a(f: Callable[[DefaultNamedArg(int, 'x')], int]): f(x=4) f(2) # E: Too many positional arguments f() @@ -1569,8 +1583,8 @@ from typing import Callable from mypy_extensions import Arg, DefaultArg int_str_fun = None # type: Callable[[int, str], str] -int_opt_str_fun = None # type: Callable[[int, DefaultArg(None, str)], str] -int_named_str_fun = None # type: Callable[[int, Arg('s', str)], str] +int_opt_str_fun = None # type: Callable[[int, DefaultArg(str, None)], str] +int_named_str_fun = None # type: Callable[[int, Arg(str, 's')], str] def isf(ii: int, ss: str) -> str: return ss @@ -1583,12 +1597,12 @@ def isf_unnamed(__i: int, __s: str) -> str: int_str_fun = isf int_str_fun = isf_unnamed -int_named_str_fun = isf_unnamed # E: Incompatible types in assignment (expression has type Callable[[int, str], str], variable has type Callable[[int, Arg('s', str)], str]) +int_named_str_fun = isf_unnamed # E: Incompatible types in assignment (expression has type Callable[[int, str], str], variable has type Callable[[int, Arg(str, 's')], str]) int_opt_str_fun = iosf int_str_fun = iosf -int_opt_str_fun = isf # E: Incompatible types in assignment (expression has type Callable[[Arg('ii', int), Arg('ss', str)], str], variable has type Callable[[int, DefaultArg(None, str)], str]) +int_opt_str_fun = isf # E: Incompatible types in assignment (expression has type Callable[[Arg(int, 'ii'), Arg(str, 'ss')], str], variable has type Callable[[int, DefaultArg(str)], str]) -int_named_str_fun = isf # E: Incompatible types in assignment (expression has type Callable[[Arg('ii', int), Arg('ss', str)], str], variable has type Callable[[int, Arg('s', str)], str]) +int_named_str_fun = isf # E: Incompatible types in assignment (expression has type Callable[[Arg(int, 'ii'), Arg(str, 'ss')], str], variable has type Callable[[int, Arg(str, 's')], str]) int_named_str_fun = iosf [builtins fixtures/dict.pyi] @@ -1642,7 +1656,7 @@ def g4(*, y: int) -> str: pass f(g1) f(g2) f(g3) -f(g4) # E: Argument 1 to "f" has incompatible type Callable[[NamedArg('y', int)], str]; expected Callable[..., int] +f(g4) # E: Argument 1 to "f" has incompatible type Callable[[NamedArg(int, 'y')], str]; expected Callable[..., int] [case testCallableWithArbitraryArgsSubtypingWithGenericFunc] from typing import Callable, TypeVar @@ -1770,7 +1784,7 @@ def g(x, y): pass def h(x): pass def j(y) -> Any: pass f = h -f = j # E: Incompatible types in assignment (expression has type Callable[[Arg('y', Any)], Any], variable has type Callable[[Arg('x', Any)], Any]) +f = j # E: Incompatible types in assignment (expression has type Callable[[Arg(Any, 'y')], Any], variable has type Callable[[Arg(Any, 'x')], Any]) f = g # E: Incompatible types in assignment (expression has type Callable[[Any, Any], Any], variable has type Callable[[Any], Any]) [case testRedefineFunction2] diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 6eb38093f61a..9226a06d714a 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -1119,7 +1119,7 @@ from typing import Callable def f(a: Callable[..., None] = lambda *a, **k: None): pass -def g(a: Callable[..., None] = lambda *a, **k: 1): # E: Incompatible types in assignment (expression has type Callable[[StarArg(Any), KwArg(Any)], int], variable has type Callable[..., None]) +def g(a: Callable[..., None] = lambda *a, **k: 1): # E: Incompatible types in assignment (expression has type Callable[[VarArg(Any), KwArg(Any)], int], variable has type Callable[..., None]) pass [builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test index 8ebf98076df4..db257478e76b 100644 --- a/test-data/unit/check-varargs.test +++ b/test-data/unit/check-varargs.test @@ -576,7 +576,7 @@ x = None # type: Callable[[int], None] def f(*x: int) -> None: pass def g(*x: str) -> None: pass x = f -x = g # E: Incompatible types in assignment (expression has type Callable[[StarArg(str)], None], variable has type Callable[[int], None]) +x = g # E: Incompatible types in assignment (expression has type Callable[[VarArg(str)], None], variable has type Callable[[int], None]) [builtins fixtures/list.pyi] [out] diff --git a/test-data/unit/lib-stub/mypy_extensions.pyi b/test-data/unit/lib-stub/mypy_extensions.pyi index 434a0faff22f..dc7d5abed641 100644 --- a/test-data/unit/lib-stub/mypy_extensions.pyi +++ b/test-data/unit/lib-stub/mypy_extensions.pyi @@ -1,20 +1,21 @@ from typing import Dict, Type, TypeVar, Optional, Any -T = TypeVar('T') +_T = TypeVar('_T') -def TypedDict(typename: str, fields: Dict[str, Type[T]]) -> Type[dict]: pass +def Arg(typ: _T = ..., name: Optional[str] = ...) -> _T: ... -def Arg(name=None, typ: T = ...) -> T: pass +def DefaultArg(typ: _T = ..., name: Optional[str] = ...) -> _T: ... -def DefaultArg(name=None, typ: T = ...) -> T: pass +def NamedArg(typ: _T = ..., name: Optional[str] = ...) -> _T: ... -def NamedArg(name=None, typ: T = ...) -> T: pass +def DefaultNamedArg(typ: _T = ..., name: Optional[str] = ...) -> _T: ... -def DefaultNamedArg(name=None, typ: T = ...) -> T: pass +def VarArg(typ: _T = ...) -> _T: ... -def StarArg(typ: T = ...) -> T: pass +def KwArg(typ: _T = ...) -> _T: ... -def KwArg(typ: T = ...) -> T: pass + +def TypedDict(typename: str, fields: Dict[str, Type[_T]]) -> Type[dict]: ... class NoReturn: pass diff --git a/test-data/unit/parse-python2.test b/test-data/unit/parse-python2.test index f3a88beb69a2..f0e4ce5ecbe8 100644 --- a/test-data/unit/parse-python2.test +++ b/test-data/unit/parse-python2.test @@ -346,12 +346,12 @@ def f(x, [x]): pass [out] main:1: error: invalid syntax -[case testTupleArgAfterStarArgInPython2] +[case testTupleArgAfterVarArgInPython2] def f(*a, (b, c)): pass [out] main:1: error: invalid syntax -[case testTupleArgAfterStarStarArgInPython2] +[case testTupleArgAfterStarVarArgInPython2] def f(*a, (b, c)): pass [out] main:1: error: invalid syntax diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index 2337d76bc1d0..bbc8c0133877 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -3121,7 +3121,7 @@ MypyFile:1( ExpressionStmt:4( IntExpr(4))) -[case testKeywordArgumentAfterStarArgumentInCall] +[case testKeywordArgumentAfterVarArgumentInCall] f(x=1, *y) [out] MypyFile:1( diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index c7e24e05bfd3..b8f0652f9d22 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -758,7 +758,7 @@ def f(*args: str) -> str: return args[0] map(f, ['x']) map(f, [1]) [out] -_program.py:4: error: Argument 1 to "map" has incompatible type Callable[[StarArg(str)], str]; expected Callable[[int], str] +_program.py:4: error: Argument 1 to "map" has incompatible type Callable[[VarArg(str)], str]; expected Callable[[int], str] [case testMapStr] import typing @@ -962,7 +962,7 @@ x = f() y = 1 y = x -[case testAppendToStarArg] +[case testAppendToVarArg] import typing def f(*x: int) -> None: x.append(1) diff --git a/typeshed b/typeshed index 359c8cc313d1..60c9439909cd 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 359c8cc313d18e309a80548c1d4643506302a525 +Subproject commit 60c9439909cdaca2a74101d458094f1c08672994 From e79c5274edc7300538f80359efd180cad4365803 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Wed, 19 Apr 2017 22:54:03 -0700 Subject: [PATCH 11/28] Change back to TypeList --- mypy/erasetype.py | 2 +- mypy/expandtype.py | 2 +- mypy/exprtotype.py | 6 +++--- mypy/fastparse.py | 6 +++--- mypy/fixup.py | 4 ++-- mypy/indirection.py | 2 +- mypy/join.py | 2 +- mypy/meet.py | 2 +- mypy/sametypes.py | 2 +- mypy/semanal.py | 6 +++--- mypy/server/astdiff.py | 2 +- mypy/subtypes.py | 2 +- mypy/typeanal.py | 8 ++++---- mypy/types.py | 20 ++++++++++---------- 14 files changed, 33 insertions(+), 33 deletions(-) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 7168e4092754..1fd4c1e1cac1 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -3,7 +3,7 @@ from mypy.types import ( Type, TypeVisitor, UnboundType, AnyType, NoneTyp, TypeVarId, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, - ErasedType, PartialType, DeletedType, TypeTranslator, ArgumentList, UninhabitedType, TypeType + ErasedType, PartialType, DeletedType, TypeTranslator, TypeList, UninhabitedType, TypeType ) from mypy import experiments diff --git a/mypy/expandtype.py b/mypy/expandtype.py index ca581fb21fa6..18301191948b 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -3,7 +3,7 @@ from mypy.types import ( Type, Instance, CallableType, TypeVisitor, UnboundType, AnyType, NoneTyp, TypeVarType, Overloaded, TupleType, TypedDictType, UnionType, - ErasedType, ArgumentList, PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId, + ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId, FunctionLike, TypeVarDef ) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index f95bdd803853..d521311f16b9 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -7,7 +7,7 @@ ) from mypy.fastparse import parse_type_comment from mypy.types import ( - Type, UnboundType, ArgumentList, EllipsisType, AnyType, Optional, CallableArgument, + Type, UnboundType, TypeList, EllipsisType, AnyType, Optional, CallableArgument, ) @@ -27,7 +27,7 @@ def _extract_str(expr: Expression) -> Optional[str]: def expr_to_unanalyzed_type(expr: Expression) -> Type: """Translate an expression to the corresponding type. - The result is not semantically analyzed. It can be UnboundType or ArgumentList. + The result is not semantically analyzed. It can be UnboundType or TypeList. Raise TypeTranslationError if the expression cannot represent a type. """ if isinstance(expr, NameExpr): @@ -78,7 +78,7 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: raise TypeTranslationError() return CallableArgument(typ, name, arg_const, expr.line, expr.column) elif isinstance(expr, ListExpr): - return ArgumentList([expr_to_unanalyzed_type(it) for it in expr.items], + return TypeList([expr_to_unanalyzed_type(it) for it in expr.items], line=expr.line, column=expr.column) elif isinstance(expr, (StrExpr, BytesExpr, UnicodeExpr)): # Parse string literal type. diff --git a/mypy/fastparse.py b/mypy/fastparse.py index eccbc7a65a3f..af97c92be534 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -23,7 +23,7 @@ check_arg_names, ) from mypy.types import ( - Type, CallableType, AnyType, UnboundType, TupleType, ArgumentList, EllipsisType, + Type, CallableType, AnyType, UnboundType, TupleType, TypeList, EllipsisType, CallableArgument, ) from mypy import defaults @@ -1022,11 +1022,11 @@ def visit_Call(self, e: ast3.Call) -> Type: return CallableArgument(typ, name, constructor, e.lineno, e.col_offset) - def translate_argument_list(self, l: Sequence[ast3.AST]) -> ArgumentList: + def translate_argument_list(self, l: Sequence[ast3.AST]) -> TypeList: types = [] # type: List[Type] names = [] # type: List[Optional[str]] kinds = [] # type: List[int] - return ArgumentList([self.visit(e) for e in l], line=self.line) + return TypeList([self.visit(e) for e in l], line=self.line) def _extract_str(self, n: ast3.expr) -> str: if isinstance(n, ast3.Str): diff --git a/mypy/fixup.py b/mypy/fixup.py index f4bc3ae50757..f1f2b4b4224a 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -10,7 +10,7 @@ ) from mypy.types import ( CallableType, EllipsisType, Instance, Overloaded, TupleType, TypedDictType, - ArgumentList, TypeVarType, UnboundType, UnionType, TypeVisitor, + TypeList, TypeVarType, UnboundType, UnionType, TypeVisitor, TypeType, CallableArgument, ) from mypy.visitor import NodeVisitor @@ -210,7 +210,7 @@ def visit_typeddict_type(self, tdt: TypedDictType) -> None: if tdt.fallback is not None: tdt.fallback.accept(self) - # def visit_type_list(self, tl: ArgumentList) -> None: + # def visit_type_list(self, tl: TypeList) -> None: # for t in tl.items: # t.accept(self) diff --git a/mypy/indirection.py b/mypy/indirection.py index ba45a8cef41b..2e69c5ebd3ff 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -42,7 +42,7 @@ def _visit(self, *typs: types.Type) -> Set[str]: def visit_unbound_type(self, t: types.UnboundType) -> Set[str]: return self._visit(*t.args) - def visit_type_list(self, t: types.ArgumentList) -> Set[str]: + def visit_type_list(self, t: types.TypeList) -> Set[str]: return self._visit(*t.items) def visit_callable_argument(self, t: types.CallableArgument) -> Set[str]: diff --git a/mypy/join.py b/mypy/join.py index 49a056a5cc44..b4866bdbc691 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -5,7 +5,7 @@ from mypy.types import ( Type, AnyType, NoneTyp, TypeVisitor, Instance, UnboundType, - TypeVarType, CallableType, TupleType, TypedDictType, ErasedType, ArgumentList, + TypeVarType, CallableType, TupleType, TypedDictType, ErasedType, TypeList, UnionType, FunctionLike, Overloaded, PartialType, DeletedType, UninhabitedType, TypeType, true_or_false ) diff --git a/mypy/meet.py b/mypy/meet.py index c9221bfe0669..aed878525eec 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -4,7 +4,7 @@ from mypy.join import is_similar_callables, combine_similar_callables, join_type_list from mypy.types import ( Type, AnyType, TypeVisitor, UnboundType, NoneTyp, TypeVarType, - Instance, CallableType, TupleType, TypedDictType, ErasedType, ArgumentList, UnionType, + Instance, CallableType, TupleType, TypedDictType, ErasedType, TypeList, UnionType, PartialType, DeletedType, UninhabitedType, TypeType ) from mypy.subtypes import is_equivalent, is_subtype diff --git a/mypy/sametypes.py b/mypy/sametypes.py index 779032f7ade3..0531ecc2d474 100644 --- a/mypy/sametypes.py +++ b/mypy/sametypes.py @@ -3,7 +3,7 @@ from mypy.types import ( Type, UnboundType, AnyType, NoneTyp, TupleType, TypedDictType, UnionType, CallableType, TypeVarType, Instance, TypeVisitor, ErasedType, - ArgumentList, Overloaded, PartialType, DeletedType, UninhabitedType, TypeType + TypeList, Overloaded, PartialType, DeletedType, UninhabitedType, TypeType ) diff --git a/mypy/semanal.py b/mypy/semanal.py index b67eb468ba39..0fd1b54e9e1d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -76,7 +76,7 @@ from mypy.messages import CANNOT_ASSIGN_TO_TYPE from mypy.types import ( NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, - FunctionLike, UnboundType, ArgumentList, TypeVarDef, TypeType, + FunctionLike, UnboundType, TypeList, TypeVarDef, TypeType, TupleType, UnionType, StarType, EllipsisType, function_type, TypedDictType, ) from mypy.nodes import implicit_module_attrs @@ -467,7 +467,7 @@ def find_type_variables_in_type(self, type: Type) -> List[Tuple[str, TypeVarExpr result.append((name, node.node)) for arg in type.args: result.extend(self.find_type_variables_in_type(arg)) - elif isinstance(type, ArgumentList): + elif isinstance(type, TypeList): for item in type.types: result.extend(self.find_type_variables_in_type(item)) elif isinstance(type, UnionType): @@ -926,7 +926,7 @@ def get_tvars(self, tp: Type) -> List[Tuple[str, TypeVarExpr]]: tvars = [] # type: List[Tuple[str, TypeVarExpr]] if isinstance(tp, UnboundType): tp_args = tp.args - elif isinstance(tp, ArgumentList): + elif isinstance(tp, TypeList): tp_args = tp.types else: return tvars diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 584fcb5cb4b2..9da9056c56b4 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -11,7 +11,7 @@ from mypy.nodes import SymbolTable, SymbolTableNode, FuncBase, TypeInfo, Var from mypy.types import ( - Type, TypeVisitor, UnboundType, ArgumentList, AnyType, NoneTyp, UninhabitedType, + Type, TypeVisitor, UnboundType, TypeList, AnyType, NoneTyp, UninhabitedType, ErasedType, DeletedType, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, PartialType, TypeType ) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 26b14702ca57..3b849abd1fac 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -3,7 +3,7 @@ from mypy.types import ( Type, AnyType, UnboundType, TypeVisitor, FormalArgument, NoneTyp, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, - ErasedType, ArgumentList, PartialType, DeletedType, UninhabitedType, TypeType, + ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance ) import mypy.applytype diff --git a/mypy/typeanal.py b/mypy/typeanal.py index ff5c1d0514af..b8ebb7716072 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -5,7 +5,7 @@ from mypy.types import ( Type, UnboundType, TypeVarType, TupleType, TypedDictType, UnionType, Instance, - AnyType, CallableType, NoneTyp, DeletedType, ArgumentList, TypeVarDef, TypeVisitor, + AnyType, CallableType, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, ArgKindException, ArgNameException, CallableArgument, get_type_vars, union_items ) @@ -320,7 +320,7 @@ def visit_uninhabited_type(self, t: UninhabitedType) -> Type: def visit_deleted_type(self, t: DeletedType) -> Type: return t - def visit_type_list(self, t: ArgumentList) -> Type: + def visit_type_list(self, t: TypeList) -> Type: self.fail('Invalid type', t) return AnyType() @@ -389,7 +389,7 @@ def analyze_callable_type(self, t: UnboundType) -> Type: is_ellipsis_args=True) elif len(t.args) == 2: ret_type = self.anal_type(t.args[1]) - if isinstance(t.args[0], ArgumentList): + if isinstance(t.args[0], TypeList): # Callable[[ARG, ...], RET] (ordinary callable type) args = [] # type: List[Type] names = [] # type: List[str] @@ -597,7 +597,7 @@ def visit_uninhabited_type(self, t: UninhabitedType) -> None: def visit_deleted_type(self, t: DeletedType) -> None: pass - def visit_type_list(self, t: ArgumentList) -> None: + def visit_type_list(self, t: TypeList) -> None: self.fail('Invalid type', t) def visit_type_var(self, t: TypeVarType) -> None: diff --git a/mypy/types.py b/mypy/types.py index 616e1773d953..6bf26dba9d50 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -254,7 +254,7 @@ def deserialize(cls, data: JsonDict) -> 'CallableArgument': name=data['name'], constructor=data['constructor']) -class ArgumentList(Type): +class TypeList(Type): """Information about argument types and names [...]. This is only used for the arguments of a Callable type, i.e. for @@ -286,15 +286,15 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: def serialize(self) -> JsonDict: return { - '.class': 'ArgumentList', + '.class': 'TypeList', 'items': [t.serialize() for t in self.items], } @classmethod - def deserialize(cls, data: JsonDict) -> 'ArgumentList': - assert data['.class'] == 'ArgumentList' or data['.class'] == 'TypeList' + def deserialize(cls, data: JsonDict) -> 'TypeList': + assert data['.class'] == 'TypeList' items = [Type.deserialize(t) for t in data['items']] - return ArgumentList(items=items) + return TypeList(items=items) class AnyType(Type): """The type 'Any'.""" @@ -1328,7 +1328,7 @@ def visit_star_type(self, t: StarType) -> T: pass @abstractmethod - def visit_type_list(self, t: ArgumentList) -> T: + def visit_type_list(self, t: TypeList) -> T: pass @abstractmethod @@ -1350,7 +1350,7 @@ class TypeTranslator(SyntheticTypeVisitor[Type]): def visit_unbound_type(self, t: UnboundType) -> Type: return t - def visit_type_list(self, t: ArgumentList) -> Type: + def visit_type_list(self, t: TypeList) -> Type: return t def visit_callable_argument(self, t: CallableArgument) -> Type: @@ -1452,8 +1452,8 @@ def visit_unbound_type(self, t: UnboundType)-> str: s += '[{}]'.format(self.list_str(t.args)) return s - def visit_type_list(self, t: ArgumentList) -> str: - return ''.format(self.list_str(t.items)) + def visit_type_list(self, t: TypeList) -> str: + return ''.format(self.list_str(t.items)) def visit_callable_argument(self, t: CallableArgument) -> str: typ = t.typ.accept(self) @@ -1611,7 +1611,7 @@ def __init__(self, strategy: Callable[[Iterable[T]], T]) -> None: def visit_unbound_type(self, t: UnboundType) -> T: return self.query_types(t.args) - def visit_type_list(self, t: ArgumentList) -> T: + def visit_type_list(self, t: TypeList) -> T: return self.query_types(t.items) def visit_callable_argument(self, t: CallableArgument) -> T: From 52ffe5cb970d10c9a68e22fc13501d769684c230 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Wed, 19 Apr 2017 23:16:30 -0700 Subject: [PATCH 12/28] Cleanups. Preparing to split into two diffs maybe? --- mypy/exprtotype.py | 4 +-- mypy/fixup.py | 10 -------- mypy/typeanal.py | 21 +++++++-------- mypy/types.py | 37 ++++----------------------- test-data/unit/check-expressions.test | 6 ++--- 5 files changed, 19 insertions(+), 59 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index d521311f16b9..6bd8fa000513 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -78,8 +78,8 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: raise TypeTranslationError() return CallableArgument(typ, name, arg_const, expr.line, expr.column) elif isinstance(expr, ListExpr): - return TypeList([expr_to_unanalyzed_type(it) for it in expr.items], - line=expr.line, column=expr.column) + return TypeList([expr_to_unanalyzed_type(t) for t in expr.items], + line=expr.line, column=expr.column) elif isinstance(expr, (StrExpr, BytesExpr, UnicodeExpr)): # Parse string literal type. try: diff --git a/mypy/fixup.py b/mypy/fixup.py index f1f2b4b4224a..259e0f511814 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -177,9 +177,6 @@ def visit_callable_type(self, ct: CallableType) -> None: val.accept(self) v.upper_bound.accept(self) -# def visit_ellipsis_type(self, e: EllipsisType) -> None: -# pass # Nothing to descend into. - def visit_overloaded(self, t: Overloaded) -> None: for ct in t.items(): ct.accept(self) @@ -210,13 +207,6 @@ def visit_typeddict_type(self, tdt: TypedDictType) -> None: if tdt.fallback is not None: tdt.fallback.accept(self) - # def visit_type_list(self, tl: TypeList) -> None: - # for t in tl.items: - # t.accept(self) - - # def visit_callable_argument(self, t: CallableArgument) -> None: - # t.typ.accept(self) - def visit_type_var(self, tvt: TypeVarType) -> None: if tvt.values: for vt in tvt.values: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index b8ebb7716072..1f5f47344d60 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -7,7 +7,7 @@ Type, UnboundType, TypeVarType, TupleType, TypedDictType, UnionType, Instance, AnyType, CallableType, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, - ArgKindException, ArgNameException, CallableArgument, get_type_vars, union_items + CallableArgument, get_type_vars, union_items ) from mypy.nodes import ( @@ -419,17 +419,14 @@ def analyze_callable_type(self, t: UnboundType) -> Type: args.append(arg) names.append(None) kinds.append(ARG_POS) - try: - check_arg_names(names, [t]*len(args), self.fail, "Callable") - check_arg_kinds(kinds, [t]*len(args), self.fail) - return CallableType(self.anal_array(args), - kinds, - names, - ret_type=ret_type, - fallback=fallback) - except (ArgKindException, ArgNameException) as e: - self.fail(e.message, t) - return AnyType() + + check_arg_names(names, [t]*len(args), self.fail, "Callable") + check_arg_kinds(kinds, [t]*len(args), self.fail) + return CallableType(self.anal_array(args), + kinds, + names, + ret_type=ret_type, + fallback=fallback) elif isinstance(t.args[0], EllipsisType): # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) return CallableType([AnyType(), AnyType()], diff --git a/mypy/types.py b/mypy/types.py index 6bf26dba9d50..8e526d6ae23c 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -272,29 +272,18 @@ def __init__(self, items: List[Type], line: int = -1, column: int = -1) -> None: def types(self) -> List[Type]: return [a.typ if isinstance(a, CallableArgument) else a for a in self.items] - @property - def names(self) -> List[Optional[str]]: - return [a.name if isinstance(a, CallableArgument) else None for a in self.items] - - @property - def constructors(self) -> List[Optional[str]]: - return [a.constructor if isinstance(a, CallableArgument) else None for a in self.items] - def accept(self, visitor: 'TypeVisitor[T]') -> T: assert isinstance(visitor, SyntheticTypeVisitor) return visitor.visit_type_list(self) def serialize(self) -> JsonDict: - return { - '.class': 'TypeList', - 'items': [t.serialize() for t in self.items], - } + return {'.class': 'TypeList', 'items': [t.serialize() for t in self.items]} @classmethod def deserialize(cls, data: JsonDict) -> 'TypeList': assert data['.class'] == 'TypeList' - items = [Type.deserialize(t) for t in data['items']] - return TypeList(items=items) + return TypeList([deserialize_type(t) for t in data['items']]) + class AnyType(Type): """The type 'Any'.""" @@ -553,22 +542,6 @@ def get_name(self) -> str: pass ('required', bool)]) -class ArgKindException(Exception): - """Raised when the argument kinds are not in the right order""" - message = None # type: str - - def __init__(self, message: str) -> None: - self.message = message - - -class ArgNameException(Exception): - """Raised when there are duplicate arg names""" - message = None # type: str - - def __init__(self, message: str) -> None: - self.message = message - - class CallableType(FunctionLike): """Type of a non-overloaded callable object (function).""" @@ -621,10 +594,10 @@ def __init__(self, assert len(arg_types) == len(arg_kinds) self.arg_types = arg_types self.arg_kinds = arg_kinds - self.is_var_arg = ARG_STAR in arg_kinds - self.is_kw_arg = ARG_STAR2 in arg_kinds self.arg_names = arg_names self.min_args = arg_kinds.count(ARG_POS) + self.is_var_arg = ARG_STAR in arg_kinds + self.is_kw_arg = ARG_STAR2 in arg_kinds self.ret_type = ret_type self.fallback = fallback assert not name or ' Date: Wed, 19 Apr 2017 23:19:57 -0700 Subject: [PATCH 13/28] update typeshed to master version --- typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/typeshed b/typeshed index 60c9439909cd..359c8cc313d1 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 60c9439909cdaca2a74101d458094f1c08672994 +Subproject commit 359c8cc313d18e309a80548c1d4643506302a525 From 398fbad1b658e9509e80d28ac607fe33a1c3fdf7 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 20 Apr 2017 00:00:42 -0700 Subject: [PATCH 14/28] more cleanups --- extensions/mypy_extensions.py | 14 ++++++++++++++ mypy/fastparse.py | 13 +++++-------- mypy/nodes.py | 7 ++++--- mypy/typeanal.py | 4 ++-- mypy/types.py | 6 ++++-- test-data/unit/parse.test | 8 ++++---- 6 files changed, 33 insertions(+), 19 deletions(-) diff --git a/extensions/mypy_extensions.py b/extensions/mypy_extensions.py index 1fcdfc9b3b9e..5c63065ebd96 100644 --- a/extensions/mypy_extensions.py +++ b/extensions/mypy_extensions.py @@ -14,6 +14,7 @@ # the (convenient) behavior of types provided by typing module. from typing import _type_check # type: ignore + def _check_fails(cls, other): try: if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools', 'typing']: @@ -93,23 +94,36 @@ class Point2D(TypedDict): syntax forms work for Python 2.7 and 3.2+ """ + def Arg(typ=Any, name=None): + """A normal positional argument""" return typ + def DefaultArg(typ=Any, name=None): + """A positional argument with a default value""" return typ + def NamedArg(typ=Any, name=None): + """A keyword-only argument""" return typ + def DefaultNamedArg(typ=Any, name=None): + """A keyword-only argument with a default value""" return typ + def VarArg(typ=Any): + """A *args-style variadic positional argument""" return typ + def KwArg(typ=Any): + """A **kwargs-style variadic keyword argument""" return typ + # Return type that indicates a function does not return class NoReturn: pass diff --git a/mypy/fastparse.py b/mypy/fastparse.py index af97c92be534..937d85caffae 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -446,7 +446,7 @@ def make_argument(arg: ast3.arg, default: Optional[ast3.expr], kind: int) -> Arg new_args.append(make_argument(args.kwarg, None, ARG_STAR2)) names.append(args.kwarg) - def fail_arg(msg: str, arg: ast3.arg): + def fail_arg(msg: str, arg: ast3.arg) -> None: self.fail(msg, arg.lineno, arg.col_offset) check_arg_names([name.arg for name in names], names, fail_arg) @@ -957,7 +957,7 @@ def __init__(self, errors: Errors, line: int = -1) -> None: self.line = line self.node_stack = [] # type: List[ast3.AST] - def visit(self, node): + def visit(self, node) -> Type: """Modified visit -- keep track of the stack of nodes""" self.node_stack.append(node) try: @@ -1021,11 +1021,7 @@ def visit_Call(self, e: ast3.Call) -> Type: value.lineno, value.col_offset) return CallableArgument(typ, name, constructor, e.lineno, e.col_offset) - def translate_argument_list(self, l: Sequence[ast3.AST]) -> TypeList: - types = [] # type: List[Type] - names = [] # type: List[Optional[str]] - kinds = [] # type: List[int] return TypeList([self.visit(e) for e in l], line=self.line) def _extract_str(self, n: ast3.expr) -> str: @@ -1033,7 +1029,8 @@ def _extract_str(self, n: ast3.expr) -> str: return n.s.strip() elif isinstance(n, ast3.NameConstant) and str(n.value) == 'None': return None - self.fail('Expected string literal for argument name, got {}'.format(type(n).__name__), self.line, 0) + self.fail('Expected string literal for argument name, got {}'.format( + type(n).__name__), self.line, 0) return None def visit_Name(self, n: ast3.Name) -> Type: @@ -1087,9 +1084,9 @@ def visit_Ellipsis(self, n: ast3.Ellipsis) -> Type: # List(expr* elts, expr_context ctx) def visit_List(self, n: ast3.List) -> Type: - l = len(n.elts) return self.translate_argument_list(n.elts) + def stringify_name(n: ast3.AST) -> Optional[str]: if isinstance(n, ast3.Name): return n.id diff --git a/mypy/nodes.py b/mypy/nodes.py index 287da97fa0bc..057dc0e241cc 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2435,6 +2435,7 @@ def get_member_expr_fullname(expr: MemberExpr) -> str: if isinstance(obj, type) and issubclass(obj, SymbolNode) and obj is not SymbolNode } + def check_arg_kinds(arg_kinds: List[int], nodes: List[T], fail: Callable[[str, T], None]) -> None: is_var_arg = False is_kw_arg = False @@ -2444,8 +2445,8 @@ def check_arg_kinds(arg_kinds: List[int], nodes: List[T], fail: Callable[[str, T if kind == ARG_POS: if is_var_arg or is_kw_arg or seen_named or seen_opt: fail("Required positional args may not appear " - "after default, named or star args", - node) + "after default, named or star args", + node) break elif kind == ARG_OPT: if is_var_arg or is_kw_arg or seen_named: @@ -2467,7 +2468,7 @@ def check_arg_kinds(arg_kinds: List[int], nodes: List[T], fail: Callable[[str, T def check_arg_names(names: List[str], nodes: List[T], fail: Callable[[str, T], None], - description: str = 'function definition') -> None: + description: str = 'function definition') -> None: seen_names = set() # type: Set[str] for name, node in zip(names, nodes): if name is not None and name in seen_names: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 1f5f47344d60..e2fd93ea2b30 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -420,8 +420,8 @@ def analyze_callable_type(self, t: UnboundType) -> Type: names.append(None) kinds.append(ARG_POS) - check_arg_names(names, [t]*len(args), self.fail, "Callable") - check_arg_kinds(kinds, [t]*len(args), self.fail) + check_arg_names(names, [t] * len(args), self.fail, "Callable") + check_arg_kinds(kinds, [t] * len(args), self.fail) return CallableType(self.anal_array(args), kinds, names, diff --git a/mypy/types.py b/mypy/types.py index 8e526d6ae23c..2b41206fc46c 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -222,6 +222,7 @@ def deserialize(cls, data: JsonDict) -> 'UnboundType': return UnboundType(data['name'], [deserialize_type(a) for a in data['args']]) + class CallableArgument(Type): typ = None # type: Type name = None # type: Optional[str] @@ -254,6 +255,7 @@ def deserialize(cls, data: JsonDict) -> 'CallableArgument': name=data['name'], constructor=data['constructor']) + class TypeList(Type): """Information about argument types and names [...]. @@ -1431,9 +1433,9 @@ def visit_type_list(self, t: TypeList) -> str: def visit_callable_argument(self, t: CallableArgument) -> str: typ = t.typ.accept(self) if t.name is None: - return "{}({})".format(t.constructor, t.typ) + return "{}({})".format(t.constructor, typ) else: - return "{}({}, {})".format(t.constructor, t.typ, t.name) + return "{}({}, {})".format(t.constructor, typ, t.name) def visit_any(self, t: AnyType) -> str: return 'Any' diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index bbc8c0133877..4198790bc52d 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -2392,7 +2392,7 @@ MypyFile:1( AssignmentStmt:1( NameExpr(f) NameExpr(None) - Callable?[, None?])) + Callable?[, None?])) [case testFunctionTypeWithArgument] f = None # type: Callable[[str], int] @@ -2401,7 +2401,7 @@ MypyFile:1( AssignmentStmt:1( NameExpr(f) NameExpr(None) - Callable?[, int?])) + Callable?[, int?])) [case testFunctionTypeWithTwoArguments] f = None # type: Callable[[a[b], x.y], List[int]] @@ -2410,7 +2410,7 @@ MypyFile:1( AssignmentStmt:1( NameExpr(f) NameExpr(None) - Callable?[, List?[int?]])) + Callable?[, List?[int?]])) [case testFunctionTypeWithExtraComma] def f(x: Callable[[str,], int]): pass @@ -2420,7 +2420,7 @@ MypyFile:1( f Args( Var(x)) - def (x: Callable?[, int?]) -> Any + def (x: Callable?[, int?]) -> Any Block:1( PassStmt:1()))) From 2c9ce025bcb7dfaaaafad07585d3c2251631d261 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 20 Apr 2017 00:06:05 -0700 Subject: [PATCH 15/28] should not have changed these test files --- test-data/unit/parse-errors.test | 2 +- test-data/unit/parse-python2.test | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test-data/unit/parse-errors.test b/test-data/unit/parse-errors.test index acfadce03233..22a3c5c50b00 100644 --- a/test-data/unit/parse-errors.test +++ b/test-data/unit/parse-errors.test @@ -272,7 +272,7 @@ file:3: error: syntax error in type comment file:3: error: Inconsistent use of '*' in function signature file:3: error: Inconsistent use of '**' in function signature -[case testPrintStatementInPython3] +[case testPrintStatementInPython3-skip] print 1 [out] file:1: error: Missing parentheses in call to 'print' diff --git a/test-data/unit/parse-python2.test b/test-data/unit/parse-python2.test index f0e4ce5ecbe8..f3a88beb69a2 100644 --- a/test-data/unit/parse-python2.test +++ b/test-data/unit/parse-python2.test @@ -346,12 +346,12 @@ def f(x, [x]): pass [out] main:1: error: invalid syntax -[case testTupleArgAfterVarArgInPython2] +[case testTupleArgAfterStarArgInPython2] def f(*a, (b, c)): pass [out] main:1: error: invalid syntax -[case testTupleArgAfterStarVarArgInPython2] +[case testTupleArgAfterStarStarArgInPython2] def f(*a, (b, c)): pass [out] main:1: error: invalid syntax From 51c6f56d5441c8c9bdc19f5d59cea997f3b179da Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 20 Apr 2017 00:10:57 -0700 Subject: [PATCH 16/28] Semanal needs to be a SyntheticTypeVisitor --- mypy/typeanal.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index e2fd93ea2b30..8679f0e51995 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -6,6 +6,7 @@ from mypy.types import ( Type, UnboundType, TypeVarType, TupleType, TypedDictType, UnionType, Instance, AnyType, CallableType, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, + SyntheticTypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, CallableArgument, get_type_vars, union_items ) @@ -100,7 +101,7 @@ def no_subscript_builtin_alias(name: str, propose_alt: bool = True) -> str: return msg -class TypeAnalyser(TypeVisitor[Type]): +class TypeAnalyser(SyntheticTypeVisitor[Type]): """Semantic analyzer for types (semantic analysis pass 2). Converts unbound types into bound types. @@ -324,6 +325,18 @@ def visit_type_list(self, t: TypeList) -> Type: self.fail('Invalid type', t) return AnyType() + def visit_callable_argument(self, t: CallableArgument) -> Type: + self.fail('Invalid type', t) + return AnyType() + + def visit_star_type(self, t: StarType) -> Type: + self.fail('Invalid type', t) + return AnyType() + + def visit_ellipsis_type(self, t: EllipsisType) -> Type: + self.fail('Invalid type', t) + return AnyType() + def visit_instance(self, t: Instance) -> Type: return t From 5e679a36ee60b8f6bc24c4696b0a1fca15715eed Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 20 Apr 2017 00:16:36 -0700 Subject: [PATCH 17/28] Annot --- mypy/fastparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 937d85caffae..fb471f2967a9 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -957,7 +957,7 @@ def __init__(self, errors: Errors, line: int = -1) -> None: self.line = line self.node_stack = [] # type: List[ast3.AST] - def visit(self, node) -> Type: + def visit(self, node: ast3.AST) -> Type: """Modified visit -- keep track of the stack of nodes""" self.node_stack.append(node) try: From 0926fe952b469c1a2abdf9250134d0241d22c064 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 20 Apr 2017 00:16:42 -0700 Subject: [PATCH 18/28] Oops --- mypy/typeanal.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 8679f0e51995..8694e3a523c9 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -329,14 +329,6 @@ def visit_callable_argument(self, t: CallableArgument) -> Type: self.fail('Invalid type', t) return AnyType() - def visit_star_type(self, t: StarType) -> Type: - self.fail('Invalid type', t) - return AnyType() - - def visit_ellipsis_type(self, t: EllipsisType) -> Type: - self.fail('Invalid type', t) - return AnyType() - def visit_instance(self, t: Instance) -> Type: return t From 288a8be5fe44a4e5b11a2ba25c16cfc0d9feb9cd Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 20 Apr 2017 00:30:23 -0700 Subject: [PATCH 19/28] Add testing for exprtotype Arg constructors in wierd places --- test-data/unit/check-functions.test | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 45b978dfde84..906280cef78c 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1447,7 +1447,7 @@ def a(f: F): [builtins fixtures/dict.pyi] [case testCallableParsingFromExpr] -from typing import Callable +from typing import Callable, List from mypy_extensions import Arg, VarArg, KwArg def WrongArg(x, y): return y @@ -1463,8 +1463,9 @@ J = Callable[[VarArg(), KwArg()], int] # ok K = Callable[[VarArg(), int], int] # E: Required positional args may not appear after default, named or star args L = Callable[[Arg(name='x', typ=int)], int] # ok # I have commented out the following test because I don't know how to expect the "defined here" note part of the error. -# M = Callable[[Arg(gnome='x', typ=int)], int] # Invalid type alias, Unexpected keyword argument "gnome" for "Arg" +# M = Callable[[Arg(gnome='x', typ=int)], int] E: Invalid type alias E: Unexpected keyword argument "gnome" for "Arg" N = Callable[[Arg(name=None, typ=int)], int] # ok +O = Callable[[List[Arg(int)]], int] # E: Invalid type [builtins fixtures/dict.pyi] From 6e67ab2710ed57f6a0e9328b0aaf6d93ac94966d Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 20 Apr 2017 09:25:04 -0700 Subject: [PATCH 20/28] Remove some ill-modified modifications to tests --- test-data/unit/parse.test | 2 +- test-data/unit/pythoneval.test | 2 +- test-data/unit/semanal-errors.test | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index 4198790bc52d..39cbd5878148 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -3121,7 +3121,7 @@ MypyFile:1( ExpressionStmt:4( IntExpr(4))) -[case testKeywordArgumentAfterVarArgumentInCall] +[case testKeywordArgumentAfterStarArgumentInCall] f(x=1, *y) [out] MypyFile:1( diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index b8f0652f9d22..8a19d0010942 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -962,7 +962,7 @@ x = f() y = 1 y = x -[case testAppendToVarArg] +[case testAppendToStarArg] import typing def f(*x: int) -> None: x.append(1) diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index a4baede20625..99584e77a977 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -807,7 +807,7 @@ Any(str, None) # E: Any(...) is no longer supported. Use cast(Any, ...) instead Any(arg=str) # E: Any(...) is no longer supported. Use cast(Any, ...) instead [out] -[case testArgumentListAsType] +[case testTypeListAsType] def f(x:[int, str]) -> None: # E: Invalid type pass [out] From 1c7d4c68f983344f0f37233d6c56ae4dff2f6cfd Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 21 Apr 2017 11:06:06 -0700 Subject: [PATCH 21/28] Jukka comments --- extensions/mypy_extensions.py | 27 ++++++++------ mypy/exprtotype.py | 41 ++++++++++++++++----- mypy/fastparse.py | 18 ++++++--- mypy/fastparse2.py | 12 +++--- mypy/fixup.py | 2 +- mypy/meet.py | 4 +- mypy/nodes.py | 8 ++-- mypy/typeanal.py | 8 ++-- mypy/types.py | 8 +++- test-data/unit/check-fastparse.test | 28 +++++++------- test-data/unit/check-functions.test | 40 +++++++++++--------- test-data/unit/check-incremental.test | 19 ++++++++++ test-data/unit/lib-stub/mypy_extensions.pyi | 12 +++--- typeshed | 2 +- 14 files changed, 144 insertions(+), 85 deletions(-) diff --git a/extensions/mypy_extensions.py b/extensions/mypy_extensions.py index 5c63065ebd96..644750095cbd 100644 --- a/extensions/mypy_extensions.py +++ b/extensions/mypy_extensions.py @@ -94,35 +94,38 @@ class Point2D(TypedDict): syntax forms work for Python 2.7 and 3.2+ """ +# Argument constructors for making more-detailed Callables. These all just +# return their type argument, to make them complete noops in terms of the +# `typing` module. -def Arg(typ=Any, name=None): +def Arg(type=Any, name=None): """A normal positional argument""" - return typ + return type -def DefaultArg(typ=Any, name=None): +def DefaultArg(type=Any, name=None): """A positional argument with a default value""" - return typ + return type -def NamedArg(typ=Any, name=None): +def NamedArg(type=Any, name=None): """A keyword-only argument""" - return typ + return type -def DefaultNamedArg(typ=Any, name=None): +def DefaultNamedArg(type=Any, name=None): """A keyword-only argument with a default value""" - return typ + return type -def VarArg(typ=Any): +def VarArg(type=Any): """A *args-style variadic positional argument""" - return typ + return type -def KwArg(typ=Any): +def KwArg(type=Any): """A **kwargs-style variadic keyword argument""" - return typ + return type # Return type that indicates a function does not return diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 6bd8fa000513..e29a25cb419c 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -3,7 +3,7 @@ from mypy.nodes import ( Expression, NameExpr, MemberExpr, IndexExpr, TupleExpr, ListExpr, StrExpr, BytesExpr, UnicodeExpr, EllipsisExpr, CallExpr, - ARG_POS, ARG_NAMED, get_member_expr_fullname + ARG_POS, ARG_NAMED, get_member_expr_fullname, UnicodeExpr, ) from mypy.fastparse import parse_type_comment from mypy.types import ( @@ -15,11 +15,13 @@ class TypeTranslationError(Exception): """Exception raised when an expression is not valid as a type.""" -def _extract_str(expr: Expression) -> Optional[str]: +def _extract_argument_name(expr: Expression) -> Optional[str]: if isinstance(expr, NameExpr) and expr.name == 'None': return None elif isinstance(expr, StrExpr): return expr.value + elif isinstance(expr, UnicodeExpr): + return expr.value else: raise TypeTranslationError() @@ -55,17 +57,38 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: else: raise TypeTranslationError() elif isinstance(expr, CallExpr): - if not isinstance(expr.callee, NameExpr): - raise TypeTranslationError() - arg_const = expr.callee.name + + c = expr.callee + names = [] + # Go through the dotted member expr chain to get the full arg + # constructor name to look up + while True: + if isinstance(c, NameExpr): + names.append(c.name) + break + elif isinstance(c, MemberExpr): + names.append(c.name) + c = c.expr + else: + raise TypeTranslationError() + arg_const = '.'.join(reversed(names)) + + # Go through the constructor args to get its name and type. name = None - typ = AnyType(implicit=True) # type: Type + default_type = AnyType(implicit=True) + typ = default_type # type: Type for i, arg in enumerate(expr.args): if expr.arg_names[i] is not None: if expr.arg_names[i] == "name": - name = _extract_str(arg) + if name is not None: + # Two names + raise TypeTranslationError() + name = _extract_argument_name(arg) continue - elif expr.arg_names[i] == "typ": + elif expr.arg_names[i] == "type": + if typ is not default_type: + # Two types + raise TypeTranslationError() typ = expr_to_unanalyzed_type(arg) continue else: @@ -73,7 +96,7 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: elif i == 0: typ = expr_to_unanalyzed_type(arg) elif i == 1: - name = _extract_str(arg) + name = _extract_argument_name(arg) else: raise TypeTranslationError() return CallableArgument(typ, name, arg_const, expr.line, expr.column) diff --git a/mypy/fastparse.py b/mypy/fastparse.py index fb471f2967a9..c9662e4869c6 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -998,22 +998,28 @@ def visit_Call(self, e: ast3.Call) -> Type: constructor = stringify_name(f) if not constructor: self.fail("Expected arg constructor name", e.lineno, e.col_offset) - constructor = "BadArgConstructor" name = None # type: Optional[str] - typ = AnyType(implicit=True) # type: Type + default_type = AnyType(implicit=True) + typ = default_type # type: Type for i, arg in enumerate(e.args): if i == 0: typ = self.visit(arg) elif i == 1: - name = self._extract_str(arg) + name = self._extract_argument_name(arg) else: self.fail("Too many arguments for argument constructor", f.lineno, f.col_offset) for k in e.keywords: value = k.value if k.arg == "name": - name = self._extract_str(value) - elif k.arg == "typ": + if name is not None: + self.fail('"{}" gets multiple values for keyword argument "name"'.format( + constructor), f.lineno, f.col_offset) + name = self._extract_argument_name(value) + elif k.arg == "type": + if typ is not default_type: + self.fail('"{}" gets multiple values for keyword argument "type"'.format( + constructor), f.lineno, f.col_offset) typ = self.visit(value) else: self.fail( @@ -1024,7 +1030,7 @@ def visit_Call(self, e: ast3.Call) -> Type: def translate_argument_list(self, l: Sequence[ast3.AST]) -> TypeList: return TypeList([self.visit(e) for e in l], line=self.line) - def _extract_str(self, n: ast3.expr) -> str: + def _extract_argument_name(self, n: ast3.expr) -> str: if isinstance(n, ast3.Str): return n.s.strip() elif isinstance(n, ast3.NameConstant) and str(n.value) == 'None': diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py index aca04187e57c..b7d5e9d400db 100644 --- a/mypy/fastparse2.py +++ b/mypy/fastparse2.py @@ -33,7 +33,7 @@ UnaryExpr, LambdaExpr, ComparisonExpr, DictionaryComprehension, SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, Argument, Expression, Statement, BackquoteExpr, PrintStmt, ExecStmt, - ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2, OverloadPart, + ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2, OverloadPart, check_arg_names, ) from mypy.types import ( Type, CallableType, AnyType, UnboundType, EllipsisType @@ -439,12 +439,10 @@ def get_type(i: int) -> Optional[Type]: new_args.append(Argument(Var(n.kwarg), typ, None, ARG_STAR2)) names.append(n.kwarg) - seen_names = set() # type: Set[str] - for name in names: - if name in seen_names: - self.fail("duplicate argument '{}' in function definition".format(name), line, 0) - break - seen_names.add(name) + # We don't have any context object to give, but we have closed around the line num + def fail_arg(msg: str, arg: None) -> None: + self.fail(msg, line, 0) + check_arg_names(names, [None] * len(names), fail_arg) return new_args, decompose_stmts diff --git a/mypy/fixup.py b/mypy/fixup.py index 259e0f511814..0d85feb86ef5 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -11,7 +11,7 @@ from mypy.types import ( CallableType, EllipsisType, Instance, Overloaded, TupleType, TypedDictType, TypeList, TypeVarType, UnboundType, UnionType, TypeVisitor, - TypeType, CallableArgument, + TypeType ) from mypy.visitor import NodeVisitor diff --git a/mypy/meet.py b/mypy/meet.py index aed878525eec..feda9bea1854 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -4,8 +4,8 @@ from mypy.join import is_similar_callables, combine_similar_callables, join_type_list from mypy.types import ( Type, AnyType, TypeVisitor, UnboundType, NoneTyp, TypeVarType, - Instance, CallableType, TupleType, TypedDictType, ErasedType, TypeList, UnionType, - PartialType, DeletedType, UninhabitedType, TypeType + Instance, CallableType, TupleType, TypedDictType, ErasedType, TypeList, UnionType, PartialType, + DeletedType, UninhabitedType, TypeType ) from mypy.subtypes import is_equivalent, is_subtype diff --git a/mypy/nodes.py b/mypy/nodes.py index 057dc0e241cc..cfd69e9a1ed9 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2445,17 +2445,17 @@ def check_arg_kinds(arg_kinds: List[int], nodes: List[T], fail: Callable[[str, T if kind == ARG_POS: if is_var_arg or is_kw_arg or seen_named or seen_opt: fail("Required positional args may not appear " - "after default, named or star args", + "after default, named or var args", node) break elif kind == ARG_OPT: if is_var_arg or is_kw_arg or seen_named: - fail("Positional default args may not appear after named or star args", node) + fail("Positional default args may not appear after named or var args", node) break seen_opt = True elif kind == ARG_STAR: if is_var_arg or is_kw_arg or seen_named: - fail("Star args may not appear after named or star args", node) + fail("Var args may not appear after named or var args", node) break is_var_arg = True elif kind == ARG_NAMED or kind == ARG_NAMED_OPT: @@ -2472,6 +2472,6 @@ def check_arg_names(names: List[str], nodes: List[T], fail: Callable[[str, T], N seen_names = set() # type: Set[str] for name, node in zip(names, nodes): if name is not None and name in seen_names: - fail("duplicate argument '{}' in {}".format(name, description), node) + fail("Duplicate argument '{}' in {}".format(name, description), node) break seen_names.add(name) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 8694e3a523c9..64c03af50f2a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -404,22 +404,22 @@ def analyze_callable_type(self, t: UnboundType) -> Type: args.append(arg.typ) names.append(arg.name) if arg.constructor is None: - kinds.append(ARG_POS) - continue + return AnyType() found = self.lookup(arg.constructor, arg) if found is None: # Looking it up already put an error message in - kinds.append(ARG_POS) + return AnyType() elif found.fullname not in ARG_KINDS_BY_CONSTRUCTOR: self.fail('Invalid argument constructor "{}"'.format( found.fullname), arg) - kinds.append(ARG_POS) + return AnyType() else: kind = ARG_KINDS_BY_CONSTRUCTOR[found.fullname] kinds.append(kind) if arg.name is not None and kind in {ARG_STAR, ARG_STAR2}: self.fail("{} arguments should not have names".format( arg.constructor), arg) + return AnyType() else: args.append(arg) names.append(None) diff --git a/mypy/types.py b/mypy/types.py index 2b41206fc46c..dc398862cd21 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -224,11 +224,15 @@ def deserialize(cls, data: JsonDict) -> 'UnboundType': class CallableArgument(Type): + """Represents a Arg(type, 'name') inside a Callable's type list. + + Note that this is a synthetic type for helping parse ASTs, not a real type. + """ typ = None # type: Type name = None # type: Optional[str] - constructor = None # type: str + constructor = None # type: Optional[str] - def __init__(self, typ: Type, name: str, constructor: str, + def __init__(self, typ: Type, name: Optional[str], constructor: Optional[str], line: int = -1, column: int = -1) -> None: super().__init__(line, column) self.typ = typ diff --git a/test-data/unit/check-fastparse.test b/test-data/unit/check-fastparse.test index 763c255bbad1..bea5efbaeb8d 100644 --- a/test-data/unit/check-fastparse.test +++ b/test-data/unit/check-fastparse.test @@ -320,47 +320,47 @@ def f(x: int): # E: Function has duplicate type signatures def f(x, y, z): pass -def g(x, y, x): # E: duplicate argument 'x' in function definition +def g(x, y, x): # E: Duplicate argument 'x' in function definition pass -def h(x, y, *x): # E: duplicate argument 'x' in function definition +def h(x, y, *x): # E: Duplicate argument 'x' in function definition pass -def i(x, y, *z, **z): # E: duplicate argument 'z' in function definition +def i(x, y, *z, **z): # E: Duplicate argument 'z' in function definition pass -def j(x: int, y: int, *, x: int = 3): # E: duplicate argument 'x' in function definition +def j(x: int, y: int, *, x: int = 3): # E: Duplicate argument 'x' in function definition pass -def k(*, y, z, y): # E: duplicate argument 'y' in function definition +def k(*, y, z, y): # E: Duplicate argument 'y' in function definition pass -lambda x, y, x: ... # E: duplicate argument 'x' in function definition +lambda x, y, x: ... # E: Duplicate argument 'x' in function definition [case testFastParserDuplicateNames_python2] def f(x, y, z): pass -def g(x, y, x): # E: duplicate argument 'x' in function definition +def g(x, y, x): # E: Duplicate argument 'x' in function definition pass -def h(x, y, *x): # E: duplicate argument 'x' in function definition +def h(x, y, *x): # E: Duplicate argument 'x' in function definition pass -def i(x, y, *z, **z): # E: duplicate argument 'z' in function definition +def i(x, y, *z, **z): # E: Duplicate argument 'z' in function definition pass -def j(x, (y, y), z): # E: duplicate argument 'y' in function definition +def j(x, (y, y), z): # E: Duplicate argument 'y' in function definition pass -def k(x, (y, x)): # E: duplicate argument 'x' in function definition +def k(x, (y, x)): # E: Duplicate argument 'x' in function definition pass -def l((x, y), (z, x)): # E: duplicate argument 'x' in function definition +def l((x, y), (z, x)): # E: Duplicate argument 'x' in function definition pass -def m(x, ((x, y), z)): # E: duplicate argument 'x' in function definition +def m(x, ((x, y), z)): # E: Duplicate argument 'x' in function definition pass -lambda x, (y, x): None # E: duplicate argument 'x' in function definition +lambda x, (y, x): None # E: Duplicate argument 'x' in function definition diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 906280cef78c..061086ccfe43 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1449,6 +1449,7 @@ def a(f: F): [case testCallableParsingFromExpr] from typing import Callable, List from mypy_extensions import Arg, VarArg, KwArg +import mypy_extensions def WrongArg(x, y): return y # Note that for this test, the 'Value of type "int" is not indexable' errors are silly, @@ -1460,12 +1461,15 @@ G = Callable[[Arg(1, 'x')], int] # E: Invalid type alias # E: Value of type "int H = Callable[[VarArg(int, 'x')], int] # E: VarArg arguments should not have names I = Callable[[VarArg(int)], int] # ok J = Callable[[VarArg(), KwArg()], int] # ok -K = Callable[[VarArg(), int], int] # E: Required positional args may not appear after default, named or star args -L = Callable[[Arg(name='x', typ=int)], int] # ok +K = Callable[[VarArg(), int], int] # E: Required positional args may not appear after default, named or var args +L = Callable[[Arg(name='x', type=int)], int] # ok # I have commented out the following test because I don't know how to expect the "defined here" note part of the error. -# M = Callable[[Arg(gnome='x', typ=int)], int] E: Invalid type alias E: Unexpected keyword argument "gnome" for "Arg" -N = Callable[[Arg(name=None, typ=int)], int] # ok +# M = Callable[[Arg(gnome='x', type=int)], int] E: Invalid type alias E: Unexpected keyword argument "gnome" for "Arg" +N = Callable[[Arg(name=None, type=int)], int] # ok O = Callable[[List[Arg(int)]], int] # E: Invalid type +P = Callable[[mypy_extensions.VarArg(int)], int] # ok +Q = Callable[[Arg(int, type=int)], int] # E: Invalid type alias # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "type" +R = Callable[[Arg(int, 'x', name='y')], int] # E: Invalid type alias # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "name" [builtins fixtures/dict.pyi] @@ -1478,9 +1482,11 @@ def WrongArg(x, y): return y def b(f: Callable[[Arg(1, 'x')], int]): pass # E: invalid type comment or annotation def d(f: Callable[[VarArg(int)], int]): pass # ok def e(f: Callable[[VarArg(), KwArg()], int]): pass # ok -def g(f: Callable[[Arg(name='x', typ=int)], int]): pass # ok -def h(f: Callable[[Arg(gnome='x', typ=int)], int]): pass # E: Unexpected argument "gnome" for argument constructor -def i(f: Callable[[Arg(name=None, typ=int)], int]): pass # ok +def g(f: Callable[[Arg(name='x', type=int)], int]): pass # ok +def h(f: Callable[[Arg(gnome='x', type=int)], int]): pass # E: Unexpected argument "gnome" for argument constructor +def i(f: Callable[[Arg(name=None, type=int)], int]): pass # ok +def j(f: Callable[[Arg(int, 'x', name='y')], int]): pass # E: "Arg" gets multiple values for keyword argument "name" +def k(f: Callable[[Arg(int, type=int)], int]): pass # E: "Arg" gets multiple values for keyword argument "type" [builtins fixtures/dict.pyi] @@ -1494,8 +1500,8 @@ def a(f: Callable[[WrongArg(int, 'x')], int]): pass # E: Invalid argument constr def b(f: Callable[[BadArg(int, 'x')], int]): pass # E: Name 'BadArg' is not defined def d(f: Callable[[ext.VarArg(int)], int]): pass # ok def e(f: Callable[[VARG(), ext.KwArg()], int]): pass # ok -def g(f: Callable[[ext.Arg(name='x', typ=int)], int]): pass # ok -def i(f: Callable[[Arg(name=None, typ=int)], int]): pass # ok +def g(f: Callable[[ext.Arg(name='x', type=int)], int]): pass # ok +def i(f: Callable[[Arg(name=None, type=int)], int]): pass # ok def f1(*args) -> int: pass def f2(*args, **kwargs) -> int: pass @@ -1524,25 +1530,25 @@ from typing import Callable from mypy_extensions import VarArg, Arg, KwArg def d(f: Callable[[VarArg(int)], int]): pass # ok def e(f: Callable[[VarArg(), KwArg()], int]): pass # ok -def g(f: Callable[[Arg(name='x', typ=int)], int]): pass # ok -def i(f: Callable[[Arg(name=None, typ=int)], int]): pass # ok +def g(f: Callable[[Arg(name='x', type=int)], int]): pass # ok +def i(f: Callable[[Arg(name=None, type=int)], int]): pass # ok [builtins fixtures/dict.pyi] [case testCallableFastParseBadArgArgName] from typing import Callable from mypy_extensions import Arg -def h(f: Callable[[Arg(gnome='x', typ=int)], int]): pass # E: Unexpected argument "gnome" for argument constructor +def h(f: Callable[[Arg(gnome='x', type=int)], int]): pass # E: Unexpected argument "gnome" for argument constructor [builtins fixtures/dict.pyi] [case testCallableKindsOrdering] from typing import Callable, Any from mypy_extensions import Arg, VarArg, KwArg, DefaultArg, NamedArg -def f(f: Callable[[VarArg(), int], int]): pass # E: Required positional args may not appear after default, named or star args -def g(f: Callable[[VarArg(), VarArg()], int]): pass # E: Star args may not appear after named or star args +def f(f: Callable[[VarArg(), int], int]): pass # E: Required positional args may not appear after default, named or var args +def g(f: Callable[[VarArg(), VarArg()], int]): pass # E: Var args may not appear after named or var args def h(f: Callable[[KwArg(), KwArg()], int]): pass # E: You may only have one **kwargs argument -def i(f: Callable[[DefaultArg(), int], int]): pass # E: Required positional args may not appear after default, named or star args -def j(f: Callable[[NamedArg(Any, 'x'), DefaultArg(int, 'y')], int]): pass # E: Positional default args may not appear after named or star args +def i(f: Callable[[DefaultArg(), int], int]): pass # E: Required positional args may not appear after default, named or var args +def j(f: Callable[[NamedArg(Any, 'x'), DefaultArg(int, 'y')], int]): pass # E: Positional default args may not appear after named or var args [builtins fixtures/dict.pyi] @@ -1550,7 +1556,7 @@ def j(f: Callable[[NamedArg(Any, 'x'), DefaultArg(int, 'y')], int]): pass # E: P from typing import Callable from mypy_extensions import Arg, VarArg, KwArg, DefaultArg -def f(f: Callable[[Arg(int, 'x'), int, Arg(int, 'x')], int]): pass # E: duplicate argument 'x' in Callable +def f(f: Callable[[Arg(int, 'x'), int, Arg(int, 'x')], int]): pass # E: Duplicate argument 'x' in Callable [builtins fixtures/dict.pyi] diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 7d3d6d862fe6..57fe88015387 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -114,6 +114,25 @@ def func1() -> A: pass [out2] tmp/mod1.py:1: error: Name 'A' is not defined +[case testIncrementalCallable] +import mod1 + +[file mod1.py] +from typing import Callable +from mypy_extensions import Arg +def func1() -> Callable[[Arg(int, 'x')], int]: pass + +[file mod1.py.next] +from typing import Callable +from mypy_extensions import Arg +def func1() -> Callable[[Arg(int, 'x')], int]: ... + + +[rechecked mod1] +[stale] + +[builtins fixtures/dict.pyi] + [case testIncrementalSameNameChange] import mod1 diff --git a/test-data/unit/lib-stub/mypy_extensions.pyi b/test-data/unit/lib-stub/mypy_extensions.pyi index dc7d5abed641..fa540b99f4cd 100644 --- a/test-data/unit/lib-stub/mypy_extensions.pyi +++ b/test-data/unit/lib-stub/mypy_extensions.pyi @@ -3,17 +3,17 @@ from typing import Dict, Type, TypeVar, Optional, Any _T = TypeVar('_T') -def Arg(typ: _T = ..., name: Optional[str] = ...) -> _T: ... +def Arg(type: _T = ..., name: Optional[str] = ...) -> _T: ... -def DefaultArg(typ: _T = ..., name: Optional[str] = ...) -> _T: ... +def DefaultArg(type: _T = ..., name: Optional[str] = ...) -> _T: ... -def NamedArg(typ: _T = ..., name: Optional[str] = ...) -> _T: ... +def NamedArg(type: _T = ..., name: Optional[str] = ...) -> _T: ... -def DefaultNamedArg(typ: _T = ..., name: Optional[str] = ...) -> _T: ... +def DefaultNamedArg(type: _T = ..., name: Optional[str] = ...) -> _T: ... -def VarArg(typ: _T = ...) -> _T: ... +def VarArg(type: _T = ...) -> _T: ... -def KwArg(typ: _T = ...) -> _T: ... +def KwArg(type: _T = ...) -> _T: ... def TypedDict(typename: str, fields: Dict[str, Type[_T]]) -> Type[dict]: ... diff --git a/typeshed b/typeshed index 8b835f95001b..5b0302e37408 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 8b835f95001b61734c6b147d3aa6eb4fbe7bce03 +Subproject commit 5b0302e3740832a47415fbe9c6202174bff3441e From f153850cd9a9ed1306cf67bb9a611f19e454ae70 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 21 Apr 2017 11:18:18 -0700 Subject: [PATCH 22/28] Synthetic types don't serialize --- mypy/types.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index dc398862cd21..9fcac23a6e32 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -244,20 +244,7 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_callable_argument(self) def serialize(self) -> JsonDict: - return { - '.class': 'CallableArgument', - 'typ': self.typ.serialize(), - 'name': self.name, - 'constructor': self.constructor, - } - - @classmethod - def deserialize(cls, data: JsonDict) -> 'CallableArgument': - assert data['.class'] == 'CallableArgument' - return CallableArgument( - typ=deserialize_type(data['typ']), - name=data['name'], - constructor=data['constructor']) + assert False, "Synthetic types don't serialize" class TypeList(Type): From be954f513ea19dbc35b98cf98eab414a93edace5 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 21 Apr 2017 11:20:46 -0700 Subject: [PATCH 23/28] Remove unused instance var --- mypy/types.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index 9fcac23a6e32..e799a5c123fc 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -543,7 +543,6 @@ class CallableType(FunctionLike): arg_names = None # type: List[str] # None if not a keyword argument min_args = 0 # Minimum number of arguments; derived from arg_kinds is_var_arg = False # Is it a varargs function? derived from arg_kinds - is_kw_arg = False ret_type = None # type: Type # Return value type name = '' # Name (may be None; for error messages) definition = None # type: SymbolNode # For error messages. May be None. @@ -590,7 +589,6 @@ def __init__(self, self.arg_names = arg_names self.min_args = arg_kinds.count(ARG_POS) self.is_var_arg = ARG_STAR in arg_kinds - self.is_kw_arg = ARG_STAR2 in arg_kinds self.ret_type = ret_type self.fallback = fallback assert not name or ' Date: Fri, 21 Apr 2017 14:56:23 -1000 Subject: [PATCH 24/28] Revert "Remove unused instance var" This reverts commit be954f513ea19dbc35b98cf98eab414a93edace5. --- mypy/types.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/types.py b/mypy/types.py index e799a5c123fc..9fcac23a6e32 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -543,6 +543,7 @@ class CallableType(FunctionLike): arg_names = None # type: List[str] # None if not a keyword argument min_args = 0 # Minimum number of arguments; derived from arg_kinds is_var_arg = False # Is it a varargs function? derived from arg_kinds + is_kw_arg = False ret_type = None # type: Type # Return value type name = '' # Name (may be None; for error messages) definition = None # type: SymbolNode # For error messages. May be None. @@ -589,6 +590,7 @@ def __init__(self, self.arg_names = arg_names self.min_args = arg_kinds.count(ARG_POS) self.is_var_arg = ARG_STAR in arg_kinds + self.is_kw_arg = ARG_STAR2 in arg_kinds self.ret_type = ret_type self.fallback = fallback assert not name or ' Date: Fri, 21 Apr 2017 15:01:27 -1000 Subject: [PATCH 25/28] Accessing TypeList types directly is not required --- mypy/types.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index 9fcac23a6e32..0310b61d0aa6 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -261,10 +261,6 @@ def __init__(self, items: List[Type], line: int = -1, column: int = -1) -> None: super().__init__(line, column) self.items = items - @property - def types(self) -> List[Type]: - return [a.typ if isinstance(a, CallableArgument) else a for a in self.items] - def accept(self, visitor: 'TypeVisitor[T]') -> T: assert isinstance(visitor, SyntheticTypeVisitor) return visitor.visit_type_list(self) From f2e3663fab64e6b0f6e636667760bfd545ce9b91 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 21 Apr 2017 15:13:22 -1000 Subject: [PATCH 26/28] Undo changes to this file they were not required --- test-data/unit/lib-stub/typing.pyi | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test-data/unit/lib-stub/typing.pyi b/test-data/unit/lib-stub/typing.pyi index 6ca469edb8eb..01ac7b14f7b9 100644 --- a/test-data/unit/lib-stub/typing.pyi +++ b/test-data/unit/lib-stub/typing.pyi @@ -11,12 +11,12 @@ Optional = 0 TypeVar = 0 Generic = 0 Tuple = 0 +Callable = 0 builtinclass = 0 _promote = 0 NamedTuple = 0 Type = 0 no_type_check = 0 -Callable = 0 ClassVar = 0 NoReturn = 0 @@ -93,16 +93,15 @@ class AsyncIterator(AsyncIterable[T], Generic[T]): def __anext__(self) -> Awaitable[T]: pass class Sequence(Iterable[T], Generic[T]): - # Use int because slice isn't defined in the default test builtins @abstractmethod - def __getitem__(self, n: int) -> T: pass + def __getitem__(self, n: Any) -> T: pass class Mapping(Generic[T, U]): pass class MutableMapping(Generic[T, U]): pass def NewType(name: str, tp: Type[T]) -> Callable[[T], T]: - def new_type(x: T) -> T: + def new_type(x): return x return new_type From 27e2a9dc66e1439c8774073614ec253863d8b7b9 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 21 Apr 2017 15:13:35 -1000 Subject: [PATCH 27/28] lint --- extensions/mypy_extensions.py | 1 + mypy/exprtotype.py | 2 +- test-data/unit/parse-python2.test | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/extensions/mypy_extensions.py b/extensions/mypy_extensions.py index 644750095cbd..82eea32a31d8 100644 --- a/extensions/mypy_extensions.py +++ b/extensions/mypy_extensions.py @@ -98,6 +98,7 @@ class Point2D(TypedDict): # return their type argument, to make them complete noops in terms of the # `typing` module. + def Arg(type=Any, name=None): """A normal positional argument""" return type diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index e29a25cb419c..8a8f1c4a7e2a 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -3,7 +3,7 @@ from mypy.nodes import ( Expression, NameExpr, MemberExpr, IndexExpr, TupleExpr, ListExpr, StrExpr, BytesExpr, UnicodeExpr, EllipsisExpr, CallExpr, - ARG_POS, ARG_NAMED, get_member_expr_fullname, UnicodeExpr, + ARG_POS, ARG_NAMED, get_member_expr_fullname ) from mypy.fastparse import parse_type_comment from mypy.types import ( diff --git a/test-data/unit/parse-python2.test b/test-data/unit/parse-python2.test index f3a88beb69a2..b654f6af7b45 100644 --- a/test-data/unit/parse-python2.test +++ b/test-data/unit/parse-python2.test @@ -374,8 +374,8 @@ def f(a, (a, b)): def g((x, (x, y))): pass [out] -main:1: error: duplicate argument 'a' in function definition -main:3: error: duplicate argument 'x' in function definition +main:1: error: Duplicate argument 'a' in function definition +main:3: error: Duplicate argument 'x' in function definition [case testBackquotesInPython2] `1 + 2` From 0780149243094f9094bd48131a2c3c35200e429c Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Mon, 1 May 2017 16:32:26 -0700 Subject: [PATCH 28/28] Disallow CallableArgument in exprtotype outside a TypeList Also add a lot more tests. --- mypy/exprtotype.py | 17 ++++++------ mypy/nodes.py | 3 ++ test-data/unit/check-functions.test | 43 +++++++++++++++++++++++++++-- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 8a8f1c4a7e2a..3eca0adca3d0 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -26,12 +26,14 @@ def _extract_argument_name(expr: Expression) -> Optional[str]: raise TypeTranslationError() -def expr_to_unanalyzed_type(expr: Expression) -> Type: +def expr_to_unanalyzed_type(expr: Expression, _parent: Optional[Expression] = None) -> Type: """Translate an expression to the corresponding type. The result is not semantically analyzed. It can be UnboundType or TypeList. Raise TypeTranslationError if the expression cannot represent a type. """ + # The `parent` paremeter is used in recursive calls to provide context for + # understanding whether an CallableArgument is ok. if isinstance(expr, NameExpr): name = expr.name return UnboundType(name, line=expr.line, column=expr.column) @@ -42,7 +44,7 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: else: raise TypeTranslationError() elif isinstance(expr, IndexExpr): - base = expr_to_unanalyzed_type(expr.base) + base = expr_to_unanalyzed_type(expr.base, expr) if isinstance(base, UnboundType): if base.args: raise TypeTranslationError() @@ -50,14 +52,13 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: args = expr.index.items else: args = [expr.index] - base.args = [expr_to_unanalyzed_type(arg) for arg in args] + base.args = [expr_to_unanalyzed_type(arg, expr) for arg in args] if not base.args: base.empty_tuple_index = True return base else: raise TypeTranslationError() - elif isinstance(expr, CallExpr): - + elif isinstance(expr, CallExpr) and isinstance(_parent, ListExpr): c = expr.callee names = [] # Go through the dotted member expr chain to get the full arg @@ -89,19 +90,19 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type: if typ is not default_type: # Two types raise TypeTranslationError() - typ = expr_to_unanalyzed_type(arg) + typ = expr_to_unanalyzed_type(arg, expr) continue else: raise TypeTranslationError() elif i == 0: - typ = expr_to_unanalyzed_type(arg) + typ = expr_to_unanalyzed_type(arg, expr) elif i == 1: name = _extract_argument_name(arg) else: raise TypeTranslationError() return CallableArgument(typ, name, arg_const, expr.line, expr.column) elif isinstance(expr, ListExpr): - return TypeList([expr_to_unanalyzed_type(t) for t in expr.items], + return TypeList([expr_to_unanalyzed_type(t, expr) for t in expr.items], line=expr.line, column=expr.column) elif isinstance(expr, (StrExpr, BytesExpr, UnicodeExpr)): # Parse string literal type. diff --git a/mypy/nodes.py b/mypy/nodes.py index 2a79d926ff17..b835623f5cda 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2470,6 +2470,9 @@ def check_arg_kinds(arg_kinds: List[int], nodes: List[T], fail: Callable[[str, T is_var_arg = True elif kind == ARG_NAMED or kind == ARG_NAMED_OPT: seen_named = True + if is_kw_arg: + fail("A **kwargs argument must be the last argument", node) + break elif kind == ARG_STAR2: if is_kw_arg: fail("You may only have one **kwargs argument", node) diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 718dd1e75f20..f730de818242 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1446,6 +1446,19 @@ def a(f: F): f("foo") # E: Argument 1 has incompatible type "str"; expected "int" [builtins fixtures/dict.pyi] +[case testCallableParsingInInheritence] + +from collections import namedtuple +class C(namedtuple('t', 'x')): + pass + +[case testCallableParsingSameName] +from typing import Callable + +def Arg(x, y): pass + +F = Callable[[Arg(int, 'x')], int] # E: Invalid argument constructor "__main__.Arg" + [case testCallableParsingFromExpr] from typing import Callable, List from mypy_extensions import Arg, VarArg, KwArg @@ -1466,7 +1479,7 @@ L = Callable[[Arg(name='x', type=int)], int] # ok # I have commented out the following test because I don't know how to expect the "defined here" note part of the error. # M = Callable[[Arg(gnome='x', type=int)], int] E: Invalid type alias E: Unexpected keyword argument "gnome" for "Arg" N = Callable[[Arg(name=None, type=int)], int] # ok -O = Callable[[List[Arg(int)]], int] # E: Invalid type +O = Callable[[List[Arg(int)]], int] # E: Invalid type alias # E: Value of type "int" is not indexable # E: Type expected within [...] # E: The type List[T] is not generic and not indexable P = Callable[[mypy_extensions.VarArg(int)], int] # ok Q = Callable[[Arg(int, type=int)], int] # E: Invalid type alias # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "type" R = Callable[[Arg(int, 'x', name='y')], int] # E: Invalid type alias # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "name" @@ -1549,7 +1562,7 @@ def g(f: Callable[[VarArg(), VarArg()], int]): pass # E: Var args may not appear def h(f: Callable[[KwArg(), KwArg()], int]): pass # E: You may only have one **kwargs argument def i(f: Callable[[DefaultArg(), int], int]): pass # E: Required positional args may not appear after default, named or var args def j(f: Callable[[NamedArg(Any, 'x'), DefaultArg(int, 'y')], int]): pass # E: Positional default args may not appear after named or var args - +def k(f: Callable[[KwArg(), NamedArg(Any, 'x')], int]): pass # E: A **kwargs argument must be the last argument [builtins fixtures/dict.pyi] [case testCallableDuplicateNames] @@ -1585,6 +1598,32 @@ def a(f: Callable[[DefaultNamedArg(int, 'x')], int]): f(x="foo") # E: Argument 1 has incompatible type "str"; expected "int" [builtins fixtures/dict.pyi] +[case testCallableWithKwargs] +from typing import Callable +from mypy_extensions import KwArg + +def a(f: Callable[[KwArg(int)], int]): + f(x=4) + f(2) # E: Too many arguments + f() + f(y=3) + f(x=4, y=3, z=10) + f(x="foo") # E: Argument 1 has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + + +[case testCallableWithVarArg] +from typing import Callable +from mypy_extensions import VarArg + +def a(f: Callable[[VarArg(int)], int]): + f(x=4) # E: Unexpected keyword argument "x" + f(2) + f() + f(3, 4, 5) + f("a") # E: Argument 1 has incompatible type "str"; expected "int" +[builtins fixtures/dict.pyi] + [case testCallableArgKindSubtyping] from typing import Callable from mypy_extensions import Arg, DefaultArg