Ensure required keyword-only arguments are provided. (#2441)

* Introduce kind ARG_NAMED_OPT for optional keyword-only arguments.

Before, mypy would assume all keyword-only arguments were optional, and not
provide an error if you failed to provide a keyword-only argument at a call
site.  Now mypy provides you with a helpful error.

* Adjust tests to understand NAMED vs NAMED_OPT

* More tests for named optionals.

Also extends the test runner to allow more than one error per line.

* Until the typeshed is fixed ignore `handlers` argument not being provided

The relevant typeshed PR is https://github.com/python/typeshed/pull/704/commits

* In test data parser, split and strip expected error comments better

Author: sixolet

mypy/checkexpr.py

@@ -666,15 +666,21 @@ def check_argument_count(self, callee: CallableType, actual_types: List[Type],
                 if messages:
                     messages.too_few_arguments(callee, context, actual_names)
                 ok = False
+            elif kind == nodes.ARG_NAMED and (not formal_to_actual[i] and
+                                              not is_unexpected_arg_error):
+                # No actual for a mandatory named formal
+                if messages:
+                    messages.missing_named_argument(callee, context, callee.arg_names[i])
+                ok = False
             elif kind in [nodes.ARG_POS, nodes.ARG_OPT,
-                          nodes.ARG_NAMED] and is_duplicate_mapping(
+                          nodes.ARG_NAMED, nodes.ARG_NAMED_OPT] and is_duplicate_mapping(
                     formal_to_actual[i], actual_kinds):
                 if (self.chk.in_checked_function() or
                         isinstance(actual_types[formal_to_actual[i][0]], TupleType)):
                     if messages:
                         messages.duplicate_argument_value(callee, i, context)
                     ok = False
-            elif (kind == nodes.ARG_NAMED and formal_to_actual[i] and
+            elif (kind in (nodes.ARG_NAMED, nodes.ARG_NAMED_OPT) and formal_to_actual[i] and
                   actual_kinds[formal_to_actual[i][0]] not in [nodes.ARG_NAMED, nodes.ARG_STAR2]):
                 # Positional argument when expecting a keyword argument.
                 if messages:
@@ -1898,7 +1904,7 @@ def map_actuals_to_formals(caller_kinds: List[int],
         if kind == nodes.ARG_POS:
             if j < ncallee:
                 if callee_kinds[j] in [nodes.ARG_POS, nodes.ARG_OPT,
-                                       nodes.ARG_NAMED]:
+                                       nodes.ARG_NAMED, nodes.ARG_NAMED_OPT]:
                     map[j].append(i)
                     j += 1
                 elif callee_kinds[j] == nodes.ARG_STAR:
@@ -1920,14 +1926,14 @@ def map_actuals_to_formals(caller_kinds: List[int],
                 # Assume that it is an iterable (if it isn't, there will be
                 # an error later).
                 while j < ncallee:
-                    if callee_kinds[j] in (nodes.ARG_NAMED, nodes.ARG_STAR2):
+                    if callee_kinds[j] in (nodes.ARG_NAMED, nodes.ARG_NAMED_OPT, nodes.ARG_STAR2):
                         break
                     else:
                         map[j].append(i)
                     if callee_kinds[j] == nodes.ARG_STAR:
                         break
                     j += 1
-        elif kind == nodes.ARG_NAMED:
+        elif kind in (nodes.ARG_NAMED, nodes.ARG_NAMED_OPT):
             name = caller_names[i]
             if name in callee_names:
                 map[callee_names.index(name)].append(i)

mypy/fastparse.py

@@ -15,7 +15,7 @@
     StarExpr, YieldFromExpr, NonlocalDecl, DictionaryComprehension,
     SetComprehension, ComplexExpr, EllipsisExpr, YieldExpr, Argument,
     AwaitExpr, TempNode, Expression, Statement,
-    ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_STAR2
+    ARG_POS, ARG_OPT, ARG_STAR, ARG_NAMED, ARG_NAMED_OPT, ARG_STAR2
 )
 from mypy.types import (
     Type, CallableType, AnyType, UnboundType, TupleType, TypeList, EllipsisType,
@@ -356,14 +356,12 @@ def make_argument(arg: ast35.arg, default: Optional[ast35.expr], kind: int) -> A
         if args.vararg is not None:
             new_args.append(make_argument(args.vararg, None, ARG_STAR))
 
-        num_no_kw_defaults = len(args.kwonlyargs) - len(args.kw_defaults)
-        # keyword-only arguments without defaults
-        for a in args.kwonlyargs[:num_no_kw_defaults]:
-            new_args.append(make_argument(a, None, ARG_NAMED))
-
         # keyword-only arguments with defaults
-        for a, d in zip(args.kwonlyargs[num_no_kw_defaults:], args.kw_defaults):
-            new_args.append(make_argument(a, d, ARG_NAMED))
+        for a, d in zip(args.kwonlyargs, args.kw_defaults):
+            new_args.append(make_argument(
+                a,
+                d,
+                ARG_NAMED if d is None else ARG_NAMED_OPT))
 
         # **kwarg
         if args.kwarg is not None:

mypy/messages.py

@@ -508,6 +508,12 @@ def too_few_arguments(self, callee: CallableType, context: Context,
                 msg += ' for {}'.format(callee.name)
         self.fail(msg, context)
 
+    def missing_named_argument(self, callee: CallableType, context: Context, name: str) -> None:
+        msg = 'Missing named argument "{}"'.format(name)
+        if callee.name:
+            msg += ' for function {}'.format(callee.name)
+        self.fail(msg, context)
+
     def too_many_arguments(self, callee: CallableType, context: Context) -> None:
         msg = 'Too many arguments'
         if callee.name:

mypy/nodes.py

@@ -1214,6 +1214,8 @@ def accept(self, visitor: NodeVisitor[T]) -> T:
 ARG_NAMED = 3  # type: int
 # **arg argument
 ARG_STAR2 = 4  # type: int
+# In an argument list, keyword-only and also optional
+ARG_NAMED_OPT = 5
 
 
 class CallExpr(Expression):

mypy/parse.py

@@ -773,7 +773,7 @@ def parse_normal_arg(self, require_named: bool,
             self.expect('=')
             initializer = self.parse_expression(precedence[','])
             if require_named:
-                kind = nodes.ARG_NAMED
+                kind = nodes.ARG_NAMED_OPT
             else:
                 kind = nodes.ARG_OPT
         else:

mypy/semanal.py

@@ -59,7 +59,7 @@
     SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr,
     FuncExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, NewTypeExpr,
     StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr,
-    ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, MroError, type_aliases,
+    ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, ARG_NAMED_OPT, MroError, type_aliases,
     YieldFromExpr, NamedTupleExpr, TypedDictExpr, NonlocalDecl, SymbolNode,
     SetComprehension, DictionaryComprehension, TYPE_ALIAS, TypeAliasExpr,
     YieldExpr, ExecStmt, Argument, BackquoteExpr, ImportBase, AwaitExpr,
@@ -1830,14 +1830,14 @@ def add_method(funcname: str, ret: Type, args: List[Argument], name=None,
                 info.names[funcname] = SymbolTableNode(MDEF, func)
 
         add_method('_replace', ret=selftype,
-                   args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED) for var in vars])
+                   args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars])
         add_method('__init__', ret=NoneTyp(), name=info.name(),
                    args=[Argument(var, var.type, None, ARG_POS) for var in vars])
         add_method('_asdict', args=[], ret=ordereddictype)
         add_method('_make', ret=selftype, is_classmethod=True,
                    args=[Argument(Var('iterable', iterable_type), iterable_type, None, ARG_POS),
-                         Argument(Var('new'), AnyType(), EllipsisExpr(), ARG_NAMED),
-                         Argument(Var('len'), AnyType(), EllipsisExpr(), ARG_NAMED)])
+                         Argument(Var('new'), AnyType(), EllipsisExpr(), ARG_NAMED_OPT),
+                         Argument(Var('len'), AnyType(), EllipsisExpr(), ARG_NAMED_OPT)])
         return info
 
     def make_argument(self, name: str, type: Type) -> Argument:

mypy/strconv.py

@@ -41,9 +41,9 @@ def func_helper(self, o: 'mypy.nodes.FuncItem') -> List[object]:
         extra = []  # type: List[Tuple[str, List[mypy.nodes.Var]]]
         for i, arg in enumerate(o.arguments):
             kind = arg.kind  # type: int
-            if kind == mypy.nodes.ARG_POS:
+            if kind in (mypy.nodes.ARG_POS, mypy.nodes.ARG_NAMED):
                 args.append(o.arguments[i].variable)
-            elif kind in (mypy.nodes.ARG_OPT, mypy.nodes.ARG_NAMED):
+            elif kind in (mypy.nodes.ARG_OPT, mypy.nodes.ARG_NAMED_OPT):
                 args.append(o.arguments[i].variable)
                 init.append(o.arguments[i].initialization_statement)
             elif kind == mypy.nodes.ARG_STAR:
@@ -108,7 +108,8 @@ def visit_import_all(self, o: 'mypy.nodes.ImportAll') -> str:
     def visit_func_def(self, o: 'mypy.nodes.FuncDef') -> str:
         a = self.func_helper(o)
         a.insert(0, o.name())
-        if mypy.nodes.ARG_NAMED in [arg.kind for arg in o.arguments]:
+        arg_kinds = {arg.kind for arg in o.arguments}
+        if len(arg_kinds & {mypy.nodes.ARG_NAMED, mypy.nodes.ARG_NAMED_OPT}) > 0:
             a.insert(1, 'MaxPos({})'.format(o.max_pos))
         if o.is_abstract:
             a.insert(-1, 'Abstract')

mypy/stubgen.py

@@ -55,7 +55,8 @@
 from mypy.nodes import (
     Expression, IntExpr, UnaryExpr, StrExpr, BytesExpr, NameExpr, FloatExpr, MemberExpr, TupleExpr,
     ListExpr, ComparisonExpr, CallExpr, ClassDef, MypyFile, Decorator, AssignmentStmt,
-    IfStmt, ImportAll, ImportFrom, Import, FuncDef, FuncBase, ARG_STAR, ARG_STAR2, ARG_NAMED
+    IfStmt, ImportAll, ImportFrom, Import, FuncDef, FuncBase,
+    ARG_STAR, ARG_STAR2, ARG_NAMED, ARG_NAMED_OPT,
 )
 from mypy.stubgenc import parse_all_signatures, find_unique_signatures, generate_stub_for_c_module
 from mypy.stubutil import is_c_module, write_header
@@ -261,7 +262,7 @@ def visit_func_def(self, o: FuncDef) -> None:
             name = var.name()
             init_stmt = arg_.initialization_statement
             if init_stmt:
-                if kind == ARG_NAMED and '*' not in args:
+                if kind in (ARG_NAMED, ARG_NAMED_OPT) and '*' not in args:
                     args.append('*')
                 typename = self.get_str_type_of_node(init_stmt.rvalue, True)
                 arg = '{}: {} = ...'.format(name, typename)

mypy/test/data.py

@@ -363,15 +363,20 @@ def expand_errors(input: List[str], output: List[str], fnam: str) -> None:
     """
 
     for i in range(len(input)):
-        m = re.search('# ([EN]):((?P<col>\d+):)? (?P<message>.*)$', input[i])
-        if m:
-            severity = 'error' if m.group(1) == 'E' else 'note'
-            col = m.group('col')
-            if col is None:
-                output.append('{}:{}: {}: {}'.format(fnam, i + 1, severity, m.group('message')))
-            else:
-                output.append('{}:{}:{}: {}: {}'.format(
-                    fnam, i + 1, col, severity, m.group('message')))
+        # The first in the split things isn't a comment
+        for possible_err_comment in input[i].split('#')[1:]:
+            m = re.search(
+                '^([EN]):((?P<col>\d+):)? (?P<message>.*)$',
+                possible_err_comment.strip())
+            if m:
+                severity = 'error' if m.group(1) == 'E' else 'note'
+                col = m.group('col')
+                if col is None:
+                    output.append(
+                        '{}:{}: {}: {}'.format(fnam, i + 1, severity, m.group('message')))
+                else:
+                    output.append('{}:{}:{}: {}: {}'.format(
+                        fnam, i + 1, col, severity, m.group('message')))
 
 
 def fix_win_path(line: str) -> str:

mypy/types.py

@@ -7,7 +7,10 @@
 )
 
 import mypy.nodes
-from mypy.nodes import INVARIANT, SymbolNode
+from mypy.nodes import (
+    INVARIANT, SymbolNode,
+    ARG_POS, ARG_OPT, ARG_STAR, ARG_STAR2, ARG_NAMED, ARG_NAMED_OPT,
+)
 
 from mypy import experiments
 
@@ -539,7 +542,7 @@ class CallableType(FunctionLike):
     """Type of a non-overloaded callable object (function)."""
 
     arg_types = None  # type: List[Type]  # Types of function arguments
-    arg_kinds = None  # type: List[int]   # mypy.nodes.ARG_ constants
+    arg_kinds = None  # type: List[int]   # ARG_ constants
     arg_names = None  # type: List[str]   # None if not a keyword argument
     min_args = 0                    # Minimum number of arguments; derived from arg_kinds
     is_var_arg = False              # Is it a varargs function?  derived from arg_kinds
@@ -581,8 +584,9 @@ def __init__(self,
         self.arg_types = arg_types
         self.arg_kinds = arg_kinds
         self.arg_names = arg_names
-        self.min_args = arg_kinds.count(mypy.nodes.ARG_POS)
-        self.is_var_arg = mypy.nodes.ARG_STAR in arg_kinds
+        self.min_args = arg_kinds.count(ARG_POS)
+        self.is_var_arg = ARG_STAR in arg_kinds
+        self.is_kw_arg = ARG_STAR2 in arg_kinds
         self.ret_type = ret_type
         self.fallback = fallback
         assert not name or '<bound method' not in name
@@ -1250,17 +1254,17 @@ def visit_callable_type(self, t: CallableType) -> str:
         for i in range(len(t.arg_types)):
             if s != '':
                 s += ', '
-            if t.arg_kinds[i] == mypy.nodes.ARG_NAMED and not bare_asterisk:
+            if t.arg_kinds[i] in (ARG_NAMED, ARG_NAMED_OPT) and not bare_asterisk:
                 s += '*, '
                 bare_asterisk = True
-            if t.arg_kinds[i] == mypy.nodes.ARG_STAR:
+            if t.arg_kinds[i] == ARG_STAR:
                 s += '*'
-            if t.arg_kinds[i] == mypy.nodes.ARG_STAR2:
+            if t.arg_kinds[i] == ARG_STAR2:
                 s += '**'
             if t.arg_names[i]:
                 s += t.arg_names[i] + ': '
             s += str(t.arg_types[i])
-            if t.arg_kinds[i] == mypy.nodes.ARG_OPT:
+            if t.arg_kinds[i] in (ARG_OPT, ARG_NAMED_OPT):
                 s += ' ='
 
         s = '({})'.format(s)

test-data/samples/crawl.py

@@ -859,5 +859,5 @@ def main() -> None:
 
 
 if __name__ == '__main__':
-    logging.basicConfig(level=logging.INFO)
+    logging.basicConfig(level=logging.INFO) # type: ignore
     main()

test-data/samples/crawl2.py

@@ -848,5 +848,5 @@ def main() -> None:
 
 
 if __name__ == '__main__':
-    logging.basicConfig(level=logging.INFO)
+    logging.basicConfig(level=logging.INFO) # type: ignore
     main()

test-data/unit/check-kwargs.test

@@ -115,10 +115,57 @@ main:4: error: Unexpected keyword argument "b"
 [case testKeywordOnlyArguments]
 import typing
 def f(a: 'A', *, b: 'B' = None) -> None: pass
+def g(a: 'A', *, b: 'B') -> None: pass
+def h(a: 'A', *, b: 'B', aa: 'A') -> None: pass
+def i(a: 'A', *, b: 'B', aa: 'A' = None) -> None: pass
 f(A(), b=B())
 f(b=B(), a=A())
 f(A())
 f(A(), B()) # E: Too many positional arguments for "f"
+g(A(), b=B())
+g(b=B(), a=A())
+g(A()) # E: Missing named argument "b" for function "g"
+g(A(), B()) # E: Too many positional arguments for "g"
+h(A()) # E: Missing named argument "b" for function "h" # E: Missing named argument "aa" for function "h"
+h(A(), b=B()) # E: Missing named argument "aa" for function "h"
+h(A(), aa=A()) # E: Missing named argument "b" for function "h"
+h(A(), b=B(), aa=A())
+h(A(), aa=A(), b=B())
+i(A()) # E: Missing named argument "b" for function "i"
+i(A(), b=B())
+i(A(), aa=A()) # E: Missing named argument "b" for function "i"
+i(A(), b=B(), aa=A())
+i(A(), aa=A(), b=B())
+
+class A: pass
+class B: pass
+
+[case testKeywordOnlyArgumentsFastparse]
+# flags: --fast-parser
+import typing
+def f(a: 'A', *, b: 'B' = None) -> None: pass
+def g(a: 'A', *, b: 'B') -> None: pass
+def h(a: 'A', *, b: 'B', aa: 'A') -> None: pass
+def i(a: 'A', *, b: 'B', aa: 'A' = None) -> None: pass
+f(A(), b=B())
+f(b=B(), a=A())
+f(A())
+f(A(), B()) # E: Too many positional arguments for "f"
+g(A(), b=B())
+g(b=B(), a=A())
+g(A()) # E: Missing named argument "b" for function "g"
+g(A(), B()) # E: Too many positional arguments for "g"
+h(A()) # E: Missing named argument "b" for function "h" # E: Missing named argument "aa" for function "h"
+h(A(), b=B()) # E: Missing named argument "aa" for function "h"
+h(A(), aa=A()) # E: Missing named argument "b" for function "h"
+h(A(), b=B(), aa=A())
+h(A(), aa=A(), b=B())
+i(A()) # E: Missing named argument "b" for function "i"
+i(A(), b=B())
+i(A(), aa=A()) # E: Missing named argument "b" for function "i"
+i(A(), b=B(), aa=A())
+i(A(), aa=A(), b=B())
+
 class A: pass
 class B: pass
 

test-data/unit/parse.test

@@ -2340,7 +2340,7 @@ MypyFile:1(
     Args(
       Var(x)
       Var(y))
-    def (x: A?, *, y: B?) -> None?
+    def (x: A?, *, y: B? =) -> None?
     Init(
       AssignmentStmt:1(
         NameExpr(y)
@@ -2357,7 +2357,7 @@ MypyFile:1(
     MaxPos(0)
     Args(
       Var(y))
-    def (*, y: B?) -> None?
+    def (*, y: B? =) -> None?
     Init(
       AssignmentStmt:1(
         NameExpr(y)
@@ -2375,7 +2375,6 @@ MypyFile:1(
     Args(
       Var(y))
     def (*, y: B?) -> None?
-    Init()
     Block:1(
       PassStmt:1())))
 

test-data/unit/semanal-types.test

@@ -1005,7 +1005,7 @@ MypyFile:1(
     MaxPos(0)
     Args(
       Var(y))
-    def (*x: builtins.int, *, y: builtins.str) -> Any
+    def (*x: builtins.int, *, y: builtins.str =) -> Any
     Init(
       AssignmentStmt:1(
         NameExpr(y [l])