TypedDict: Recognize creation of TypedDict instance. Define TypedDictType. (#2342)

Notable visitor implementations added:
* Subtype
* Join
* Meet
* Constraint Solve

Also:
* Fix support for using dict(...) in TypedDict instance constructor.
* Allow instantiation of empty TypedDict.
* Disallow underscore prefix on TypedDict item names.
* TypeAnalyser: Resolve an unbound reference to a typeddict as a named TypedDictType 
  rather than as an Instance.

Author: davidfstr

mypy/checkexpr.py

@@ -1,10 +1,11 @@
 """Expression type checker. This file is conceptually part of TypeChecker."""
 
+from collections import OrderedDict
 from typing import cast, Dict, Set, List, Iterable, Tuple, Callable, Union, Optional
 
 from mypy.types import (
     Type, AnyType, CallableType, Overloaded, NoneTyp, Void, TypeVarDef,
-    TupleType, Instance, TypeVarId, TypeVarType, ErasedType, UnionType,
+    TupleType, TypedDictType, Instance, TypeVarId, TypeVarType, ErasedType, UnionType,
     PartialType, DeletedType, UnboundType, UninhabitedType, TypeType,
     true_only, false_only, is_named_instance, function_type,
     get_typ_args, set_typ_args,
@@ -169,6 +170,10 @@ def visit_call_expr(self, e: CallExpr) -> Type:
         if e.analyzed:
             # It's really a special form that only looks like a call.
             return self.accept(e.analyzed, self.chk.type_context[-1])
+        if isinstance(e.callee, NameExpr) and isinstance(e.callee.node, TypeInfo) and \
+                e.callee.node.typeddict_type is not None:
+            return self.check_typeddict_call(e.callee.node.typeddict_type,
+                                             e.arg_kinds, e.arg_names, e.args, e)
         self.try_infer_partial_type(e)
         callee_type = self.accept(e.callee)
         if (self.chk.options.disallow_untyped_calls and
@@ -178,6 +183,80 @@ def visit_call_expr(self, e: CallExpr) -> Type:
             return self.msg.untyped_function_call(callee_type, e)
         return self.check_call_expr_with_callee_type(callee_type, e)
 
+    def check_typeddict_call(self, callee: TypedDictType,
+                             arg_kinds: List[int],
+                             arg_names: List[str],
+                             args: List[Expression],
+                             context: Context) -> Type:
+        if len(args) >= 1 and all([ak == ARG_NAMED for ak in arg_kinds]):
+            # ex: Point(x=42, y=1337)
+            item_names = arg_names
+            item_args = args
+            return self.check_typeddict_call_with_kwargs(
+                callee, OrderedDict(zip(item_names, item_args)), context)
+
+        if len(args) == 1 and arg_kinds[0] == ARG_POS:
+            unique_arg = args[0]
+            if isinstance(unique_arg, DictExpr):
+                # ex: Point({'x': 42, 'y': 1337})
+                return self.check_typeddict_call_with_dict(callee, unique_arg, context)
+            if isinstance(unique_arg, CallExpr) and isinstance(unique_arg.analyzed, DictExpr):
+                # ex: Point(dict(x=42, y=1337))
+                return self.check_typeddict_call_with_dict(callee, unique_arg.analyzed, context)
+
+        if len(args) == 0:
+            # ex: EmptyDict()
+            return self.check_typeddict_call_with_kwargs(
+                callee, OrderedDict(), context)
+
+        self.chk.fail(messages.INVALID_TYPEDDICT_ARGS, context)
+        return AnyType()
+
+    def check_typeddict_call_with_dict(self, callee: TypedDictType,
+                                       kwargs: DictExpr,
+                                       context: Context) -> Type:
+        item_name_exprs = [item[0] for item in kwargs.items]
+        item_args = [item[1] for item in kwargs.items]
+
+        item_names = []  # List[str]
+        for item_name_expr in item_name_exprs:
+            if not isinstance(item_name_expr, StrExpr):
+                self.chk.fail(messages.TYPEDDICT_ITEM_NAME_MUST_BE_STRING_LITERAL, item_name_expr)
+                return AnyType()
+            item_names.append(item_name_expr.value)
+
+        return self.check_typeddict_call_with_kwargs(
+            callee, OrderedDict(zip(item_names, item_args)), context)
+
+    def check_typeddict_call_with_kwargs(self, callee: TypedDictType,
+                                         kwargs: 'OrderedDict[str, Expression]',
+                                         context: Context) -> Type:
+        if callee.items.keys() != kwargs.keys():
+            callee_item_names = callee.items.keys()
+            kwargs_item_names = kwargs.keys()
+
+            self.msg.typeddict_instantiated_with_unexpected_items(
+                expected_item_names=list(callee_item_names),
+                actual_item_names=list(kwargs_item_names),
+                context=context)
+            return AnyType()
+
+        items = OrderedDict()  # type: OrderedDict[str, Type]
+        for (item_name, item_expected_type) in callee.items.items():
+            item_value = kwargs[item_name]
+
+            item_actual_type = self.chk.check_simple_assignment(
+                lvalue_type=item_expected_type, rvalue=item_value, context=item_value,
+                msg=messages.INCOMPATIBLE_TYPES,
+                lvalue_name='TypedDict item "{}"'.format(item_name),
+                rvalue_name='expression')
+            items[item_name] = item_actual_type
+
+        mapping_value_type = join.join_type_list(list(items.values()))
+        fallback = self.chk.named_generic_type('typing.Mapping',
+                                               [self.chk.str_type(), mapping_value_type])
+        return TypedDictType(items, fallback)
+
     # Types and methods that can be used to infer partial types.
     item_args = {'builtins.list': ['append'],
                  'builtins.set': ['add', 'discard'],

mypy/checkmember.py

@@ -3,7 +3,7 @@
 from typing import cast, Callable, List, Optional, TypeVar
 
 from mypy.types import (
-    Type, Instance, AnyType, TupleType, CallableType, FunctionLike, TypeVarDef,
+    Type, Instance, AnyType, TupleType, TypedDictType, CallableType, FunctionLike, TypeVarDef,
     Overloaded, TypeVarType, UnionType, PartialType,
     DeletedType, NoneTyp, TypeType, function_type
 )
@@ -116,6 +116,11 @@ def analyze_member_access(name: str,
         return analyze_member_access(name, typ.fallback, node, is_lvalue, is_super,
                                      is_operator, builtin_type, not_ready_callback, msg,
                                      original_type=original_type, chk=chk)
+    elif isinstance(typ, TypedDictType):
+        # Actually look up from the fallback instance type.
+        return analyze_member_access(name, typ.fallback, node, is_lvalue, is_super,
+                                     is_operator, builtin_type, not_ready_callback, msg,
+                                     original_type=original_type, chk=chk)
     elif isinstance(typ, FunctionLike) and typ.is_type_obj():
         # Class attribute.
         # TODO super?

mypy/constraints.py

@@ -1,11 +1,11 @@
 """Type inference constraints."""
 
-from typing import List, Optional
+from typing import Iterable, List, Optional
 
 from mypy.types import (
     CallableType, Type, TypeVisitor, UnboundType, AnyType, Void, NoneTyp, TypeVarType,
-    Instance, TupleType, UnionType, Overloaded, ErasedType, PartialType, DeletedType,
-    UninhabitedType, TypeType, TypeVarId, is_named_instance
+    Instance, TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType,
+    DeletedType, UninhabitedType, TypeType, TypeVarId, is_named_instance
 )
 from mypy.maptype import map_instance_to_supertype
 from mypy import nodes
@@ -342,11 +342,27 @@ def visit_tuple_type(self, template: TupleType) -> List[Constraint]:
         else:
             return []
 
+    def visit_typeddict_type(self, template: TypedDictType) -> List[Constraint]:
+        actual = self.actual
+        if isinstance(actual, TypedDictType):
+            res = []  # type: List[Constraint]
+            # NOTE: Non-matching keys are ignored. Compatibility is checked
+            #       elsewhere so this shouldn't be unsafe.
+            for (item_name, template_item_type, actual_item_type) in template.zip(actual):
+                res.extend(infer_constraints(template_item_type,
+                                             actual_item_type,
+                                             self.direction))
+            return res
+        elif isinstance(actual, AnyType):
+            return self.infer_against_any(template.items.values())
+        else:
+            return []
+
     def visit_union_type(self, template: UnionType) -> List[Constraint]:
         assert False, ("Unexpected UnionType in ConstraintBuilderVisitor"
                        " (should have been handled in infer_constraints)")
 
-    def infer_against_any(self, types: List[Type]) -> List[Constraint]:
+    def infer_against_any(self, types: Iterable[Type]) -> List[Constraint]:
         res = []  # type: List[Constraint]
         for t in types:
             res.extend(infer_constraints(t, AnyType(), self.direction))

mypy/erasetype.py

@@ -2,8 +2,8 @@
 
 from mypy.types import (
     Type, TypeVisitor, UnboundType, ErrorType, AnyType, Void, NoneTyp, TypeVarId,
-    Instance, TypeVarType, CallableType, TupleType, UnionType, Overloaded, ErasedType,
-    PartialType, DeletedType, TypeTranslator, TypeList, UninhabitedType, TypeType
+    Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded,
+    ErasedType, PartialType, DeletedType, TypeTranslator, TypeList, UninhabitedType, TypeType
 )
 from mypy import experiments
 
@@ -78,6 +78,9 @@ def visit_overloaded(self, t: Overloaded) -> Type:
     def visit_tuple_type(self, t: TupleType) -> Type:
         return t.fallback.accept(self)
 
+    def visit_typeddict_type(self, t: TypedDictType) -> Type:
+        return t.fallback.accept(self)
+
     def visit_union_type(self, t: UnionType) -> Type:
         return AnyType()        # XXX: return underlying type if only one?
 

mypy/expandtype.py

@@ -1,9 +1,9 @@
-from typing import Dict, List
+from typing import Dict, Iterable, List
 
 from mypy.types import (
     Type, Instance, CallableType, TypeVisitor, UnboundType, ErrorType, AnyType,
-    Void, NoneTyp, TypeVarType, Overloaded, TupleType, UnionType, ErasedType, TypeList,
-    PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId
+    Void, NoneTyp, TypeVarType, Overloaded, TupleType, TypedDictType, UnionType,
+    ErasedType, TypeList, PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId
 )
 
 
@@ -93,6 +93,9 @@ def visit_overloaded(self, t: Overloaded) -> Type:
     def visit_tuple_type(self, t: TupleType) -> Type:
         return t.copy_modified(items=self.expand_types(t.items))
 
+    def visit_typeddict_type(self, t: TypedDictType) -> Type:
+        return t.copy_modified(item_types=self.expand_types(t.items.values()))
+
     def visit_union_type(self, t: UnionType) -> Type:
         # After substituting for type variables in t.items,
         # some of the resulting types might be subtypes of others.
@@ -108,7 +111,7 @@ def visit_type_type(self, t: TypeType) -> Type:
         item = t.item.accept(self)
         return TypeType(item)
 
-    def expand_types(self, types: List[Type]) -> List[Type]:
+    def expand_types(self, types: Iterable[Type]) -> List[Type]:
         a = []  # type: List[Type]
         for t in types:
             a.append(t.accept(self))

mypy/fixup.py

@@ -2,13 +2,17 @@
 
 from typing import Any, Dict, Optional
 
-from mypy.nodes import (MypyFile, SymbolNode, SymbolTable, SymbolTableNode,
-                        TypeInfo, FuncDef, OverloadedFuncDef, Decorator, Var,
-                        TypeVarExpr, ClassDef,
-                        LDEF, MDEF, GDEF)
-from mypy.types import (CallableType, EllipsisType, Instance, Overloaded, TupleType,
-                        TypeList, TypeVarType, UnboundType, UnionType, TypeVisitor,
-                        TypeType)
+from mypy.nodes import (
+    MypyFile, SymbolNode, SymbolTable, SymbolTableNode,
+    TypeInfo, FuncDef, OverloadedFuncDef, Decorator, Var,
+    TypeVarExpr, ClassDef,
+    LDEF, MDEF, GDEF
+)
+from mypy.types import (
+    CallableType, EllipsisType, Instance, Overloaded, TupleType, TypedDictType,
+    TypeList, TypeVarType, UnboundType, UnionType, TypeVisitor,
+    TypeType
+)
 from mypy.visitor import NodeVisitor
 
 
@@ -192,6 +196,13 @@ def visit_tuple_type(self, tt: TupleType) -> None:
         if tt.fallback is not None:
             tt.fallback.accept(self)
 
+    def visit_typeddict_type(self, tdt: TypedDictType) -> None:
+        if tdt.items:
+            for it in tdt.items.values():
+                it.accept(self)
+        if tdt.fallback is not None:
+            tdt.fallback.accept(self)
+
     def visit_type_list(self, tl: TypeList) -> None:
         for t in tl.items:
             t.accept(self)

mypy/indirection.py

@@ -87,6 +87,9 @@ def visit_overloaded(self, t: types.Overloaded) -> Set[str]:
     def visit_tuple_type(self, t: types.TupleType) -> Set[str]:
         return self._visit(*t.items) | self._visit(t.fallback)
 
+    def visit_typeddict_type(self, t: types.TypedDictType) -> Set[str]:
+        return self._visit(*t.items.values()) | self._visit(t.fallback)
+
     def visit_star_type(self, t: types.StarType) -> Set[str]:
         return set()
 

mypy/join.py

@@ -1,10 +1,11 @@
 """Calculation of the least upper bound types (joins)."""
 
-from typing import List
+from collections import OrderedDict
+from typing import cast, List
 
 from mypy.types import (
     Type, AnyType, NoneTyp, Void, TypeVisitor, Instance, UnboundType,
-    ErrorType, TypeVarType, CallableType, TupleType, ErasedType, TypeList,
+    ErrorType, TypeVarType, CallableType, TupleType, TypedDictType, ErasedType, TypeList,
     UnionType, FunctionLike, Overloaded, PartialType, DeletedType,
     UninhabitedType, TypeType, true_or_false
 )
@@ -170,6 +171,8 @@ def visit_instance(self, t: Instance) -> Type:
             return join_types(t, self.s.fallback)
         elif isinstance(self.s, TypeType):
             return join_types(t, self.s)
+        elif isinstance(self.s, TypedDictType):
+            return join_types(t, self.s)
         else:
             return self.default(self.s)
 
@@ -234,13 +237,27 @@ def visit_tuple_type(self, t: TupleType) -> Type:
             items = []  # type: List[Type]
             for i in range(t.length()):
                 items.append(self.join(t.items[i], self.s.items[i]))
-            # join fallback types if they are different
             fallback = join_instances(self.s.fallback, t.fallback)
             assert isinstance(fallback, Instance)
             return TupleType(items, fallback)
         else:
             return self.default(self.s)
 
+    def visit_typeddict_type(self, t: TypedDictType) -> Type:
+        if isinstance(self.s, TypedDictType):
+            items = OrderedDict([
+                (item_name, s_item_type)
+                for (item_name, s_item_type, t_item_type) in self.s.zip(t)
+                if is_equivalent(s_item_type, t_item_type)
+            ])
+            mapping_value_type = join_type_list(list(items.values()))
+            fallback = self.s.create_anonymous_fallback(value_type=mapping_value_type)
+            return TypedDictType(items, fallback)
+        elif isinstance(self.s, Instance):
+            return join_instances(self.s, t.fallback)
+        else:
+            return self.default(self.s)
+
     def visit_partial_type(self, t: PartialType) -> Type:
         # We only have partial information so we can't decide the join result. We should
         # never get here.
@@ -266,6 +283,8 @@ def default(self, typ: Type) -> Type:
             return ErrorType()
         elif isinstance(typ, TupleType):
             return self.default(typ.fallback)
+        elif isinstance(typ, TypedDictType):
+            return self.default(typ.fallback)
         elif isinstance(typ, FunctionLike):
             return self.default(typ.fallback)
         elif isinstance(typ, TypeVarType):

mypy/meet.py

@@ -1,12 +1,13 @@
-from typing import List
+from collections import OrderedDict
+from typing import List, Optional
 
-from mypy.join import is_similar_callables, combine_similar_callables
+from mypy.join import is_similar_callables, combine_similar_callables, join_type_list
 from mypy.types import (
     Type, AnyType, TypeVisitor, UnboundType, Void, ErrorType, NoneTyp, TypeVarType,
-    Instance, CallableType, TupleType, ErasedType, TypeList, UnionType, PartialType,
+    Instance, CallableType, TupleType, TypedDictType, ErasedType, TypeList, UnionType, PartialType,
     DeletedType, UninhabitedType, TypeType
 )
-from mypy.subtypes import is_subtype
+from mypy.subtypes import is_equivalent, is_subtype
 
 from mypy import experiments
 
@@ -252,6 +253,21 @@ def visit_tuple_type(self, t: TupleType) -> Type:
         else:
             return self.default(self.s)
 
+    def visit_typeddict_type(self, t: TypedDictType) -> Type:
+        if isinstance(self.s, TypedDictType):
+            for (_, l, r) in self.s.zip(t):
+                if not is_equivalent(l, r):
+                    return self.default(self.s)
+            items = OrderedDict([
+                (item_name, s_item_type or t_item_type)
+                for (item_name, s_item_type, t_item_type) in self.s.zipall(t)
+            ])
+            mapping_value_type = join_type_list(list(items.values()))
+            fallback = self.s.create_anonymous_fallback(value_type=mapping_value_type)
+            return TypedDictType(items, fallback)
+        else:
+            return self.default(self.s)
+
     def visit_partial_type(self, t: PartialType) -> Type:
         # We can't determine the meet of partial types. We should never get here.
         assert False, 'Internal error'