New analyzer: resolve name clashes between generated and existing nodes (python#6972)

Fixes python#6454.
Fixes python#6973.

Note that the solution doesn't guarantee the order of numbers N in 
foo-redefinitionN coincides with the textual order of nodes. But it looks 
like we don't need this for anything.

Author: ilevkivskyi

mypy/newsemanal/semanal_namedtuple.py

@@ -20,6 +20,7 @@
 )
 from mypy.options import Options
 from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError
+from mypy.util import get_unique_redefinition_name
 
 MYPY = False
 if MYPY:
@@ -493,10 +494,15 @@ def save_namedtuple_body(self, named_tuple_info: TypeInfo) -> Iterator[None]:
         # Restore the names in the original symbol table. This ensures that the symbol
         # table contains the field objects created by build_namedtuple_typeinfo. Exclude
         # __doc__, which can legally be overwritten by the class.
-        named_tuple_info.names.update({
-            key: value for key, value in nt_names.items()
-            if key not in named_tuple_info.names or key != '__doc__'
-        })
+        for key, value in nt_names.items():
+            if key in named_tuple_info.names:
+                if key == '__doc__':
+                    continue
+                # Keep existing (user-provided) definitions under mangled names, so they
+                # get semantically analyzed.
+                r_key = get_unique_redefinition_name(key, named_tuple_info.names)
+                named_tuple_info.names[r_key] = named_tuple_info.names[key]
+            named_tuple_info.names[key] = value
 
     # Helpers
 

mypy/plugins/common.py

@@ -8,6 +8,7 @@
 from mypy.semanal import set_callable_name
 from mypy.types import CallableType, Overloaded, Type, TypeVarDef, LiteralType, Instance
 from mypy.typevars import fill_typevars
+from mypy.util import get_unique_redefinition_name
 
 
 def _get_decorator_bool_argument(
@@ -117,6 +118,13 @@ def add_method(
     func._fullname = info.fullname() + '.' + name
     func.line = info.line
 
+    # NOTE: we would like the plugin generated node to dominate, but we still
+    # need to keep any existing definitions so they get semantically analyzed.
+    if name in info.names:
+        # Get a nice unique name instead.
+        r_name = get_unique_redefinition_name(name, info.names)
+        info.names[r_name] = info.names[name]
+
     info.names[name] = SymbolTableNode(MDEF, func, plugin_generated=True)
     info.defn.defs.body.append(func)
 

mypy/util.py

@@ -4,7 +4,7 @@
 import re
 import subprocess
 import sys
-from typing import TypeVar, List, Tuple, Optional, Dict, Sequence, Iterable
+from typing import TypeVar, List, Tuple, Optional, Dict, Sequence, Iterable, Container
 
 MYPY = False
 if MYPY:
@@ -290,6 +290,22 @@ def unmangle(name: str) -> str:
     return name.rstrip("'")
 
 
+def get_unique_redefinition_name(name: str, existing: Container[str]) -> str:
+    """Get a simple redefinition name not present among existing.
+
+    For example, for name 'foo' we try 'foo-redefinition', 'foo-redefinition2',
+    'foo-redefinition3', etc. until we find one that is not in existing.
+    """
+    r_name = name + '-redefinition'
+    if r_name not in existing:
+        return r_name
+
+    i = 2
+    while r_name + str(i) in existing:
+        i += 1
+    return r_name + str(i)
+
+
 def check_python_version(program: str) -> None:
     """Report issues with the Python used to run mypy, dmypy, or stubgen"""
     # Check for known bad Python versions.

test-data/unit/check-class-namedtuple.test

@@ -587,9 +587,7 @@ reveal_type(takes_base(Base(1)))  # E: Revealed type is 'builtins.int'
 reveal_type(takes_base(Child(1)))  # E: Revealed type is 'builtins.int'
 [builtins fixtures/tuple.pyi]
 
--- Depends on collecting functions via AST, not symbol tables.
 [case testNewNamedTupleIllegalNames]
-# flags: --no-new-semantic-analyzer
 from typing import Callable, NamedTuple
 
 class XMethBad(NamedTuple):
@@ -614,16 +612,16 @@ class AnnotationsAsAMethod(NamedTuple):
 
 class ReuseNames(NamedTuple):
     x: int
-    def x(self) -> str:  # E: Name 'x' already defined on line 23
+    def x(self) -> str:  # E: Name 'x' already defined on line 22
         return ''
 
     def y(self) -> int:
         return 0
-    y: str  # E: Name 'y' already defined on line 27
+    y: str  # E: Name 'y' already defined on line 26
 
 class ReuseCallableNamed(NamedTuple):
     z: Callable[[ReuseNames], int]
-    def z(self) -> int:  # E: Name 'z' already defined on line 32
+    def z(self) -> int:  # E: Name 'z' already defined on line 31
         return 0
 
 [builtins fixtures/dict.pyi]

test-data/unit/check-dataclasses.test

@@ -393,9 +393,8 @@ class Application:  # E: eq must be True if order is True
 
 [builtins fixtures/list.pyi]
 
--- Blocked by #6454
 [case testDataclassOrderingWithCustomMethods]
-# flags: --python-version 3.6  --no-new-semantic-analyzer
+# flags: --python-version 3.6
 from dataclasses import dataclass
 
 @dataclass(order=True)