gh-54781: Move Lib/lib2to3/tests/ to Lib/test/test_lib2to3/ (#94049)

* Move Lib/lib2to3/tests/ to Lib/test/test_lib2to3/.
* Remove Lib/test/test_lib2to3.py.
* Update imports.
* all_project_files(): use different paths and sort files
  to make the tests more reproducible.
* Update references to tests.

Author: vstinner

Lib/test/test_lib2to3.py

@@ -17,9 +17,9 @@
 import logging
 
 # Local imports
-from .. import pytree
-from .. import pgen2
-from ..pgen2 import driver
+from lib2to3 import pytree
+from lib2to3 import pgen2
+from lib2to3.pgen2 import driver
 
 logging.basicConfig()
 

Lib/lib2to3/tests/__init__.py renamed to Lib/test/test_lib2to3/__init__.py

@@ -8,12 +8,14 @@
 from textwrap import dedent
 
 # Local imports
+import lib2to3
 from lib2to3 import pytree, refactor
 from lib2to3.pgen2 import driver as pgen2_driver
 
+lib2to3_dir = os.path.dirname(lib2to3.__file__)
 test_dir = os.path.dirname(__file__)
 proj_dir = os.path.normpath(os.path.join(test_dir, ".."))
-grammar_path = os.path.join(test_dir, "..", "Grammar.txt")
+grammar_path = os.path.join(lib2to3_dir, "Grammar.txt")
 grammar = pgen2_driver.load_grammar(grammar_path)
 grammar_no_print_statement = pgen2_driver.load_grammar(grammar_path)
 del grammar_no_print_statement.keywords["print"]
@@ -49,10 +51,19 @@ def get_refactorer(fixer_pkg="lib2to3", fixers=None, options=None):
     options = options or {}
     return refactor.RefactoringTool(fixers, options, explicit=True)
 
-def all_project_files():
-    for dirpath, dirnames, filenames in os.walk(proj_dir):
+def _all_project_files(root, files):
+    for dirpath, dirnames, filenames in os.walk(root):
         for filename in filenames:
-            if filename.endswith(".py"):
-                yield os.path.join(dirpath, filename)
+            if not filename.endswith(".py"):
+                continue
+            files.append(os.path.join(dirpath, filename))
+
+def all_project_files():
+    files = []
+    _all_project_files(lib2to3_dir, files)
+    _all_project_files(test_dir, files)
+    # Sort to get more reproducible tests
+    files.sort()
+    return files
 
 TestCase = unittest.TestCase

Lib/lib2to3/tests/__main__.py renamed to Lib/test/test_lib2to3/__main__.py

@@ -7,7 +7,7 @@
 
 # Local imports
 from lib2to3 import pygram, fixer_util
-from lib2to3.tests import support
+from test.test_lib2to3 import support
 
 
 class FixerTestCase(support.TestCase):
@@ -1791,7 +1791,7 @@ def f():
 
 class Test_imports(FixerTestCase, ImportsFixerTests):
     fixer = "imports"
-    from ..fixes.fix_imports import MAPPING as modules
+    from lib2to3.fixes.fix_imports import MAPPING as modules
 
     def test_multiple_imports(self):
         b = """import urlparse, cStringIO"""
@@ -1812,16 +1812,16 @@ def test_multiple_imports_as(self):
 
 class Test_imports2(FixerTestCase, ImportsFixerTests):
     fixer = "imports2"
-    from ..fixes.fix_imports2 import MAPPING as modules
+    from lib2to3.fixes.fix_imports2 import MAPPING as modules
 
 
 class Test_imports_fixer_order(FixerTestCase, ImportsFixerTests):
 
     def setUp(self):
         super(Test_imports_fixer_order, self).setUp(['imports', 'imports2'])
-        from ..fixes.fix_imports2 import MAPPING as mapping2
+        from lib2to3.fixes.fix_imports2 import MAPPING as mapping2
         self.modules = mapping2.copy()
-        from ..fixes.fix_imports import MAPPING as mapping1
+        from lib2to3.fixes.fix_imports import MAPPING as mapping1
         for key in ('dbhash', 'dumbdbm', 'dbm', 'gdbm'):
             self.modules[key] = mapping1[key]
 
@@ -1833,7 +1833,7 @@ def test_after_local_imports_refactoring(self):
 
 class Test_urllib(FixerTestCase):
     fixer = "urllib"
-    from ..fixes.fix_urllib import MAPPING as modules
+    from lib2to3.fixes.fix_urllib import MAPPING as modules
 
     def test_import_module(self):
         for old, changes in self.modules.items():

Lib/lib2to3/tests/data/README renamed to Lib/test/test_lib2to3/data/README

@@ -26,7 +26,7 @@
 # Local imports
 from lib2to3.pgen2 import driver as pgen2_driver
 from lib2to3.pgen2 import tokenize
-from ..pgen2.parse import ParseError
+from lib2to3.pgen2.parse import ParseError
 from lib2to3.pygram import python_symbols as syms
 
 

Lib/lib2to3/tests/data/bom.py renamed to Lib/test/test_lib2to3/data/bom.py

@@ -8,12 +8,12 @@ FWK="/Library/Frameworks/Python.framework/Versions/@PYVER@"
 
 "${FWK}/bin/python@PYVER@" -E -s -Wi \
     "${FWK}/lib/python${PYVER}/compileall.py" -q -j0 \
-    -f -x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
+    -f -x 'bad_coding|badsyntax|site-packages|test/test_lib2to3/data' \
     "${FWK}/lib/python${PYVER}"
 
 "${FWK}/bin/python@PYVER@" -E -s -Wi -O \
     "${FWK}/lib/python${PYVER}/compileall.py" -q -j0 \
-    -f -x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
+    -f -x 'bad_coding|badsyntax|site-packages|test/test_lib2to3/data' \
     "${FWK}/lib/python${PYVER}"
 
 "${FWK}/bin/python@PYVER@" -E -s -Wi \

Lib/lib2to3/tests/data/crlf.py renamed to Lib/test/test_lib2to3/data/crlf.py

@@ -1921,10 +1921,6 @@ LIBSUBDIRS=	asyncio \
 		__phello__
 TESTSUBDIRS=	distutils/tests \
 		idlelib/idle_test \
-		lib2to3/tests \
-		lib2to3/tests/data \
-		lib2to3/tests/data/fixers \
-		lib2to3/tests/data/fixers/myfixes \
 		test test/audiodata \
 		test/capath test/cjkencodings \
 		test/data test/decimaltestdata \
@@ -1986,6 +1982,10 @@ TESTSUBDIRS=	distutils/tests \
 		test/test_importlib/zipdata01 \
 		test/test_importlib/zipdata02 \
 		test/test_json \
+		test/test_lib2to3 \
+		test/test_lib2to3/data \
+		test/test_lib2to3/data/fixers \
+		test/test_lib2to3/data/fixers/myfixes \
 		test/test_peg_generator \
 		test/test_tools \
 		test/test_warnings test/test_warnings/data \
@@ -2073,17 +2073,17 @@ libinstall:	all $(srcdir)/Modules/xxmodule.c
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST)  $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
 		-j0 -d $(LIBDEST) -f \
-		-x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
+		-x 'bad_coding|badsyntax|site-packages|test/test_lib2to3/data' \
 		$(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
 		-j0 -d $(LIBDEST) -f \
-		-x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
+		-x 'bad_coding|badsyntax|site-packages|test/test_lib2to3/data' \
 		$(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi -OO $(DESTDIR)$(LIBDEST)/compileall.py \
 		-j0 -d $(LIBDEST) -f \
-		-x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
+		-x 'bad_coding|badsyntax|site-packages|test/test_lib2to3/data' \
 		$(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \

Lib/lib2to3/tests/data/different_encoding.py renamed to Lib/test/test_lib2to3/data/different_encoding.py

@@ -561,33 +561,6 @@
     <Compile Include="lib2to3\pygram.py" />
     <Compile Include="lib2to3\pytree.py" />
     <Compile Include="lib2to3\refactor.py" />
-    <Compile Include="lib2to3\tests\data\bom.py" />
-    <Compile Include="lib2to3\tests\data\crlf.py" />
-    <Compile Include="lib2to3\tests\data\different_encoding.py" />
-    <Compile Include="lib2to3\tests\data\false_encoding.py" />
-    <Compile Include="lib2to3\tests\data\fixers\bad_order.py" />
-    <Compile Include="lib2to3\tests\data\fixers\myfixes\fix_explicit.py" />
-    <Compile Include="lib2to3\tests\data\fixers\myfixes\fix_first.py" />
-    <Compile Include="lib2to3\tests\data\fixers\myfixes\fix_last.py" />
-    <Compile Include="lib2to3\tests\data\fixers\myfixes\fix_parrot.py" />
-    <Compile Include="lib2to3\tests\data\fixers\myfixes\fix_preorder.py" />
-    <Compile Include="lib2to3\tests\data\fixers\myfixes\__init__.py" />
-    <Compile Include="lib2to3\tests\data\fixers\no_fixer_cls.py" />
-    <Compile Include="lib2to3\tests\data\fixers\parrot_example.py" />
-    <Compile Include="lib2to3\tests\data\infinite_recursion.py" />
-    <Compile Include="lib2to3\tests\data\py2_test_grammar.py" />
-    <Compile Include="lib2to3\tests\data\py3_test_grammar.py" />
-    <Compile Include="lib2to3\tests\pytree_idempotency.py" />
-    <Compile Include="lib2to3\tests\support.py" />
-    <Compile Include="lib2to3\tests\test_all_fixers.py" />
-    <Compile Include="lib2to3\tests\test_fixers.py" />
-    <Compile Include="lib2to3\tests\test_main.py" />
-    <Compile Include="lib2to3\tests\test_parser.py" />
-    <Compile Include="lib2to3\tests\test_pytree.py" />
-    <Compile Include="lib2to3\tests\test_refactor.py" />
-    <Compile Include="lib2to3\tests\test_util.py" />
-    <Compile Include="lib2to3\tests\__init__.py" />
-    <Compile Include="lib2to3\tests\__main__.py" />
     <Compile Include="lib2to3\__init__.py" />
     <Compile Include="lib2to3\__main__.py" />
     <Compile Include="linecache.py" />
@@ -1157,7 +1130,33 @@
     <Compile Include="test\test_keywordonlyarg.py" />
     <Compile Include="test\test_kqueue.py" />
     <Compile Include="test\test_largefile.py" />
-    <Compile Include="test\test_lib2to3.py" />
+    <Compile Include="test\test_lib2to3\data\bom.py" />
+    <Compile Include="test\test_lib2to3\data\crlf.py" />
+    <Compile Include="test\test_lib2to3\data\different_encoding.py" />
+    <Compile Include="test\test_lib2to3\data\false_encoding.py" />
+    <Compile Include="test\test_lib2to3\data\fixers\bad_order.py" />
+    <Compile Include="test\test_lib2to3\data\fixers\myfixes\fix_explicit.py" />
+    <Compile Include="test\test_lib2to3\data\fixers\myfixes\fix_first.py" />
+    <Compile Include="test\test_lib2to3\data\fixers\myfixes\fix_last.py" />
+    <Compile Include="test\test_lib2to3\data\fixers\myfixes\fix_parrot.py" />
+    <Compile Include="test\test_lib2to3\data\fixers\myfixes\fix_preorder.py" />
+    <Compile Include="test\test_lib2to3\data\fixers\myfixes\__init__.py" />
+    <Compile Include="test\test_lib2to3\data\fixers\no_fixer_cls.py" />
+    <Compile Include="test\test_lib2to3\data\fixers\parrot_example.py" />
+    <Compile Include="test\test_lib2to3\data\infinite_recursion.py" />
+    <Compile Include="test\test_lib2to3\data\py2_test_grammar.py" />
+    <Compile Include="test\test_lib2to3\data\py3_test_grammar.py" />
+    <Compile Include="test\test_lib2to3\pytree_idempotency.py" />
+    <Compile Include="test\test_lib2to3\support.py" />
+    <Compile Include="test\test_lib2to3\test_all_fixers.py" />
+    <Compile Include="test\test_lib2to3\test_fixers.py" />
+    <Compile Include="test\test_lib2to3\test_main.py" />
+    <Compile Include="test\test_lib2to3\test_parser.py" />
+    <Compile Include="test\test_lib2to3\test_pytree.py" />
+    <Compile Include="test\test_lib2to3\test_refactor.py" />
+    <Compile Include="test\test_lib2to3\test_util.py" />
+    <Compile Include="test\test_lib2to3\__init__.py" />
+    <Compile Include="test\test_lib2to3\__main__.py" />
     <Compile Include="test\test_linecache.py" />
     <Compile Include="test\test_list.py" />
     <Compile Include="test\test_listcomps.py" />
@@ -1743,10 +1742,6 @@
     <Folder Include="lib2to3" />
     <Folder Include="lib2to3\fixes" />
     <Folder Include="lib2to3\pgen2" />
-    <Folder Include="lib2to3\tests" />
-    <Folder Include="lib2to3\tests\data" />
-    <Folder Include="lib2to3\tests\data\fixers" />
-    <Folder Include="lib2to3\tests\data\fixers\myfixes" />
     <Folder Include="logging" />
     <Folder Include="msilib" />
     <Folder Include="multiprocessing" />
@@ -1802,6 +1797,10 @@
     <Folder Include="test\test_import\data\package" />
     <Folder Include="test\test_import\data\package2" />
     <Folder Include="test\test_json" />
+    <Folder Include="test\test_lib2to3" />
+    <Folder Include="test\test_lib2to3\data" />
+    <Folder Include="test\test_lib2to3\data\fixers" />
+    <Folder Include="test\test_lib2to3\data\fixers\myfixes" />
     <Folder Include="test\test_peg_generator" />
     <Folder Include="test\test_tools" />
     <Folder Include="test\test_unittest" />

Lib/lib2to3/tests/data/false_encoding.py renamed to Lib/test/test_lib2to3/data/false_encoding.py

@@ -88,7 +88,7 @@ time_stdlib: $(CPYTHON) venv
 		-d $(CPYTHON) \
 		$(TESTFLAGS) \
 		--exclude "*/bad*" \
-		--exclude "*/lib2to3/tests/data/*"
+		--exclude "*/test/test_lib2to3/data/*"
 
 mypy: regen-metaparser
 	$(MYPY)  # For list of files, see mypy.ini

Lib/lib2to3/tests/data/fixers/bad_order.py renamed to Lib/test/test_lib2to3/data/fixers/bad_order.py

@@ -78,7 +78,7 @@ def run_benchmark_stdlib(subcommand):
             verbose=False,
             excluded_files=[
                 "*/bad*",
-                "*/lib2to3/tests/data/*",
+                "*/test/test_lib2to3/data/*",
             ],
             short=True,
             mode=modes[subcommand],

Lib/lib2to3/tests/data/fixers/myfixes/__init__.py renamed to Lib/test/test_lib2to3/data/fixers/myfixes/__init__.py

@@ -35,7 +35,6 @@
     '_xxtestfuzz',
     'distutils.tests',
     'idlelib.idle_test',
-    'lib2to3.tests',
     'test',
     'xxlimited',
     'xxlimited_35',