Re-introduce the 'stubtest' key for third-party packages (#7351)

Some distributions can't be tested with stubtest for a variety of
reasons or because of bugs in stubtest. This key:

* let's us keep metadata about a distribution in one place,
* prevents us from modifying the scripts because of issues with a
  specific distribution, and
* will trigger tests if only the key is changed.

Author: srittau

CONTRIBUTING.md

@@ -176,6 +176,9 @@ supported:
   [removing obsolete third-party libraries](#third-party-library-removal-policy).
   It contains the first version of the corresponding library that ships
   its own `py.typed` file.
+* `stubtest` (default: `true`): Whether stubtest should be run against this
+  package. Please avoid setting this to `false`, and add a comment if you have
+  to.
 * `stubtest_apt_dependencies` (default: `[]`): A list of Ubuntu APT packages
   that need to be installed for stubtest to run successfully. These are
   usually packages needed to pip install the implementation distribution.

stubs/SQLAlchemy/METADATA.toml

@@ -3,3 +3,4 @@ extra_description = """\
   The `sqlalchemy-stubs` package is an alternative to this package and also \
   includes a mypy plugin for more precise types.\
 """
+stubtest = false  # https://github.com/python/typeshed/issues/7307

tests/check_consistent.py

@@ -18,7 +18,7 @@
 import tomli
 
 consistent_files = [{"stdlib/@python2/builtins.pyi", "stdlib/@python2/__builtin__.pyi"}]
-metadata_keys = {"version", "python2", "requires", "extra_description", "obsolete_since", "stubtest_apt_dependencies"}
+metadata_keys = {"version", "python2", "requires", "extra_description", "obsolete_since", "stubtest", "stubtest_apt_dependencies"}
 allowed_files = {"README.md"}
 
 

tests/stubtest_third_party.py

@@ -11,7 +11,7 @@
 import venv
 from glob import glob
 from pathlib import Path
-from typing import NoReturn
+from typing import Any, NoReturn
 
 import tomli
 
@@ -32,7 +32,7 @@ def run_stubtest(dist: Path) -> bool:
     with open(dist / "METADATA.toml") as f:
         metadata = dict(tomli.loads(f.read()))
 
-    if not has_py3_stubs(dist):
+    if not run_stubtest_for(metadata, dist):
         print(f"Skipping stubtest for {dist.name}\n\n")
         return True
 
@@ -115,6 +115,10 @@ def run_stubtest(dist: Path) -> bool:
     return True
 
 
+def run_stubtest_for(metadata: dict[str, Any], dist: Path) -> bool:
+    return has_py3_stubs(dist) and metadata.get("stubtest", True)
+
+
 # Keep this in sync with mypy_test.py
 def has_py3_stubs(dist: Path) -> bool:
     return len(glob(f"{dist}/*.pyi")) > 0 or len(glob(f"{dist}/[!@]*/__init__.pyi")) > 0
@@ -137,9 +141,6 @@ def main() -> NoReturn:
     for i, dist in enumerate(dists):
         if i % args.num_shards != args.shard_index:
             continue
-        if dist.name == "SQLAlchemy":
-            # See https://github.com/python/typeshed/issues/7307
-            continue
         if not run_stubtest(dist):
             result = 1
     sys.exit(result)