Generate distutils-stubs on install

build_with_distutils_stubs.py

@@ -0,0 +1,74 @@
+"""Generate distutils stub files inside the source directory before packaging.
+We have to do this as a custom build backend for PEP 660 editable installs.
+Doing it this way also allows us to point local type-checkers to types/distutils,
+overriding the stdlib types even on Python < 3.12."""
+
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+
+from setuptools._path import StrPath
+from setuptools.build_meta import *  # noqa: F403 # expose everything
+from setuptools.build_meta import (
+    _ConfigSettings,
+    build_editable as _build_editable,
+    build_sdist as _build_sdist,
+    build_wheel as _build_wheel,
+)
+
+_vendored_distutils_path = Path(__file__).parent / "setuptools" / "_distutils"
+_distutils_stubs_path = Path(__file__).parent / "distutils-stubs"
+
+
+def _regenerate_distutils_stubs() -> None:
+    shutil.rmtree(_distutils_stubs_path, ignore_errors=True)
+    _distutils_stubs_path.mkdir(parents=True)
+    (_distutils_stubs_path / ".gitignore").write_text("*")
+    (_distutils_stubs_path / "ruff.toml").write_text('[lint]\nignore = ["F403"]')
+    (_distutils_stubs_path / "py.typed").write_text("\n")
+    for path in _vendored_distutils_path.rglob("*.py"):
+        relative_path = path.relative_to(_vendored_distutils_path)
+        if "tests" in relative_path.parts:
+            continue
+        stub_path = _distutils_stubs_path / relative_path.with_suffix(".pyi")
+        stub_path.parent.mkdir(parents=True, exist_ok=True)
+        module = "setuptools._distutils." + str(relative_path.with_suffix("")).replace(
+            os.sep, "."
+        ).removesuffix(".__init__")
+        if str(relative_path) == "__init__.py":
+            # Work around python/mypy#18775
+            stub_path.write_text("""\
+from typing import Final
+
+__version__: Final[str]
+""")
+        else:
+            stub_path.write_text(f"from {module} import *\n")
+
+
+def build_wheel(  # type: ignore[no-redef]
+    wheel_directory: StrPath,
+    config_settings: _ConfigSettings = None,
+    metadata_directory: StrPath | None = None,
+) -> str:
+    _regenerate_distutils_stubs()
+    return _build_wheel(wheel_directory, config_settings, metadata_directory)
+
+
+def build_sdist(  # type: ignore[no-redef]
+    sdist_directory: StrPath,
+    config_settings: _ConfigSettings = None,
+) -> str:
+    _regenerate_distutils_stubs()
+    return _build_sdist(sdist_directory, config_settings)
+
+
+def build_editable(  # type: ignore[no-redef]
+    wheel_directory: StrPath,
+    config_settings: _ConfigSettings = None,
+    metadata_directory: StrPath | None = None,
+) -> str:
+    _regenerate_distutils_stubs()
+    return _build_editable(wheel_directory, config_settings, metadata_directory)

mypy.ini

@@ -5,7 +5,7 @@
 strict = False
 
 # Early opt-in even when strict = False
-# warn_unused_ignores = True # Disabled until we have distutils stubs for Python 3.12+
+warn_unused_ignores = True
 warn_redundant_casts = True
 enable_error_code = ignore-without-code
 
@@ -48,14 +48,6 @@ disable_error_code =
 [mypy-pkg_resources.tests.*]
 disable_error_code = import-not-found
 
-# - distutils doesn't exist on Python 3.12, unfortunately, this means typing
-#   will be missing for subclasses of distutils on Python 3.12 until either:
-#     - support for `SETUPTOOLS_USE_DISTUTILS=stdlib` is dropped (#3625)
-#       for setuptools to import `_distutils` directly
-#     - or non-stdlib distutils typings are exposed
-[mypy-distutils.*]
-ignore_missing_imports = True
-
 #  - wheel: does not intend on exposing a programmatic API https://github.com/pypa/wheel/pull/610#issuecomment-2081687671
 [mypy-wheel.*]
 follow_untyped_imports = True

newsfragments/4861.feature.rst

@@ -0,0 +1 @@
+``setuptools`` now provide its own ``distutils-stubs`` instead of relying on typeshed -- by :user:`Avasam`

pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 requires = []
-build-backend = "setuptools.build_meta"
+build-backend = "build_with_distutils_stubs"
 backend-path = ["."]
 
 [project]
@@ -199,6 +199,7 @@ include-package-data = true
 include = [
 	"setuptools*",
 	"pkg_resources*",
+	"distutils-stubs*",
 	"_distutils_hack*",
 ]
 exclude = [

pyrightconfig.json

@@ -12,6 +12,8 @@
 	],
 	// Our testing setup doesn't allow passing CLI arguments, so local devs have to set this manually.
 	// "pythonVersion": "3.9",
+	// Allow using distutils-stubs on Python 3.12+
+	"reportMissingModuleSource": false,
 	// For now we don't mind if mypy's `type: ignore` comments accidentally suppresses pyright issues
 	"enableTypeIgnoreComments": true,
 	"typeCheckingMode": "basic",

setuptools/__init__.py

@@ -1,9 +1,4 @@
 """Extensions to the 'distutils' for large or complex distributions"""
-# mypy: disable_error_code=override
-# Command.reinitialize_command has an extra **kw param that distutils doesn't have
-# Can't disable on the exact line because distutils doesn't exists on Python 3.12
-# and mypy isn't aware of distutils_hack, causing distutils.core.Command to be Any,
-# and a [unused-ignore] to be raised on 3.12+
 
 from __future__ import annotations
 
@@ -224,7 +219,7 @@ def reinitialize_command(
     ) -> _Command:
         cmd = _Command.reinitialize_command(self, command, reinit_subcommands)
         vars(cmd).update(kw)
-        return cmd  # pyright: ignore[reportReturnType] # pypa/distutils#307
+        return cmd
 
     @abstractmethod
     def initialize_options(self) -> None:

setuptools/_distutils/command/install.py

@@ -264,12 +264,12 @@ def initialize_options(self) -> None:
         # supplied by the user, they are filled in using the installation
         # scheme implied by prefix/exec-prefix/home and the contents of
         # that installation scheme.
-        self.install_purelib = None  # for pure module distributions
-        self.install_platlib = None  # non-pure (dists w/ extensions)
-        self.install_headers = None  # for C/C++ headers
+        self.install_purelib: str | None = None  # for pure module distributions
+        self.install_platlib: str | None = None  # non-pure (dists w/ extensions)
+        self.install_headers: str | None = None  # for C/C++ headers
         self.install_lib: str | None = None  # set to either purelib or platlib
-        self.install_scripts = None
-        self.install_data = None
+        self.install_scripts: str | None = None
+        self.install_data: str | None = None
         self.install_userbase = USER_BASE
         self.install_usersite = USER_SITE
 

setuptools/_distutils/dist.py

@@ -45,6 +45,7 @@
 
     # type-only import because of mutual dependence between these modules
     from .cmd import Command
+    from .extension import Extension
 
 _CommandT = TypeVar("_CommandT", bound="Command")
 _OptionsList: TypeAlias = list[
@@ -220,18 +221,18 @@ def __init__(self, attrs: MutableMapping[str, Any] | None = None) -> None:  # no
         # These options are really the business of various commands, rather
         # than of the Distribution itself.  We provide aliases for them in
         # Distribution as a convenience to the developer.
-        self.packages = None
+        self.packages: list[str] | None = None
         self.package_data: dict[str, list[str]] = {}
-        self.package_dir = None
-        self.py_modules = None
+        self.package_dir: dict[str, str] | None = None
+        self.py_modules: list[str] | None = None
         self.libraries = None
         self.headers = None
-        self.ext_modules = None
+        self.ext_modules: list[Extension] | None = None
         self.ext_package = None
         self.include_dirs = None
         self.extra_path = None
         self.scripts = None
-        self.data_files = None
+        self.data_files: list[str | tuple] | None = None
         self.password = ''
 
         # And now initialize bookkeeping stuff that can't be supplied by

setuptools/_path.py

@@ -11,7 +11,9 @@
     from typing_extensions import TypeAlias
 
 StrPath: TypeAlias = Union[str, os.PathLike[str]]  #  Same as _typeshed.StrPath
-StrPathT = TypeVar("StrPathT", bound=Union[str, os.PathLike[str]])
+StrPathT = TypeVar("StrPathT", bound=StrPath)
+BytesPath: TypeAlias = Union[bytes, os.PathLike[bytes]]  #  Same as _typeshed.BytesPath
+BytesPathT = TypeVar("BytesPathT", bound=BytesPath)
 
 
 def ensure_directory(path):

setuptools/build_meta.py

@@ -88,11 +88,11 @@ def patch(cls):
         for the duration of this context.
         """
         orig = distutils.core.Distribution
-        distutils.core.Distribution = cls  # type: ignore[misc] # monkeypatching
+        distutils.core.Distribution = cls
         try:
             yield
         finally:
-            distutils.core.Distribution = orig  # type: ignore[misc] # monkeypatching
+            distutils.core.Distribution = orig
 
 
 @contextlib.contextmanager

setuptools/command/bdist_egg.py

@@ -132,7 +132,7 @@ def do_install_data(self) -> None:
         site_packages = os.path.normcase(os.path.realpath(_get_purelib()))
         old, self.distribution.data_files = self.distribution.data_files, []
 
-        for item in old:
+        for item in old or ():
             if isinstance(item, tuple) and len(item) == 2:
                 if os.path.isabs(item[0]):
                     realpath = os.path.realpath(item[0])

setuptools/command/build_ext.py

@@ -248,7 +248,7 @@ def setup_shlib_compiler(self):
             compiler.set_link_objects(self.link_objects)
 
         # hack so distutils' build_extension() builds a library instead
-        compiler.link_shared_object = link_shared_object.__get__(compiler)  # type: ignore[method-assign]
+        compiler.link_shared_object = link_shared_object.__get__(compiler)
 
     def get_export_symbols(self, ext):
         if isinstance(ext, Library):

setuptools/command/build_py.py

@@ -9,10 +9,11 @@
 from functools import partial
 from glob import glob
 from pathlib import Path
+from typing import Literal, overload
 
 from more_itertools import unique_everseen
 
-from .._path import StrPath, StrPathT
+from .._path import BytesPath, BytesPathT, StrPath, StrPathT
 from ..dist import Distribution
 from ..warnings import SetuptoolsDeprecationWarning
 
@@ -48,21 +49,50 @@ def finalize_options(self):
         if 'data_files' in self.__dict__:
             del self.__dict__['data_files']
 
-    def copy_file(  # type: ignore[override] # No overload, no bytes support
+    @overload  # type: ignore[override] # Truthy link with bytes is not supported, unlike supertype
+    def copy_file(
         self,
         infile: StrPath,
         outfile: StrPathT,
         preserve_mode: bool = True,
         preserve_times: bool = True,
         link: str | None = None,
-        level: object = 1,
-    ) -> tuple[StrPathT | str, bool]:
+        level: int = 1,
+    ) -> tuple[StrPathT | str, bool]: ...
+    @overload
+    def copy_file(
+        self,
+        infile: BytesPath,
+        outfile: BytesPathT,
+        preserve_mode: bool = True,
+        preserve_times: bool = True,
+        link: Literal[""] | None = None,
+        level: int = 1,
+    ) -> tuple[BytesPathT | bytes, bool]: ...
+    def copy_file(
+        self,
+        infile: StrPath | BytesPath,
+        outfile: StrPath | BytesPath,
+        preserve_mode: bool = True,
+        preserve_times: bool = True,
+        link: str | None = None,
+        level: int = 1,
+    ) -> tuple[StrPath | BytesPath, bool]:
         # Overwrite base class to allow using links
         if link:
-            infile = str(Path(infile).resolve())
-            outfile = str(Path(outfile).resolve())  # type: ignore[assignment] # Re-assigning a str when outfile is StrPath is ok
-        return super().copy_file(  # pyright: ignore[reportReturnType] # pypa/distutils#309
-            infile, outfile, preserve_mode, preserve_times, link, level
+            # NOTE: Explanation for the type ignores:
+            # 1. If link is truthy, then we only allow infile and outfile to be StrPath
+            # 2. Re-assigning a str when outfile is StrPath is ok
+            # We can't easily check for PathLike[str], so ignoring instead of asserting.
+            infile = str(Path(infile).resolve())  # type: ignore[arg-type]
+            outfile = str(Path(outfile).resolve())  # type: ignore[arg-type]
+        return super().copy_file(  # type: ignore[misc, type-var] # pyright: ignore[reportCallIssue]
+            infile,  # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
+            outfile,  # pyright: ignore[reportArgumentType]
+            preserve_mode,
+            preserve_times,
+            link,
+            level,
         )
 
     def run(self) -> None:
@@ -135,7 +165,7 @@ def find_data_files(self, package, src_dir):
         )
         return self.exclude_data_files(package, src_dir, files)
 
-    def get_outputs(self, include_bytecode: bool = True) -> list[str]:  # type: ignore[override] # Using a real boolean instead of 0|1
+    def get_outputs(self, include_bytecode: bool = True) -> list[str]:
         """See :class:`setuptools.commands.build.SubCommand`"""
         if self.editable_mode:
             return list(self.get_output_mapping().keys())

setuptools/command/install_lib.py

@@ -95,10 +95,9 @@ def copy_tree(
         self,
         infile: StrPath,
         outfile: str,
-        # override: Using actual booleans
-        preserve_mode: bool = True,  # type: ignore[override]
-        preserve_times: bool = True,  # type: ignore[override]
-        preserve_symlinks: bool = False,  # type: ignore[override]
+        preserve_mode: bool = True,
+        preserve_times: bool = True,
+        preserve_symlinks: bool = False,
         level: object = 1,
     ) -> list[str]:
         assert preserve_mode

setuptools/command/setopt.py

@@ -37,7 +37,7 @@ def edit_config(filename, settings, dry_run=False):
     """
     log.debug("Reading configuration from %s", filename)
     opts = configparser.RawConfigParser()
-    opts.optionxform = lambda optionstr: optionstr  # type: ignore[method-assign] # overriding method
+    opts.optionxform = lambda optionstr: optionstr
     _cfg_read_utf8_with_fallback(opts, filename)
 
     for section, options in settings.items():

setuptools/config/setupcfg.py

@@ -17,7 +17,7 @@
 from collections import defaultdict
 from collections.abc import Iterable, Iterator
 from functools import partial, wraps
-from typing import TYPE_CHECKING, Any, Callable, ClassVar, Generic, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Callable, ClassVar, Generic, TypeVar
 
 from packaging.markers import default_environment as marker_env
 from packaging.requirements import InvalidRequirement, Requirement
@@ -101,8 +101,7 @@ def _apply(
     filenames = [*other_files, filepath]
 
     try:
-        # TODO: Temporary cast until mypy 1.12 is released with upstream fixes from typeshed
-        _Distribution.parse_config_files(dist, filenames=cast(list[str], filenames))
+        _Distribution.parse_config_files(dist, filenames=filenames)
         handlers = parse_configuration(
             dist, dist.command_options, ignore_option_errors=ignore_option_errors
         )

setuptools/dist.py

@@ -48,6 +48,8 @@
 
     from pkg_resources import Distribution as _pkg_resources_Distribution
 
+    from .extension import Extension
+
 
 __all__ = ['Distribution']
 
@@ -301,6 +303,9 @@ class Distribution(_Distribution):
     # Used by build_py, editable_wheel and install_lib commands for legacy namespaces
     namespace_packages: list[str]  #: :meta private: DEPRECATED
 
+    # override distutils.extension.Extension with setuptools.extension.Extension
+    ext_modules: list[Extension] | None  # type: ignore[assignment]
+
     # Any: Dynamic assignment results in Incompatible types in assignment
     def __init__(self, attrs: MutableMapping[str, Any] | None = None) -> None:
         have_package_data = hasattr(self, "package_data")
@@ -832,7 +837,7 @@ def fetch_build_egg(self, req):
 
         return fetch_build_egg(self, req)
 
-    def get_command_class(self, command: str) -> type[distutils.cmd.Command]:  # type: ignore[override] # Not doing complex overrides yet
+    def get_command_class(self, command: str) -> type[distutils.cmd.Command]:
         """Pluggable version of get_command_class()"""
         if command in self.cmdclass:
             return self.cmdclass[command]