diff --git a/build_with_distutils_stubs.py b/build_with_distutils_stubs.py new file mode 100644 index 0000000000..aafc266d42 --- /dev/null +++ b/build_with_distutils_stubs.py @@ -0,0 +1,74 @@ +"""Generate distutils stub files inside the source directory before packaging. +We have to do this as a custom build backend for PEP 660 editable installs. +Doing it this way also allows us to point local type-checkers to types/distutils, +overriding the stdlib types even on Python < 3.12.""" + +from __future__ import annotations + +import os +import shutil +from pathlib import Path + +from setuptools._path import StrPath +from setuptools.build_meta import * # noqa: F403 # expose everything +from setuptools.build_meta import ( + _ConfigSettings, + build_editable as _build_editable, + build_sdist as _build_sdist, + build_wheel as _build_wheel, +) + +_vendored_distutils_path = Path(__file__).parent / "setuptools" / "_distutils" +_distutils_stubs_path = Path(__file__).parent / "distutils-stubs" + + +def _regenerate_distutils_stubs() -> None: + shutil.rmtree(_distutils_stubs_path, ignore_errors=True) + _distutils_stubs_path.mkdir(parents=True) + (_distutils_stubs_path / ".gitignore").write_text("*") + (_distutils_stubs_path / "ruff.toml").write_text('[lint]\nignore = ["F403"]') + (_distutils_stubs_path / "py.typed").write_text("\n") + for path in _vendored_distutils_path.rglob("*.py"): + relative_path = path.relative_to(_vendored_distutils_path) + if "tests" in relative_path.parts: + continue + stub_path = _distutils_stubs_path / relative_path.with_suffix(".pyi") + stub_path.parent.mkdir(parents=True, exist_ok=True) + module = "setuptools._distutils." + str(relative_path.with_suffix("")).replace( + os.sep, "." + ).removesuffix(".__init__") + if str(relative_path) == "__init__.py": + # Work around python/mypy#18775 + stub_path.write_text("""\ +from typing import Final + +__version__: Final[str] +""") + else: + stub_path.write_text(f"from {module} import *\n") + + +def build_wheel( # type: ignore[no-redef] + wheel_directory: StrPath, + config_settings: _ConfigSettings = None, + metadata_directory: StrPath | None = None, +) -> str: + _regenerate_distutils_stubs() + return _build_wheel(wheel_directory, config_settings, metadata_directory) + + +def build_sdist( # type: ignore[no-redef] + sdist_directory: StrPath, + config_settings: _ConfigSettings = None, +) -> str: + _regenerate_distutils_stubs() + return _build_sdist(sdist_directory, config_settings) + + +def build_editable( # type: ignore[no-redef] + wheel_directory: StrPath, + config_settings: _ConfigSettings = None, + metadata_directory: StrPath | None = None, +) -> str: + _regenerate_distutils_stubs() + return _build_editable(wheel_directory, config_settings, metadata_directory) diff --git a/mypy.ini b/mypy.ini index c1d01a42c3..823c6b47f1 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,7 +5,7 @@ strict = False # Early opt-in even when strict = False -# warn_unused_ignores = True # Disabled until we have distutils stubs for Python 3.12+ +warn_unused_ignores = True warn_redundant_casts = True enable_error_code = ignore-without-code @@ -48,14 +48,6 @@ disable_error_code = [mypy-pkg_resources.tests.*] disable_error_code = import-not-found -# - distutils doesn't exist on Python 3.12, unfortunately, this means typing -# will be missing for subclasses of distutils on Python 3.12 until either: -# - support for `SETUPTOOLS_USE_DISTUTILS=stdlib` is dropped (#3625) -# for setuptools to import `_distutils` directly -# - or non-stdlib distutils typings are exposed -[mypy-distutils.*] -ignore_missing_imports = True - # - wheel: does not intend on exposing a programmatic API https://github.com/pypa/wheel/pull/610#issuecomment-2081687671 [mypy-wheel.*] follow_untyped_imports = True diff --git a/newsfragments/4861.feature.rst b/newsfragments/4861.feature.rst new file mode 100644 index 0000000000..37541b8599 --- /dev/null +++ b/newsfragments/4861.feature.rst @@ -0,0 +1 @@ +``setuptools`` now provide its own ``distutils-stubs`` instead of relying on typeshed -- by :user:`Avasam` diff --git a/pyproject.toml b/pyproject.toml index 3ba37aa59d..e6d1c9837c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] requires = [] -build-backend = "setuptools.build_meta" +build-backend = "build_with_distutils_stubs" backend-path = ["."] [project] @@ -199,6 +199,7 @@ include-package-data = true include = [ "setuptools*", "pkg_resources*", + "distutils-stubs*", "_distutils_hack*", ] exclude = [ diff --git a/pyrightconfig.json b/pyrightconfig.json index da3cd978ce..6123957847 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -12,6 +12,8 @@ ], // Our testing setup doesn't allow passing CLI arguments, so local devs have to set this manually. // "pythonVersion": "3.9", + // Allow using distutils-stubs on Python 3.12+ + "reportMissingModuleSource": false, // For now we don't mind if mypy's `type: ignore` comments accidentally suppresses pyright issues "enableTypeIgnoreComments": true, "typeCheckingMode": "basic", diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 64464dfaa3..befbff939e 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -1,9 +1,4 @@ """Extensions to the 'distutils' for large or complex distributions""" -# mypy: disable_error_code=override -# Command.reinitialize_command has an extra **kw param that distutils doesn't have -# Can't disable on the exact line because distutils doesn't exists on Python 3.12 -# and mypy isn't aware of distutils_hack, causing distutils.core.Command to be Any, -# and a [unused-ignore] to be raised on 3.12+ from __future__ import annotations @@ -224,7 +219,7 @@ def reinitialize_command( ) -> _Command: cmd = _Command.reinitialize_command(self, command, reinit_subcommands) vars(cmd).update(kw) - return cmd # pyright: ignore[reportReturnType] # pypa/distutils#307 + return cmd @abstractmethod def initialize_options(self) -> None: diff --git a/setuptools/_distutils/command/install.py b/setuptools/_distutils/command/install.py index dc17e56a80..50c04f1cb8 100644 --- a/setuptools/_distutils/command/install.py +++ b/setuptools/_distutils/command/install.py @@ -264,12 +264,12 @@ def initialize_options(self) -> None: # supplied by the user, they are filled in using the installation # scheme implied by prefix/exec-prefix/home and the contents of # that installation scheme. - self.install_purelib = None # for pure module distributions - self.install_platlib = None # non-pure (dists w/ extensions) - self.install_headers = None # for C/C++ headers + self.install_purelib: str | None = None # for pure module distributions + self.install_platlib: str | None = None # non-pure (dists w/ extensions) + self.install_headers: str | None = None # for C/C++ headers self.install_lib: str | None = None # set to either purelib or platlib - self.install_scripts = None - self.install_data = None + self.install_scripts: str | None = None + self.install_data: str | None = None self.install_userbase = USER_BASE self.install_usersite = USER_SITE diff --git a/setuptools/_distutils/dist.py b/setuptools/_distutils/dist.py index 37b788df92..f75b691910 100644 --- a/setuptools/_distutils/dist.py +++ b/setuptools/_distutils/dist.py @@ -45,6 +45,7 @@ # type-only import because of mutual dependence between these modules from .cmd import Command + from .extension import Extension _CommandT = TypeVar("_CommandT", bound="Command") _OptionsList: TypeAlias = list[ @@ -220,18 +221,18 @@ def __init__(self, attrs: MutableMapping[str, Any] | None = None) -> None: # no # These options are really the business of various commands, rather # than of the Distribution itself. We provide aliases for them in # Distribution as a convenience to the developer. - self.packages = None + self.packages: list[str] | None = None self.package_data: dict[str, list[str]] = {} - self.package_dir = None - self.py_modules = None + self.package_dir: dict[str, str] | None = None + self.py_modules: list[str] | None = None self.libraries = None self.headers = None - self.ext_modules = None + self.ext_modules: list[Extension] | None = None self.ext_package = None self.include_dirs = None self.extra_path = None self.scripts = None - self.data_files = None + self.data_files: list[str | tuple] | None = None self.password = '' # And now initialize bookkeeping stuff that can't be supplied by diff --git a/setuptools/_path.py b/setuptools/_path.py index 0d99b0f539..5a396709cc 100644 --- a/setuptools/_path.py +++ b/setuptools/_path.py @@ -11,7 +11,9 @@ from typing_extensions import TypeAlias StrPath: TypeAlias = Union[str, os.PathLike[str]] # Same as _typeshed.StrPath -StrPathT = TypeVar("StrPathT", bound=Union[str, os.PathLike[str]]) +StrPathT = TypeVar("StrPathT", bound=StrPath) +BytesPath: TypeAlias = Union[bytes, os.PathLike[bytes]] # Same as _typeshed.BytesPath +BytesPathT = TypeVar("BytesPathT", bound=BytesPath) def ensure_directory(path): diff --git a/setuptools/build_meta.py b/setuptools/build_meta.py index 8f2e930c73..69e6b62cc0 100644 --- a/setuptools/build_meta.py +++ b/setuptools/build_meta.py @@ -88,11 +88,11 @@ def patch(cls): for the duration of this context. """ orig = distutils.core.Distribution - distutils.core.Distribution = cls # type: ignore[misc] # monkeypatching + distutils.core.Distribution = cls try: yield finally: - distutils.core.Distribution = orig # type: ignore[misc] # monkeypatching + distutils.core.Distribution = orig @contextlib.contextmanager diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index 7f66c3ba6a..83f6a804a1 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -132,7 +132,7 @@ def do_install_data(self) -> None: site_packages = os.path.normcase(os.path.realpath(_get_purelib())) old, self.distribution.data_files = self.distribution.data_files, [] - for item in old: + for item in old or (): if isinstance(item, tuple) and len(item) == 2: if os.path.isabs(item[0]): realpath = os.path.realpath(item[0]) diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index be833a379c..8e75fb408d 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -248,7 +248,7 @@ def setup_shlib_compiler(self): compiler.set_link_objects(self.link_objects) # hack so distutils' build_extension() builds a library instead - compiler.link_shared_object = link_shared_object.__get__(compiler) # type: ignore[method-assign] + compiler.link_shared_object = link_shared_object.__get__(compiler) def get_export_symbols(self, ext): if isinstance(ext, Library): diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 2f6fcb7cdc..76b65b95cc 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -9,10 +9,11 @@ from functools import partial from glob import glob from pathlib import Path +from typing import Literal, overload from more_itertools import unique_everseen -from .._path import StrPath, StrPathT +from .._path import BytesPath, BytesPathT, StrPath, StrPathT from ..dist import Distribution from ..warnings import SetuptoolsDeprecationWarning @@ -48,21 +49,50 @@ def finalize_options(self): if 'data_files' in self.__dict__: del self.__dict__['data_files'] - def copy_file( # type: ignore[override] # No overload, no bytes support + @overload # type: ignore[override] # Truthy link with bytes is not supported, unlike supertype + def copy_file( self, infile: StrPath, outfile: StrPathT, preserve_mode: bool = True, preserve_times: bool = True, link: str | None = None, - level: object = 1, - ) -> tuple[StrPathT | str, bool]: + level: int = 1, + ) -> tuple[StrPathT | str, bool]: ... + @overload + def copy_file( + self, + infile: BytesPath, + outfile: BytesPathT, + preserve_mode: bool = True, + preserve_times: bool = True, + link: Literal[""] | None = None, + level: int = 1, + ) -> tuple[BytesPathT | bytes, bool]: ... + def copy_file( + self, + infile: StrPath | BytesPath, + outfile: StrPath | BytesPath, + preserve_mode: bool = True, + preserve_times: bool = True, + link: str | None = None, + level: int = 1, + ) -> tuple[StrPath | BytesPath, bool]: # Overwrite base class to allow using links if link: - infile = str(Path(infile).resolve()) - outfile = str(Path(outfile).resolve()) # type: ignore[assignment] # Re-assigning a str when outfile is StrPath is ok - return super().copy_file( # pyright: ignore[reportReturnType] # pypa/distutils#309 - infile, outfile, preserve_mode, preserve_times, link, level + # NOTE: Explanation for the type ignores: + # 1. If link is truthy, then we only allow infile and outfile to be StrPath + # 2. Re-assigning a str when outfile is StrPath is ok + # We can't easily check for PathLike[str], so ignoring instead of asserting. + infile = str(Path(infile).resolve()) # type: ignore[arg-type] + outfile = str(Path(outfile).resolve()) # type: ignore[arg-type] + return super().copy_file( # type: ignore[misc, type-var] # pyright: ignore[reportCallIssue] + infile, # type: ignore[arg-type] # pyright: ignore[reportArgumentType] + outfile, # pyright: ignore[reportArgumentType] + preserve_mode, + preserve_times, + link, + level, ) def run(self) -> None: @@ -135,7 +165,7 @@ def find_data_files(self, package, src_dir): ) return self.exclude_data_files(package, src_dir, files) - def get_outputs(self, include_bytecode: bool = True) -> list[str]: # type: ignore[override] # Using a real boolean instead of 0|1 + def get_outputs(self, include_bytecode: bool = True) -> list[str]: """See :class:`setuptools.commands.build.SubCommand`""" if self.editable_mode: return list(self.get_output_mapping().keys()) diff --git a/setuptools/command/install_lib.py b/setuptools/command/install_lib.py index 8e1e072710..ef76286d4f 100644 --- a/setuptools/command/install_lib.py +++ b/setuptools/command/install_lib.py @@ -95,10 +95,9 @@ def copy_tree( self, infile: StrPath, outfile: str, - # override: Using actual booleans - preserve_mode: bool = True, # type: ignore[override] - preserve_times: bool = True, # type: ignore[override] - preserve_symlinks: bool = False, # type: ignore[override] + preserve_mode: bool = True, + preserve_times: bool = True, + preserve_symlinks: bool = False, level: object = 1, ) -> list[str]: assert preserve_mode diff --git a/setuptools/command/setopt.py b/setuptools/command/setopt.py index 678a0593d6..a9be2418e0 100644 --- a/setuptools/command/setopt.py +++ b/setuptools/command/setopt.py @@ -37,7 +37,7 @@ def edit_config(filename, settings, dry_run=False): """ log.debug("Reading configuration from %s", filename) opts = configparser.RawConfigParser() - opts.optionxform = lambda optionstr: optionstr # type: ignore[method-assign] # overriding method + opts.optionxform = lambda optionstr: optionstr _cfg_read_utf8_with_fallback(opts, filename) for section, options in settings.items(): diff --git a/setuptools/config/setupcfg.py b/setuptools/config/setupcfg.py index 633aa9d45d..16db3c1b9d 100644 --- a/setuptools/config/setupcfg.py +++ b/setuptools/config/setupcfg.py @@ -17,7 +17,7 @@ from collections import defaultdict from collections.abc import Iterable, Iterator from functools import partial, wraps -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Generic, TypeVar, cast +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Generic, TypeVar from packaging.markers import default_environment as marker_env from packaging.requirements import InvalidRequirement, Requirement @@ -101,8 +101,7 @@ def _apply( filenames = [*other_files, filepath] try: - # TODO: Temporary cast until mypy 1.12 is released with upstream fixes from typeshed - _Distribution.parse_config_files(dist, filenames=cast(list[str], filenames)) + _Distribution.parse_config_files(dist, filenames=filenames) handlers = parse_configuration( dist, dist.command_options, ignore_option_errors=ignore_option_errors ) diff --git a/setuptools/dist.py b/setuptools/dist.py index 8d972cc49b..8593f31675 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -48,6 +48,8 @@ from pkg_resources import Distribution as _pkg_resources_Distribution + from .extension import Extension + __all__ = ['Distribution'] @@ -301,6 +303,9 @@ class Distribution(_Distribution): # Used by build_py, editable_wheel and install_lib commands for legacy namespaces namespace_packages: list[str] #: :meta private: DEPRECATED + # override distutils.extension.Extension with setuptools.extension.Extension + ext_modules: list[Extension] | None # type: ignore[assignment] + # Any: Dynamic assignment results in Incompatible types in assignment def __init__(self, attrs: MutableMapping[str, Any] | None = None) -> None: have_package_data = hasattr(self, "package_data") @@ -832,7 +837,7 @@ def fetch_build_egg(self, req): return fetch_build_egg(self, req) - def get_command_class(self, command: str) -> type[distutils.cmd.Command]: # type: ignore[override] # Not doing complex overrides yet + def get_command_class(self, command: str) -> type[distutils.cmd.Command]: """Pluggable version of get_command_class()""" if command in self.cmdclass: return self.cmdclass[command] diff --git a/setuptools/errors.py b/setuptools/errors.py index 990ecbf4e2..fa580085ac 100644 --- a/setuptools/errors.py +++ b/setuptools/errors.py @@ -3,6 +3,8 @@ Provides exceptions used by setuptools modules. """ +# Odd mypy issue with this specific import, alias and base classes on 3.12+ +# mypy: disable-error-code="valid-type,misc" from __future__ import annotations from distutils import errors as _distutils_errors @@ -30,15 +32,15 @@ BaseError = _distutils_errors.DistutilsError -class InvalidConfigError(OptionError): # type: ignore[valid-type, misc] # distutils imports are `Any` on python 3.12+ +class InvalidConfigError(OptionError): """Error used for invalid configurations.""" -class RemovedConfigError(OptionError): # type: ignore[valid-type, misc] # distutils imports are `Any` on python 3.12+ +class RemovedConfigError(OptionError): """Error used for configurations that were deprecated and removed.""" -class RemovedCommandError(BaseError, RuntimeError): # type: ignore[valid-type, misc] # distutils imports are `Any` on python 3.12+ +class RemovedCommandError(BaseError, RuntimeError): """Error used for commands that have been removed in setuptools. Since ``setuptools`` is built on ``distutils``, simply removing a command @@ -48,7 +50,7 @@ class RemovedCommandError(BaseError, RuntimeError): # type: ignore[valid-type, """ -class PackageDiscoveryError(BaseError, RuntimeError): # type: ignore[valid-type, misc] # distutils imports are `Any` on python 3.12+ +class PackageDiscoveryError(BaseError, RuntimeError): """Impossible to perform automatic discovery of packages and/or modules. The current project layout or given discovery options can lead to problems when diff --git a/setuptools/extension.py b/setuptools/extension.py index 76e03d9d6b..e6acf93c31 100644 --- a/setuptools/extension.py +++ b/setuptools/extension.py @@ -151,12 +151,7 @@ def __init__( # The *args is needed for compatibility as calls may use positional # arguments. py_limited_api may be set only via keyword. self.py_limited_api = py_limited_api - super().__init__( - name, - sources, # type: ignore[arg-type] # Vendored version of setuptools supports PathLike - *args, - **kw, - ) + super().__init__(name, sources, *args, **kw) def _convert_pyx_sources_to_lang(self): """ diff --git a/setuptools/logging.py b/setuptools/logging.py index 532da899f7..7601ded6e8 100644 --- a/setuptools/logging.py +++ b/setuptools/logging.py @@ -32,7 +32,7 @@ def configure() -> None: # and then loaded again when patched, # implying: id(distutils.log) != id(distutils.dist.log). # Make sure the same module object is used everywhere: - distutils.dist.log = distutils.log + distutils.dist.log = distutils.log # type: ignore[assignment] def set_threshold(level: int) -> int: diff --git a/setuptools/monkey.py b/setuptools/monkey.py index 6ad1abac29..177ec05e66 100644 --- a/setuptools/monkey.py +++ b/setuptools/monkey.py @@ -73,7 +73,7 @@ def patch_all(): import setuptools # we can't patch distutils.cmd, alas - distutils.core.Command = setuptools.Command # type: ignore[misc,assignment] # monkeypatching + distutils.core.Command = setuptools.Command _patch_distribution_metadata() @@ -82,8 +82,8 @@ def patch_all(): module.Distribution = setuptools.dist.Distribution # Install the patched Extension - distutils.core.Extension = setuptools.extension.Extension # type: ignore[misc,assignment] # monkeypatching - distutils.extension.Extension = setuptools.extension.Extension # type: ignore[misc,assignment] # monkeypatching + distutils.core.Extension = setuptools.extension.Extension + distutils.extension.Extension = setuptools.extension.Extension if 'distutils.command.build_ext' in sys.modules: sys.modules[ 'distutils.command.build_ext' diff --git a/setuptools/tests/config/test_apply_pyprojecttoml.py b/setuptools/tests/config/test_apply_pyprojecttoml.py index 489fd98e26..365530a349 100644 --- a/setuptools/tests/config/test_apply_pyprojecttoml.py +++ b/setuptools/tests/config/test_apply_pyprojecttoml.py @@ -548,6 +548,7 @@ def test_pyproject_sets_attribute(self, tmp_path, monkeypatch): pyproject.write_text(cleandoc(toml_config), encoding="utf-8") with pytest.warns(pyprojecttoml._ExperimentalConfiguration): dist = pyprojecttoml.apply_configuration(Distribution({}), pyproject) + assert dist.ext_modules assert len(dist.ext_modules) == 1 assert dist.ext_modules[0].name == "my.ext" assert set(dist.ext_modules[0].sources) == {"hello.c", "world.c"} diff --git a/setuptools/tests/test_config_discovery.py b/setuptools/tests/test_config_discovery.py index b5df8203cd..18d7bd57ea 100644 --- a/setuptools/tests/test_config_discovery.py +++ b/setuptools/tests/test_config_discovery.py @@ -389,6 +389,7 @@ def test_skip_discovery_with_setupcfg_metadata(self, tmp_path): assert dist.get_version() == "42" assert dist.py_modules is None assert dist.packages is None + assert dist.ext_modules assert len(dist.ext_modules) == 1 assert dist.ext_modules[0].name == "proj"