venv/lib/python3.11/site-packages/setuptools/discovery.py

   1 """Automatic discovery of Python modules and packages (for inclusion in the
   2 distribution) and other config values.
   3
   4 For the purposes of this module, the following nomenclature is used:
   5
   6 - "src-layout": a directory representing a Python project that contains a "src"
   7   folder. Everything under the "src" folder is meant to be included in the
   8   distribution when packaging the project. Example::
   9
  10     .
  11     ├── tox.ini
  12     ├── pyproject.toml
  13     └── src/
  14         └── mypkg/
  15             ├── __init__.py
  16             ├── mymodule.py
  17             └── my_data_file.txt
  18
  19 - "flat-layout": a Python project that does not use "src-layout" but instead
  20   have a directory under the project root for each package::
  21
  22     .
  23     ├── tox.ini
  24     ├── pyproject.toml
  25     └── mypkg/
  26         ├── __init__.py
  27         ├── mymodule.py
  28         └── my_data_file.txt
  29
  30 - "single-module": a project that contains a single Python script direct under
  31   the project root (no directory used)::
  32
  33     .
  34     ├── tox.ini
  35     ├── pyproject.toml
  36     └── mymodule.py
  37
  38 """
  39
  40 import itertools
  41 import os
  42 from fnmatch import fnmatchcase
  43 from glob import glob
  44 from pathlib import Path
  45 from typing import (
  46     TYPE_CHECKING,
  47     Dict,
  48     Iterable,
  49     Iterator,
  50     List,
  51     Mapping,
  52     Optional,
  53     Tuple,
  54     Union,
  55 )
  56
  57 import _distutils_hack.override  # noqa: F401
  58
  59 from distutils import log
  60 from distutils.util import convert_path
  61
  62 _Path = Union[str, os.PathLike]
  63 StrIter = Iterator[str]
  64
  65 chain_iter = itertools.chain.from_iterable
  66
  67 if TYPE_CHECKING:
  68     from setuptools import Distribution  # noqa
  69
  70
  71 def _valid_name(path: _Path) -> bool:
  72     # Ignore invalid names that cannot be imported directly
  73     return os.path.basename(path).isidentifier()
  74
  75
  76 class _Filter:
  77     """
  78     Given a list of patterns, create a callable that will be true only if
  79     the input matches at least one of the patterns.
  80     """
  81
  82     def __init__(self, *patterns: str):
  83         self._patterns = dict.fromkeys(patterns)
  84
  85     def __call__(self, item: str) -> bool:
  86         return any(fnmatchcase(item, pat) for pat in self._patterns)
  87
  88     def __contains__(self, item: str) -> bool:
  89         return item in self._patterns
  90
  91
  92 class _Finder:
  93     """Base class that exposes functionality for module/package finders"""
  94
  95     ALWAYS_EXCLUDE: Tuple[str, ...] = ()
  96     DEFAULT_EXCLUDE: Tuple[str, ...] = ()
  97
  98     @classmethod
  99     def find(
 100         cls,
 101         where: _Path = '.',
 102         exclude: Iterable[str] = (),
 103         include: Iterable[str] = ('*',),
 104     ) -> List[str]:
 105         """Return a list of all Python items (packages or modules, depending on
 106         the finder implementation) found within directory 'where'.
 107
 108         'where' is the root directory which will be searched.
 109         It should be supplied as a "cross-platform" (i.e. URL-style) path;
 110         it will be converted to the appropriate local path syntax.
 111
 112         'exclude' is a sequence of names to exclude; '*' can be used
 113         as a wildcard in the names.
 114         When finding packages, 'foo.*' will exclude all subpackages of 'foo'
 115         (but not 'foo' itself).
 116
 117         'include' is a sequence of names to include.
 118         If it's specified, only the named items will be included.
 119         If it's not specified, all found items will be included.
 120         'include' can contain shell style wildcard patterns just like
 121         'exclude'.
 122         """
 123
 124         exclude = exclude or cls.DEFAULT_EXCLUDE
 125         return list(
 126             cls._find_iter(
 127                 convert_path(str(where)),
 128                 _Filter(*cls.ALWAYS_EXCLUDE, *exclude),
 129                 _Filter(*include),
 130             )
 131         )
 132
 133     @classmethod
 134     def _find_iter(cls, where: _Path, exclude: _Filter, include: _Filter) -> StrIter:
 135         raise NotImplementedError
 136
 137
 138 class PackageFinder(_Finder):
 139     """
 140     Generate a list of all Python packages found within a directory
 141     """
 142
 143     ALWAYS_EXCLUDE = ("ez_setup", "*__pycache__")
 144
 145     @classmethod
 146     def _find_iter(cls, where: _Path, exclude: _Filter, include: _Filter) -> StrIter:
 147         """
 148         All the packages found in 'where' that pass the 'include' filter, but
 149         not the 'exclude' filter.
 150         """
 151         for root, dirs, files in os.walk(str(where), followlinks=True):
 152             # Copy dirs to iterate over it, then empty dirs.
 153             all_dirs = dirs[:]
 154             dirs[:] = []
 155
 156             for dir in all_dirs:
 157                 full_path = os.path.join(root, dir)
 158                 rel_path = os.path.relpath(full_path, where)
 159                 package = rel_path.replace(os.path.sep, '.')
 160
 161                 # Skip directory trees that are not valid packages
 162                 if '.' in dir or not cls._looks_like_package(full_path, package):
 163                     continue
 164
 165                 # Should this package be included?
 166                 if include(package) and not exclude(package):
 167                     yield package
 168
 169                 # Early pruning if there is nothing else to be scanned
 170                 if f"{package}*" in exclude or f"{package}.*" in exclude:
 171                     continue
 172
 173                 # Keep searching subdirectories, as there may be more packages
 174                 # down there, even if the parent was excluded.
 175                 dirs.append(dir)
 176
 177     @staticmethod
 178     def _looks_like_package(path: _Path, _package_name: str) -> bool:
 179         """Does a directory look like a package?"""
 180         return os.path.isfile(os.path.join(path, '__init__.py'))
 181
 182
 183 class PEP420PackageFinder(PackageFinder):
 184     @staticmethod
 185     def _looks_like_package(_path: _Path, _package_name: str) -> bool:
 186         return True
 187
 188
 189 class ModuleFinder(_Finder):
 190     """Find isolated Python modules.
 191     This function will **not** recurse subdirectories.
 192     """
 193
 194     @classmethod
 195     def _find_iter(cls, where: _Path, exclude: _Filter, include: _Filter) -> StrIter:
 196         for file in glob(os.path.join(where, "*.py")):
 197             module, _ext = os.path.splitext(os.path.basename(file))
 198
 199             if not cls._looks_like_module(module):
 200                 continue
 201
 202             if include(module) and not exclude(module):
 203                 yield module
 204
 205     _looks_like_module = staticmethod(_valid_name)
 206
 207
 208 # We have to be extra careful in the case of flat layout to not include files
 209 # and directories not meant for distribution (e.g. tool-related)
 210
 211
 212 class FlatLayoutPackageFinder(PEP420PackageFinder):
 213     _EXCLUDE = (
 214         "ci",
 215         "bin",
 216         "debian",
 217         "doc",
 218         "docs",
 219         "documentation",
 220         "manpages",
 221         "news",
 222         "newsfragments",
 223         "changelog",
 224         "test",
 225         "tests",
 226         "unit_test",
 227         "unit_tests",
 228         "example",
 229         "examples",
 230         "scripts",
 231         "tools",
 232         "util",
 233         "utils",
 234         "python",
 235         "build",
 236         "dist",
 237         "venv",
 238         "env",
 239         "requirements",
 240         # ---- Task runners / Build tools ----
 241         "tasks",  # invoke
 242         "fabfile",  # fabric
 243         "site_scons",  # SCons
 244         # ---- Other tools ----
 245         "benchmark",
 246         "benchmarks",
 247         "exercise",
 248         "exercises",
 249         "htmlcov",  # Coverage.py
 250         # ---- Hidden directories/Private packages ----
 251         "[._]*",
 252     )
 253
 254     DEFAULT_EXCLUDE = tuple(chain_iter((p, f"{p}.*") for p in _EXCLUDE))
 255     """Reserved package names"""
 256
 257     @staticmethod
 258     def _looks_like_package(_path: _Path, package_name: str) -> bool:
 259         names = package_name.split('.')
 260         # Consider PEP 561
 261         root_pkg_is_valid = names[0].isidentifier() or names[0].endswith("-stubs")
 262         return root_pkg_is_valid and all(name.isidentifier() for name in names[1:])
 263
 264
 265 class FlatLayoutModuleFinder(ModuleFinder):
 266     DEFAULT_EXCLUDE = (
 267         "setup",
 268         "conftest",
 269         "test",
 270         "tests",
 271         "example",
 272         "examples",
 273         "build",
 274         # ---- Task runners ----
 275         "toxfile",
 276         "noxfile",
 277         "pavement",
 278         "dodo",
 279         "tasks",
 280         "fabfile",
 281         # ---- Other tools ----
 282         "[Ss][Cc]onstruct",  # SCons
 283         "conanfile",  # Connan: C/C++ build tool
 284         "manage",  # Django
 285         "benchmark",
 286         "benchmarks",
 287         "exercise",
 288         "exercises",
 289         # ---- Hidden files/Private modules ----
 290         "[._]*",
 291     )
 292     """Reserved top-level module names"""
 293
 294
 295 def _find_packages_within(root_pkg: str, pkg_dir: _Path) -> List[str]:
 296     nested = PEP420PackageFinder.find(pkg_dir)
 297     return [root_pkg] + [".".join((root_pkg, n)) for n in nested]
 298
 299
 300 class ConfigDiscovery:
 301     """Fill-in metadata and options that can be automatically derived
 302     (from other metadata/options, the file system or conventions)
 303     """
 304
 305     def __init__(self, distribution: "Distribution"):
 306         self.dist = distribution
 307         self._called = False
 308         self._disabled = False
 309         self._skip_ext_modules = False
 310
 311     def _disable(self):
 312         """Internal API to disable automatic discovery"""
 313         self._disabled = True
 314
 315     def _ignore_ext_modules(self):
 316         """Internal API to disregard ext_modules.
 317
 318         Normally auto-discovery would not be triggered if ``ext_modules`` are set
 319         (this is done for backward compatibility with existing packages relying on
 320         ``setup.py`` or ``setup.cfg``). However, ``setuptools`` can call this function
 321         to ignore given ``ext_modules`` and proceed with the auto-discovery if
 322         ``packages`` and ``py_modules`` are not given (e.g. when using pyproject.toml
 323         metadata).
 324         """
 325         self._skip_ext_modules = True
 326
 327     @property
 328     def _root_dir(self) -> _Path:
 329         # The best is to wait until `src_root` is set in dist, before using _root_dir.
 330         return self.dist.src_root or os.curdir
 331
 332     @property
 333     def _package_dir(self) -> Dict[str, str]:
 334         if self.dist.package_dir is None:
 335             return {}
 336         return self.dist.package_dir
 337
 338     def __call__(self, force=False, name=True, ignore_ext_modules=False):
 339         """Automatically discover missing configuration fields
 340         and modifies the given ``distribution`` object in-place.
 341
 342         Note that by default this will only have an effect the first time the
 343         ``ConfigDiscovery`` object is called.
 344
 345         To repeatedly invoke automatic discovery (e.g. when the project
 346         directory changes), please use ``force=True`` (or create a new
 347         ``ConfigDiscovery`` instance).
 348         """
 349         if force is False and (self._called or self._disabled):
 350             # Avoid overhead of multiple calls
 351             return
 352
 353         self._analyse_package_layout(ignore_ext_modules)
 354         if name:
 355             self.analyse_name()  # depends on ``packages`` and ``py_modules``
 356
 357         self._called = True
 358
 359     def _explicitly_specified(self, ignore_ext_modules: bool) -> bool:
 360         """``True`` if the user has specified some form of package/module listing"""
 361         ignore_ext_modules = ignore_ext_modules or self._skip_ext_modules
 362         ext_modules = not (self.dist.ext_modules is None or ignore_ext_modules)
 363         return (
 364             self.dist.packages is not None
 365             or self.dist.py_modules is not None
 366             or ext_modules
 367             or hasattr(self.dist, "configuration")
 368             and self.dist.configuration
 369             # ^ Some projects use numpy.distutils.misc_util.Configuration
 370         )
 371
 372     def _analyse_package_layout(self, ignore_ext_modules: bool) -> bool:
 373         if self._explicitly_specified(ignore_ext_modules):
 374             # For backward compatibility, just try to find modules/packages
 375             # when nothing is given
 376             return True
 377
 378         log.debug(
 379             "No `packages` or `py_modules` configuration, performing "
 380             "automatic discovery."
 381         )
 382
 383         return (
 384             self._analyse_explicit_layout()
 385             or self._analyse_src_layout()
 386             # flat-layout is the trickiest for discovery so it should be last
 387             or self._analyse_flat_layout()
 388         )
 389
 390     def _analyse_explicit_layout(self) -> bool:
 391         """The user can explicitly give a package layout via ``package_dir``"""
 392         package_dir = self._package_dir.copy()  # don't modify directly
 393         package_dir.pop("", None)  # This falls under the "src-layout" umbrella
 394         root_dir = self._root_dir
 395
 396         if not package_dir:
 397             return False
 398
 399         log.debug(f"`explicit-layout` detected -- analysing {package_dir}")
 400         pkgs = chain_iter(
 401             _find_packages_within(pkg, os.path.join(root_dir, parent_dir))
 402             for pkg, parent_dir in package_dir.items()
 403         )
 404         self.dist.packages = list(pkgs)
 405         log.debug(f"discovered packages -- {self.dist.packages}")
 406         return True
 407
 408     def _analyse_src_layout(self) -> bool:
 409         """Try to find all packages or modules under the ``src`` directory
 410         (or anything pointed by ``package_dir[""]``).
 411
 412         The "src-layout" is relatively safe for automatic discovery.
 413         We assume that everything within is meant to be included in the
 414         distribution.
 415
 416         If ``package_dir[""]`` is not given, but the ``src`` directory exists,
 417         this function will set ``package_dir[""] = "src"``.
 418         """
 419         package_dir = self._package_dir
 420         src_dir = os.path.join(self._root_dir, package_dir.get("", "src"))
 421         if not os.path.isdir(src_dir):
 422             return False
 423
 424         log.debug(f"`src-layout` detected -- analysing {src_dir}")
 425         package_dir.setdefault("", os.path.basename(src_dir))
 426         self.dist.package_dir = package_dir  # persist eventual modifications
 427         self.dist.packages = PEP420PackageFinder.find(src_dir)
 428         self.dist.py_modules = ModuleFinder.find(src_dir)
 429         log.debug(f"discovered packages -- {self.dist.packages}")
 430         log.debug(f"discovered py_modules -- {self.dist.py_modules}")
 431         return True
 432
 433     def _analyse_flat_layout(self) -> bool:
 434         """Try to find all packages and modules under the project root.
 435
 436         Since the ``flat-layout`` is more dangerous in terms of accidentally including
 437         extra files/directories, this function is more conservative and will raise an
 438         error if multiple packages or modules are found.
 439
 440         This assumes that multi-package dists are uncommon and refuse to support that
 441         use case in order to be able to prevent unintended errors.
 442         """
 443         log.debug(f"`flat-layout` detected -- analysing {self._root_dir}")
 444         return self._analyse_flat_packages() or self._analyse_flat_modules()
 445
 446     def _analyse_flat_packages(self) -> bool:
 447         self.dist.packages = FlatLayoutPackageFinder.find(self._root_dir)
 448         top_level = remove_nested_packages(remove_stubs(self.dist.packages))
 449         log.debug(f"discovered packages -- {self.dist.packages}")
 450         self._ensure_no_accidental_inclusion(top_level, "packages")
 451         return bool(top_level)
 452
 453     def _analyse_flat_modules(self) -> bool:
 454         self.dist.py_modules = FlatLayoutModuleFinder.find(self._root_dir)
 455         log.debug(f"discovered py_modules -- {self.dist.py_modules}")
 456         self._ensure_no_accidental_inclusion(self.dist.py_modules, "modules")
 457         return bool(self.dist.py_modules)
 458
 459     def _ensure_no_accidental_inclusion(self, detected: List[str], kind: str):
 460         if len(detected) > 1:
 461             from inspect import cleandoc
 462
 463             from setuptools.errors import PackageDiscoveryError
 464
 465             msg = f"""Multiple top-level {kind} discovered in a flat-layout: {detected}.
 466
 467             To avoid accidental inclusion of unwanted files or directories,
 468             setuptools will not proceed with this build.
 469
 470             If you are trying to create a single distribution with multiple {kind}
 471             on purpose, you should not rely on automatic discovery.
 472             Instead, consider the following options:
 473
 474             1. set up custom discovery (`find` directive with `include` or `exclude`)
 475             2. use a `src-layout`
 476             3. explicitly set `py_modules` or `packages` with a list of names
 477
 478             To find more information, look for "package discovery" on setuptools docs.
 479             """
 480             raise PackageDiscoveryError(cleandoc(msg))
 481
 482     def analyse_name(self):
 483         """The packages/modules are the essential contribution of the author.
 484         Therefore the name of the distribution can be derived from them.
 485         """
 486         if self.dist.metadata.name or self.dist.name:
 487             # get_name() is not reliable (can return "UNKNOWN")
 488             return None
 489
 490         log.debug("No `name` configuration, performing automatic discovery")
 491
 492         name = (
 493             self._find_name_single_package_or_module()
 494             or self._find_name_from_packages()
 495         )
 496         if name:
 497             self.dist.metadata.name = name
 498
 499     def _find_name_single_package_or_module(self) -> Optional[str]:
 500         """Exactly one module or package"""
 501         for field in ('packages', 'py_modules'):
 502             items = getattr(self.dist, field, None) or []
 503             if items and len(items) == 1:
 504                 log.debug(f"Single module/package detected, name: {items[0]}")
 505                 return items[0]
 506
 507         return None
 508
 509     def _find_name_from_packages(self) -> Optional[str]:
 510         """Try to find the root package that is not a PEP 420 namespace"""
 511         if not self.dist.packages:
 512             return None
 513
 514         packages = remove_stubs(sorted(self.dist.packages, key=len))
 515         package_dir = self.dist.package_dir or {}
 516
 517         parent_pkg = find_parent_package(packages, package_dir, self._root_dir)
 518         if parent_pkg:
 519             log.debug(f"Common parent package detected, name: {parent_pkg}")
 520             return parent_pkg
 521
 522         log.warn("No parent package detected, impossible to derive `name`")
 523         return None
 524
 525
 526 def remove_nested_packages(packages: List[str]) -> List[str]:
 527     """Remove nested packages from a list of packages.
 528
 529     >>> remove_nested_packages(["a", "a.b1", "a.b2", "a.b1.c1"])
 530     ['a']
 531     >>> remove_nested_packages(["a", "b", "c.d", "c.d.e.f", "g.h", "a.a1"])
 532     ['a', 'b', 'c.d', 'g.h']
 533     """
 534     pkgs = sorted(packages, key=len)
 535     top_level = pkgs[:]
 536     size = len(pkgs)
 537     for i, name in enumerate(reversed(pkgs)):
 538         if any(name.startswith(f"{other}.") for other in top_level):
 539             top_level.pop(size - i - 1)
 540
 541     return top_level
 542
 543
 544 def remove_stubs(packages: List[str]) -> List[str]:
 545     """Remove type stubs (:pep:`561`) from a list of packages.
 546
 547     >>> remove_stubs(["a", "a.b", "a-stubs", "a-stubs.b.c", "b", "c-stubs"])
 548     ['a', 'a.b', 'b']
 549     """
 550     return [pkg for pkg in packages if not pkg.split(".")[0].endswith("-stubs")]
 551
 552
 553 def find_parent_package(
 554     packages: List[str], package_dir: Mapping[str, str], root_dir: _Path
 555 ) -> Optional[str]:
 556     """Find the parent package that is not a namespace."""
 557     packages = sorted(packages, key=len)
 558     common_ancestors = []
 559     for i, name in enumerate(packages):
 560         if not all(n.startswith(f"{name}.") for n in packages[i + 1 :]):
 561             # Since packages are sorted by length, this condition is able
 562             # to find a list of all common ancestors.
 563             # When there is divergence (e.g. multiple root packages)
 564             # the list will be empty
 565             break
 566         common_ancestors.append(name)
 567
 568     for name in common_ancestors:
 569         pkg_path = find_package_path(name, package_dir, root_dir)
 570         init = os.path.join(pkg_path, "__init__.py")
 571         if os.path.isfile(init):
 572             return name
 573
 574     return None
 575
 576
 577 def find_package_path(
 578     name: str, package_dir: Mapping[str, str], root_dir: _Path
 579 ) -> str:
 580     """Given a package name, return the path where it should be found on
 581     disk, considering the ``package_dir`` option.
 582
 583     >>> path = find_package_path("my.pkg", {"": "root/is/nested"}, ".")
 584     >>> path.replace(os.sep, "/")
 585     './root/is/nested/my/pkg'
 586
 587     >>> path = find_package_path("my.pkg", {"my": "root/is/nested"}, ".")
 588     >>> path.replace(os.sep, "/")
 589     './root/is/nested/pkg'
 590
 591     >>> path = find_package_path("my.pkg", {"my.pkg": "root/is/nested"}, ".")
 592     >>> path.replace(os.sep, "/")
 593     './root/is/nested'
 594
 595     >>> path = find_package_path("other.pkg", {"my.pkg": "root/is/nested"}, ".")
 596     >>> path.replace(os.sep, "/")
 597     './other/pkg'
 598     """
 599     parts = name.split(".")
 600     for i in range(len(parts), 0, -1):
 601         # Look backwards, the most specific package_dir first
 602         partial_name = ".".join(parts[:i])
 603         if partial_name in package_dir:
 604             parent = package_dir[partial_name]
 605             return os.path.join(root_dir, parent, *parts[i:])
 606
 607     parent = package_dir.get("") or ""
 608     return os.path.join(root_dir, *parent.split("/"), *parts)
 609
 610
 611 def construct_package_dir(packages: List[str], package_path: _Path) -> Dict[str, str]:
 612     parent_pkgs = remove_nested_packages(packages)
 613     prefix = Path(package_path).parts
 614     return {pkg: "/".join([*prefix, *pkg.split(".")]) for pkg in parent_pkgs}