venv/lib/python3.11/site-packages/setuptools/command/build_py.py

   1 from functools import partial
   2 from glob import glob
   3 from distutils.util import convert_path
   4 import distutils.command.build_py as orig
   5 import os
   6 import fnmatch
   7 import textwrap
   8 import io
   9 import distutils.errors
  10 import itertools
  11 import stat
  12 from pathlib import Path
  13 from typing import Dict, Iterable, Iterator, List, Optional, Tuple
  14
  15 from ..extern.more_itertools import unique_everseen
  16 from ..warnings import SetuptoolsDeprecationWarning
  17
  18
  19 def make_writable(target):
  20     os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE)
  21
  22
  23 class build_py(orig.build_py):
  24     """Enhanced 'build_py' command that includes data files with packages
  25
  26     The data files are specified via a 'package_data' argument to 'setup()'.
  27     See 'setuptools.dist.Distribution' for more details.
  28
  29     Also, this version of the 'build_py' command allows you to specify both
  30     'py_modules' and 'packages' in the same setup operation.
  31     """
  32
  33     editable_mode: bool = False
  34     existing_egg_info_dir: Optional[str] = None  #: Private API, internal use only.
  35
  36     def finalize_options(self):
  37         orig.build_py.finalize_options(self)
  38         self.package_data = self.distribution.package_data
  39         self.exclude_package_data = self.distribution.exclude_package_data or {}
  40         if 'data_files' in self.__dict__:
  41             del self.__dict__['data_files']
  42         self.__updated_files = []
  43
  44     def copy_file(
  45         self, infile, outfile, preserve_mode=1, preserve_times=1, link=None, level=1
  46     ):
  47         # Overwrite base class to allow using links
  48         if link:
  49             infile = str(Path(infile).resolve())
  50             outfile = str(Path(outfile).resolve())
  51         return super().copy_file(
  52             infile, outfile, preserve_mode, preserve_times, link, level
  53         )
  54
  55     def run(self):
  56         """Build modules, packages, and copy data files to build directory"""
  57         if not (self.py_modules or self.packages) or self.editable_mode:
  58             return
  59
  60         if self.py_modules:
  61             self.build_modules()
  62
  63         if self.packages:
  64             self.build_packages()
  65             self.build_package_data()
  66
  67         # Only compile actual .py files, using our base class' idea of what our
  68         # output files are.
  69         self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0))
  70
  71     def __getattr__(self, attr):
  72         "lazily compute data files"
  73         if attr == 'data_files':
  74             self.data_files = self._get_data_files()
  75             return self.data_files
  76         return orig.build_py.__getattr__(self, attr)
  77
  78     def build_module(self, module, module_file, package):
  79         outfile, copied = orig.build_py.build_module(self, module, module_file, package)
  80         if copied:
  81             self.__updated_files.append(outfile)
  82         return outfile, copied
  83
  84     def _get_data_files(self):
  85         """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
  86         self.analyze_manifest()
  87         return list(map(self._get_pkg_data_files, self.packages or ()))
  88
  89     def get_data_files_without_manifest(self):
  90         """
  91         Generate list of ``(package,src_dir,build_dir,filenames)`` tuples,
  92         but without triggering any attempt to analyze or build the manifest.
  93         """
  94         # Prevent eventual errors from unset `manifest_files`
  95         # (that would otherwise be set by `analyze_manifest`)
  96         self.__dict__.setdefault('manifest_files', {})
  97         return list(map(self._get_pkg_data_files, self.packages or ()))
  98
  99     def _get_pkg_data_files(self, package):
 100         # Locate package source directory
 101         src_dir = self.get_package_dir(package)
 102
 103         # Compute package build directory
 104         build_dir = os.path.join(*([self.build_lib] + package.split('.')))
 105
 106         # Strip directory from globbed filenames
 107         filenames = [
 108             os.path.relpath(file, src_dir)
 109             for file in self.find_data_files(package, src_dir)
 110         ]
 111         return package, src_dir, build_dir, filenames
 112
 113     def find_data_files(self, package, src_dir):
 114         """Return filenames for package's data files in 'src_dir'"""
 115         patterns = self._get_platform_patterns(
 116             self.package_data,
 117             package,
 118             src_dir,
 119         )
 120         globs_expanded = map(partial(glob, recursive=True), patterns)
 121         # flatten the expanded globs into an iterable of matches
 122         globs_matches = itertools.chain.from_iterable(globs_expanded)
 123         glob_files = filter(os.path.isfile, globs_matches)
 124         files = itertools.chain(
 125             self.manifest_files.get(package, []),
 126             glob_files,
 127         )
 128         return self.exclude_data_files(package, src_dir, files)
 129
 130     def get_outputs(self, include_bytecode=1) -> List[str]:
 131         """See :class:`setuptools.commands.build.SubCommand`"""
 132         if self.editable_mode:
 133             return list(self.get_output_mapping().keys())
 134         return super().get_outputs(include_bytecode)
 135
 136     def get_output_mapping(self) -> Dict[str, str]:
 137         """See :class:`setuptools.commands.build.SubCommand`"""
 138         mapping = itertools.chain(
 139             self._get_package_data_output_mapping(),
 140             self._get_module_mapping(),
 141         )
 142         return dict(sorted(mapping, key=lambda x: x[0]))
 143
 144     def _get_module_mapping(self) -> Iterator[Tuple[str, str]]:
 145         """Iterate over all modules producing (dest, src) pairs."""
 146         for package, module, module_file in self.find_all_modules():
 147             package = package.split('.')
 148             filename = self.get_module_outfile(self.build_lib, package, module)
 149             yield (filename, module_file)
 150
 151     def _get_package_data_output_mapping(self) -> Iterator[Tuple[str, str]]:
 152         """Iterate over package data producing (dest, src) pairs."""
 153         for package, src_dir, build_dir, filenames in self.data_files:
 154             for filename in filenames:
 155                 target = os.path.join(build_dir, filename)
 156                 srcfile = os.path.join(src_dir, filename)
 157                 yield (target, srcfile)
 158
 159     def build_package_data(self):
 160         """Copy data files into build directory"""
 161         for target, srcfile in self._get_package_data_output_mapping():
 162             self.mkpath(os.path.dirname(target))
 163             _outf, _copied = self.copy_file(srcfile, target)
 164             make_writable(target)
 165
 166     def analyze_manifest(self):
 167         self.manifest_files = mf = {}
 168         if not self.distribution.include_package_data:
 169             return
 170         src_dirs = {}
 171         for package in self.packages or ():
 172             # Locate package source directory
 173             src_dirs[assert_relative(self.get_package_dir(package))] = package
 174
 175         if (
 176             getattr(self, 'existing_egg_info_dir', None)
 177             and Path(self.existing_egg_info_dir, "SOURCES.txt").exists()
 178         ):
 179             egg_info_dir = self.existing_egg_info_dir
 180             manifest = Path(egg_info_dir, "SOURCES.txt")
 181             files = manifest.read_text(encoding="utf-8").splitlines()
 182         else:
 183             self.run_command('egg_info')
 184             ei_cmd = self.get_finalized_command('egg_info')
 185             egg_info_dir = ei_cmd.egg_info
 186             files = ei_cmd.filelist.files
 187
 188         check = _IncludePackageDataAbuse()
 189         for path in self._filter_build_files(files, egg_info_dir):
 190             d, f = os.path.split(assert_relative(path))
 191             prev = None
 192             oldf = f
 193             while d and d != prev and d not in src_dirs:
 194                 prev = d
 195                 d, df = os.path.split(d)
 196                 f = os.path.join(df, f)
 197             if d in src_dirs:
 198                 if f == oldf:
 199                     if check.is_module(f):
 200                         continue  # it's a module, not data
 201                 else:
 202                     importable = check.importable_subpackage(src_dirs[d], f)
 203                     if importable:
 204                         check.warn(importable)
 205                 mf.setdefault(src_dirs[d], []).append(path)
 206
 207     def _filter_build_files(self, files: Iterable[str], egg_info: str) -> Iterator[str]:
 208         """
 209         ``build_meta`` may try to create egg_info outside of the project directory,
 210         and this can be problematic for certain plugins (reported in issue #3500).
 211
 212         Extensions might also include between their sources files created on the
 213         ``build_lib`` and ``build_temp`` directories.
 214
 215         This function should filter this case of invalid files out.
 216         """
 217         build = self.get_finalized_command("build")
 218         build_dirs = (egg_info, self.build_lib, build.build_temp, build.build_base)
 219         norm_dirs = [os.path.normpath(p) for p in build_dirs if p]
 220
 221         for file in files:
 222             norm_path = os.path.normpath(file)
 223             if not os.path.isabs(file) or all(d not in norm_path for d in norm_dirs):
 224                 yield file
 225
 226     def get_data_files(self):
 227         pass  # Lazily compute data files in _get_data_files() function.
 228
 229     def check_package(self, package, package_dir):
 230         """Check namespace packages' __init__ for declare_namespace"""
 231         try:
 232             return self.packages_checked[package]
 233         except KeyError:
 234             pass
 235
 236         init_py = orig.build_py.check_package(self, package, package_dir)
 237         self.packages_checked[package] = init_py
 238
 239         if not init_py or not self.distribution.namespace_packages:
 240             return init_py
 241
 242         for pkg in self.distribution.namespace_packages:
 243             if pkg == package or pkg.startswith(package + '.'):
 244                 break
 245         else:
 246             return init_py
 247
 248         with io.open(init_py, 'rb') as f:
 249             contents = f.read()
 250         if b'declare_namespace' not in contents:
 251             raise distutils.errors.DistutilsError(
 252                 "Namespace package problem: %s is a namespace package, but "
 253                 "its\n__init__.py does not call declare_namespace()! Please "
 254                 'fix it.\n(See the setuptools manual under '
 255                 '"Namespace Packages" for details.)\n"' % (package,)
 256             )
 257         return init_py
 258
 259     def initialize_options(self):
 260         self.packages_checked = {}
 261         orig.build_py.initialize_options(self)
 262         self.editable_mode = False
 263         self.existing_egg_info_dir = None
 264
 265     def get_package_dir(self, package):
 266         res = orig.build_py.get_package_dir(self, package)
 267         if self.distribution.src_root is not None:
 268             return os.path.join(self.distribution.src_root, res)
 269         return res
 270
 271     def exclude_data_files(self, package, src_dir, files):
 272         """Filter filenames for package's data files in 'src_dir'"""
 273         files = list(files)
 274         patterns = self._get_platform_patterns(
 275             self.exclude_package_data,
 276             package,
 277             src_dir,
 278         )
 279         match_groups = (fnmatch.filter(files, pattern) for pattern in patterns)
 280         # flatten the groups of matches into an iterable of matches
 281         matches = itertools.chain.from_iterable(match_groups)
 282         bad = set(matches)
 283         keepers = (fn for fn in files if fn not in bad)
 284         # ditch dupes
 285         return list(unique_everseen(keepers))
 286
 287     @staticmethod
 288     def _get_platform_patterns(spec, package, src_dir):
 289         """
 290         yield platform-specific path patterns (suitable for glob
 291         or fn_match) from a glob-based spec (such as
 292         self.package_data or self.exclude_package_data)
 293         matching package in src_dir.
 294         """
 295         raw_patterns = itertools.chain(
 296             spec.get('', []),
 297             spec.get(package, []),
 298         )
 299         return (
 300             # Each pattern has to be converted to a platform-specific path
 301             os.path.join(src_dir, convert_path(pattern))
 302             for pattern in raw_patterns
 303         )
 304
 305
 306 def assert_relative(path):
 307     if not os.path.isabs(path):
 308         return path
 309     from distutils.errors import DistutilsSetupError
 310
 311     msg = (
 312         textwrap.dedent(
 313             """
 314         Error: setup script specifies an absolute path:
 315
 316             %s
 317
 318         setup() arguments must *always* be /-separated paths relative to the
 319         setup.py directory, *never* absolute paths.
 320         """
 321         ).lstrip()
 322         % path
 323     )
 324     raise DistutilsSetupError(msg)
 325
 326
 327 class _IncludePackageDataAbuse:
 328     """Inform users that package or module is included as 'data file'"""
 329
 330     class _Warning(SetuptoolsDeprecationWarning):
 331         _SUMMARY = """
 332         Package {importable!r} is absent from the `packages` configuration.
 333         """
 334
 335         _DETAILS = """
 336         ############################
 337         # Package would be ignored #
 338         ############################
 339         Python recognizes {importable!r} as an importable package[^1],
 340         but it is absent from setuptools' `packages` configuration.
 341
 342         This leads to an ambiguous overall configuration. If you want to distribute this
 343         package, please make sure that {importable!r} is explicitly added
 344         to the `packages` configuration field.
 345
 346         Alternatively, you can also rely on setuptools' discovery methods
 347         (for example by using `find_namespace_packages(...)`/`find_namespace:`
 348         instead of `find_packages(...)`/`find:`).
 349
 350         You can read more about "package discovery" on setuptools documentation page:
 351
 352         - https://setuptools.pypa.io/en/latest/userguide/package_discovery.html
 353
 354         If you don't want {importable!r} to be distributed and are
 355         already explicitly excluding {importable!r} via
 356         `find_namespace_packages(...)/find_namespace` or `find_packages(...)/find`,
 357         you can try to use `exclude_package_data`, or `include-package-data=False` in
 358         combination with a more fine grained `package-data` configuration.
 359
 360         You can read more about "package data files" on setuptools documentation page:
 361
 362         - https://setuptools.pypa.io/en/latest/userguide/datafiles.html
 363
 364
 365         [^1]: For Python, any directory (with suitable naming) can be imported,
 366               even if it does not contain any `.py` files.
 367               On the other hand, currently there is no concept of package data
 368               directory, all directories are treated like packages.
 369         """
 370         # _DUE_DATE: still not defined as this is particularly controversial.
 371         # Warning initially introduced in May 2022. See issue #3340 for discussion.
 372
 373     def __init__(self):
 374         self._already_warned = set()
 375
 376     def is_module(self, file):
 377         return file.endswith(".py") and file[: -len(".py")].isidentifier()
 378
 379     def importable_subpackage(self, parent, file):
 380         pkg = Path(file).parent
 381         parts = list(itertools.takewhile(str.isidentifier, pkg.parts))
 382         if parts:
 383             return ".".join([parent, *parts])
 384         return None
 385
 386     def warn(self, importable):
 387         if importable not in self._already_warned:
 388             self._Warning.emit(importable=importable)
 389             self._already_warned.add(importable)