venv/lib/python3.11/site-packages/pip/_vendor/distlib/index.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # Copyright (C) 2013 Vinay Sajip.
   4 # Licensed to the Python Software Foundation under a contributor agreement.
   5 # See LICENSE.txt and CONTRIBUTORS.txt.
   6 #
   7 import hashlib
   8 import logging
   9 import os
  10 import shutil
  11 import subprocess
  12 import tempfile
  13 try:
  14     from threading import Thread
  15 except ImportError:  # pragma: no cover
  16     from dummy_threading import Thread
  17
  18 from . import DistlibException
  19 from .compat import (HTTPBasicAuthHandler, Request, HTTPPasswordMgr,
  20                      urlparse, build_opener, string_types)
  21 from .util import zip_dir, ServerProxy
  22
  23 logger = logging.getLogger(__name__)
  24
  25 DEFAULT_INDEX = 'https://pypi.org/pypi'
  26 DEFAULT_REALM = 'pypi'
  27
  28 class PackageIndex(object):
  29     """
  30     This class represents a package index compatible with PyPI, the Python
  31     Package Index.
  32     """
  33
  34     boundary = b'----------ThIs_Is_tHe_distlib_index_bouNdaRY_$'
  35
  36     def __init__(self, url=None):
  37         """
  38         Initialise an instance.
  39
  40         :param url: The URL of the index. If not specified, the URL for PyPI is
  41                     used.
  42         """
  43         self.url = url or DEFAULT_INDEX
  44         self.read_configuration()
  45         scheme, netloc, path, params, query, frag = urlparse(self.url)
  46         if params or query or frag or scheme not in ('http', 'https'):
  47             raise DistlibException('invalid repository: %s' % self.url)
  48         self.password_handler = None
  49         self.ssl_verifier = None
  50         self.gpg = None
  51         self.gpg_home = None
  52         with open(os.devnull, 'w') as sink:
  53             # Use gpg by default rather than gpg2, as gpg2 insists on
  54             # prompting for passwords
  55             for s in ('gpg', 'gpg2'):
  56                 try:
  57                     rc = subprocess.check_call([s, '--version'], stdout=sink,
  58                                                stderr=sink)
  59                     if rc == 0:
  60                         self.gpg = s
  61                         break
  62                 except OSError:
  63                     pass
  64
  65     def _get_pypirc_command(self):
  66         """
  67         Get the distutils command for interacting with PyPI configurations.
  68         :return: the command.
  69         """
  70         from .util import _get_pypirc_command as cmd
  71         return cmd()
  72
  73     def read_configuration(self):
  74         """
  75         Read the PyPI access configuration as supported by distutils. This populates
  76         ``username``, ``password``, ``realm`` and ``url`` attributes from the
  77         configuration.
  78         """
  79         from .util import _load_pypirc
  80         cfg = _load_pypirc(self)
  81         self.username = cfg.get('username')
  82         self.password = cfg.get('password')
  83         self.realm = cfg.get('realm', 'pypi')
  84         self.url = cfg.get('repository', self.url)
  85
  86     def save_configuration(self):
  87         """
  88         Save the PyPI access configuration. You must have set ``username`` and
  89         ``password`` attributes before calling this method.
  90         """
  91         self.check_credentials()
  92         from .util import _store_pypirc
  93         _store_pypirc(self)
  94
  95     def check_credentials(self):
  96         """
  97         Check that ``username`` and ``password`` have been set, and raise an
  98         exception if not.
  99         """
 100         if self.username is None or self.password is None:
 101             raise DistlibException('username and password must be set')
 102         pm = HTTPPasswordMgr()
 103         _, netloc, _, _, _, _ = urlparse(self.url)
 104         pm.add_password(self.realm, netloc, self.username, self.password)
 105         self.password_handler = HTTPBasicAuthHandler(pm)
 106
 107     def register(self, metadata):  # pragma: no cover
 108         """
 109         Register a distribution on PyPI, using the provided metadata.
 110
 111         :param metadata: A :class:`Metadata` instance defining at least a name
 112                          and version number for the distribution to be
 113                          registered.
 114         :return: The HTTP response received from PyPI upon submission of the
 115                 request.
 116         """
 117         self.check_credentials()
 118         metadata.validate()
 119         d = metadata.todict()
 120         d[':action'] = 'verify'
 121         request = self.encode_request(d.items(), [])
 122         response = self.send_request(request)
 123         d[':action'] = 'submit'
 124         request = self.encode_request(d.items(), [])
 125         return self.send_request(request)
 126
 127     def _reader(self, name, stream, outbuf):
 128         """
 129         Thread runner for reading lines of from a subprocess into a buffer.
 130
 131         :param name: The logical name of the stream (used for logging only).
 132         :param stream: The stream to read from. This will typically a pipe
 133                        connected to the output stream of a subprocess.
 134         :param outbuf: The list to append the read lines to.
 135         """
 136         while True:
 137             s = stream.readline()
 138             if not s:
 139                 break
 140             s = s.decode('utf-8').rstrip()
 141             outbuf.append(s)
 142             logger.debug('%s: %s' % (name, s))
 143         stream.close()
 144
 145     def get_sign_command(self, filename, signer, sign_password, keystore=None):  # pragma: no cover
 146         """
 147         Return a suitable command for signing a file.
 148
 149         :param filename: The pathname to the file to be signed.
 150         :param signer: The identifier of the signer of the file.
 151         :param sign_password: The passphrase for the signer's
 152                               private key used for signing.
 153         :param keystore: The path to a directory which contains the keys
 154                          used in verification. If not specified, the
 155                          instance's ``gpg_home`` attribute is used instead.
 156         :return: The signing command as a list suitable to be
 157                  passed to :class:`subprocess.Popen`.
 158         """
 159         cmd = [self.gpg, '--status-fd', '2', '--no-tty']
 160         if keystore is None:
 161             keystore = self.gpg_home
 162         if keystore:
 163             cmd.extend(['--homedir', keystore])
 164         if sign_password is not None:
 165             cmd.extend(['--batch', '--passphrase-fd', '0'])
 166         td = tempfile.mkdtemp()
 167         sf = os.path.join(td, os.path.basename(filename) + '.asc')
 168         cmd.extend(['--detach-sign', '--armor', '--local-user',
 169                     signer, '--output', sf, filename])
 170         logger.debug('invoking: %s', ' '.join(cmd))
 171         return cmd, sf
 172
 173     def run_command(self, cmd, input_data=None):
 174         """
 175         Run a command in a child process , passing it any input data specified.
 176
 177         :param cmd: The command to run.
 178         :param input_data: If specified, this must be a byte string containing
 179                            data to be sent to the child process.
 180         :return: A tuple consisting of the subprocess' exit code, a list of
 181                  lines read from the subprocess' ``stdout``, and a list of
 182                  lines read from the subprocess' ``stderr``.
 183         """
 184         kwargs = {
 185             'stdout': subprocess.PIPE,
 186             'stderr': subprocess.PIPE,
 187         }
 188         if input_data is not None:
 189             kwargs['stdin'] = subprocess.PIPE
 190         stdout = []
 191         stderr = []
 192         p = subprocess.Popen(cmd, **kwargs)
 193         # We don't use communicate() here because we may need to
 194         # get clever with interacting with the command
 195         t1 = Thread(target=self._reader, args=('stdout', p.stdout, stdout))
 196         t1.start()
 197         t2 = Thread(target=self._reader, args=('stderr', p.stderr, stderr))
 198         t2.start()
 199         if input_data is not None:
 200             p.stdin.write(input_data)
 201             p.stdin.close()
 202
 203         p.wait()
 204         t1.join()
 205         t2.join()
 206         return p.returncode, stdout, stderr
 207
 208     def sign_file(self, filename, signer, sign_password, keystore=None):  # pragma: no cover
 209         """
 210         Sign a file.
 211
 212         :param filename: The pathname to the file to be signed.
 213         :param signer: The identifier of the signer of the file.
 214         :param sign_password: The passphrase for the signer's
 215                               private key used for signing.
 216         :param keystore: The path to a directory which contains the keys
 217                          used in signing. If not specified, the instance's
 218                          ``gpg_home`` attribute is used instead.
 219         :return: The absolute pathname of the file where the signature is
 220                  stored.
 221         """
 222         cmd, sig_file = self.get_sign_command(filename, signer, sign_password,
 223                                               keystore)
 224         rc, stdout, stderr = self.run_command(cmd,
 225                                               sign_password.encode('utf-8'))
 226         if rc != 0:
 227             raise DistlibException('sign command failed with error '
 228                                    'code %s' % rc)
 229         return sig_file
 230
 231     def upload_file(self, metadata, filename, signer=None, sign_password=None,
 232                     filetype='sdist', pyversion='source', keystore=None):
 233         """
 234         Upload a release file to the index.
 235
 236         :param metadata: A :class:`Metadata` instance defining at least a name
 237                          and version number for the file to be uploaded.
 238         :param filename: The pathname of the file to be uploaded.
 239         :param signer: The identifier of the signer of the file.
 240         :param sign_password: The passphrase for the signer's
 241                               private key used for signing.
 242         :param filetype: The type of the file being uploaded. This is the
 243                         distutils command which produced that file, e.g.
 244                         ``sdist`` or ``bdist_wheel``.
 245         :param pyversion: The version of Python which the release relates
 246                           to. For code compatible with any Python, this would
 247                           be ``source``, otherwise it would be e.g. ``3.2``.
 248         :param keystore: The path to a directory which contains the keys
 249                          used in signing. If not specified, the instance's
 250                          ``gpg_home`` attribute is used instead.
 251         :return: The HTTP response received from PyPI upon submission of the
 252                 request.
 253         """
 254         self.check_credentials()
 255         if not os.path.exists(filename):
 256             raise DistlibException('not found: %s' % filename)
 257         metadata.validate()
 258         d = metadata.todict()
 259         sig_file = None
 260         if signer:
 261             if not self.gpg:
 262                 logger.warning('no signing program available - not signed')
 263             else:
 264                 sig_file = self.sign_file(filename, signer, sign_password,
 265                                           keystore)
 266         with open(filename, 'rb') as f:
 267             file_data = f.read()
 268         md5_digest = hashlib.md5(file_data).hexdigest()
 269         sha256_digest = hashlib.sha256(file_data).hexdigest()
 270         d.update({
 271             ':action': 'file_upload',
 272             'protocol_version': '1',
 273             'filetype': filetype,
 274             'pyversion': pyversion,
 275             'md5_digest': md5_digest,
 276             'sha256_digest': sha256_digest,
 277         })
 278         files = [('content', os.path.basename(filename), file_data)]
 279         if sig_file:
 280             with open(sig_file, 'rb') as f:
 281                 sig_data = f.read()
 282             files.append(('gpg_signature', os.path.basename(sig_file),
 283                          sig_data))
 284             shutil.rmtree(os.path.dirname(sig_file))
 285         request = self.encode_request(d.items(), files)
 286         return self.send_request(request)
 287
 288     def upload_documentation(self, metadata, doc_dir):  # pragma: no cover
 289         """
 290         Upload documentation to the index.
 291
 292         :param metadata: A :class:`Metadata` instance defining at least a name
 293                          and version number for the documentation to be
 294                          uploaded.
 295         :param doc_dir: The pathname of the directory which contains the
 296                         documentation. This should be the directory that
 297                         contains the ``index.html`` for the documentation.
 298         :return: The HTTP response received from PyPI upon submission of the
 299                 request.
 300         """
 301         self.check_credentials()
 302         if not os.path.isdir(doc_dir):
 303             raise DistlibException('not a directory: %r' % doc_dir)
 304         fn = os.path.join(doc_dir, 'index.html')
 305         if not os.path.exists(fn):
 306             raise DistlibException('not found: %r' % fn)
 307         metadata.validate()
 308         name, version = metadata.name, metadata.version
 309         zip_data = zip_dir(doc_dir).getvalue()
 310         fields = [(':action', 'doc_upload'),
 311                   ('name', name), ('version', version)]
 312         files = [('content', name, zip_data)]
 313         request = self.encode_request(fields, files)
 314         return self.send_request(request)
 315
 316     def get_verify_command(self, signature_filename, data_filename,
 317                            keystore=None):
 318         """
 319         Return a suitable command for verifying a file.
 320
 321         :param signature_filename: The pathname to the file containing the
 322                                    signature.
 323         :param data_filename: The pathname to the file containing the
 324                               signed data.
 325         :param keystore: The path to a directory which contains the keys
 326                          used in verification. If not specified, the
 327                          instance's ``gpg_home`` attribute is used instead.
 328         :return: The verifying command as a list suitable to be
 329                  passed to :class:`subprocess.Popen`.
 330         """
 331         cmd = [self.gpg, '--status-fd', '2', '--no-tty']
 332         if keystore is None:
 333             keystore = self.gpg_home
 334         if keystore:
 335             cmd.extend(['--homedir', keystore])
 336         cmd.extend(['--verify', signature_filename, data_filename])
 337         logger.debug('invoking: %s', ' '.join(cmd))
 338         return cmd
 339
 340     def verify_signature(self, signature_filename, data_filename,
 341                          keystore=None):
 342         """
 343         Verify a signature for a file.
 344
 345         :param signature_filename: The pathname to the file containing the
 346                                    signature.
 347         :param data_filename: The pathname to the file containing the
 348                               signed data.
 349         :param keystore: The path to a directory which contains the keys
 350                          used in verification. If not specified, the
 351                          instance's ``gpg_home`` attribute is used instead.
 352         :return: True if the signature was verified, else False.
 353         """
 354         if not self.gpg:
 355             raise DistlibException('verification unavailable because gpg '
 356                                    'unavailable')
 357         cmd = self.get_verify_command(signature_filename, data_filename,
 358                                       keystore)
 359         rc, stdout, stderr = self.run_command(cmd)
 360         if rc not in (0, 1):
 361             raise DistlibException('verify command failed with error '
 362                              'code %s' % rc)
 363         return rc == 0
 364
 365     def download_file(self, url, destfile, digest=None, reporthook=None):
 366         """
 367         This is a convenience method for downloading a file from an URL.
 368         Normally, this will be a file from the index, though currently
 369         no check is made for this (i.e. a file can be downloaded from
 370         anywhere).
 371
 372         The method is just like the :func:`urlretrieve` function in the
 373         standard library, except that it allows digest computation to be
 374         done during download and checking that the downloaded data
 375         matched any expected value.
 376
 377         :param url: The URL of the file to be downloaded (assumed to be
 378                     available via an HTTP GET request).
 379         :param destfile: The pathname where the downloaded file is to be
 380                          saved.
 381         :param digest: If specified, this must be a (hasher, value)
 382                        tuple, where hasher is the algorithm used (e.g.
 383                        ``'md5'``) and ``value`` is the expected value.
 384         :param reporthook: The same as for :func:`urlretrieve` in the
 385                            standard library.
 386         """
 387         if digest is None:
 388             digester = None
 389             logger.debug('No digest specified')
 390         else:
 391             if isinstance(digest, (list, tuple)):
 392                 hasher, digest = digest
 393             else:
 394                 hasher = 'md5'
 395             digester = getattr(hashlib, hasher)()
 396             logger.debug('Digest specified: %s' % digest)
 397         # The following code is equivalent to urlretrieve.
 398         # We need to do it this way so that we can compute the
 399         # digest of the file as we go.
 400         with open(destfile, 'wb') as dfp:
 401             # addinfourl is not a context manager on 2.x
 402             # so we have to use try/finally
 403             sfp = self.send_request(Request(url))
 404             try:
 405                 headers = sfp.info()
 406                 blocksize = 8192
 407                 size = -1
 408                 read = 0
 409                 blocknum = 0
 410                 if "content-length" in headers:
 411                     size = int(headers["Content-Length"])
 412                 if reporthook:
 413                     reporthook(blocknum, blocksize, size)
 414                 while True:
 415                     block = sfp.read(blocksize)
 416                     if not block:
 417                         break
 418                     read += len(block)
 419                     dfp.write(block)
 420                     if digester:
 421                         digester.update(block)
 422                     blocknum += 1
 423                     if reporthook:
 424                         reporthook(blocknum, blocksize, size)
 425             finally:
 426                 sfp.close()
 427
 428         # check that we got the whole file, if we can
 429         if size >= 0 and read < size:
 430             raise DistlibException(
 431                 'retrieval incomplete: got only %d out of %d bytes'
 432                 % (read, size))
 433         # if we have a digest, it must match.
 434         if digester:
 435             actual = digester.hexdigest()
 436             if digest != actual:
 437                 raise DistlibException('%s digest mismatch for %s: expected '
 438                                        '%s, got %s' % (hasher, destfile,
 439                                                        digest, actual))
 440             logger.debug('Digest verified: %s', digest)
 441
 442     def send_request(self, req):
 443         """
 444         Send a standard library :class:`Request` to PyPI and return its
 445         response.
 446
 447         :param req: The request to send.
 448         :return: The HTTP response from PyPI (a standard library HTTPResponse).
 449         """
 450         handlers = []
 451         if self.password_handler:
 452             handlers.append(self.password_handler)
 453         if self.ssl_verifier:
 454             handlers.append(self.ssl_verifier)
 455         opener = build_opener(*handlers)
 456         return opener.open(req)
 457
 458     def encode_request(self, fields, files):
 459         """
 460         Encode fields and files for posting to an HTTP server.
 461
 462         :param fields: The fields to send as a list of (fieldname, value)
 463                        tuples.
 464         :param files: The files to send as a list of (fieldname, filename,
 465                       file_bytes) tuple.
 466         """
 467         # Adapted from packaging, which in turn was adapted from
 468         # http://code.activestate.com/recipes/146306
 469
 470         parts = []
 471         boundary = self.boundary
 472         for k, values in fields:
 473             if not isinstance(values, (list, tuple)):
 474                 values = [values]
 475
 476             for v in values:
 477                 parts.extend((
 478                     b'--' + boundary,
 479                     ('Content-Disposition: form-data; name="%s"' %
 480                      k).encode('utf-8'),
 481                     b'',
 482                     v.encode('utf-8')))
 483         for key, filename, value in files:
 484             parts.extend((
 485                 b'--' + boundary,
 486                 ('Content-Disposition: form-data; name="%s"; filename="%s"' %
 487                  (key, filename)).encode('utf-8'),
 488                 b'',
 489                 value))
 490
 491         parts.extend((b'--' + boundary + b'--', b''))
 492
 493         body = b'\r\n'.join(parts)
 494         ct = b'multipart/form-data; boundary=' + boundary
 495         headers = {
 496             'Content-type': ct,
 497             'Content-length': str(len(body))
 498         }
 499         return Request(self.url, body, headers)
 500
 501     def search(self, terms, operator=None):  # pragma: no cover
 502         if isinstance(terms, string_types):
 503             terms = {'name': terms}
 504         rpc_proxy = ServerProxy(self.url, timeout=3.0)
 505         try:
 506             return rpc_proxy.search(terms, operator or 'and')
 507         finally:
 508             rpc_proxy('close')()