]>
Commit | Line | Data |
---|---|---|
e0df8241 JR |
1 | """Backing implementation for InstallRequirement's various constructors |
2 | ||
3 | The idea here is that these formed a major chunk of InstallRequirement's size | |
4 | so, moving them and support code dedicated to them outside of that class | |
5 | helps creates for better understandability for the rest of the code. | |
6 | ||
7 | These are meant to be used elsewhere within pip to create instances of | |
8 | InstallRequirement. | |
9 | """ | |
10 | ||
11 | import logging | |
12 | import os | |
13 | import re | |
14 | from typing import Dict, List, Optional, Set, Tuple, Union | |
15 | ||
16 | from pip._vendor.packaging.markers import Marker | |
17 | from pip._vendor.packaging.requirements import InvalidRequirement, Requirement | |
18 | from pip._vendor.packaging.specifiers import Specifier | |
19 | ||
20 | from pip._internal.exceptions import InstallationError | |
21 | from pip._internal.models.index import PyPI, TestPyPI | |
22 | from pip._internal.models.link import Link | |
23 | from pip._internal.models.wheel import Wheel | |
24 | from pip._internal.req.req_file import ParsedRequirement | |
25 | from pip._internal.req.req_install import InstallRequirement | |
26 | from pip._internal.utils.filetypes import is_archive_file | |
27 | from pip._internal.utils.misc import is_installable_dir | |
28 | from pip._internal.utils.packaging import get_requirement | |
29 | from pip._internal.utils.urls import path_to_url | |
30 | from pip._internal.vcs import is_url, vcs | |
31 | ||
32 | __all__ = [ | |
33 | "install_req_from_editable", | |
34 | "install_req_from_line", | |
35 | "parse_editable", | |
36 | ] | |
37 | ||
38 | logger = logging.getLogger(__name__) | |
39 | operators = Specifier._operators.keys() | |
40 | ||
41 | ||
42 | def _strip_extras(path: str) -> Tuple[str, Optional[str]]: | |
43 | m = re.match(r"^(.+)(\[[^\]]+\])$", path) | |
44 | extras = None | |
45 | if m: | |
46 | path_no_extras = m.group(1) | |
47 | extras = m.group(2) | |
48 | else: | |
49 | path_no_extras = path | |
50 | ||
51 | return path_no_extras, extras | |
52 | ||
53 | ||
54 | def convert_extras(extras: Optional[str]) -> Set[str]: | |
55 | if not extras: | |
56 | return set() | |
57 | return get_requirement("placeholder" + extras.lower()).extras | |
58 | ||
59 | ||
60 | def parse_editable(editable_req: str) -> Tuple[Optional[str], str, Set[str]]: | |
61 | """Parses an editable requirement into: | |
62 | - a requirement name | |
63 | - an URL | |
64 | - extras | |
65 | - editable options | |
66 | Accepted requirements: | |
67 | svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir | |
68 | .[some_extra] | |
69 | """ | |
70 | ||
71 | url = editable_req | |
72 | ||
73 | # If a file path is specified with extras, strip off the extras. | |
74 | url_no_extras, extras = _strip_extras(url) | |
75 | ||
76 | if os.path.isdir(url_no_extras): | |
77 | # Treating it as code that has already been checked out | |
78 | url_no_extras = path_to_url(url_no_extras) | |
79 | ||
80 | if url_no_extras.lower().startswith("file:"): | |
81 | package_name = Link(url_no_extras).egg_fragment | |
82 | if extras: | |
83 | return ( | |
84 | package_name, | |
85 | url_no_extras, | |
86 | get_requirement("placeholder" + extras.lower()).extras, | |
87 | ) | |
88 | else: | |
89 | return package_name, url_no_extras, set() | |
90 | ||
91 | for version_control in vcs: | |
92 | if url.lower().startswith(f"{version_control}:"): | |
93 | url = f"{version_control}+{url}" | |
94 | break | |
95 | ||
96 | link = Link(url) | |
97 | ||
98 | if not link.is_vcs: | |
99 | backends = ", ".join(vcs.all_schemes) | |
100 | raise InstallationError( | |
101 | f"{editable_req} is not a valid editable requirement. " | |
102 | f"It should either be a path to a local project or a VCS URL " | |
103 | f"(beginning with {backends})." | |
104 | ) | |
105 | ||
106 | package_name = link.egg_fragment | |
107 | if not package_name: | |
108 | raise InstallationError( | |
109 | "Could not detect requirement name for '{}', please specify one " | |
110 | "with #egg=your_package_name".format(editable_req) | |
111 | ) | |
112 | return package_name, url, set() | |
113 | ||
114 | ||
115 | def check_first_requirement_in_file(filename: str) -> None: | |
116 | """Check if file is parsable as a requirements file. | |
117 | ||
118 | This is heavily based on ``pkg_resources.parse_requirements``, but | |
119 | simplified to just check the first meaningful line. | |
120 | ||
121 | :raises InvalidRequirement: If the first meaningful line cannot be parsed | |
122 | as an requirement. | |
123 | """ | |
124 | with open(filename, encoding="utf-8", errors="ignore") as f: | |
125 | # Create a steppable iterator, so we can handle \-continuations. | |
126 | lines = ( | |
127 | line | |
128 | for line in (line.strip() for line in f) | |
129 | if line and not line.startswith("#") # Skip blank lines/comments. | |
130 | ) | |
131 | ||
132 | for line in lines: | |
133 | # Drop comments -- a hash without a space may be in a URL. | |
134 | if " #" in line: | |
135 | line = line[: line.find(" #")] | |
136 | # If there is a line continuation, drop it, and append the next line. | |
137 | if line.endswith("\\"): | |
138 | line = line[:-2].strip() + next(lines, "") | |
139 | Requirement(line) | |
140 | return | |
141 | ||
142 | ||
143 | def deduce_helpful_msg(req: str) -> str: | |
144 | """Returns helpful msg in case requirements file does not exist, | |
145 | or cannot be parsed. | |
146 | ||
147 | :params req: Requirements file path | |
148 | """ | |
149 | if not os.path.exists(req): | |
150 | return f" File '{req}' does not exist." | |
151 | msg = " The path does exist. " | |
152 | # Try to parse and check if it is a requirements file. | |
153 | try: | |
154 | check_first_requirement_in_file(req) | |
155 | except InvalidRequirement: | |
156 | logger.debug("Cannot parse '%s' as requirements file", req) | |
157 | else: | |
158 | msg += ( | |
159 | f"The argument you provided " | |
160 | f"({req}) appears to be a" | |
161 | f" requirements file. If that is the" | |
162 | f" case, use the '-r' flag to install" | |
163 | f" the packages specified within it." | |
164 | ) | |
165 | return msg | |
166 | ||
167 | ||
168 | class RequirementParts: | |
169 | def __init__( | |
170 | self, | |
171 | requirement: Optional[Requirement], | |
172 | link: Optional[Link], | |
173 | markers: Optional[Marker], | |
174 | extras: Set[str], | |
175 | ): | |
176 | self.requirement = requirement | |
177 | self.link = link | |
178 | self.markers = markers | |
179 | self.extras = extras | |
180 | ||
181 | ||
182 | def parse_req_from_editable(editable_req: str) -> RequirementParts: | |
183 | name, url, extras_override = parse_editable(editable_req) | |
184 | ||
185 | if name is not None: | |
186 | try: | |
187 | req: Optional[Requirement] = Requirement(name) | |
188 | except InvalidRequirement: | |
189 | raise InstallationError(f"Invalid requirement: '{name}'") | |
190 | else: | |
191 | req = None | |
192 | ||
193 | link = Link(url) | |
194 | ||
195 | return RequirementParts(req, link, None, extras_override) | |
196 | ||
197 | ||
198 | # ---- The actual constructors follow ---- | |
199 | ||
200 | ||
201 | def install_req_from_editable( | |
202 | editable_req: str, | |
203 | comes_from: Optional[Union[InstallRequirement, str]] = None, | |
204 | *, | |
205 | use_pep517: Optional[bool] = None, | |
206 | isolated: bool = False, | |
207 | global_options: Optional[List[str]] = None, | |
208 | hash_options: Optional[Dict[str, List[str]]] = None, | |
209 | constraint: bool = False, | |
210 | user_supplied: bool = False, | |
211 | permit_editable_wheels: bool = False, | |
212 | config_settings: Optional[Dict[str, Union[str, List[str]]]] = None, | |
213 | ) -> InstallRequirement: | |
214 | parts = parse_req_from_editable(editable_req) | |
215 | ||
216 | return InstallRequirement( | |
217 | parts.requirement, | |
218 | comes_from=comes_from, | |
219 | user_supplied=user_supplied, | |
220 | editable=True, | |
221 | permit_editable_wheels=permit_editable_wheels, | |
222 | link=parts.link, | |
223 | constraint=constraint, | |
224 | use_pep517=use_pep517, | |
225 | isolated=isolated, | |
226 | global_options=global_options, | |
227 | hash_options=hash_options, | |
228 | config_settings=config_settings, | |
229 | extras=parts.extras, | |
230 | ) | |
231 | ||
232 | ||
233 | def _looks_like_path(name: str) -> bool: | |
234 | """Checks whether the string "looks like" a path on the filesystem. | |
235 | ||
236 | This does not check whether the target actually exists, only judge from the | |
237 | appearance. | |
238 | ||
239 | Returns true if any of the following conditions is true: | |
240 | * a path separator is found (either os.path.sep or os.path.altsep); | |
241 | * a dot is found (which represents the current directory). | |
242 | """ | |
243 | if os.path.sep in name: | |
244 | return True | |
245 | if os.path.altsep is not None and os.path.altsep in name: | |
246 | return True | |
247 | if name.startswith("."): | |
248 | return True | |
249 | return False | |
250 | ||
251 | ||
252 | def _get_url_from_path(path: str, name: str) -> Optional[str]: | |
253 | """ | |
254 | First, it checks whether a provided path is an installable directory. If it | |
255 | is, returns the path. | |
256 | ||
257 | If false, check if the path is an archive file (such as a .whl). | |
258 | The function checks if the path is a file. If false, if the path has | |
259 | an @, it will treat it as a PEP 440 URL requirement and return the path. | |
260 | """ | |
261 | if _looks_like_path(name) and os.path.isdir(path): | |
262 | if is_installable_dir(path): | |
263 | return path_to_url(path) | |
264 | # TODO: The is_installable_dir test here might not be necessary | |
265 | # now that it is done in load_pyproject_toml too. | |
266 | raise InstallationError( | |
267 | f"Directory {name!r} is not installable. Neither 'setup.py' " | |
268 | "nor 'pyproject.toml' found." | |
269 | ) | |
270 | if not is_archive_file(path): | |
271 | return None | |
272 | if os.path.isfile(path): | |
273 | return path_to_url(path) | |
274 | urlreq_parts = name.split("@", 1) | |
275 | if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]): | |
276 | # If the path contains '@' and the part before it does not look | |
277 | # like a path, try to treat it as a PEP 440 URL req instead. | |
278 | return None | |
279 | logger.warning( | |
280 | "Requirement %r looks like a filename, but the file does not exist", | |
281 | name, | |
282 | ) | |
283 | return path_to_url(path) | |
284 | ||
285 | ||
286 | def parse_req_from_line(name: str, line_source: Optional[str]) -> RequirementParts: | |
287 | if is_url(name): | |
288 | marker_sep = "; " | |
289 | else: | |
290 | marker_sep = ";" | |
291 | if marker_sep in name: | |
292 | name, markers_as_string = name.split(marker_sep, 1) | |
293 | markers_as_string = markers_as_string.strip() | |
294 | if not markers_as_string: | |
295 | markers = None | |
296 | else: | |
297 | markers = Marker(markers_as_string) | |
298 | else: | |
299 | markers = None | |
300 | name = name.strip() | |
301 | req_as_string = None | |
302 | path = os.path.normpath(os.path.abspath(name)) | |
303 | link = None | |
304 | extras_as_string = None | |
305 | ||
306 | if is_url(name): | |
307 | link = Link(name) | |
308 | else: | |
309 | p, extras_as_string = _strip_extras(path) | |
310 | url = _get_url_from_path(p, name) | |
311 | if url is not None: | |
312 | link = Link(url) | |
313 | ||
314 | # it's a local file, dir, or url | |
315 | if link: | |
316 | # Handle relative file URLs | |
317 | if link.scheme == "file" and re.search(r"\.\./", link.url): | |
318 | link = Link(path_to_url(os.path.normpath(os.path.abspath(link.path)))) | |
319 | # wheel file | |
320 | if link.is_wheel: | |
321 | wheel = Wheel(link.filename) # can raise InvalidWheelFilename | |
322 | req_as_string = f"{wheel.name}=={wheel.version}" | |
323 | else: | |
324 | # set the req to the egg fragment. when it's not there, this | |
325 | # will become an 'unnamed' requirement | |
326 | req_as_string = link.egg_fragment | |
327 | ||
328 | # a requirement specifier | |
329 | else: | |
330 | req_as_string = name | |
331 | ||
332 | extras = convert_extras(extras_as_string) | |
333 | ||
334 | def with_source(text: str) -> str: | |
335 | if not line_source: | |
336 | return text | |
337 | return f"{text} (from {line_source})" | |
338 | ||
339 | def _parse_req_string(req_as_string: str) -> Requirement: | |
340 | try: | |
341 | req = get_requirement(req_as_string) | |
342 | except InvalidRequirement: | |
343 | if os.path.sep in req_as_string: | |
344 | add_msg = "It looks like a path." | |
345 | add_msg += deduce_helpful_msg(req_as_string) | |
346 | elif "=" in req_as_string and not any( | |
347 | op in req_as_string for op in operators | |
348 | ): | |
349 | add_msg = "= is not a valid operator. Did you mean == ?" | |
350 | else: | |
351 | add_msg = "" | |
352 | msg = with_source(f"Invalid requirement: {req_as_string!r}") | |
353 | if add_msg: | |
354 | msg += f"\nHint: {add_msg}" | |
355 | raise InstallationError(msg) | |
356 | else: | |
357 | # Deprecate extras after specifiers: "name>=1.0[extras]" | |
358 | # This currently works by accident because _strip_extras() parses | |
359 | # any extras in the end of the string and those are saved in | |
360 | # RequirementParts | |
361 | for spec in req.specifier: | |
362 | spec_str = str(spec) | |
363 | if spec_str.endswith("]"): | |
364 | msg = f"Extras after version '{spec_str}'." | |
365 | raise InstallationError(msg) | |
366 | return req | |
367 | ||
368 | if req_as_string is not None: | |
369 | req: Optional[Requirement] = _parse_req_string(req_as_string) | |
370 | else: | |
371 | req = None | |
372 | ||
373 | return RequirementParts(req, link, markers, extras) | |
374 | ||
375 | ||
376 | def install_req_from_line( | |
377 | name: str, | |
378 | comes_from: Optional[Union[str, InstallRequirement]] = None, | |
379 | *, | |
380 | use_pep517: Optional[bool] = None, | |
381 | isolated: bool = False, | |
382 | global_options: Optional[List[str]] = None, | |
383 | hash_options: Optional[Dict[str, List[str]]] = None, | |
384 | constraint: bool = False, | |
385 | line_source: Optional[str] = None, | |
386 | user_supplied: bool = False, | |
387 | config_settings: Optional[Dict[str, Union[str, List[str]]]] = None, | |
388 | ) -> InstallRequirement: | |
389 | """Creates an InstallRequirement from a name, which might be a | |
390 | requirement, directory containing 'setup.py', filename, or URL. | |
391 | ||
392 | :param line_source: An optional string describing where the line is from, | |
393 | for logging purposes in case of an error. | |
394 | """ | |
395 | parts = parse_req_from_line(name, line_source) | |
396 | ||
397 | return InstallRequirement( | |
398 | parts.requirement, | |
399 | comes_from, | |
400 | link=parts.link, | |
401 | markers=parts.markers, | |
402 | use_pep517=use_pep517, | |
403 | isolated=isolated, | |
404 | global_options=global_options, | |
405 | hash_options=hash_options, | |
406 | config_settings=config_settings, | |
407 | constraint=constraint, | |
408 | extras=parts.extras, | |
409 | user_supplied=user_supplied, | |
410 | ) | |
411 | ||
412 | ||
413 | def install_req_from_req_string( | |
414 | req_string: str, | |
415 | comes_from: Optional[InstallRequirement] = None, | |
416 | isolated: bool = False, | |
417 | use_pep517: Optional[bool] = None, | |
418 | user_supplied: bool = False, | |
419 | ) -> InstallRequirement: | |
420 | try: | |
421 | req = get_requirement(req_string) | |
422 | except InvalidRequirement: | |
423 | raise InstallationError(f"Invalid requirement: '{req_string}'") | |
424 | ||
425 | domains_not_allowed = [ | |
426 | PyPI.file_storage_domain, | |
427 | TestPyPI.file_storage_domain, | |
428 | ] | |
429 | if ( | |
430 | req.url | |
431 | and comes_from | |
432 | and comes_from.link | |
433 | and comes_from.link.netloc in domains_not_allowed | |
434 | ): | |
435 | # Explicitly disallow pypi packages that depend on external urls | |
436 | raise InstallationError( | |
437 | "Packages installed from PyPI cannot depend on packages " | |
438 | "which are not also hosted on PyPI.\n" | |
439 | "{} depends on {} ".format(comes_from.name, req) | |
440 | ) | |
441 | ||
442 | return InstallRequirement( | |
443 | req, | |
444 | comes_from, | |
445 | isolated=isolated, | |
446 | use_pep517=use_pep517, | |
447 | user_supplied=user_supplied, | |
448 | ) | |
449 | ||
450 | ||
451 | def install_req_from_parsed_requirement( | |
452 | parsed_req: ParsedRequirement, | |
453 | isolated: bool = False, | |
454 | use_pep517: Optional[bool] = None, | |
455 | user_supplied: bool = False, | |
456 | config_settings: Optional[Dict[str, Union[str, List[str]]]] = None, | |
457 | ) -> InstallRequirement: | |
458 | if parsed_req.is_editable: | |
459 | req = install_req_from_editable( | |
460 | parsed_req.requirement, | |
461 | comes_from=parsed_req.comes_from, | |
462 | use_pep517=use_pep517, | |
463 | constraint=parsed_req.constraint, | |
464 | isolated=isolated, | |
465 | user_supplied=user_supplied, | |
466 | config_settings=config_settings, | |
467 | ) | |
468 | ||
469 | else: | |
470 | req = install_req_from_line( | |
471 | parsed_req.requirement, | |
472 | comes_from=parsed_req.comes_from, | |
473 | use_pep517=use_pep517, | |
474 | isolated=isolated, | |
475 | global_options=( | |
476 | parsed_req.options.get("global_options", []) | |
477 | if parsed_req.options | |
478 | else [] | |
479 | ), | |
480 | hash_options=( | |
481 | parsed_req.options.get("hashes", {}) if parsed_req.options else {} | |
482 | ), | |
483 | constraint=parsed_req.constraint, | |
484 | line_source=parsed_req.line_source, | |
485 | user_supplied=user_supplied, | |
486 | config_settings=config_settings, | |
487 | ) | |
488 | return req | |
489 | ||
490 | ||
491 | def install_req_from_link_and_ireq( | |
492 | link: Link, ireq: InstallRequirement | |
493 | ) -> InstallRequirement: | |
494 | return InstallRequirement( | |
495 | req=ireq.req, | |
496 | comes_from=ireq.comes_from, | |
497 | editable=ireq.editable, | |
498 | link=link, | |
499 | markers=ireq.markers, | |
500 | use_pep517=ireq.use_pep517, | |
501 | isolated=ireq.isolated, | |
502 | global_options=ireq.global_options, | |
503 | hash_options=ireq.hash_options, | |
504 | config_settings=ireq.config_settings, | |
505 | user_supplied=ireq.user_supplied, | |
506 | ) |