Воркед

This commit is contained in:
kromskii2 committed 2025-03-15 09:36:28 +07:00
1 parent ac32c55ab9
commit ee94d8e529
1138 files changed
+55771 -77901

No files matched your search

@@ -4,8 +4,18 @@ Contains interface (MultiDomainBasicAuth) and associated glue code for
providing credentials in the context of network requests.
"""
import logging
import os
import shutil
import subprocess
import sysconfig
import typing
import urllib.parse
from typing import Any, Dict, List, Optional, Tuple
from abc import ABC, abstractmethod
from functools import lru_cache
from os.path import commonprefix
from pathlib import Path
from typing import Any, Dict, List, NamedTuple, Optional, Tuple
from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
from pip._vendor.requests.models import Request, Response
@@ -23,59 +33,204 @@ from pip._internal.vcs.versioncontrol import AuthInfo
logger = getLogger(__name__)
Credentials = Tuple[str, str, str]
try:
import keyring
except ImportError:
keyring = None # type: ignore[assignment]
except Exception as exc:
logger.warning(
"Keyring is skipped due to an exception: %s",
str(exc),
)
keyring = None # type: ignore[assignment]
KEYRING_DISABLED = False
def get_keyring_auth(url: Optional[str], username: Optional[str]) -> Optional[AuthInfo]:
"""Return the tuple auth for a given url from keyring."""
global keyring
if not url or not keyring:
class Credentials(NamedTuple):
url: str
username: str
password: str
class KeyRingBaseProvider(ABC):
"""Keyring base provider interface"""
has_keyring: bool
@abstractmethod
def get_auth_info(
self, url: str, username: Optional[str]
) -> Optional[AuthInfo]: ...
@abstractmethod
def save_auth_info(self, url: str, username: str, password: str) -> None: ...
class KeyRingNullProvider(KeyRingBaseProvider):
"""Keyring null provider"""
has_keyring = False
def get_auth_info(self, url: str, username: Optional[str]) -> Optional[AuthInfo]:
return None
try:
try:
get_credential = keyring.get_credential
except AttributeError:
pass
else:
def save_auth_info(self, url: str, username: str, password: str) -> None:
return None
class KeyRingPythonProvider(KeyRingBaseProvider):
"""Keyring interface which uses locally imported `keyring`"""
has_keyring = True
def __init__(self) -> None:
import keyring
self.keyring = keyring
def get_auth_info(self, url: str, username: Optional[str]) -> Optional[AuthInfo]:
# Support keyring's get_credential interface which supports getting
# credentials without a username. This is only available for
# keyring>=15.2.0.
if hasattr(self.keyring, "get_credential"):
logger.debug("Getting credentials from keyring for %s", url)
cred = get_credential(url, username)
cred = self.keyring.get_credential(url, username)
if cred is not None:
return cred.username, cred.password
return None
if username:
if username is not None:
logger.debug("Getting password from keyring for %s", url)
password = keyring.get_password(url, username)
password = self.keyring.get_password(url, username)
if password:
return username, password
return None
except Exception as exc:
logger.warning(
"Keyring is skipped due to an exception: %s",
str(exc),
def save_auth_info(self, url: str, username: str, password: str) -> None:
self.keyring.set_password(url, username, password)
class KeyRingCliProvider(KeyRingBaseProvider):
"""Provider which uses `keyring` cli
Instead of calling the keyring package installed alongside pip
we call keyring on the command line which will enable pip to
use which ever installation of keyring is available first in
PATH.
"""
has_keyring = True
def __init__(self, cmd: str) -> None:
self.keyring = cmd
def get_auth_info(self, url: str, username: Optional[str]) -> Optional[AuthInfo]:
# This is the default implementation of keyring.get_credential
# https://github.com/jaraco/keyring/blob/97689324abcf01bd1793d49063e7ca01e03d7d07/keyring/backend.py#L134-L139
if username is not None:
password = self._get_password(url, username)
if password is not None:
return username, password
return None
def save_auth_info(self, url: str, username: str, password: str) -> None:
return self._set_password(url, username, password)
def _get_password(self, service_name: str, username: str) -> Optional[str]:
"""Mirror the implementation of keyring.get_password using cli"""
if self.keyring is None:
return None
cmd = [self.keyring, "get", service_name, username]
env = os.environ.copy()
env["PYTHONIOENCODING"] = "utf-8"
res = subprocess.run(
cmd,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
env=env,
)
keyring = None # type: ignore[assignment]
return None
if res.returncode:
return None
return res.stdout.decode("utf-8").strip(os.linesep)
def _set_password(self, service_name: str, username: str, password: str) -> None:
"""Mirror the implementation of keyring.set_password using cli"""
if self.keyring is None:
return None
env = os.environ.copy()
env["PYTHONIOENCODING"] = "utf-8"
subprocess.run(
[self.keyring, "set", service_name, username],
input=f"{password}{os.linesep}".encode(),
env=env,
check=True,
)
return None
@lru_cache(maxsize=None)
def get_keyring_provider(provider: str) -> KeyRingBaseProvider:
logger.verbose("Keyring provider requested: %s", provider)
# keyring has previously failed and been disabled
if KEYRING_DISABLED:
provider = "disabled"
if provider in ["import", "auto"]:
try:
impl = KeyRingPythonProvider()
logger.verbose("Keyring provider set: import")
return impl
except ImportError:
pass
except Exception as exc:
# In the event of an unexpected exception
# we should warn the user
msg = "Installed copy of keyring fails with exception %s"
if provider == "auto":
msg = msg + ", trying to find a keyring executable as a fallback"
logger.warning(msg, exc, exc_info=logger.isEnabledFor(logging.DEBUG))
if provider in ["subprocess", "auto"]:
cli = shutil.which("keyring")
if cli and cli.startswith(sysconfig.get_path("scripts")):
# all code within this function is stolen from shutil.which implementation
@typing.no_type_check
def PATH_as_shutil_which_determines_it() -> str:
path = os.environ.get("PATH", None)
if path is None:
try:
path = os.confstr("CS_PATH")
except (AttributeError, ValueError):
# os.confstr() or CS_PATH is not available
path = os.defpath
# bpo-35755: Don't use os.defpath if the PATH environment variable is
# set to an empty string
return path
scripts = Path(sysconfig.get_path("scripts"))
paths = []
for path in PATH_as_shutil_which_determines_it().split(os.pathsep):
p = Path(path)
try:
if not p.samefile(scripts):
paths.append(path)
except FileNotFoundError:
pass
path = os.pathsep.join(paths)
cli = shutil.which("keyring", path=path)
if cli:
logger.verbose("Keyring provider set: subprocess with executable %s", cli)
return KeyRingCliProvider(cli)
logger.verbose("Keyring provider set: disabled")
return KeyRingNullProvider()
class MultiDomainBasicAuth(AuthBase):
def __init__(
self, prompting: bool = True, index_urls: Optional[List[str]] = None
self,
prompting: bool = True,
index_urls: Optional[List[str]] = None,
keyring_provider: str = "auto",
) -> None:
self.prompting = prompting
self.index_urls = index_urls
self.keyring_provider = keyring_provider # type: ignore[assignment]
self.passwords: Dict[str, AuthInfo] = {}
# When the user is prompted to enter credentials and keyring is
# available, we will offer to save them. If the user accepts,
@@ -84,6 +239,51 @@ class MultiDomainBasicAuth(AuthBase):
# ``save_credentials`` to save these.
self._credentials_to_save: Optional[Credentials] = None
@property
def keyring_provider(self) -> KeyRingBaseProvider:
return get_keyring_provider(self._keyring_provider)
@keyring_provider.setter
def keyring_provider(self, provider: str) -> None:
# The free function get_keyring_provider has been decorated with
# functools.cache. If an exception occurs in get_keyring_auth that
# cache will be cleared and keyring disabled, take that into account
# if you want to remove this indirection.
self._keyring_provider = provider
@property
def use_keyring(self) -> bool:
# We won't use keyring when --no-input is passed unless
# a specific provider is requested because it might require
# user interaction
return self.prompting or self._keyring_provider not in ["auto", "disabled"]
def _get_keyring_auth(
self,
url: Optional[str],
username: Optional[str],
) -> Optional[AuthInfo]:
"""Return the tuple auth for a given url from keyring."""
# Do nothing if no url was provided
if not url:
return None
try:
return self.keyring_provider.get_auth_info(url, username)
except Exception as exc:
# Log the full exception (with stacktrace) at debug, so it'll only
# show up when running in verbose mode.
logger.debug("Keyring is skipped due to an exception", exc_info=True)
# Always log a shortened version of the exception.
logger.warning(
"Keyring is skipped due to an exception: %s",
str(exc),
)
global KEYRING_DISABLED
KEYRING_DISABLED = True
get_keyring_provider.cache_clear()
return None
def _get_index_url(self, url: str) -> Optional[str]:
"""Return the original index URL matching the requested URL.
@@ -100,15 +300,42 @@ class MultiDomainBasicAuth(AuthBase):
if not url or not self.index_urls:
return None
for u in self.index_urls:
prefix = remove_auth_from_url(u).rstrip("/") + "/"
if url.startswith(prefix):
return u
return None
url = remove_auth_from_url(url).rstrip("/") + "/"
parsed_url = urllib.parse.urlsplit(url)
candidates = []
for index in self.index_urls:
index = index.rstrip("/") + "/"
parsed_index = urllib.parse.urlsplit(remove_auth_from_url(index))
if parsed_url == parsed_index:
return index
if parsed_url.netloc != parsed_index.netloc:
continue
candidate = urllib.parse.urlsplit(index)
candidates.append(candidate)
if not candidates:
return None
candidates.sort(
reverse=True,
key=lambda candidate: commonprefix(
[
parsed_url.path,
candidate.path,
]
).rfind("/"),
)
return urllib.parse.urlunsplit(candidates[0])
def _get_new_credentials(
self,
original_url: str,
*,
allow_netrc: bool = True,
allow_keyring: bool = False,
) -> AuthInfo:
@@ -152,8 +379,8 @@ class MultiDomainBasicAuth(AuthBase):
# The index url is more specific than the netloc, so try it first
# fmt: off
kr_auth = (
get_keyring_auth(index_url, username) or
get_keyring_auth(netloc, username)
self._get_keyring_auth(index_url, username) or
self._get_keyring_auth(netloc, username)
)
# fmt: on
if kr_auth:
@@ -230,18 +457,23 @@ class MultiDomainBasicAuth(AuthBase):
def _prompt_for_password(
self, netloc: str
) -> Tuple[Optional[str], Optional[str], bool]:
username = ask_input(f"User for {netloc}: ")
username = ask_input(f"User for {netloc}: ") if self.prompting else None
if not username:
return None, None, False
auth = get_keyring_auth(netloc, username)
if auth and auth[0] is not None and auth[1] is not None:
return auth[0], auth[1], False
if self.use_keyring:
auth = self._get_keyring_auth(netloc, username)
if auth and auth[0] is not None and auth[1] is not None:
return auth[0], auth[1], False
password = ask_password("Password: ")
return username, password, True
# Factored out to allow for easy patching in tests
def _should_save_password_to_keyring(self) -> bool:
if not keyring:
if (
not self.prompting
or not self.use_keyring
or not self.keyring_provider.has_keyring
):
return False
return ask("Save credentials to keyring [y/N]: ", ["y", "n"]) == "y"
@@ -251,19 +483,22 @@ class MultiDomainBasicAuth(AuthBase):
if resp.status_code != 401:
return resp
username, password = None, None
# Query the keyring for credentials:
if self.use_keyring:
username, password = self._get_new_credentials(
resp.url,
allow_netrc=False,
allow_keyring=True,
)
# We are not able to prompt the user so simply return the response
if not self.prompting:
if not self.prompting and not username and not password:
return resp
parsed = urllib.parse.urlparse(resp.url)
# Query the keyring for credentials:
username, password = self._get_new_credentials(
resp.url,
allow_netrc=False,
allow_keyring=True,
)
# Prompt the user for a new username and password
save = False
if not username and not password:
@@ -276,11 +511,17 @@ class MultiDomainBasicAuth(AuthBase):
# Prompt to save the password to keyring
if save and self._should_save_password_to_keyring():
self._credentials_to_save = (parsed.netloc, username, password)
self._credentials_to_save = Credentials(
url=parsed.netloc,
username=username,
password=password,
)
# Consume content and release the original connection to allow our new
# request to reuse the same one.
resp.content
# The result of the assignment isn't used, it's just needed to consume
# the content.
_ = resp.content
resp.raw.release_conn()
# Add our new username and password to the request
@@ -309,15 +550,17 @@ class MultiDomainBasicAuth(AuthBase):
def save_credentials(self, resp: Response, **kwargs: Any) -> None:
"""Response callback to save credentials on success."""
assert keyring is not None, "should never reach here without keyring"
if not keyring:
return
assert (
self.keyring_provider.has_keyring
), "should never reach here without keyring"
creds = self._credentials_to_save
self._credentials_to_save = None
if creds and resp.status_code < 400:
try:
logger.info("Saving credentials to keyring")
keyring.set_password(*creds)
self.keyring_provider.save_auth_info(
creds.url, creds.username, creds.password
)
except Exception:
logger.exception("Failed to save credentials")
@@ -3,10 +3,11 @@
import os
from contextlib import contextmanager
from typing import Generator, Optional
from datetime import datetime
from typing import BinaryIO, Generator, Optional, Union
from pip._vendor.cachecontrol.cache import BaseCache
from pip._vendor.cachecontrol.caches import FileCache
from pip._vendor.cachecontrol.cache import SeparateBodyBaseCache
from pip._vendor.cachecontrol.caches import SeparateBodyFileCache
from pip._vendor.requests.models import Response
from pip._internal.utils.filesystem import adjacent_tmp_file, replace
@@ -28,10 +29,22 @@ def suppressed_cache_errors() -> Generator[None, None, None]:
pass
class SafeFileCache(BaseCache):
class SafeFileCache(SeparateBodyBaseCache):
"""
A file based cache which is safe to use even when the target directory may
not be accessible or writable.
There is a race condition when two processes try to write and/or read the
same entry at the same time, since each entry consists of two separate
files (https://github.com/psf/cachecontrol/issues/324). We therefore have
additional logic that makes sure that both files to be present before
returning an entry; this fixes the read side of the race condition.
For the write side, we assume that the server will only ever return the
same data for the same URL, which ought to be the case for files pip is
downloading. PyPI does not have a mechanism to swap out a wheel for
another wheel, for example. If this assumption is not true, the
CacheControl issue will need to be fixed.
"""
def __init__(self, directory: str) -> None:
@@ -43,27 +56,63 @@ class SafeFileCache(BaseCache):
# From cachecontrol.caches.file_cache.FileCache._fn, brought into our
# class for backwards-compatibility and to avoid using a non-public
# method.
hashed = FileCache.encode(name)
hashed = SeparateBodyFileCache.encode(name)
parts = list(hashed[:5]) + [hashed]
return os.path.join(self.directory, *parts)
def get(self, key: str) -> Optional[bytes]:
path = self._get_cache_path(key)
# The cache entry is only valid if both metadata and body exist.
metadata_path = self._get_cache_path(key)
body_path = metadata_path + ".body"
if not (os.path.exists(metadata_path) and os.path.exists(body_path)):
return None
with suppressed_cache_errors():
with open(path, "rb") as f:
with open(metadata_path, "rb") as f:
return f.read()
def set(self, key: str, value: bytes, expires: Optional[int] = None) -> None:
path = self._get_cache_path(key)
def _write(self, path: str, data: bytes) -> None:
with suppressed_cache_errors():
ensure_dir(os.path.dirname(path))
with adjacent_tmp_file(path) as f:
f.write(value)
f.write(data)
# Inherit the read/write permissions of the cache directory
# to enable multi-user cache use-cases.
mode = (
os.stat(self.directory).st_mode
& 0o666 # select read/write permissions of cache directory
| 0o600 # set owner read/write permissions
)
# Change permissions only if there is no risk of following a symlink.
if os.chmod in os.supports_fd:
os.chmod(f.fileno(), mode)
elif os.chmod in os.supports_follow_symlinks:
os.chmod(f.name, mode, follow_symlinks=False)
replace(f.name, path)
def set(
self, key: str, value: bytes, expires: Union[int, datetime, None] = None
) -> None:
path = self._get_cache_path(key)
self._write(path, value)
def delete(self, key: str) -> None:
path = self._get_cache_path(key)
with suppressed_cache_errors():
os.remove(path)
with suppressed_cache_errors():
os.remove(path + ".body")
def get_body(self, key: str) -> Optional[BinaryIO]:
# The cache entry is only valid if both metadata and body exist.
metadata_path = self._get_cache_path(key)
body_path = metadata_path + ".body"
if not (os.path.exists(metadata_path) and os.path.exists(body_path)):
return None
with suppressed_cache_errors():
return open(body_path, "rb")
def set_body(self, key: str, body: bytes) -> None:
path = self._get_cache_path(key) + ".body"
self._write(path, body)
@@ -1,12 +1,13 @@
"""Download files with progress indicators.
"""
import email.message
import logging
import mimetypes
import os
from typing import Iterable, Optional, Tuple
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
from pip._vendor.requests.models import Response
from pip._internal.cli.progress_bars import get_download_progress_renderer
from pip._internal.exceptions import NetworkConnectionError
@@ -42,7 +43,7 @@ def _prepare_download(
logged_url = redact_auth_from_url(url)
if total_length:
logged_url = "{} ({})".format(logged_url, format_size(total_length))
logged_url = f"{logged_url} ({format_size(total_length)})"
if is_from_cache(resp):
logger.info("Using cached %s", logged_url)
@@ -55,12 +56,12 @@ def _prepare_download(
show_progress = False
elif not total_length:
show_progress = True
elif total_length > (40 * 1000):
elif total_length > (512 * 1024):
show_progress = True
else:
show_progress = False
chunks = response_chunks(resp, CONTENT_CHUNK_SIZE)
chunks = response_chunks(resp)
if not show_progress:
return chunks
@@ -6,7 +6,7 @@ from bisect import bisect_left, bisect_right
from contextlib import contextmanager
from tempfile import NamedTemporaryFile
from typing import Any, Dict, Generator, List, Optional, Tuple
from zipfile import BadZipfile, ZipFile
from zipfile import BadZipFile, ZipFile
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
@@ -23,7 +23,7 @@ class HTTPRangeRequestUnsupported(Exception):
def dist_from_wheel_url(name: str, url: str, session: PipSession) -> BaseDistribution:
"""Return a distribution object from the given wheel URL.
This uses HTTP range requests to only fetch the potion of the wheel
This uses HTTP range requests to only fetch the portion of the wheel
containing metadata, just enough for the object to be constructed.
If such requests are not supported, HTTPRangeRequestUnsupported
is raised.
@@ -159,8 +159,8 @@ class LazyZipOverHTTP:
try:
# For read-only ZIP files, ZipFile only needs
# methods read, seek, seekable and tell.
ZipFile(self) # type: ignore
except BadZipfile:
ZipFile(self)
except BadZipFile:
pass
else:
break
@@ -3,6 +3,7 @@ network request configuration and behavior.
"""
import email.utils
import functools
import io
import ipaddress
import json
@@ -106,6 +107,7 @@ def looks_like_ci() -> bool:
return any(name in os.environ for name in CI_ENVIRONMENT_VARIABLES)
@functools.lru_cache(maxsize=1)
def user_agent() -> str:
"""
Return a string representing the user agent.
@@ -230,7 +232,7 @@ class LocalFSAdapter(BaseAdapter):
# to return a better error message:
resp.status_code = 404
resp.reason = type(exc).__name__
resp.raw = io.BytesIO(f"{resp.reason}: {exc}".encode("utf8"))
resp.raw = io.BytesIO(f"{resp.reason}: {exc}".encode())
else:
modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
content_type = mimetypes.guess_type(pathname)[0] or "text/plain"
@@ -316,7 +318,6 @@ class InsecureCacheControlAdapter(CacheControlAdapter):
class PipSession(requests.Session):
timeout: Optional[int] = None
def __init__(
@@ -338,6 +339,7 @@ class PipSession(requests.Session):
# Namespace the attribute with "pip_" just in case to prevent
# possible conflicts with the base class.
self.pip_trusted_origins: List[Tuple[str, Optional[int]]] = []
self.pip_proxy = None
# Attach our User Agent to the request
self.headers["User-Agent"] = user_agent()
@@ -356,8 +358,9 @@ class PipSession(requests.Session):
# is typically considered a transient error so we'll go ahead and
# retry it.
# A 500 may indicate transient error in Amazon S3
# A 502 may be a transient error from a CDN like CloudFlare or CloudFront
# A 520 or 527 - may indicate transient error in CloudFlare
status_forcelist=[500, 503, 520, 527],
status_forcelist=[500, 502, 503, 520, 527],
# Add a small amount of back off between failed requests in
# order to prevent hammering the service.
backoff_factor=0.25,
@@ -420,15 +423,17 @@ class PipSession(requests.Session):
msg += f" (from {source})"
logger.info(msg)
host_port = parse_netloc(host)
if host_port not in self.pip_trusted_origins:
self.pip_trusted_origins.append(host_port)
parsed_host, parsed_port = parse_netloc(host)
if parsed_host is None:
raise ValueError(f"Trusted host URL must include a host part: {host!r}")
if (parsed_host, parsed_port) not in self.pip_trusted_origins:
self.pip_trusted_origins.append((parsed_host, parsed_port))
self.mount(
build_url_from_netloc(host, scheme="http") + "/", self._trusted_host_adapter
)
self.mount(build_url_from_netloc(host) + "/", self._trusted_host_adapter)
if not host_port[1]:
if not parsed_port:
self.mount(
build_url_from_netloc(host, scheme="http") + ":",
self._trusted_host_adapter,
@@ -1,6 +1,6 @@
from typing import Dict, Generator
from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
from pip._vendor.requests.models import Response
from pip._internal.exceptions import NetworkConnectionError
@@ -25,6 +25,8 @@ from pip._internal.exceptions import NetworkConnectionError
# possible to make this work.
HEADERS: Dict[str, str] = {"Accept-Encoding": "identity"}
DOWNLOAD_CHUNK_SIZE = 256 * 1024
def raise_for_status(resp: Response) -> None:
http_error_msg = ""
@@ -55,7 +57,7 @@ def raise_for_status(resp: Response) -> None:
def response_chunks(
response: Response, chunk_size: int = CONTENT_CHUNK_SIZE
response: Response, chunk_size: int = DOWNLOAD_CHUNK_SIZE
) -> Generator[bytes, None, None]:
"""Given a requests Response, provide the data chunks."""
try:
@@ -13,6 +13,8 @@ from pip._internal.network.utils import raise_for_status
if TYPE_CHECKING:
from xmlrpc.client import _HostType, _Marshallable
from _typeshed import SizedBuffer
logger = logging.getLogger(__name__)
@@ -33,7 +35,7 @@ class PipXmlrpcTransport(xmlrpc.client.Transport):
self,
host: "_HostType",
handler: str,
request_body: bytes,
request_body: "SizedBuffer",
verbose: bool = False,
) -> Tuple["_Marshallable", ...]:
assert isinstance(host, str)