[core] Workaround erroneous urllib Windows proxy parsing (#7092)

Convert proxies extracted from windows registry to http for older Python versions.
See: https://github.com/python/cpython/issues/86793

Authored by: coletdjnz

diff --git a/Makefile b/Makefile
index f03fe205239aa7fa2f85cf3c07cb8e6de6419f62..b1ac0e7d68466c6388dd0437318678cb48e73dcf 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -74,7 +74,7 @@ offlinetest: codetest
        $(PYTHON) -m pytest -k "not download"
 
 # XXX: This is hard to maintain
-CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/utils yt_dlp/dependencies
+CODE_FOLDERS = yt_dlp yt_dlp/downloader yt_dlp/extractor yt_dlp/postprocessor yt_dlp/compat yt_dlp/compat/urllib yt_dlp/utils yt_dlp/dependencies
 yt-dlp: yt_dlp/*.py yt_dlp/*/*.py
        mkdir -p zip
        for d in $(CODE_FOLDERS) ; do \

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index f69bc98c55a9201de3790e584635336c001c31b8..f49dbf07dabe4dd48b8023edb1ed40865631a64c 100644 (file)
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -21,9 +21,9 @@
 import tokenize
 import traceback
 import unicodedata
-import urllib.request
 
 from .cache import Cache
+from .compat import urllib  # isort: split
 from .compat import compat_os_name, compat_shlex_quote
 from .cookies import load_cookies
 from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name

diff --git a/yt_dlp/compat/urllib/__init__.py b/yt_dlp/compat/urllib/__init__.py
new file mode 100644 (file)
index 0000000..6b6b8e1
--- /dev/null
+++ b/yt_dlp/compat/urllib/__init__.py
@@ -0,0 +1,7 @@
+# flake8: noqa: F405
+from urllib import *  # noqa: F403
+
+from ..compat_utils import passthrough_module
+
+passthrough_module(__name__, 'urllib')
+del passthrough_module

diff --git a/yt_dlp/compat/urllib/request.py b/yt_dlp/compat/urllib/request.py
new file mode 100644 (file)
index 0000000..ff63b2f
--- /dev/null
+++ b/yt_dlp/compat/urllib/request.py
@@ -0,0 +1,40 @@
+# flake8: noqa: F405
+from urllib.request import *  # noqa: F403
+
+from ..compat_utils import passthrough_module
+
+passthrough_module(__name__, 'urllib.request')
+del passthrough_module
+
+
+from .. import compat_os_name
+
+if compat_os_name == 'nt':
+    # On older python versions, proxies are extracted from Windows registry erroneously. [1]
+    # If the https proxy in the registry does not have a scheme, urllib will incorrectly add https:// to it. [2]
+    # It is unlikely that the user has actually set it to be https, so we should be fine to safely downgrade
+    # it to http on these older python versions to avoid issues
+    # This also applies for ftp proxy type, as ftp:// proxy scheme is not supported.
+    # 1: https://github.com/python/cpython/issues/86793
+    # 2: https://github.com/python/cpython/blob/51f1ae5ceb0673316c4e4b0175384e892e33cc6e/Lib/urllib/request.py#L2683-L2698
+    import sys
+    from urllib.request import getproxies_environment, getproxies_registry
+
+    def getproxies_registry_patched():
+        proxies = getproxies_registry()
+        if (
+            sys.version_info >= (3, 10, 5)  # https://docs.python.org/3.10/whatsnew/changelog.html#python-3-10-5-final
+            or (3, 9, 13) <= sys.version_info < (3, 10)  # https://docs.python.org/3.9/whatsnew/changelog.html#python-3-9-13-final
+        ):
+            return proxies
+
+        for scheme in ('https', 'ftp'):
+            if scheme in proxies and proxies[scheme].startswith(f'{scheme}://'):
+                proxies[scheme] = 'http' + proxies[scheme][len(scheme):]
+
+        return proxies
+
+    def getproxies():
+        return getproxies_environment() or getproxies_registry_patched()
+
+del compat_os_name