[downloader/http] Avoid infinite loop when no data is received

[yt-dlp.git] / yt_dlp / downloader / http.py

diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py
index c6b6627a56723baf8d35f840a68c8efea72780f8..7c5daea8592339771965bda83fa6b6193b59207c 100644 (file)
--- a/yt_dlp/downloader/http.py
+++ b/yt_dlp/downloader/http.py
@@ -1,13 +1,15 @@
+import http.client
 import os
 import random
 import socket
 import ssl
 import time
+import urllib.error
 
 from .common import FileDownloader
-from ..compat import compat_http_client, compat_urllib_error
 from ..utils import (
     ContentTooShortError,
+    RetryManager,
     ThrottledDownload,
     XAttrMetadataError,
     XAttrUnavailableError,
@@ -24,7 +26,7 @@
     socket.timeout,  # compat: py < 3.10
     ConnectionError,
     ssl.SSLError,
-    compat_http_client.HTTPException
+    http.client.HTTPException
 )
 
 
@@ -43,8 +45,8 @@ class DownloadContext(dict):
         ctx.tmpfilename = self.temp_name(filename)
         ctx.stream = None
 
-        # Do not include the Accept-Encoding header
-        headers = {'Youtubedl-no-compression': 'True'}
+        # Disable compression
+        headers = {'Accept-Encoding': 'identity'}
         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)
@@ -71,9 +73,6 @@ class DownloadContext(dict):
 
         ctx.is_resume = ctx.resume_len > 0
 
-        count = 0
-        retries = self.params.get('retries', 0)
-
         class SucceedDownload(Exception):
             pass
 
@@ -151,11 +150,12 @@ def establish_connection():
                     # Content-Range is either not present or invalid. Assuming remote webserver is
                     # trying to send the whole file, resume is not possible, so wiping the local file
                     # and performing entire redownload
-                    self.report_unable_to_resume()
+                    elif range_start > 0:
+                        self.report_unable_to_resume()
                     ctx.resume_len = 0
                     ctx.open_mode = 'wb'
                 ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))
-            except compat_urllib_error.HTTPError as err:
+            except urllib.error.HTTPError as err:
                 if err.code == 416:
                     # Unable to resume (requested range not satisfiable)
                     try:
@@ -163,7 +163,7 @@ def establish_connection():
                         ctx.data = self.ydl.urlopen(
                             sanitized_Request(url, request_data, headers))
                         content_length = ctx.data.info()['Content-Length']
-                    except compat_urllib_error.HTTPError as err:
+                    except urllib.error.HTTPError as err:
                         if err.code < 500 or err.code >= 600:
                             raise
                     else:
@@ -196,7 +196,7 @@ def establish_connection():
                     # Unexpected HTTP error
                     raise
                 raise RetryDownload(err)
-            except compat_urllib_error.URLError as err:
+            except urllib.error.URLError as err:
                 if isinstance(err.reason, ssl.CertificateError):
                     raise
                 raise RetryDownload(err)
@@ -205,8 +205,19 @@ def establish_connection():
             except RESPONSE_READ_EXCEPTIONS as err:
                 raise RetryDownload(err)
 
+        def close_stream():
+            if ctx.stream is not None:
+                if not ctx.tmpfilename == '-':
+                    ctx.stream.close()
+                ctx.stream = None
+
         def download():
-            data_len = ctx.data.info().get('Content-length', None)
+            data_len = ctx.data.info().get('Content-length')
+
+            if ctx.data.info().get('Content-encoding'):
+                # Content-encoding is present, Content-length is not reliable anymore as we are
+                # doing auto decompression. (See: https://github.com/yt-dlp/yt-dlp/pull/6176)
+                data_len = None
 
             # Range HTTP header may be ignored/unsupported by a webserver
             # (e.g. extractor/scivee.py, extractor/bambuser.py).
@@ -238,12 +249,9 @@ def download():
             before = start  # start measuring
 
             def retry(e):
-                to_stdout = ctx.tmpfilename == '-'
-                if ctx.stream is not None:
-                    if not to_stdout:
-                        ctx.stream.close()
-                    ctx.stream = None
-                ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename))
+                close_stream()
+                ctx.resume_len = (byte_counter if ctx.tmpfilename == '-'
+                                  else os.path.getsize(encodeFilename(ctx.tmpfilename)))
                 raise RetryDownload(e)
 
             while True:
@@ -331,23 +339,21 @@ def retry(e):
                 elif speed:
                     ctx.throttle_start = None
 
-            if not is_test and ctx.chunk_size and ctx.content_len is not None and byte_counter < ctx.content_len:
-                ctx.resume_len = byte_counter
-                # ctx.block_size = block_size
-                raise NextFragment()
-
             if ctx.stream is None:
                 self.to_stderr('\n')
                 self.report_error('Did not get any data blocks')
                 return False
+
+            if not is_test and ctx.chunk_size and ctx.content_len is not None and byte_counter < ctx.content_len:
+                ctx.resume_len = byte_counter
+                raise NextFragment()
+
             if ctx.tmpfilename != '-':
                 ctx.stream.close()
 
             if data_len is not None and byte_counter != data_len:
                 err = ContentTooShortError(byte_counter, int(data_len))
-                if count <= retries:
-                    retry(err)
-                raise err
+                retry(err)
 
             self.try_rename(ctx.tmpfilename, ctx.filename)
 
@@ -366,21 +372,20 @@ def retry(e):
 
             return True
 
-        while count <= retries:
+        for retry in RetryManager(self.params.get('retries'), self.report_retry):
             try:
                 establish_connection()
                 return download()
-            except RetryDownload as e:
-                count += 1
-                if count <= retries:
-                    self.report_retry(e.source_error, count, retries)
-                else:
-                    self.to_screen(f'[download] Got server HTTP error: {e.source_error}')
+            except RetryDownload as err:
+                retry.error = err.source_error
                 continue
             except NextFragment:
+                retry.error = None
+                retry.attempt -= 1
                 continue
             except SucceedDownload:
                 return True
-
-        self.report_error('giving up after %s retries' % retries)
+            except:  # noqa: E722
+                close_stream()
+                raise
         return False