Add option `--sleep-requests` to sleep b/w requests (Closes #106)

* Also fix documentation of `sleep_interval_subtitles`

Related issues:
https://github.com/blackjack4494/yt-dlc/issues/158
https://github.com/blackjack4494/youtube-dlc/issues/195
https://github.com/ytdl-org/youtube-dl/pull/28270
https://github.com/ytdl-org/youtube-dl/pull/28144
https://github.com/ytdl-org/youtube-dl/issues/27767
https://github.com/ytdl-org/youtube-dl/issues/23638
https://github.com/ytdl-org/youtube-dl/issues/26287
https://github.com/ytdl-org/youtube-dl/issues/26319

diff --git a/README.md b/README.md
index 69813c2b75e89b549bfb8eef4368d8c5957ec414..8def97e4c1202ea945493ca88f6b68d0084ee2e4 100644 (file)
--- a/README.md
+++ b/README.md
@@ -491,6 +491,8 @@ ## Workarounds:
     --bidi-workaround                Work around terminals that lack
                                      bidirectional text support. Requires bidiv
                                      or fribidi executable in PATH
+    --sleep-requests SECONDS         Number of seconds to sleep between requests
+                                     during data extraction
     --sleep-interval SECONDS         Number of seconds to sleep before each
                                      download when used alone or a lower bound
                                      of a range for randomized sleep before each
@@ -501,7 +503,8 @@ ## Workarounds:
                                      before each download (maximum possible
                                      number of seconds to sleep). Must only be
                                      used along with --min-sleep-interval
-    --sleep-subtitles SECONDS        Enforce sleep interval on subtitles as well
+    --sleep-subtitles SECONDS        Number of seconds to sleep before each
+                                     subtitle download
 
 ## Video Format Options:
     -f, --format FORMAT              Video format code, see "FORMAT SELECTION"

diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index a54755e158383d7248280bbe680337e0be56e8f2..3c53f4cd88a3edeb97b9ba8448822fcbece4f0b8 100644 (file)
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -324,6 +324,8 @@ class YoutubeDL(object):
     source_address:    Client-side IP address to bind to.
     call_home:         Boolean, true iff we are allowed to contact the
                        yt-dlp servers for debugging. (BROKEN)
+    sleep_interval_requests: Number of seconds to sleep between requests
+                       during extraction
     sleep_interval:    Number of seconds to sleep before each download when
                        used alone or a lower bound of a range for randomized
                        sleep before each download (minimum possible number
@@ -334,6 +336,7 @@ class YoutubeDL(object):
                        Must only be used along with sleep_interval.
                        Actual sleep time will be a random float from range
                        [sleep_interval; max_sleep_interval].
+    sleep_interval_subtitles: Number of seconds to sleep before each subtitle download
     listformats:       Print an overview of available video formats and exit.
     list_thumbnails:   Print a table of all thumbnails and exit.
     match_filter:      A function that gets called with the info_dict of
@@ -406,6 +409,7 @@ class YoutubeDL(object):
     _ies = []
     _pps = {'beforedl': [], 'aftermove': [], 'normal': []}
     __prepare_filename_warned = False
+    _first_webpage_request = True
     _download_retcode = None
     _num_downloads = None
     _playlist_level = 0
@@ -420,6 +424,7 @@ def __init__(self, params=None, auto_init=True):
         self._ies_instances = {}
         self._pps = {'beforedl': [], 'aftermove': [], 'normal': []}
         self.__prepare_filename_warned = False
+        self._first_webpage_request = True
         self._post_hooks = []
         self._progress_hooks = []
         self._download_retcode = 0
@@ -2166,15 +2171,6 @@ def dl(name, info, subtitle=False):
                     else:
                         try:
                             dl(sub_filename, sub_info, subtitle=True)
-                            '''
-                            if self.params.get('sleep_interval_subtitles', False):
-                                dl(sub_filename, sub_info)
-                            else:
-                                sub_data = ie._request_webpage(
-                                    sub_info['url'], info_dict['id'], note=False).read()
-                                with io.open(encodeFilename(sub_filename), 'wb') as subfile:
-                                    subfile.write(sub_data)
-                            '''
                             files_to_move[sub_filename] = sub_filename_final
                         except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                             self.report_warning('Unable to download subtitle for "%s": %s' %

diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 91539daaf8e3a70be1f4397471460f32e208651e..15605ab31ee522ac7c82c5fba86648aa0aea1f23 100644 (file)
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -169,6 +169,12 @@ def _real_main(argv=None):
             parser.error('max sleep interval must be greater than or equal to min sleep interval')
     else:
         opts.max_sleep_interval = opts.sleep_interval
+    if opts.sleep_interval_subtitles is not None:
+        if opts.sleep_interval_subtitles < 0:
+            parser.error('subtitles sleep interval must be positive or 0')
+    if opts.sleep_interval_requests is not None:
+        if opts.sleep_interval_requests < 0:
+            parser.error('requests sleep interval must be positive or 0')
     if opts.ap_mso and opts.ap_mso not in MSO_INFO:
         parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers')
     if opts.overwrites:
@@ -524,6 +530,7 @@ def report_args_compat(arg, name):
         'fixup': opts.fixup,
         'source_address': opts.source_address,
         'call_home': opts.call_home,
+        'sleep_interval_requests': opts.sleep_interval_requests,
         'sleep_interval': opts.sleep_interval,
         'max_sleep_interval': opts.max_sleep_interval,
         'sleep_interval_subtitles': opts.sleep_interval_subtitles,

diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 677a61b8679e21c9cd567c852da5ef0afe9bb154..47b91a00a7a6e0c18c434ce8dcf0c8ca48785055 100644 (file)
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -606,6 +606,14 @@ def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fa
 
         See _download_webpage docstring for arguments specification.
         """
+        if not self._downloader._first_webpage_request:
+            sleep_interval = float_or_none(self._downloader.params.get('sleep_interval_requests')) or 0
+            if sleep_interval > 0:
+                self.to_screen('Sleeping %s seconds...' % sleep_interval)
+                time.sleep(sleep_interval)
+        else:
+            self._downloader._first_webpage_request = False
+
         if note is None:
             self.report_download_webpage(video_id)
         elif note is not False:

diff --git a/yt_dlp/options.py b/yt_dlp/options.py
index 5f200fb8f7e39e151912d53976c40b64d6524883..ae11e6b8bc5ecbda35d4a4916b0cf20e47f2ce60 100644 (file)
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@@ -696,6 +696,10 @@ def _dict_from_multiple_values_options_callback(
         '--bidi-workaround',
         dest='bidi_workaround', action='store_true',
         help='Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
+    workarounds.add_option(
+        '--sleep-requests', metavar='SECONDS',
+        dest='sleep_interval_requests', type=float,
+        help='Number of seconds to sleep between requests during data extraction')
     workarounds.add_option(
         '--sleep-interval', '--min-sleep-interval', metavar='SECONDS',
         dest='sleep_interval', type=float,
@@ -714,7 +718,7 @@ def _dict_from_multiple_values_options_callback(
     workarounds.add_option(
         '--sleep-subtitles', metavar='SECONDS',
         dest='sleep_interval_subtitles', default=0, type=int,
-        help='Enforce sleep interval on subtitles as well')
+        help='Number of seconds to sleep before each subtitle download')
 
     verbosity = optparse.OptionGroup(parser, 'Verbosity and Simulation Options')
     verbosity.add_option(