[misc] Add `hatch`, `ruff`, `pre-commit` and improve dev docs (#7409)

[yt-dlp.git] / yt_dlp / extractor / itv.py

diff --git a/yt_dlp/extractor/itv.py b/yt_dlp/extractor/itv.py
index d69782b78240fc4d8df3b84a656f7b4b4e894c1a..55c4165215c1fc6a445f28b2bac2f7b8b535e20d 100644 (file)
--- a/yt_dlp/extractor/itv.py
+++ b/yt_dlp/extractor/itv.py
@@ -1,26 +1,22 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
 import json
 
-from .common import InfoExtractor
 from .brightcove import BrightcoveNewIE
-
+from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
+    JSON_LD_RE,
+    ExtractorError,
     base_url,
     clean_html,
     determine_ext,
     extract_attributes,
-    ExtractorError,
     get_element_by_class,
-    JSON_LD_RE,
     merge_dicts,
     parse_duration,
     smuggle_url,
     try_get,
-    url_or_none,
     url_basename,
+    url_or_none,
     urljoin,
 )
 
@@ -117,7 +113,7 @@ def _get_subtitles(self, video_id, variants, ios_playlist_url, headers, *args, *
         # See: https://github.com/yt-dlp/yt-dlp/issues/986
         platform_tag_subs, featureset_subs = next(
             ((platform_tag, featureset)
-             for platform_tag, featuresets in reversed(variants.items()) for featureset in featuresets
+             for platform_tag, featuresets in reversed(list(variants.items())) for featureset in featuresets
              if try_get(featureset, lambda x: x[2]) == 'outband-webvtt'),
             (None, None))
 
@@ -146,8 +142,8 @@ def _real_extract(self, url):
         # See: https://github.com/yt-dlp/yt-dlp/issues/986
         platform_tag_video, featureset_video = next(
             ((platform_tag, featureset)
-             for platform_tag, featuresets in reversed(variants.items()) for featureset in featuresets
-             if try_get(featureset, lambda x: x[:2]) == ['hls', 'aes']),
+             for platform_tag, featuresets in reversed(list(variants.items())) for featureset in featuresets
+             if set(try_get(featureset, lambda x: x[:2]) or []) == {'aes', 'hls'}),
             (None, None))
         if not platform_tag_video or not featureset_video:
             raise ExtractorError('No downloads available', expected=True, video_id=video_id)
@@ -175,7 +171,6 @@ def _real_extract(self, url):
                 formats.append({
                     'url': href,
                 })
-        self._sort_formats(formats)
         info = self._search_json_ld(webpage, video_id, default={})
         if not info:
             json_ld = self._parse_json(self._search_regex(
@@ -220,35 +215,42 @@ def _real_extract(self, url):
 
 
 class ITVBTCCIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?itv\.com/btcc/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:news|btcc)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _TESTS = [{
         'url': 'https://www.itv.com/btcc/articles/btcc-2019-brands-hatch-gp-race-action',
         'info_dict': {
             'id': 'btcc-2019-brands-hatch-gp-race-action',
             'title': 'BTCC 2019: Brands Hatch GP race action',
         },
         'playlist_count': 12,
-    }
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1582188683001/HkiHLnNRx_default/index.html?videoId=%s'
+    }, {
+        'url': 'https://www.itv.com/news/2021-10-27/i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike',
+        'info_dict': {
+            'id': 'i-have-to-protect-the-country-says-rishi-sunak-as-uk-faces-interest-rate-hike',
+            'title': 'md5:6ef054dd9f069330db3dcc66cb772d32'
+        },
+        'playlist_count': 4
+    }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
 
-        json_map = try_get(self._parse_json(self._html_search_regex(
-            '(?s)<script[^>]+id=[\'"]__NEXT_DATA__[^>]*>([^<]+)</script>', webpage, 'json_map'), playlist_id),
+        json_map = try_get(
+            self._search_nextjs_data(webpage, playlist_id),
             lambda x: x['props']['pageProps']['article']['body']['content']) or []
 
-        # Discard empty objects
-        video_ids = []
+        entries = []
         for video in json_map:
-            if video['data'].get('id'):
-                video_ids.append(video['data']['id'])
-
-        entries = [
-            self.url_result(
-                smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {
+            if not any(video['data'].get(attr) == 'Brightcove' for attr in ('name', 'type')):
+                continue
+            video_id = video['data']['id']
+            account_id = video['data']['accountId']
+            player_id = video['data']['playerId']
+            entries.append(self.url_result(
+                smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id), {
                     # ITV does not like some GB IP ranges, so here are some
                     # IP blocks it accepts
                     'geo_ip_blocks': [
@@ -256,8 +258,7 @@ def _real_extract(self, url):
                     ],
                     'referrer': url,
                 }),
-                ie=BrightcoveNewIE.ie_key(), video_id=video_id)
-            for video_id in video_ids]
+                ie=BrightcoveNewIE.ie_key(), video_id=video_id))
 
         title = self._og_search_title(webpage, fatal=False)