[extractor] Framework for embed detection (#4307)

[yt-dlp.git] / yt_dlp / extractor / spotify.py

diff --git a/yt_dlp/extractor/spotify.py b/yt_dlp/extractor/spotify.py
index 826f98cffc7a0fbcac20382a3eaee55be8d9fe8b..f476b7022c7e787724a8f27502f6039af86fe688 100644 (file)
--- a/yt_dlp/extractor/spotify.py
+++ b/yt_dlp/extractor/spotify.py
@@ -1,15 +1,15 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
+import functools
 import json
 import re
 
 from .common import InfoExtractor
 from ..utils import (
+    OnDemandPagedList,
     clean_podcast_url,
     float_or_none,
     int_or_none,
     strip_or_none,
+    traverse_obj,
     try_get,
     unified_strdate,
 )
@@ -22,13 +22,13 @@ class SpotifyBaseIE(InfoExtractor):
         'MinimalShow': '13ee079672fad3f858ea45a55eb109553b4fb0969ed793185b2e34cbb6ee7cc0',
         'ShowEpisodes': 'e0e5ce27bd7748d2c59b4d44ba245a8992a05be75d6fabc3b20753fc8857444d',
     }
-    _VALID_URL_TEMPL = r'https?://open\.spotify\.com/%s/(?P<id>[^/?&#]+)'
+    _VALID_URL_TEMPL = r'https?://open\.spotify\.com/(?:embed-podcast/|embed/|)%s/(?P<id>[^/?&#]+)'
 
     def _real_initialize(self):
         self._ACCESS_TOKEN = self._download_json(
             'https://open.spotify.com/get_access_token', None)['accessToken']
 
-    def _call_api(self, operation, video_id, variables):
+    def _call_api(self, operation, video_id, variables, **kwargs):
         return self._download_json(
             'https://api-partner.spotify.com/pathfinder/v1/query', video_id, query={
                 'operationName': 'query' + operation,
@@ -38,7 +38,8 @@ def _call_api(self, operation, video_id, variables):
                         'sha256Hash': self._OPERATION_HASHES[operation],
                     },
                 })
-            }, headers={'authorization': 'Bearer ' + self._ACCESS_TOKEN})['data']
+            }, headers={'authorization': 'Bearer ' + self._ACCESS_TOKEN},
+            **kwargs)['data']
 
     def _extract_episode(self, episode, series):
         episode_id = episode['id']
@@ -96,11 +97,18 @@ def _extract_episode(self, episode, series):
             'series': series,
         }
 
+    @classmethod
+    def _extract_urls(cls, webpage):
+        return re.findall(
+            r'<iframe[^>]+src="(https?://open\.spotify.com/embed/[^"]+)"',
+            webpage)
+
 
 class SpotifyIE(SpotifyBaseIE):
     IE_NAME = 'spotify'
+    IE_DESC = 'Spotify episodes'
     _VALID_URL = SpotifyBaseIE._VALID_URL_TEMPL % 'episode'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://open.spotify.com/episode/4Z7GAJ50bgctf6uclHlWKo',
         'md5': '74010a1e3fa4d9e1ab3aa7ad14e42d3b',
         'info_dict': {
@@ -112,7 +120,10 @@ class SpotifyIE(SpotifyBaseIE):
             'release_date': '20201217',
             'series': "The Guardian's Audio Long Reads",
         }
-    }
+    }, {
+        'url': 'https://open.spotify.com/embed/episode/4TvCsKKs2thXmarHigWvXE?si=7eatS8AbQb6RxqO2raIuWA',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         episode_id = self._match_id(url)
@@ -125,6 +136,7 @@ def _real_extract(self, url):
 
 class SpotifyShowIE(SpotifyBaseIE):
     IE_NAME = 'spotify:show'
+    IE_DESC = 'Spotify shows'
     _VALID_URL = SpotifyBaseIE._VALID_URL_TEMPL % 'show'
     _TEST = {
         'url': 'https://open.spotify.com/show/4PM9Ke6l66IRNpottHKV9M',
@@ -135,22 +147,25 @@ class SpotifyShowIE(SpotifyBaseIE):
         },
         'playlist_mincount': 36,
     }
+    _PER_PAGE = 100
+
+    def _fetch_page(self, show_id, page=0):
+        return self._call_api('ShowEpisodes', show_id, {
+            'limit': 100,
+            'offset': page * self._PER_PAGE,
+            'uri': f'spotify:show:{show_id}',
+        }, note=f'Downloading page {page + 1} JSON metadata')['podcast']
 
     def _real_extract(self, url):
         show_id = self._match_id(url)
-        podcast = self._call_api('ShowEpisodes', show_id, {
-            'limit': 1000000000,
-            'offset': 0,
-            'uri': 'spotify:show:' + show_id,
-        })['podcast']
-        podcast_name = podcast.get('name')
-
-        entries = []
-        for item in (try_get(podcast, lambda x: x['episodes']['items']) or []):
-            episode = item.get('episode')
-            if not episode:
-                continue
-            entries.append(self._extract_episode(episode, podcast_name))
+        first_page = self._fetch_page(show_id)
+
+        def _entries(page):
+            podcast = self._fetch_page(show_id, page) if page else first_page
+            yield from map(
+                functools.partial(self._extract_episode, series=podcast.get('name')),
+                traverse_obj(podcast, ('episodes', 'items', ..., 'episode')))
 
         return self.playlist_result(
-            entries, show_id, podcast_name, podcast.get('description'))
+            OnDemandPagedList(_entries, self._PER_PAGE),
+            show_id, first_page.get('name'), first_page.get('description'))