yt_dlp/extractor/banbye.py

   1 import math
   2 import urllib.parse
   3
   4 from .common import InfoExtractor
   5 from ..utils import (
   6     InAdvancePagedList,
   7     determine_ext,
   8     format_field,
   9     int_or_none,
  10     join_nonempty,
  11     traverse_obj,
  12     unified_timestamp,
  13     url_or_none,
  14 )
  15
  16
  17 class BanByeBaseIE(InfoExtractor):
  18     _API_BASE = 'https://api.banbye.com'
  19     _CDN_BASE = 'https://cdn.banbye.com'
  20     _VIDEO_BASE = 'https://banbye.com/watch'
  21
  22     @staticmethod
  23     def _extract_playlist_id(url, param='playlist'):
  24         return urllib.parse.parse_qs(
  25             urllib.parse.urlparse(url).query).get(param, [None])[0]
  26
  27     def _extract_playlist(self, playlist_id):
  28         data = self._download_json(f'{self._API_BASE}/playlists/{playlist_id}', playlist_id)
  29         return self.playlist_result([
  30             self.url_result(f'{self._VIDEO_BASE}/{video_id}', BanByeIE)
  31             for video_id in data['videoIds']], playlist_id, data.get('name'))
  32
  33
  34 class BanByeIE(BanByeBaseIE):
  35     _VALID_URL = r'https?://(?:www\.)?banbye\.com/(?:en/)?watch/(?P<id>[\w-]+)'
  36     _TESTS = [{
  37         # ['src']['mp4']['levels'] direct mp4 urls only
  38         'url': 'https://banbye.com/watch/v_ytfmvkVYLE8T',
  39         'md5': '2f4ea15c5ca259a73d909b2cfd558eb5',
  40         'info_dict': {
  41             'id': 'v_ytfmvkVYLE8T',
  42             'ext': 'mp4',
  43             'title': 'md5:5ec098f88a0d796f987648de6322ba0f',
  44             'description': 'md5:4d94836e73396bc18ef1fa0f43e5a63a',
  45             'uploader': 'wRealu24',
  46             'channel_id': 'ch_wrealu24',
  47             'channel_url': 'https://banbye.com/channel/ch_wrealu24',
  48             'timestamp': 1647604800,
  49             'upload_date': '20220318',
  50             'duration': 1931,
  51             'thumbnail': r're:https?://.*\.webp',
  52             'tags': 'count:5',
  53             'like_count': int,
  54             'dislike_count': int,
  55             'view_count': int,
  56             'comment_count': int,
  57         },
  58     }, {
  59         'url': 'https://banbye.com/watch/v_2JjQtqjKUE_F?playlistId=p_Ld82N6gBw_OJ',
  60         'info_dict': {
  61             'title': 'Krzysztof Karoń',
  62             'id': 'p_Ld82N6gBw_OJ',
  63         },
  64         'playlist_mincount': 9,
  65     }, {
  66         # ['src']['mp4']['levels'] direct mp4 urls only
  67         'url': 'https://banbye.com/watch/v_kb6_o1Kyq-CD',
  68         'info_dict': {
  69             'id': 'v_kb6_o1Kyq-CD',
  70             'ext': 'mp4',
  71             'title': 'Co tak naprawdę dzieje się we Francji?! Czy Warszawa a potem cała Polska będzie drugim Paryżem?!🤔🇵🇱',
  72             'description': 'md5:82be4c0e13eae8ea1ca8b9f2e07226a8',
  73             'uploader': 'Marcin Rola - MOIM ZDANIEM!🇵🇱',
  74             'channel_id': 'ch_QgWnHvDG2fo5',
  75             'channel_url': 'https://banbye.com/channel/ch_QgWnHvDG2fo5',
  76             'duration': 597,
  77             'timestamp': 1688642656,
  78             'upload_date': '20230706',
  79             'thumbnail': 'https://cdn.banbye.com/video/v_kb6_o1Kyq-CD/96.webp',
  80             'tags': ['Paryż', 'Francja', 'Polska', 'Imigranci', 'Morawiecki', 'Tusk'],
  81             'like_count': int,
  82             'dislike_count': int,
  83             'view_count': int,
  84             'comment_count': int,
  85         },
  86     }, {
  87         # ['src']['hls']['levels'] variant m3u8 urls only; master m3u8 is 404
  88         'url': 'https://banbye.com/watch/v_a_gPFuC9LoW5',
  89         'info_dict': {
  90             'id': 'v_a_gPFuC9LoW5',
  91             'ext': 'mp4',
  92             'title': 'md5:183524056bebdfa245fd6d214f63c0fe',
  93             'description': 'md5:943ac87287ca98d28d8b8797719827c6',
  94             'uploader': 'wRealu24',
  95             'channel_id': 'ch_wrealu24',
  96             'channel_url': 'https://banbye.com/channel/ch_wrealu24',
  97             'upload_date': '20231113',
  98             'timestamp': 1699874062,
  99             'view_count': int,
 100             'like_count': int,
 101             'dislike_count': int,
 102             'comment_count': int,
 103             'thumbnail': 'https://cdn.banbye.com/video/v_a_gPFuC9LoW5/96.webp',
 104             'tags': ['jaszczur', 'sejm', 'lewica', 'polska', 'ukrainizacja', 'pierwszeposiedzeniesejmu'],
 105         },
 106         'expected_warnings': ['Failed to download m3u8'],
 107     }, {
 108         # ['src']['hls']['masterPlaylist'] m3u8 only
 109         'url': 'https://banbye.com/watch/v_B0rsKWsr-aaa',
 110         'info_dict': {
 111             'id': 'v_B0rsKWsr-aaa',
 112             'ext': 'mp4',
 113             'title': 'md5:00b254164b82101b3f9e5326037447ed',
 114             'description': 'md5:3fd8b48aa81954ba024bc60f5de6e167',
 115             'uploader': 'PSTV Piotr Szlachtowicz ',
 116             'channel_id': 'ch_KV9EVObkB9wB',
 117             'channel_url': 'https://banbye.com/channel/ch_KV9EVObkB9wB',
 118             'upload_date': '20240629',
 119             'timestamp': 1719646816,
 120             'duration': 2377,
 121             'view_count': int,
 122             'like_count': int,
 123             'dislike_count': int,
 124             'comment_count': int,
 125             'thumbnail': 'https://cdn.banbye.com/video/v_B0rsKWsr-aaa/96.webp',
 126             'tags': ['Biden', 'Trump', 'Wybory', 'USA'],
 127         },
 128     }]
 129
 130     def _real_extract(self, url):
 131         video_id = self._match_id(url)
 132         playlist_id = self._extract_playlist_id(url, 'playlistId')
 133
 134         if self._yes_playlist(playlist_id, video_id):
 135             return self._extract_playlist(playlist_id)
 136
 137         data = self._download_json(f'{self._API_BASE}/videos/{video_id}', video_id)
 138         thumbnails = [{
 139             'id': f'{quality}p',
 140             'url': f'{self._CDN_BASE}/video/{video_id}/{quality}.webp',
 141         } for quality in [48, 96, 144, 240, 512, 1080]]
 142
 143         formats = []
 144         url_data = self._download_json(f'{self._API_BASE}/videos/{video_id}/url', video_id, data=b'')
 145         if master_url := traverse_obj(url_data, ('src', 'hls', 'masterPlaylist', {url_or_none})):
 146             formats = self._extract_m3u8_formats(master_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
 147
 148         for format_id, format_url in traverse_obj(url_data, (
 149                 'src', ('mp4', 'hls'), 'levels', {dict.items}, lambda _, v: url_or_none(v[1]))):
 150             ext = determine_ext(format_url)
 151             is_hls = ext == 'm3u8'
 152             formats.append({
 153                 'url': format_url,
 154                 'ext': 'mp4' if is_hls else ext,
 155                 'format_id': join_nonempty(is_hls and 'hls', format_id),
 156                 'protocol': 'm3u8_native' if is_hls else 'https',
 157                 'height': int_or_none(format_id),
 158             })
 159         self._remove_duplicate_formats(formats)
 160
 161         return {
 162             'id': video_id,
 163             'title': data.get('title'),
 164             'description': data.get('desc'),
 165             'uploader': traverse_obj(data, ('channel', 'name')),
 166             'channel_id': data.get('channelId'),
 167             'channel_url': format_field(data, 'channelId', 'https://banbye.com/channel/%s'),
 168             'timestamp': unified_timestamp(data.get('publishedAt')),
 169             'duration': data.get('duration'),
 170             'tags': data.get('tags'),
 171             'formats': formats,
 172             'thumbnails': thumbnails,
 173             'like_count': data.get('likes'),
 174             'dislike_count': data.get('dislikes'),
 175             'view_count': data.get('views'),
 176             'comment_count': data.get('commentCount'),
 177         }
 178
 179
 180 class BanByeChannelIE(BanByeBaseIE):
 181     _VALID_URL = r'https?://(?:www\.)?banbye\.com/(?:en/)?channel/(?P<id>\w+)'
 182     _TESTS = [{
 183         'url': 'https://banbye.com/channel/ch_wrealu24',
 184         'info_dict': {
 185             'title': 'wRealu24',
 186             'id': 'ch_wrealu24',
 187             'description': 'md5:da54e48416b74dfdde20a04867c0c2f6',
 188         },
 189         'playlist_mincount': 791,
 190     }, {
 191         'url': 'https://banbye.com/channel/ch_wrealu24?playlist=p_Ld82N6gBw_OJ',
 192         'info_dict': {
 193             'title': 'Krzysztof Karoń',
 194             'id': 'p_Ld82N6gBw_OJ',
 195         },
 196         'playlist_count': 9,
 197     }]
 198     _PAGE_SIZE = 100
 199
 200     def _real_extract(self, url):
 201         channel_id = self._match_id(url)
 202         playlist_id = self._extract_playlist_id(url)
 203
 204         if playlist_id:
 205             return self._extract_playlist(playlist_id)
 206
 207         def page_func(page_num):
 208             data = self._download_json(f'{self._API_BASE}/videos', channel_id, query={
 209                 'channelId': channel_id,
 210                 'sort': 'new',
 211                 'limit': self._PAGE_SIZE,
 212                 'offset': page_num * self._PAGE_SIZE,
 213             }, note=f'Downloading page {page_num + 1}')
 214             return [
 215                 self.url_result(f"{self._VIDEO_BASE}/{video['_id']}", BanByeIE)
 216                 for video in data['items']
 217             ]
 218
 219         channel_data = self._download_json(f'{self._API_BASE}/channels/{channel_id}', channel_id)
 220         entries = InAdvancePagedList(
 221             page_func,
 222             math.ceil(channel_data['videoCount'] / self._PAGE_SIZE),
 223             self._PAGE_SIZE)
 224
 225         return self.playlist_result(
 226             entries, channel_id, channel_data.get('name'), channel_data.get('description'))