yt_dlp/extractor/svt.py

   1 import json
   2 import re
   3
   4 from .common import InfoExtractor
   5 from ..compat import compat_str
   6 from ..utils import (
   7     determine_ext,
   8     dict_get,
   9     int_or_none,
  10     str_or_none,
  11     strip_or_none,
  12     traverse_obj,
  13     try_get,
  14     unified_timestamp,
  15 )
  16
  17
  18 class SVTBaseIE(InfoExtractor):
  19     _GEO_COUNTRIES = ['SE']
  20
  21     def _extract_video(self, video_info, video_id):
  22         is_live = dict_get(video_info, ('live', 'simulcast'), default=False)
  23         m3u8_protocol = 'm3u8' if is_live else 'm3u8_native'
  24         formats = []
  25         subtitles = {}
  26         for vr in video_info['videoReferences']:
  27             player_type = vr.get('playerType') or vr.get('format')
  28             vurl = vr['url']
  29             ext = determine_ext(vurl)
  30             if ext == 'm3u8':
  31                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
  32                     vurl, video_id,
  33                     ext='mp4', entry_protocol=m3u8_protocol,
  34                     m3u8_id=player_type, fatal=False)
  35                 formats.extend(fmts)
  36                 self._merge_subtitles(subs, target=subtitles)
  37             elif ext == 'f4m':
  38                 formats.extend(self._extract_f4m_formats(
  39                     vurl + '?hdcore=3.3.0', video_id,
  40                     f4m_id=player_type, fatal=False))
  41             elif ext == 'mpd':
  42                 fmts, subs = self._extract_mpd_formats_and_subtitles(
  43                     vurl, video_id, mpd_id=player_type, fatal=False)
  44                 formats.extend(fmts)
  45                 self._merge_subtitles(subs, target=subtitles)
  46             else:
  47                 formats.append({
  48                     'format_id': player_type,
  49                     'url': vurl,
  50                 })
  51         rights = try_get(video_info, lambda x: x['rights'], dict) or {}
  52         if not formats and rights.get('geoBlockedSweden'):
  53             self.raise_geo_restricted(
  54                 'This video is only available in Sweden',
  55                 countries=self._GEO_COUNTRIES, metadata_available=True)
  56
  57         subtitle_references = dict_get(video_info, ('subtitles', 'subtitleReferences'))
  58         if isinstance(subtitle_references, list):
  59             for sr in subtitle_references:
  60                 subtitle_url = sr.get('url')
  61                 subtitle_lang = sr.get('language', 'sv')
  62                 if subtitle_url:
  63                     sub = {
  64                         'url': subtitle_url,
  65                     }
  66                     if determine_ext(subtitle_url) == 'm3u8':
  67                         # XXX: no way of testing, is it ever hit?
  68                         sub['ext'] = 'vtt'
  69                     subtitles.setdefault(subtitle_lang, []).append(sub)
  70
  71         title = video_info.get('title')
  72
  73         series = video_info.get('programTitle')
  74         season_number = int_or_none(video_info.get('season'))
  75         episode = video_info.get('episodeTitle')
  76         episode_number = int_or_none(video_info.get('episodeNumber'))
  77
  78         timestamp = unified_timestamp(rights.get('validFrom'))
  79         duration = int_or_none(dict_get(video_info, ('materialLength', 'contentDuration')))
  80         age_limit = None
  81         adult = dict_get(
  82             video_info, ('inappropriateForChildren', 'blockedForChildren'),
  83             skip_false_values=False)
  84         if adult is not None:
  85             age_limit = 18 if adult else 0
  86
  87         return {
  88             'id': video_id,
  89             'title': title,
  90             'formats': formats,
  91             'subtitles': subtitles,
  92             'duration': duration,
  93             'timestamp': timestamp,
  94             'age_limit': age_limit,
  95             'series': series,
  96             'season_number': season_number,
  97             'episode': episode,
  98             'episode_number': episode_number,
  99             'is_live': is_live,
 100         }
 101
 102
 103 class SVTIE(SVTBaseIE):
 104     _VALID_URL = r'https?://(?:www\.)?svt\.se/wd\?(?:.*?&)?widgetId=(?P<widget_id>\d+)&.*?\barticleId=(?P<id>\d+)'
 105     _EMBED_REGEX = [r'(?:<iframe src|href)="(?P<url>%s[^"]*)"' % _VALID_URL]
 106     _TEST = {
 107         'url': 'http://www.svt.se/wd?widgetId=23991&sectionId=541&articleId=2900353&type=embed&contextSectionId=123&autostart=false',
 108         'md5': '33e9a5d8f646523ce0868ecfb0eed77d',
 109         'info_dict': {
 110             'id': '2900353',
 111             'ext': 'mp4',
 112             'title': 'Stjärnorna skojar till det - under SVT-intervjun',
 113             'duration': 27,
 114             'age_limit': 0,
 115         },
 116     }
 117
 118     def _real_extract(self, url):
 119         mobj = self._match_valid_url(url)
 120         widget_id = mobj.group('widget_id')
 121         article_id = mobj.group('id')
 122
 123         info = self._download_json(
 124             'http://www.svt.se/wd?widgetId=%s&articleId=%s&format=json&type=embed&output=json' % (widget_id, article_id),
 125             article_id)
 126
 127         info_dict = self._extract_video(info['video'], article_id)
 128         info_dict['title'] = info['context']['title']
 129         return info_dict
 130
 131
 132 class SVTPlayBaseIE(SVTBaseIE):
 133     _SVTPLAY_RE = r'root\s*\[\s*(["\'])_*svtplay\1\s*\]\s*=\s*(?P<json>{.+?})\s*;\s*\n'
 134
 135
 136 class SVTPlayIE(SVTPlayBaseIE):
 137     IE_DESC = 'SVT Play and Öppet arkiv'
 138     _VALID_URL = r'''(?x)
 139                     (?:
 140                         (?:
 141                             svt:|
 142                             https?://(?:www\.)?svt\.se/barnkanalen/barnplay/[^/]+/
 143                         )
 144                         (?P<svt_id>[^/?#&]+)|
 145                         https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/(?:video|klipp|kanaler)/(?P<id>[^/?#&]+)
 146                         (?:.*?(?:modalId|id)=(?P<modal_id>[\da-zA-Z-]+))?
 147                     )
 148                     '''
 149     _TESTS = [{
 150         'url': 'https://www.svtplay.se/video/30479064',
 151         'md5': '2382036fd6f8c994856c323fe51c426e',
 152         'info_dict': {
 153             'id': '8zVbDPA',
 154             'ext': 'mp4',
 155             'title': 'Designdrömmar i Stenungsund',
 156             'timestamp': 1615770000,
 157             'upload_date': '20210315',
 158             'duration': 3519,
 159             'thumbnail': r're:^https?://(?:.*[\.-]jpg|www.svtstatic.se/image/.*)$',
 160             'age_limit': 0,
 161             'subtitles': {
 162                 'sv': [{
 163                     'ext': 'vtt',
 164                 }]
 165             },
 166         },
 167         'params': {
 168             'skip_download': 'm3u8',
 169         },
 170         'skip': 'Episode is no longer available',
 171     }, {
 172         'url': 'https://www.svtplay.se/video/emBxBQj',
 173         'md5': '2382036fd6f8c994856c323fe51c426e',
 174         'info_dict': {
 175             'id': 'eyBd9aj',
 176             'ext': 'mp4',
 177             'title': '1. Farlig kryssning',
 178             'timestamp': 1491019200,
 179             'upload_date': '20170401',
 180             'duration': 2566,
 181             'thumbnail': r're:^https?://(?:.*[\.-]jpg|www.svtstatic.se/image/.*)$',
 182             'age_limit': 0,
 183             'episode': '1. Farlig kryssning',
 184             'series': 'Rederiet',
 185             'subtitles': {
 186                 'sv': 'count:3'
 187             },
 188         },
 189         'params': {
 190             'skip_download': 'm3u8',
 191         },
 192     }, {
 193         'url': 'https://www.svtplay.se/video/jz2rYz7/anders-hansen-moter/james-fallon?info=visa',
 194         'info_dict': {
 195             'id': 'jvXAGVb',
 196             'ext': 'mp4',
 197             'title': 'James Fallon',
 198             'timestamp': 1673917200,
 199             'upload_date': '20230117',
 200             'duration': 1081,
 201             'thumbnail': r're:^https?://(?:.*[\.-]jpg|www.svtstatic.se/image/.*)$',
 202             'age_limit': 0,
 203             'episode': 'James Fallon',
 204             'series': 'Anders Hansen möter...',
 205         },
 206         'params': {
 207             'skip_download': 'dash',
 208         },
 209     }, {
 210         'url': 'https://www.svtplay.se/video/30479064/husdrommar/husdrommar-sasong-8-designdrommar-i-stenungsund?modalId=8zVbDPA',
 211         'only_matching': True,
 212     }, {
 213         'url': 'https://www.svtplay.se/video/30684086/rapport/rapport-24-apr-18-00-7?id=e72gVpa',
 214         'only_matching': True,
 215     }, {
 216         # geo restricted to Sweden
 217         'url': 'http://www.oppetarkiv.se/video/5219710/trollflojten',
 218         'only_matching': True,
 219     }, {
 220         'url': 'http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg',
 221         'only_matching': True,
 222     }, {
 223         'url': 'https://www.svtplay.se/kanaler/svt1',
 224         'only_matching': True,
 225     }, {
 226         'url': 'svt:1376446-003A',
 227         'only_matching': True,
 228     }, {
 229         'url': 'svt:14278044',
 230         'only_matching': True,
 231     }, {
 232         'url': 'https://www.svt.se/barnkanalen/barnplay/kar/eWv5MLX/',
 233         'only_matching': True,
 234     }, {
 235         'url': 'svt:eWv5MLX',
 236         'only_matching': True,
 237     }]
 238
 239     def _extract_by_video_id(self, video_id, webpage=None):
 240         data = self._download_json(
 241             'https://api.svt.se/videoplayer-api/video/%s' % video_id,
 242             video_id, headers=self.geo_verification_headers())
 243         info_dict = self._extract_video(data, video_id)
 244         if not info_dict.get('title'):
 245             title = dict_get(info_dict, ('episode', 'series'))
 246             if not title and webpage:
 247                 title = re.sub(
 248                     r'\s*\|\s*.+?$', '', self._og_search_title(webpage))
 249             if not title:
 250                 title = video_id
 251             info_dict['title'] = title
 252         return info_dict
 253
 254     def _real_extract(self, url):
 255         mobj = self._match_valid_url(url)
 256         video_id = mobj.group('id')
 257         svt_id = mobj.group('svt_id') or mobj.group('modal_id')
 258
 259         if svt_id:
 260             return self._extract_by_video_id(svt_id)
 261
 262         webpage = self._download_webpage(url, video_id)
 263
 264         data = self._parse_json(
 265             self._search_regex(
 266                 self._SVTPLAY_RE, webpage, 'embedded data', default='{}',
 267                 group='json'),
 268             video_id, fatal=False)
 269
 270         thumbnail = self._og_search_thumbnail(webpage)
 271
 272         if data:
 273             video_info = try_get(
 274                 data, lambda x: x['context']['dispatcher']['stores']['VideoTitlePageStore']['data']['video'],
 275                 dict)
 276             if video_info:
 277                 info_dict = self._extract_video(video_info, video_id)
 278                 info_dict.update({
 279                     'title': data['context']['dispatcher']['stores']['MetaStore']['title'],
 280                     'thumbnail': thumbnail,
 281                 })
 282                 return info_dict
 283
 284             svt_id = try_get(
 285                 data, lambda x: x['statistics']['dataLake']['content']['id'],
 286                 compat_str)
 287
 288         if not svt_id:
 289             nextjs_data = self._search_nextjs_data(webpage, video_id, fatal=False)
 290             svt_id = traverse_obj(nextjs_data, (
 291                 'props', 'urqlState', ..., 'data', {json.loads}, 'detailsPageByPath',
 292                 'video', 'svtId', {str}), get_all=False)
 293
 294         if not svt_id:
 295             svt_id = self._search_regex(
 296                 (r'<video[^>]+data-video-id=["\']([\da-zA-Z-]+)',
 297                  r'<[^>]+\bdata-rt=["\']top-area-play-button["\'][^>]+\bhref=["\'][^"\']*video/[\w-]+/[^"\']*\b(?:modalId|id)=([\w-]+)'),
 298                 webpage, 'video id')
 299
 300         info_dict = self._extract_by_video_id(svt_id, webpage)
 301         info_dict['thumbnail'] = thumbnail
 302
 303         return info_dict
 304
 305
 306 class SVTSeriesIE(SVTPlayBaseIE):
 307     _VALID_URL = r'https?://(?:www\.)?svtplay\.se/(?P<id>[^/?&#]+)(?:.+?\btab=(?P<season_slug>[^&#]+))?'
 308     _TESTS = [{
 309         'url': 'https://www.svtplay.se/rederiet',
 310         'info_dict': {
 311             'id': '14445680',
 312             'title': 'Rederiet',
 313             'description': 'md5:d9fdfff17f5d8f73468176ecd2836039',
 314         },
 315         'playlist_mincount': 318,
 316     }, {
 317         'url': 'https://www.svtplay.se/rederiet?tab=season-2-14445680',
 318         'info_dict': {
 319             'id': 'season-2-14445680',
 320             'title': 'Rederiet - Säsong 2',
 321             'description': 'md5:d9fdfff17f5d8f73468176ecd2836039',
 322         },
 323         'playlist_mincount': 12,
 324     }]
 325
 326     @classmethod
 327     def suitable(cls, url):
 328         return False if SVTIE.suitable(url) or SVTPlayIE.suitable(url) else super(SVTSeriesIE, cls).suitable(url)
 329
 330     def _real_extract(self, url):
 331         series_slug, season_id = self._match_valid_url(url).groups()
 332
 333         series = self._download_json(
 334             'https://api.svt.se/contento/graphql', series_slug,
 335             'Downloading series page', query={
 336                 'query': '''{
 337   listablesBySlug(slugs: ["%s"]) {
 338     associatedContent(include: [productionPeriod, season]) {
 339       items {
 340         item {
 341           ... on Episode {
 342             videoSvtId
 343           }
 344         }
 345       }
 346       id
 347       name
 348     }
 349     id
 350     longDescription
 351     name
 352     shortDescription
 353   }
 354 }''' % series_slug,
 355             })['data']['listablesBySlug'][0]
 356
 357         season_name = None
 358
 359         entries = []
 360         for season in series['associatedContent']:
 361             if not isinstance(season, dict):
 362                 continue
 363             if season_id:
 364                 if season.get('id') != season_id:
 365                     continue
 366                 season_name = season.get('name')
 367             items = season.get('items')
 368             if not isinstance(items, list):
 369                 continue
 370             for item in items:
 371                 video = item.get('item') or {}
 372                 content_id = video.get('videoSvtId')
 373                 if not content_id or not isinstance(content_id, compat_str):
 374                     continue
 375                 entries.append(self.url_result(
 376                     'svt:' + content_id, SVTPlayIE.ie_key(), content_id))
 377
 378         title = series.get('name')
 379         season_name = season_name or season_id
 380
 381         if title and season_name:
 382             title = '%s - %s' % (title, season_name)
 383         elif season_id:
 384             title = season_id
 385
 386         return self.playlist_result(
 387             entries, season_id or series.get('id'), title,
 388             dict_get(series, ('longDescription', 'shortDescription')))
 389
 390
 391 class SVTPageIE(InfoExtractor):
 392     _VALID_URL = r'https?://(?:www\.)?svt\.se/(?P<path>(?:[^/]+/)*(?P<id>[^/?&#]+))'
 393     _TESTS = [{
 394         'url': 'https://www.svt.se/sport/ishockey/bakom-masken-lehners-kamp-mot-mental-ohalsa',
 395         'info_dict': {
 396             'id': '25298267',
 397             'title': 'Bakom masken – Lehners kamp mot mental ohälsa',
 398         },
 399         'playlist_count': 4,
 400     }, {
 401         'url': 'https://www.svt.se/nyheter/utrikes/svenska-andrea-ar-en-mil-fran-branderna-i-kalifornien',
 402         'info_dict': {
 403             'id': '24243746',
 404             'title': 'Svenska Andrea redo att fly sitt hem i Kalifornien',
 405         },
 406         'playlist_count': 2,
 407     }, {
 408         # only programTitle
 409         'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
 410         'info_dict': {
 411             'id': '8439V2K',
 412             'ext': 'mp4',
 413             'title': 'Stjärnorna skojar till det - under SVT-intervjun',
 414             'duration': 27,
 415             'age_limit': 0,
 416         },
 417     }, {
 418         'url': 'https://www.svt.se/nyheter/lokalt/vast/svt-testar-tar-nagon-upp-skrapet-1',
 419         'only_matching': True,
 420     }, {
 421         'url': 'https://www.svt.se/vader/manadskronikor/maj2018',
 422         'only_matching': True,
 423     }]
 424
 425     @classmethod
 426     def suitable(cls, url):
 427         return False if SVTIE.suitable(url) or SVTPlayIE.suitable(url) else super(SVTPageIE, cls).suitable(url)
 428
 429     def _real_extract(self, url):
 430         path, display_id = self._match_valid_url(url).groups()
 431
 432         article = self._download_json(
 433             'https://api.svt.se/nss-api/page/' + path, display_id,
 434             query={'q': 'articles'})['articles']['content'][0]
 435
 436         entries = []
 437
 438         def _process_content(content):
 439             if content.get('_type') in ('VIDEOCLIP', 'VIDEOEPISODE'):
 440                 video_id = compat_str(content['image']['svtId'])
 441                 entries.append(self.url_result(
 442                     'svt:' + video_id, SVTPlayIE.ie_key(), video_id))
 443
 444         for media in article.get('media', []):
 445             _process_content(media)
 446
 447         for obj in article.get('structuredBody', []):
 448             _process_content(obj.get('content') or {})
 449
 450         return self.playlist_result(
 451             entries, str_or_none(article.get('id')),
 452             strip_or_none(article.get('title')))