yt_dlp/extractor/afreecatv.py

   1 import functools
   2 import re
   3
   4 from .common import InfoExtractor
   5 from ..utils import (
   6     ExtractorError,
   7     OnDemandPagedList,
   8     date_from_str,
   9     determine_ext,
  10     int_or_none,
  11     qualities,
  12     traverse_obj,
  13     unified_strdate,
  14     unified_timestamp,
  15     update_url_query,
  16     url_or_none,
  17     urlencode_postdata,
  18     xpath_text,
  19 )
  20
  21
  22 class AfreecaTVIE(InfoExtractor):
  23     IE_NAME = 'afreecatv'
  24     IE_DESC = 'afreecatv.com'
  25     _VALID_URL = r'''(?x)
  26                     https?://
  27                         (?:
  28                             (?:(?:live|afbbs|www)\.)?afreeca(?:tv)?\.com(?::\d+)?
  29                             (?:
  30                                 /app/(?:index|read_ucc_bbs)\.cgi|
  31                                 /player/[Pp]layer\.(?:swf|html)
  32                             )\?.*?\bnTitleNo=|
  33                             vod\.afreecatv\.com/(PLAYER/STATION|player)/
  34                         )
  35                         (?P<id>\d+)
  36                     '''
  37     _NETRC_MACHINE = 'afreecatv'
  38     _TESTS = [{
  39         'url': 'http://live.afreecatv.com:8079/app/index.cgi?szType=read_ucc_bbs&szBjId=dailyapril&nStationNo=16711924&nBbsNo=18605867&nTitleNo=36164052&szSkin=',
  40         'md5': 'f72c89fe7ecc14c1b5ce506c4996046e',
  41         'info_dict': {
  42             'id': '36164052',
  43             'ext': 'mp4',
  44             'title': '데일리 에이프릴 요정들의 시상식!',
  45             'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
  46             'uploader': 'dailyapril',
  47             'uploader_id': 'dailyapril',
  48             'upload_date': '20160503',
  49         },
  50         'skip': 'Video is gone',
  51     }, {
  52         'url': 'http://afbbs.afreecatv.com:8080/app/read_ucc_bbs.cgi?nStationNo=16711924&nTitleNo=36153164&szBjId=dailyapril&nBbsNo=18605867',
  53         'info_dict': {
  54             'id': '36153164',
  55             'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
  56             'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
  57             'uploader': 'dailyapril',
  58             'uploader_id': 'dailyapril',
  59         },
  60         'playlist_count': 2,
  61         'playlist': [{
  62             'md5': 'd8b7c174568da61d774ef0203159bf97',
  63             'info_dict': {
  64                 'id': '36153164_1',
  65                 'ext': 'mp4',
  66                 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
  67                 'upload_date': '20160502',
  68             },
  69         }, {
  70             'md5': '58f2ce7f6044e34439ab2d50612ab02b',
  71             'info_dict': {
  72                 'id': '36153164_2',
  73                 'ext': 'mp4',
  74                 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
  75                 'upload_date': '20160502',
  76             },
  77         }],
  78         'skip': 'Video is gone',
  79     }, {
  80         'url': 'http://vod.afreecatv.com/PLAYER/STATION/18650793',
  81         'info_dict': {
  82             'id': '18650793',
  83             'ext': 'mp4',
  84             'title': '오늘은 다르다! 쏘님의 우월한 위아래~ 댄스리액션!',
  85             'thumbnail': r're:^https?://.*\.jpg$',
  86             'uploader': '윈아디',
  87             'uploader_id': 'badkids',
  88             'duration': 107,
  89         },
  90         'params': {
  91             'skip_download': True,
  92         },
  93     }, {
  94         'url': 'http://vod.afreecatv.com/PLAYER/STATION/10481652',
  95         'info_dict': {
  96             'id': '10481652',
  97             'title': "BJ유트루와 함께하는 '팅커벨 메이크업!'",
  98             'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
  99             'uploader': 'dailyapril',
 100             'uploader_id': 'dailyapril',
 101             'duration': 6492,
 102         },
 103         'playlist_count': 2,
 104         'playlist': [{
 105             'md5': 'd8b7c174568da61d774ef0203159bf97',
 106             'info_dict': {
 107                 'id': '20160502_c4c62b9d_174361386_1',
 108                 'ext': 'mp4',
 109                 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 1)",
 110                 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
 111                 'uploader': 'dailyapril',
 112                 'uploader_id': 'dailyapril',
 113                 'upload_date': '20160502',
 114                 'duration': 3601,
 115             },
 116         }, {
 117             'md5': '58f2ce7f6044e34439ab2d50612ab02b',
 118             'info_dict': {
 119                 'id': '20160502_39e739bb_174361386_2',
 120                 'ext': 'mp4',
 121                 'title': "BJ유트루와 함께하는 '팅커벨 메이크업!' (part 2)",
 122                 'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
 123                 'uploader': 'dailyapril',
 124                 'uploader_id': 'dailyapril',
 125                 'upload_date': '20160502',
 126                 'duration': 2891,
 127             },
 128         }],
 129         'params': {
 130             'skip_download': True,
 131         },
 132     }, {
 133         # non standard key
 134         'url': 'http://vod.afreecatv.com/PLAYER/STATION/20515605',
 135         'info_dict': {
 136             'id': '20170411_BE689A0E_190960999_1_2_h',
 137             'ext': 'mp4',
 138             'title': '혼자사는여자집',
 139             'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
 140             'uploader': '♥이슬이',
 141             'uploader_id': 'dasl8121',
 142             'upload_date': '20170411',
 143             'duration': 213,
 144         },
 145         'params': {
 146             'skip_download': True,
 147         },
 148     }, {
 149         # PARTIAL_ADULT
 150         'url': 'http://vod.afreecatv.com/PLAYER/STATION/32028439',
 151         'info_dict': {
 152             'id': '20180327_27901457_202289533_1',
 153             'ext': 'mp4',
 154             'title': '[생]빨개요♥ (part 1)',
 155             'thumbnail': 're:^https?://(?:video|st)img.afreecatv.com/.*$',
 156             'uploader': '[SA]서아',
 157             'uploader_id': 'bjdyrksu',
 158             'upload_date': '20180327',
 159             'duration': 3601,
 160         },
 161         'params': {
 162             'skip_download': True,
 163         },
 164         'expected_warnings': ['adult content'],
 165     }, {
 166         'url': 'http://www.afreecatv.com/player/Player.swf?szType=szBjId=djleegoon&nStationNo=11273158&nBbsNo=13161095&nTitleNo=36327652',
 167         'only_matching': True,
 168     }, {
 169         'url': 'http://vod.afreecatv.com/PLAYER/STATION/15055030',
 170         'only_matching': True,
 171     }, {
 172         'url': 'http://vod.afreecatv.com/player/15055030',
 173         'only_matching': True,
 174     }]
 175
 176     @staticmethod
 177     def parse_video_key(key):
 178         video_key = {}
 179         m = re.match(r'^(?P<upload_date>\d{8})_\w+_(?P<part>\d+)$', key)
 180         if m:
 181             video_key['upload_date'] = m.group('upload_date')
 182             video_key['part'] = int(m.group('part'))
 183         return video_key
 184
 185     def _perform_login(self, username, password):
 186         login_form = {
 187             'szWork': 'login',
 188             'szType': 'json',
 189             'szUid': username,
 190             'szPassword': password,
 191             'isSaveId': 'false',
 192             'szScriptVar': 'oLoginRet',
 193             'szAction': '',
 194         }
 195
 196         response = self._download_json(
 197             'https://login.afreecatv.com/app/LoginAction.php', None,
 198             'Logging in', data=urlencode_postdata(login_form))
 199
 200         _ERRORS = {
 201             -4: 'Your account has been suspended due to a violation of our terms and policies.',
 202             -5: 'https://member.afreecatv.com/app/user_delete_progress.php',
 203             -6: 'https://login.afreecatv.com/membership/changeMember.php',
 204             -8: "Hello! AfreecaTV here.\nThe username you have entered belongs to \n an account that requires a legal guardian's consent. \nIf you wish to use our services without restriction, \nplease make sure to go through the necessary verification process.",
 205             -9: 'https://member.afreecatv.com/app/pop_login_block.php',
 206             -11: 'https://login.afreecatv.com/afreeca/second_login.php',
 207             -12: 'https://member.afreecatv.com/app/user_security.php',
 208             0: 'The username does not exist or you have entered the wrong password.',
 209             -1: 'The username does not exist or you have entered the wrong password.',
 210             -3: 'You have entered your username/password incorrectly.',
 211             -7: 'You cannot use your Global AfreecaTV account to access Korean AfreecaTV.',
 212             -10: 'Sorry for the inconvenience. \nYour account has been blocked due to an unauthorized access. \nPlease contact our Help Center for assistance.',
 213             -32008: 'You have failed to log in. Please contact our Help Center.',
 214         }
 215
 216         result = int_or_none(response.get('RESULT'))
 217         if result != 1:
 218             error = _ERRORS.get(result, 'You have failed to log in.')
 219             raise ExtractorError(
 220                 'Unable to login: %s said: %s' % (self.IE_NAME, error),
 221                 expected=True)
 222
 223     def _real_extract(self, url):
 224         video_id = self._match_id(url)
 225
 226         webpage = self._download_webpage(url, video_id)
 227
 228         if re.search(r'alert\(["\']This video has been deleted', webpage):
 229             raise ExtractorError(
 230                 'Video %s has been deleted' % video_id, expected=True)
 231
 232         station_id = self._search_regex(
 233             r'nStationNo\s*=\s*(\d+)', webpage, 'station')
 234         bbs_id = self._search_regex(
 235             r'nBbsNo\s*=\s*(\d+)', webpage, 'bbs')
 236         video_id = self._search_regex(
 237             r'nTitleNo\s*=\s*(\d+)', webpage, 'title', default=video_id)
 238
 239         partial_view = False
 240         adult_view = False
 241         for _ in range(2):
 242             query = {
 243                 'nTitleNo': video_id,
 244                 'nStationNo': station_id,
 245                 'nBbsNo': bbs_id,
 246             }
 247             if partial_view:
 248                 query['partialView'] = 'SKIP_ADULT'
 249             if adult_view:
 250                 query['adultView'] = 'ADULT_VIEW'
 251             video_xml = self._download_xml(
 252                 'http://afbbs.afreecatv.com:8080/api/video/get_video_info.php',
 253                 video_id, 'Downloading video info XML%s'
 254                 % (' (skipping adult)' if partial_view else ''),
 255                 video_id, headers={
 256                     'Referer': url,
 257                 }, query=query)
 258
 259             flag = xpath_text(video_xml, './track/flag', 'flag', default=None)
 260             if flag and flag == 'SUCCEED':
 261                 break
 262             if flag == 'PARTIAL_ADULT':
 263                 self.report_warning(
 264                     'In accordance with local laws and regulations, underage users are restricted from watching adult content. '
 265                     'Only content suitable for all ages will be downloaded. '
 266                     'Provide account credentials if you wish to download restricted content.')
 267                 partial_view = True
 268                 continue
 269             elif flag == 'ADULT':
 270                 if not adult_view:
 271                     adult_view = True
 272                     continue
 273                 error = 'Only users older than 19 are able to watch this video. Provide account credentials to download this content.'
 274             else:
 275                 error = flag
 276             raise ExtractorError(
 277                 '%s said: %s' % (self.IE_NAME, error), expected=True)
 278         else:
 279             raise ExtractorError('Unable to download video info')
 280
 281         video_element = video_xml.findall('./track/video')[-1]
 282         if video_element is None or video_element.text is None:
 283             raise ExtractorError(
 284                 'Video %s does not exist' % video_id, expected=True)
 285
 286         video_url = video_element.text.strip()
 287
 288         title = xpath_text(video_xml, './track/title', 'title', fatal=True)
 289
 290         uploader = xpath_text(video_xml, './track/nickname', 'uploader')
 291         uploader_id = xpath_text(video_xml, './track/bj_id', 'uploader id')
 292         duration = int_or_none(xpath_text(
 293             video_xml, './track/duration', 'duration'))
 294         thumbnail = xpath_text(video_xml, './track/titleImage', 'thumbnail')
 295
 296         common_entry = {
 297             'uploader': uploader,
 298             'uploader_id': uploader_id,
 299             'thumbnail': thumbnail,
 300         }
 301
 302         info = common_entry.copy()
 303         info.update({
 304             'id': video_id,
 305             'title': title,
 306             'duration': duration,
 307         })
 308
 309         if not video_url:
 310             entries = []
 311             file_elements = video_element.findall('./file')
 312             one = len(file_elements) == 1
 313             for file_num, file_element in enumerate(file_elements, start=1):
 314                 file_url = url_or_none(file_element.text)
 315                 if not file_url:
 316                     continue
 317                 key = file_element.get('key', '')
 318                 upload_date = unified_strdate(self._search_regex(
 319                     r'^(\d{8})_', key, 'upload date', default=None))
 320                 if upload_date is not None:
 321                     # sometimes the upload date isn't included in the file name
 322                     # instead, another random ID is, which may parse as a valid
 323                     # date but be wildly out of a reasonable range
 324                     parsed_date = date_from_str(upload_date)
 325                     if parsed_date.year < 2000 or parsed_date.year >= 2100:
 326                         upload_date = None
 327                 file_duration = int_or_none(file_element.get('duration'))
 328                 format_id = key if key else '%s_%s' % (video_id, file_num)
 329                 if determine_ext(file_url) == 'm3u8':
 330                     formats = self._extract_m3u8_formats(
 331                         file_url, video_id, 'mp4', entry_protocol='m3u8_native',
 332                         m3u8_id='hls',
 333                         note='Downloading part %d m3u8 information' % file_num)
 334                 else:
 335                     formats = [{
 336                         'url': file_url,
 337                         'format_id': 'http',
 338                     }]
 339                 if not formats and not self.get_param('ignore_no_formats'):
 340                     continue
 341                 file_info = common_entry.copy()
 342                 file_info.update({
 343                     'id': format_id,
 344                     'title': title if one else '%s (part %d)' % (title, file_num),
 345                     'upload_date': upload_date,
 346                     'duration': file_duration,
 347                     'formats': formats,
 348                 })
 349                 entries.append(file_info)
 350             entries_info = info.copy()
 351             entries_info.update({
 352                 '_type': 'multi_video',
 353                 'entries': entries,
 354             })
 355             return entries_info
 356
 357         info = {
 358             'id': video_id,
 359             'title': title,
 360             'uploader': uploader,
 361             'uploader_id': uploader_id,
 362             'duration': duration,
 363             'thumbnail': thumbnail,
 364         }
 365
 366         if determine_ext(video_url) == 'm3u8':
 367             info['formats'] = self._extract_m3u8_formats(
 368                 video_url, video_id, 'mp4', entry_protocol='m3u8_native',
 369                 m3u8_id='hls')
 370         else:
 371             app, playpath = video_url.split('mp4:')
 372             info.update({
 373                 'url': app,
 374                 'ext': 'flv',
 375                 'play_path': 'mp4:' + playpath,
 376                 'rtmp_live': True,  # downloading won't end without this
 377             })
 378
 379         return info
 380
 381
 382 class AfreecaTVLiveIE(AfreecaTVIE):  # XXX: Do not subclass from concrete IE
 383
 384     IE_NAME = 'afreecatv:live'
 385     _VALID_URL = r'https?://play\.afreeca(?:tv)?\.com/(?P<id>[^/]+)(?:/(?P<bno>\d+))?'
 386     _TESTS = [{
 387         'url': 'https://play.afreecatv.com/pyh3646/237852185',
 388         'info_dict': {
 389             'id': '237852185',
 390             'ext': 'mp4',
 391             'title': '【 우루과이 오늘은 무슨일이? 】',
 392             'uploader': '박진우[JINU]',
 393             'uploader_id': 'pyh3646',
 394             'timestamp': 1640661495,
 395             'is_live': True,
 396         },
 397         'skip': 'Livestream has ended',
 398     }, {
 399         'url': 'http://play.afreeca.com/pyh3646/237852185',
 400         'only_matching': True,
 401     }, {
 402         'url': 'http://play.afreeca.com/pyh3646',
 403         'only_matching': True,
 404     }]
 405
 406     _LIVE_API_URL = 'https://live.afreecatv.com/afreeca/player_live_api.php'
 407
 408     _QUALITIES = ('sd', 'hd', 'hd2k', 'original')
 409
 410     def _real_extract(self, url):
 411         broadcaster_id, broadcast_no = self._match_valid_url(url).group('id', 'bno')
 412         password = self.get_param('videopassword')
 413
 414         info = self._download_json(self._LIVE_API_URL, broadcaster_id, fatal=False,
 415                                    data=urlencode_postdata({'bid': broadcaster_id})) or {}
 416         channel_info = info.get('CHANNEL') or {}
 417         broadcaster_id = channel_info.get('BJID') or broadcaster_id
 418         broadcast_no = channel_info.get('BNO') or broadcast_no
 419         password_protected = channel_info.get('BPWD')
 420         if not broadcast_no:
 421             raise ExtractorError(f'Unable to extract broadcast number ({broadcaster_id} may not be live)', expected=True)
 422         if password_protected == 'Y' and password is None:
 423             raise ExtractorError(
 424                 'This livestream is protected by a password, use the --video-password option',
 425                 expected=True)
 426
 427         formats = []
 428         quality_key = qualities(self._QUALITIES)
 429         for quality_str in self._QUALITIES:
 430             params = {
 431                 'bno': broadcast_no,
 432                 'stream_type': 'common',
 433                 'type': 'aid',
 434                 'quality': quality_str,
 435             }
 436             if password is not None:
 437                 params['pwd'] = password
 438             aid_response = self._download_json(
 439                 self._LIVE_API_URL, broadcast_no, fatal=False,
 440                 data=urlencode_postdata(params),
 441                 note=f'Downloading access token for {quality_str} stream',
 442                 errnote=f'Unable to download access token for {quality_str} stream')
 443             aid = traverse_obj(aid_response, ('CHANNEL', 'AID'))
 444             if not aid:
 445                 continue
 446
 447             stream_base_url = channel_info.get('RMD') or 'https://livestream-manager.afreecatv.com'
 448             stream_info = self._download_json(
 449                 f'{stream_base_url}/broad_stream_assign.html', broadcast_no, fatal=False,
 450                 query={
 451                     'return_type': channel_info.get('CDN', 'gcp_cdn'),
 452                     'broad_key': f'{broadcast_no}-common-{quality_str}-hls',
 453                 },
 454                 note=f'Downloading metadata for {quality_str} stream',
 455                 errnote=f'Unable to download metadata for {quality_str} stream') or {}
 456
 457             if stream_info.get('view_url'):
 458                 formats.append({
 459                     'format_id': quality_str,
 460                     'url': update_url_query(stream_info['view_url'], {'aid': aid}),
 461                     'ext': 'mp4',
 462                     'protocol': 'm3u8',
 463                     'quality': quality_key(quality_str),
 464                 })
 465
 466         station_info = self._download_json(
 467             'https://st.afreecatv.com/api/get_station_status.php', broadcast_no,
 468             query={'szBjId': broadcaster_id}, fatal=False,
 469             note='Downloading channel metadata', errnote='Unable to download channel metadata') or {}
 470
 471         return {
 472             'id': broadcast_no,
 473             'title': channel_info.get('TITLE') or station_info.get('station_title'),
 474             'uploader': channel_info.get('BJNICK') or station_info.get('station_name'),
 475             'uploader_id': broadcaster_id,
 476             'timestamp': unified_timestamp(station_info.get('broad_start')),
 477             'formats': formats,
 478             'is_live': True,
 479         }
 480
 481
 482 class AfreecaTVUserIE(InfoExtractor):
 483     IE_NAME = 'afreecatv:user'
 484     _VALID_URL = r'https?://bj\.afreeca(?:tv)?\.com/(?P<id>[^/]+)/vods/?(?P<slug_type>[^/]+)?'
 485     _TESTS = [{
 486         'url': 'https://bj.afreecatv.com/ryuryu24/vods/review',
 487         'info_dict': {
 488             '_type': 'playlist',
 489             'id': 'ryuryu24',
 490             'title': 'ryuryu24 - review',
 491         },
 492         'playlist_count': 218,
 493     }, {
 494         'url': 'https://bj.afreecatv.com/parang1995/vods/highlight',
 495         'info_dict': {
 496             '_type': 'playlist',
 497             'id': 'parang1995',
 498             'title': 'parang1995 - highlight',
 499         },
 500         'playlist_count': 997,
 501     }, {
 502         'url': 'https://bj.afreecatv.com/ryuryu24/vods',
 503         'info_dict': {
 504             '_type': 'playlist',
 505             'id': 'ryuryu24',
 506             'title': 'ryuryu24 - all',
 507         },
 508         'playlist_count': 221,
 509     }, {
 510         'url': 'https://bj.afreecatv.com/ryuryu24/vods/balloonclip',
 511         'info_dict': {
 512             '_type': 'playlist',
 513             'id': 'ryuryu24',
 514             'title': 'ryuryu24 - balloonclip',
 515         },
 516         'playlist_count': 0,
 517     }]
 518     _PER_PAGE = 60
 519
 520     def _fetch_page(self, user_id, user_type, page):
 521         page += 1
 522         info = self._download_json(f'https://bjapi.afreecatv.com/api/{user_id}/vods/{user_type}', user_id,
 523                                    query={'page': page, 'per_page': self._PER_PAGE, 'orderby': 'reg_date'},
 524                                    note=f'Downloading {user_type} video page {page}')
 525         for item in info['data']:
 526             yield self.url_result(
 527                 f'https://vod.afreecatv.com/player/{item["title_no"]}/', AfreecaTVIE, item['title_no'])
 528
 529     def _real_extract(self, url):
 530         user_id, user_type = self._match_valid_url(url).group('id', 'slug_type')
 531         user_type = user_type or 'all'
 532         entries = OnDemandPagedList(functools.partial(self._fetch_page, user_id, user_type), self._PER_PAGE)
 533         return self.playlist_result(entries, user_id, f'{user_id} - {user_type}')