[yt-dlp.git] / yt_dlp / extractor / adn.py

# coding: utf-8
from __future__ import unicode_literals

import base64
import binascii
import json
import os
import random

from .common import InfoExtractor
from ..aes import aes_cbc_decrypt
from ..compat import (
    compat_HTTPError,
    compat_b64decode,
    compat_ord,
)
from ..utils import (
    ass_subtitles_timecode,
    bytes_to_intlist,
    bytes_to_long,
    ExtractorError,
    float_or_none,
    int_or_none,
    intlist_to_bytes,
    long_to_bytes,
    pkcs1pad,
    strip_or_none,
    try_get,
    unified_strdate,
    urlencode_postdata,
)


class ADNIE(InfoExtractor):
    IE_DESC = 'Anime Digital Network'
    _VALID_URL = r'https?://(?:www\.)?animedigitalnetwork\.fr/video/[^/]+/(?P<id>\d+)'
    _TEST = {
        'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites',
        'md5': '0319c99885ff5547565cacb4f3f9348d',
        'info_dict': {
            'id': '7778',
            'ext': 'mp4',
            'title': 'Blue Exorcist - Kyôto Saga - Episode 1',
            'description': 'md5:2f7b5aa76edbc1a7a92cedcda8a528d5',
            'series': 'Blue Exorcist - Kyôto Saga',
            'duration': 1467,
            'release_date': '20170106',
            'comment_count': int,
            'average_rating': float,
            'season_number': 2,
            'episode': 'Début des hostilités',
            'episode_number': 1,
        }
    }

    _NETRC_MACHINE = 'animedigitalnetwork'
    _BASE_URL = 'http://animedigitalnetwork.fr'
    _API_BASE_URL = 'https://gw.api.animedigitalnetwork.fr/'
    _PLAYER_BASE_URL = _API_BASE_URL + 'player/'
    _HEADERS = {}
    _LOGIN_ERR_MESSAGE = 'Unable to log in'
    _RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537)
    _POS_ALIGN_MAP = {
        'start': 1,
        'end': 3,
    }
    _LINE_ALIGN_MAP = {
        'middle': 8,
        'end': 4,
    }

    def _get_subtitles(self, sub_url, video_id):
        if not sub_url:
            return None

        enc_subtitles = self._download_webpage(
            sub_url, video_id, 'Downloading subtitles location', fatal=False) or '{}'
        subtitle_location = (self._parse_json(enc_subtitles, video_id, fatal=False) or {}).get('location')
        if subtitle_location:
            enc_subtitles = self._download_webpage(
                subtitle_location, video_id, 'Downloading subtitles data',
                fatal=False, headers={'Origin': 'https://animedigitalnetwork.fr'})
        if not enc_subtitles:
            return None

        # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
        dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
            bytes_to_intlist(compat_b64decode(enc_subtitles[24:])),
            bytes_to_intlist(binascii.unhexlify(self._K + 'ab9f52f5baae7c72')),
            bytes_to_intlist(compat_b64decode(enc_subtitles[:24]))
        ))
        subtitles_json = self._parse_json(
            dec_subtitles[:-compat_ord(dec_subtitles[-1])].decode(),
            None, fatal=False)
        if not subtitles_json:
            return None

        subtitles = {}
        for sub_lang, sub in subtitles_json.items():
            ssa = '''[Script Info]
ScriptType:V4.00
[V4 Styles]
Format: Name,Fontname,Fontsize,PrimaryColour,SecondaryColour,TertiaryColour,BackColour,Bold,Italic,BorderStyle,Outline,Shadow,Alignment,MarginL,MarginR,MarginV,AlphaLevel,Encoding
Style: Default,Arial,18,16777215,16777215,16777215,0,-1,0,1,1,0,2,20,20,20,0,0
[Events]
Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
            for current in sub:
                start, end, text, line_align, position_align = (
                    float_or_none(current.get('startTime')),
                    float_or_none(current.get('endTime')),
                    current.get('text'), current.get('lineAlign'),
                    current.get('positionAlign'))
                if start is None or end is None or text is None:
                    continue
                alignment = self._POS_ALIGN_MAP.get(position_align, 2) + self._LINE_ALIGN_MAP.get(line_align, 0)
                ssa += os.linesep + 'Dialogue: Marked=0,%s,%s,Default,,0,0,0,,%s%s' % (
                    ass_subtitles_timecode(start),
                    ass_subtitles_timecode(end),
                    '{\\a%d}' % alignment if alignment != 2 else '',
                    text.replace('\n', '\\N').replace('<i>', '{\\i1}').replace('</i>', '{\\i0}'))

            if sub_lang == 'vostf':
                sub_lang = 'fr'
            subtitles.setdefault(sub_lang, []).extend([{
                'ext': 'json',
                'data': json.dumps(sub),
            }, {
                'ext': 'ssa',
                'data': ssa,
            }])
        return subtitles

    def _real_initialize(self):
        username, password = self._get_login_info()
        if not username:
            return
        try:
            access_token = (self._download_json(
                self._API_BASE_URL + 'authentication/login', None,
                'Logging in', self._LOGIN_ERR_MESSAGE, fatal=False,
                data=urlencode_postdata({
                    'password': password,
                    'rememberMe': False,
                    'source': 'Web',
                    'username': username,
                })) or {}).get('accessToken')
            if access_token:
                self._HEADERS = {'authorization': 'Bearer ' + access_token}
        except ExtractorError as e:
            message = None
            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
                resp = self._parse_json(
                    e.cause.read().decode(), None, fatal=False) or {}
                message = resp.get('message') or resp.get('code')
            self.report_warning(message or self._LOGIN_ERR_MESSAGE)

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_base_url = self._PLAYER_BASE_URL + 'video/%s/' % video_id
        player = self._download_json(
            video_base_url + 'configuration', video_id,
            'Downloading player config JSON metadata',
            headers=self._HEADERS)['player']
        options = player['options']

        user = options['user']
        if not user.get('hasAccess'):
            self.raise_login_required()

        token = self._download_json(
            user.get('refreshTokenUrl') or (self._PLAYER_BASE_URL + 'refresh/token'),
            video_id, 'Downloading access token', headers={
                'x-player-refresh-token': user['refreshToken']
            }, data=b'')['token']

        links_url = try_get(options, lambda x: x['video']['url']) or (video_base_url + 'link')
        self._K = ''.join([random.choice('0123456789abcdef') for _ in range(16)])
        message = bytes_to_intlist(json.dumps({
            'k': self._K,
            't': token,
        }))

        # Sometimes authentication fails for no good reason, retry with
        # a different random padding
        links_data = None
        for _ in range(3):
            padded_message = intlist_to_bytes(pkcs1pad(message, 128))
            n, e = self._RSA_KEY
            encrypted_message = long_to_bytes(pow(bytes_to_long(padded_message), e, n))
            authorization = base64.b64encode(encrypted_message).decode()

            try:
                links_data = self._download_json(
                    links_url, video_id, 'Downloading links JSON metadata', headers={
                        'X-Player-Token': authorization
                    }, query={
                        'freeWithAds': 'true',
                        'adaptive': 'false',
                        'withMetadata': 'true',
                        'source': 'Web'
                    })
                break
            except ExtractorError as e:
                if not isinstance(e.cause, compat_HTTPError):
                    raise e

                if e.cause.code == 401:
                    # This usually goes away with a different random pkcs1pad, so retry
                    continue

                error = self._parse_json(e.cause.read(), video_id)
                message = error.get('message')
                if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country':
                    self.raise_geo_restricted(msg=message)
                raise ExtractorError(message)
        else:
            raise ExtractorError('Giving up retrying')

        links = links_data.get('links') or {}
        metas = links_data.get('metadata') or {}
        sub_url = (links.get('subtitles') or {}).get('all')
        video_info = links_data.get('video') or {}
        title = metas['title']

        formats = []
        for format_id, qualities in (links.get('streaming') or {}).items():
            if not isinstance(qualities, dict):
                continue
            for quality, load_balancer_url in qualities.items():
                load_balancer_data = self._download_json(
                    load_balancer_url, video_id,
                    'Downloading %s %s JSON metadata' % (format_id, quality),
                    fatal=False) or {}
                m3u8_url = load_balancer_data.get('location')
                if not m3u8_url:
                    continue
                m3u8_formats = self._extract_m3u8_formats(
                    m3u8_url, video_id, 'mp4', 'm3u8_native',
                    m3u8_id=format_id, fatal=False)
                if format_id == 'vf':
                    for f in m3u8_formats:
                        f['language'] = 'fr'
                formats.extend(m3u8_formats)
        self._sort_formats(formats)

        video = (self._download_json(
            self._API_BASE_URL + 'video/%s' % video_id, video_id,
            'Downloading additional video metadata', fatal=False) or {}).get('video') or {}
        show = video.get('show') or {}

        return {
            'id': video_id,
            'title': title,
            'description': strip_or_none(metas.get('summary') or video.get('summary')),
            'thumbnail': video_info.get('image') or player.get('image'),
            'formats': formats,
            'subtitles': self.extract_subtitles(sub_url, video_id),
            'episode': metas.get('subtitle') or video.get('name'),
            'episode_number': int_or_none(video.get('shortNumber')),
            'series': show.get('title'),
            'season_number': int_or_none(video.get('season')),
            'duration': int_or_none(video_info.get('duration') or video.get('duration')),
            'release_date': unified_strdate(video.get('releaseDate')),
            'average_rating': float_or_none(video.get('rating') or metas.get('rating')),
            'comment_count': int_or_none(video.get('commentsCount')),
        }
Commit	Line	Data
82be732b RA	1	# coding: utf-8
	2	from __future__ import unicode_literals
	3
1ea559c4 RA	4	import base64
1ea559c4 RA	5	import binascii
82be732b RA	6	import json
82be732b RA	7	import os
1ea559c4	8	import random
82be732b RA	9
	10	from .common import InfoExtractor
	11	from ..aes import aes_cbc_decrypt
cf282071	12	from ..compat import (
30a074c2	13	compat_HTTPError,
cf282071 S	14	compat_b64decode,
	15	compat_ord,
	16	)
82be732b	17	from ..utils import (
aa7785f8	18	ass_subtitles_timecode,
82be732b	19	bytes_to_intlist,
1ea559c4	20	bytes_to_long,
82be732b RA	21	ExtractorError,
82be732b RA	22	float_or_none,
30a074c2	23	int_or_none,
82be732b	24	intlist_to_bytes,
1ea559c4 RA	25	long_to_bytes,
1ea559c4 RA	26	pkcs1pad,
82be732b	27	strip_or_none,
30a074c2	28	try_get,
30a074c2	29	unified_strdate,
2181983a	30	urlencode_postdata,
82be732b RA	31	)
	32
	33
	34	class ADNIE(InfoExtractor):
	35	IE_DESC = 'Anime Digital Network'
	36	_VALID_URL = r'https?://(?:www\.)?animedigitalnetwork\.fr/video/[^/]+/(?P<id>\d+)'
	37	_TEST = {
	38	'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites',
30a074c2	39	'md5': '0319c99885ff5547565cacb4f3f9348d',
82be732b RA	40	'info_dict': {
	41	'id': '7778',
	42	'ext': 'mp4',
30a074c2	43	'title': 'Blue Exorcist - Kyôto Saga - Episode 1',
82be732b	44	'description': 'md5:2f7b5aa76edbc1a7a92cedcda8a528d5',
30a074c2	45	'series': 'Blue Exorcist - Kyôto Saga',
	46	'duration': 1467,
	47	'release_date': '20170106',
	48	'comment_count': int,
	49	'average_rating': float,
	50	'season_number': 2,
	51	'episode': 'Début des hostilités',
	52	'episode_number': 1,
82be732b RA	53	}
82be732b RA	54	}
30a074c2	55
2181983a	56	_NETRC_MACHINE = 'animedigitalnetwork'
20e2c9de	57	_BASE_URL = 'http://animedigitalnetwork.fr'
30a074c2	58	_API_BASE_URL = 'https://gw.api.animedigitalnetwork.fr/'
30a074c2	59	_PLAYER_BASE_URL = _API_BASE_URL + 'player/'
2181983a	60	_HEADERS = {}
2181983a	61	_LOGIN_ERR_MESSAGE = 'Unable to log in'
30a074c2	62	_RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537)
b966740c RA	63	_POS_ALIGN_MAP = {
	64	'start': 1,
	65	'end': 3,
	66	}
	67	_LINE_ALIGN_MAP = {
	68	'middle': 8,
	69	'end': 4,
	70	}
	71
30a074c2	72	def _get_subtitles(self, sub_url, video_id):
30a074c2	73	if not sub_url:
82be732b RA	74	return None
	75
	76	enc_subtitles = self._download_webpage(
30a074c2	77	sub_url, video_id, 'Downloading subtitles location', fatal=False) or '{}'
e6c9ae31 RA	78	subtitle_location = (self._parse_json(enc_subtitles, video_id, fatal=False) or {}).get('location')
	79	if subtitle_location:
	80	enc_subtitles = self._download_webpage(
30a074c2	81	subtitle_location, video_id, 'Downloading subtitles data',
30a074c2	82	fatal=False, headers={'Origin': 'https://animedigitalnetwork.fr'})
82be732b RA	83	if not enc_subtitles:
	84	return None
	85
	86	# http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
	87	dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
cf282071	88	bytes_to_intlist(compat_b64decode(enc_subtitles[24:])),
30a074c2	89	bytes_to_intlist(binascii.unhexlify(self._K + 'ab9f52f5baae7c72')),
cf282071	90	bytes_to_intlist(compat_b64decode(enc_subtitles[:24]))
82be732b RA	91	))
82be732b RA	92	subtitles_json = self._parse_json(
20e2c9de	93	dec_subtitles[:-compat_ord(dec_subtitles[-1])].decode(),
82be732b RA	94	None, fatal=False)
	95	if not subtitles_json:
	96	return None
	97
	98	subtitles = {}
	99	for sub_lang, sub in subtitles_json.items():
b966740c RA	100	ssa = '''[Script Info]
	101	ScriptType:V4.00
	102	[V4 Styles]
2bbde1d0 RA	103	Format: Name,Fontname,Fontsize,PrimaryColour,SecondaryColour,TertiaryColour,BackColour,Bold,Italic,BorderStyle,Outline,Shadow,Alignment,MarginL,MarginR,MarginV,AlphaLevel,Encoding
2bbde1d0 RA	104	Style: Default,Arial,18,16777215,16777215,16777215,0,-1,0,1,1,0,2,20,20,20,0,0
b966740c	105	[Events]
2bbde1d0	106	Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
b966740c RA	107	for current in sub:
b966740c RA	108	start, end, text, line_align, position_align = (
82be732b RA	109	float_or_none(current.get('startTime')),
82be732b RA	110	float_or_none(current.get('endTime')),
b966740c RA	111	current.get('text'), current.get('lineAlign'),
b966740c RA	112	current.get('positionAlign'))
82be732b RA	113	if start is None or end is None or text is None:
82be732b RA	114	continue
b966740c	115	alignment = self._POS_ALIGN_MAP.get(position_align, 2) + self._LINE_ALIGN_MAP.get(line_align, 0)
2bbde1d0	116	ssa += os.linesep + 'Dialogue: Marked=0,%s,%s,Default,,0,0,0,,%s%s' % (
aa7785f8	117	ass_subtitles_timecode(start),
aa7785f8	118	ass_subtitles_timecode(end),
b966740c RA	119	'{\\a%d}' % alignment if alignment != 2 else '',
b966740c RA	120	text.replace('\n', '\\N').replace('<i>', '{\\i1}').replace('</i>', '{\\i0}'))
82be732b RA	121
	122	if sub_lang == 'vostf':
	123	sub_lang = 'fr'
	124	subtitles.setdefault(sub_lang, []).extend([{
	125	'ext': 'json',
	126	'data': json.dumps(sub),
	127	}, {
b966740c RA	128	'ext': 'ssa',
b966740c RA	129	'data': ssa,
82be732b RA	130	}])
	131	return subtitles
	132
2181983a	133	def _real_initialize(self):
	134	username, password = self._get_login_info()
	135	if not username:
	136	return
	137	try:
	138	access_token = (self._download_json(
	139	self._API_BASE_URL + 'authentication/login', None,
	140	'Logging in', self._LOGIN_ERR_MESSAGE, fatal=False,
	141	data=urlencode_postdata({
	142	'password': password,
	143	'rememberMe': False,
	144	'source': 'Web',
	145	'username': username,
	146	})) or {}).get('accessToken')
	147	if access_token:
	148	self._HEADERS = {'authorization': 'Bearer ' + access_token}
	149	except ExtractorError as e:
	150	message = None
	151	if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
	152	resp = self._parse_json(
	153	e.cause.read().decode(), None, fatal=False) or {}
	154	message = resp.get('message') or resp.get('code')
	155	self.report_warning(message or self._LOGIN_ERR_MESSAGE)
	156
82be732b RA	157	def _real_extract(self, url):
82be732b RA	158	video_id = self._match_id(url)
30a074c2	159	video_base_url = self._PLAYER_BASE_URL + 'video/%s/' % video_id
	160	player = self._download_json(
	161	video_base_url + 'configuration', video_id,
2181983a	162	'Downloading player config JSON metadata',
2181983a	163	headers=self._HEADERS)['player']
30a074c2	164	options = player['options']
	165
	166	user = options['user']
	167	if not user.get('hasAccess'):
2181983a	168	self.raise_login_required()
30a074c2	169
	170	token = self._download_json(
	171	user.get('refreshTokenUrl') or (self._PLAYER_BASE_URL + 'refresh/token'),
	172	video_id, 'Downloading access token', headers={
	173	'x-player-refresh-token': user['refreshToken']
	174	}, data=b'')['token']
	175
	176	links_url = try_get(options, lambda x: x['video']['url']) or (video_base_url + 'link')
	177	self._K = ''.join([random.choice('0123456789abcdef') for _ in range(16)])
	178	message = bytes_to_intlist(json.dumps({
	179	'k': self._K,
	180	't': token,
	181	}))
	182
	183	# Sometimes authentication fails for no good reason, retry with
	184	# a different random padding
	185	links_data = None
	186	for _ in range(3):
1ea559c4 RA	187	padded_message = intlist_to_bytes(pkcs1pad(message, 128))
	188	n, e = self._RSA_KEY
	189	encrypted_message = long_to_bytes(pow(bytes_to_long(padded_message), e, n))
	190	authorization = base64.b64encode(encrypted_message).decode()
30a074c2	191
	192	try:
	193	links_data = self._download_json(
	194	links_url, video_id, 'Downloading links JSON metadata', headers={
	195	'X-Player-Token': authorization
	196	}, query={
	197	'freeWithAds': 'true',
	198	'adaptive': 'false',
	199	'withMetadata': 'true',
	200	'source': 'Web'
	201	})
	202	break
	203	except ExtractorError as e:
	204	if not isinstance(e.cause, compat_HTTPError):
	205	raise e
	206
	207	if e.cause.code == 401:
	208	# This usually goes away with a different random pkcs1pad, so retry
	209	continue
	210
	211	error = self._parse_json(e.cause.read(), video_id)
	212	message = error.get('message')
	213	if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country':
	214	self.raise_geo_restricted(msg=message)
2181983a	215	raise ExtractorError(message)
30a074c2	216	else:
	217	raise ExtractorError('Giving up retrying')
	218
	219	links = links_data.get('links') or {}
	220	metas = links_data.get('metadata') or {}
	221	sub_url = (links.get('subtitles') or {}).get('all')
	222	video_info = links_data.get('video') or {}
	223	title = metas['title']
82be732b RA	224
82be732b RA	225	formats = []
30a074c2	226	for format_id, qualities in (links.get('streaming') or {}).items():
20e2c9de RA	227	if not isinstance(qualities, dict):
20e2c9de RA	228	continue
b966740c	229	for quality, load_balancer_url in qualities.items():
82be732b	230	load_balancer_data = self._download_json(
b966740c RA	231	load_balancer_url, video_id,
	232	'Downloading %s %s JSON metadata' % (format_id, quality),
	233	fatal=False) or {}
82be732b RA	234	m3u8_url = load_balancer_data.get('location')
	235	if not m3u8_url:
	236	continue
	237	m3u8_formats = self._extract_m3u8_formats(
	238	m3u8_url, video_id, 'mp4', 'm3u8_native',
	239	m3u8_id=format_id, fatal=False)
	240	if format_id == 'vf':
	241	for f in m3u8_formats:
	242	f['language'] = 'fr'
	243	formats.extend(m3u8_formats)
82be732b RA	244	self._sort_formats(formats)
82be732b RA	245
30a074c2	246	video = (self._download_json(
	247	self._API_BASE_URL + 'video/%s' % video_id, video_id,
	248	'Downloading additional video metadata', fatal=False) or {}).get('video') or {}
	249	show = video.get('show') or {}
	250
82be732b RA	251	return {
	252	'id': video_id,
	253	'title': title,
30a074c2	254	'description': strip_or_none(metas.get('summary') or video.get('summary')),
30a074c2	255	'thumbnail': video_info.get('image') or player.get('image'),
82be732b	256	'formats': formats,
30a074c2	257	'subtitles': self.extract_subtitles(sub_url, video_id),
	258	'episode': metas.get('subtitle') or video.get('name'),
	259	'episode_number': int_or_none(video.get('shortNumber')),
	260	'series': show.get('title'),
	261	'season_number': int_or_none(video.get('season')),
	262	'duration': int_or_none(video_info.get('duration') or video.get('duration')),
	263	'release_date': unified_strdate(video.get('releaseDate')),
	264	'average_rating': float_or_none(video.get('rating') or metas.get('rating')),
	265	'comment_count': int_or_none(video.get('commentsCount')),
82be732b	266	}