[yt-dlp.git] / youtube_dlc / downloader / ism.py

from __future__ import unicode_literals

import time
import binascii
import io

from .fragment import FragmentFD
from ..compat import (
    compat_Struct,
    compat_urllib_error,
)


u8 = compat_Struct('>B')
u88 = compat_Struct('>Bx')
u16 = compat_Struct('>H')
u1616 = compat_Struct('>Hxx')
u32 = compat_Struct('>I')
u64 = compat_Struct('>Q')

s88 = compat_Struct('>bx')
s16 = compat_Struct('>h')
s1616 = compat_Struct('>hxx')
s32 = compat_Struct('>i')

unity_matrix = (s32.pack(0x10000) + s32.pack(0) * 3) * 2 + s32.pack(0x40000000)

TRACK_ENABLED = 0x1
TRACK_IN_MOVIE = 0x2
TRACK_IN_PREVIEW = 0x4

SELF_CONTAINED = 0x1


def box(box_type, payload):
    return u32.pack(8 + len(payload)) + box_type + payload


def full_box(box_type, version, flags, payload):
    return box(box_type, u8.pack(version) + u32.pack(flags)[1:] + payload)


def write_piff_header(stream, params):
    track_id = params['track_id']
    fourcc = params['fourcc']
    duration = params['duration']
    timescale = params.get('timescale', 10000000)
    language = params.get('language', 'und')
    height = params.get('height', 0)
    width = params.get('width', 0)
    is_audio = width == 0 and height == 0
    creation_time = modification_time = int(time.time())

    ftyp_payload = b'isml'  # major brand
    ftyp_payload += u32.pack(1)  # minor version
    ftyp_payload += b'piff' + b'iso2'  # compatible brands
    stream.write(box(b'ftyp', ftyp_payload))  # File Type Box

    mvhd_payload = u64.pack(creation_time)
    mvhd_payload += u64.pack(modification_time)
    mvhd_payload += u32.pack(timescale)
    mvhd_payload += u64.pack(duration)
    mvhd_payload += s1616.pack(1)  # rate
    mvhd_payload += s88.pack(1)  # volume
    mvhd_payload += u16.pack(0)  # reserved
    mvhd_payload += u32.pack(0) * 2  # reserved
    mvhd_payload += unity_matrix
    mvhd_payload += u32.pack(0) * 6  # pre defined
    mvhd_payload += u32.pack(0xffffffff)  # next track id
    moov_payload = full_box(b'mvhd', 1, 0, mvhd_payload)  # Movie Header Box

    tkhd_payload = u64.pack(creation_time)
    tkhd_payload += u64.pack(modification_time)
    tkhd_payload += u32.pack(track_id)  # track id
    tkhd_payload += u32.pack(0)  # reserved
    tkhd_payload += u64.pack(duration)
    tkhd_payload += u32.pack(0) * 2  # reserved
    tkhd_payload += s16.pack(0)  # layer
    tkhd_payload += s16.pack(0)  # alternate group
    tkhd_payload += s88.pack(1 if is_audio else 0)  # volume
    tkhd_payload += u16.pack(0)  # reserved
    tkhd_payload += unity_matrix
    tkhd_payload += u1616.pack(width)
    tkhd_payload += u1616.pack(height)
    trak_payload = full_box(b'tkhd', 1, TRACK_ENABLED | TRACK_IN_MOVIE | TRACK_IN_PREVIEW, tkhd_payload)  # Track Header Box

    mdhd_payload = u64.pack(creation_time)
    mdhd_payload += u64.pack(modification_time)
    mdhd_payload += u32.pack(timescale)
    mdhd_payload += u64.pack(duration)
    mdhd_payload += u16.pack(((ord(language[0]) - 0x60) << 10) | ((ord(language[1]) - 0x60) << 5) | (ord(language[2]) - 0x60))
    mdhd_payload += u16.pack(0)  # pre defined
    mdia_payload = full_box(b'mdhd', 1, 0, mdhd_payload)  # Media Header Box

    hdlr_payload = u32.pack(0)  # pre defined
    hdlr_payload += b'soun' if is_audio else b'vide'  # handler type
    hdlr_payload += u32.pack(0) * 3  # reserved
    hdlr_payload += (b'Sound' if is_audio else b'Video') + b'Handler\0'  # name
    mdia_payload += full_box(b'hdlr', 0, 0, hdlr_payload)  # Handler Reference Box

    if is_audio:
        smhd_payload = s88.pack(0)  # balance
        smhd_payload += u16.pack(0)  # reserved
        media_header_box = full_box(b'smhd', 0, 0, smhd_payload)  # Sound Media Header
    else:
        vmhd_payload = u16.pack(0)  # graphics mode
        vmhd_payload += u16.pack(0) * 3  # opcolor
        media_header_box = full_box(b'vmhd', 0, 1, vmhd_payload)  # Video Media Header
    minf_payload = media_header_box

    dref_payload = u32.pack(1)  # entry count
    dref_payload += full_box(b'url ', 0, SELF_CONTAINED, b'')  # Data Entry URL Box
    dinf_payload = full_box(b'dref', 0, 0, dref_payload)  # Data Reference Box
    minf_payload += box(b'dinf', dinf_payload)  # Data Information Box

    stsd_payload = u32.pack(1)  # entry count

    sample_entry_payload = u8.pack(0) * 6  # reserved
    sample_entry_payload += u16.pack(1)  # data reference index
    if is_audio:
        sample_entry_payload += u32.pack(0) * 2  # reserved
        sample_entry_payload += u16.pack(params.get('channels', 2))
        sample_entry_payload += u16.pack(params.get('bits_per_sample', 16))
        sample_entry_payload += u16.pack(0)  # pre defined
        sample_entry_payload += u16.pack(0)  # reserved
        sample_entry_payload += u1616.pack(params['sampling_rate'])

        if fourcc == 'AACL':
            sample_entry_box = box(b'mp4a', sample_entry_payload)
    else:
        sample_entry_payload += u16.pack(0)  # pre defined
        sample_entry_payload += u16.pack(0)  # reserved
        sample_entry_payload += u32.pack(0) * 3  # pre defined
        sample_entry_payload += u16.pack(width)
        sample_entry_payload += u16.pack(height)
        sample_entry_payload += u1616.pack(0x48)  # horiz resolution 72 dpi
        sample_entry_payload += u1616.pack(0x48)  # vert resolution 72 dpi
        sample_entry_payload += u32.pack(0)  # reserved
        sample_entry_payload += u16.pack(1)  # frame count
        sample_entry_payload += u8.pack(0) * 32  # compressor name
        sample_entry_payload += u16.pack(0x18)  # depth
        sample_entry_payload += s16.pack(-1)  # pre defined

        codec_private_data = binascii.unhexlify(params['codec_private_data'].encode('utf-8'))
        if fourcc in ('H264', 'AVC1'):
            sps, pps = codec_private_data.split(u32.pack(1))[1:]
            avcc_payload = u8.pack(1)  # configuration version
            avcc_payload += sps[1:4]  # avc profile indication + profile compatibility + avc level indication
            avcc_payload += u8.pack(0xfc | (params.get('nal_unit_length_field', 4) - 1))  # complete representation (1) + reserved (11111) + length size minus one
            avcc_payload += u8.pack(1)  # reserved (0) + number of sps (0000001)
            avcc_payload += u16.pack(len(sps))
            avcc_payload += sps
            avcc_payload += u8.pack(1)  # number of pps
            avcc_payload += u16.pack(len(pps))
            avcc_payload += pps
            sample_entry_payload += box(b'avcC', avcc_payload)  # AVC Decoder Configuration Record
            sample_entry_box = box(b'avc1', sample_entry_payload)  # AVC Simple Entry
    stsd_payload += sample_entry_box

    stbl_payload = full_box(b'stsd', 0, 0, stsd_payload)  # Sample Description Box

    stts_payload = u32.pack(0)  # entry count
    stbl_payload += full_box(b'stts', 0, 0, stts_payload)  # Decoding Time to Sample Box

    stsc_payload = u32.pack(0)  # entry count
    stbl_payload += full_box(b'stsc', 0, 0, stsc_payload)  # Sample To Chunk Box

    stco_payload = u32.pack(0)  # entry count
    stbl_payload += full_box(b'stco', 0, 0, stco_payload)  # Chunk Offset Box

    minf_payload += box(b'stbl', stbl_payload)  # Sample Table Box

    mdia_payload += box(b'minf', minf_payload)  # Media Information Box

    trak_payload += box(b'mdia', mdia_payload)  # Media Box

    moov_payload += box(b'trak', trak_payload)  # Track Box

    mehd_payload = u64.pack(duration)
    mvex_payload = full_box(b'mehd', 1, 0, mehd_payload)  # Movie Extends Header Box

    trex_payload = u32.pack(track_id)  # track id
    trex_payload += u32.pack(1)  # default sample description index
    trex_payload += u32.pack(0)  # default sample duration
    trex_payload += u32.pack(0)  # default sample size
    trex_payload += u32.pack(0)  # default sample flags
    mvex_payload += full_box(b'trex', 0, 0, trex_payload)  # Track Extends Box

    moov_payload += box(b'mvex', mvex_payload)  # Movie Extends Box
    stream.write(box(b'moov', moov_payload))  # Movie Box


def extract_box_data(data, box_sequence):
    data_reader = io.BytesIO(data)
    while True:
        box_size = u32.unpack(data_reader.read(4))[0]
        box_type = data_reader.read(4)
        if box_type == box_sequence[0]:
            box_data = data_reader.read(box_size - 8)
            if len(box_sequence) == 1:
                return box_data
            return extract_box_data(box_data, box_sequence[1:])
        data_reader.seek(box_size - 8, 1)


class IsmFD(FragmentFD):
    """
    Download segments in a ISM manifest
    """

    FD_NAME = 'ism'

    def real_download(self, filename, info_dict):
        segments = info_dict['fragments'][:1] if self.params.get(
            'test', False) else info_dict['fragments']

        ctx = {
            'filename': filename,
            'total_frags': len(segments),
        }

        self._prepare_and_start_frag_download(ctx)

        fragment_retries = self.params.get('fragment_retries', 0)
        skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)

        track_written = False
        frag_index = 0
        for i, segment in enumerate(segments):
            frag_index += 1
            if frag_index <= ctx['fragment_index']:
                continue
            count = 0
            while count <= fragment_retries:
                try:
                    success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
                    if not success:
                        return False
                    if not track_written:
                        tfhd_data = extract_box_data(frag_content, [b'moof', b'traf', b'tfhd'])
                        info_dict['_download_params']['track_id'] = u32.unpack(tfhd_data[4:8])[0]
                        write_piff_header(ctx['dest_stream'], info_dict['_download_params'])
                        track_written = True
                    self._append_fragment(ctx, frag_content)
                    break
                except compat_urllib_error.HTTPError as err:
                    count += 1
                    if count <= fragment_retries:
                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
            if count > fragment_retries:
                if skip_unavailable_fragments:
                    self.report_skip_fragment(frag_index)
                    continue
                self.report_error('giving up after %s fragment retries' % fragment_retries)
                return False

        self._finish_frag_download(ctx)

        return True
Commit	Line	Data
b2758123 RA	1	from __future__ import unicode_literals
b2758123 RA	2
b2758123	3	import time
b2758123 RA	4	import binascii
	5	import io
	6
	7	from .fragment import FragmentFD
65220c3b S	8	from ..compat import (
	9	compat_Struct,
	10	compat_urllib_error,
	11	)
	12
	13
	14	u8 = compat_Struct('>B')
	15	u88 = compat_Struct('>Bx')
	16	u16 = compat_Struct('>H')
	17	u1616 = compat_Struct('>Hxx')
	18	u32 = compat_Struct('>I')
	19	u64 = compat_Struct('>Q')
	20
	21	s88 = compat_Struct('>bx')
	22	s16 = compat_Struct('>h')
	23	s1616 = compat_Struct('>hxx')
	24	s32 = compat_Struct('>i')
b2758123 RA	25
	26	unity_matrix = (s32.pack(0x10000) + s32.pack(0) * 3) * 2 + s32.pack(0x40000000)
	27
	28	TRACK_ENABLED = 0x1
	29	TRACK_IN_MOVIE = 0x2
	30	TRACK_IN_PREVIEW = 0x4
	31
	32	SELF_CONTAINED = 0x1
	33
	34
	35	def box(box_type, payload):
	36	return u32.pack(8 + len(payload)) + box_type + payload
	37
	38
	39	def full_box(box_type, version, flags, payload):
	40	return box(box_type, u8.pack(version) + u32.pack(flags)[1:] + payload)
	41
	42
	43	def write_piff_header(stream, params):
	44	track_id = params['track_id']
	45	fourcc = params['fourcc']
	46	duration = params['duration']
	47	timescale = params.get('timescale', 10000000)
	48	language = params.get('language', 'und')
	49	height = params.get('height', 0)
	50	width = params.get('width', 0)
	51	is_audio = width == 0 and height == 0
	52	creation_time = modification_time = int(time.time())
	53
	54	ftyp_payload = b'isml' # major brand
	55	ftyp_payload += u32.pack(1) # minor version
	56	ftyp_payload += b'piff' + b'iso2' # compatible brands
	57	stream.write(box(b'ftyp', ftyp_payload)) # File Type Box
	58
	59	mvhd_payload = u64.pack(creation_time)
	60	mvhd_payload += u64.pack(modification_time)
	61	mvhd_payload += u32.pack(timescale)
	62	mvhd_payload += u64.pack(duration)
	63	mvhd_payload += s1616.pack(1) # rate
	64	mvhd_payload += s88.pack(1) # volume
	65	mvhd_payload += u16.pack(0) # reserved
	66	mvhd_payload += u32.pack(0) * 2 # reserved
	67	mvhd_payload += unity_matrix
	68	mvhd_payload += u32.pack(0) * 6 # pre defined
	69	mvhd_payload += u32.pack(0xffffffff) # next track id
	70	moov_payload = full_box(b'mvhd', 1, 0, mvhd_payload) # Movie Header Box
	71
	72	tkhd_payload = u64.pack(creation_time)
	73	tkhd_payload += u64.pack(modification_time)
	74	tkhd_payload += u32.pack(track_id) # track id
	75	tkhd_payload += u32.pack(0) # reserved
	76	tkhd_payload += u64.pack(duration)
	77	tkhd_payload += u32.pack(0) * 2 # reserved
	78	tkhd_payload += s16.pack(0) # layer
	79	tkhd_payload += s16.pack(0) # alternate group
	80	tkhd_payload += s88.pack(1 if is_audio else 0) # volume
	81	tkhd_payload += u16.pack(0) # reserved
	82	tkhd_payload += unity_matrix
	83	tkhd_payload += u1616.pack(width)
	84	tkhd_payload += u1616.pack(height)
	85	trak_payload = full_box(b'tkhd', 1, TRACK_ENABLED \| TRACK_IN_MOVIE \| TRACK_IN_PREVIEW, tkhd_payload) # Track Header Box
	86
	87	mdhd_payload = u64.pack(creation_time)
	88	mdhd_payload += u64.pack(modification_time)
89	mdhd_payload += u32.pack(timescale)
90	mdhd_payload += u64.pack(duration)
91	mdhd_payload += u16.pack(((ord(language[0]) - 0x60) << 10) \| ((ord(language[1]) - 0x60) << 5) \| (ord(language[2]) - 0x60))
92	mdhd_payload += u16.pack(0) # pre defined
93	mdia_payload = full_box(b'mdhd', 1, 0, mdhd_payload) # Media Header Box
94
95	hdlr_payload = u32.pack(0) # pre defined
96	hdlr_payload += b'soun' if is_audio else b'vide' # handler type
97	hdlr_payload += u32.pack(0) * 3 # reserved
98	hdlr_payload += (b'Sound' if is_audio else b'Video') + b'Handler\0' # name
99	mdia_payload += full_box(b'hdlr', 0, 0, hdlr_payload) # Handler Reference Box
100
101	if is_audio:
102	smhd_payload = s88.pack(0) # balance
2583c0b5	103	smhd_payload += u16.pack(0) # reserved
b2758123 RA	104	media_header_box = full_box(b'smhd', 0, 0, smhd_payload) # Sound Media Header
	105	else:
	106	vmhd_payload = u16.pack(0) # graphics mode
	107	vmhd_payload += u16.pack(0) * 3 # opcolor
	108	media_header_box = full_box(b'vmhd', 0, 1, vmhd_payload) # Video Media Header
	109	minf_payload = media_header_box
	110
	111	dref_payload = u32.pack(1) # entry count
	112	dref_payload += full_box(b'url ', 0, SELF_CONTAINED, b'') # Data Entry URL Box
	113	dinf_payload = full_box(b'dref', 0, 0, dref_payload) # Data Reference Box
	114	minf_payload += box(b'dinf', dinf_payload) # Data Information Box
	115
	116	stsd_payload = u32.pack(1) # entry count
	117
	118	sample_entry_payload = u8.pack(0) * 6 # reserved
	119	sample_entry_payload += u16.pack(1) # data reference index
	120	if is_audio:
	121	sample_entry_payload += u32.pack(0) * 2 # reserved
	122	sample_entry_payload += u16.pack(params.get('channels', 2))
	123	sample_entry_payload += u16.pack(params.get('bits_per_sample', 16))
	124	sample_entry_payload += u16.pack(0) # pre defined
	125	sample_entry_payload += u16.pack(0) # reserved
	126	sample_entry_payload += u1616.pack(params['sampling_rate'])
	127
	128	if fourcc == 'AACL':
9d64e1dc	129	sample_entry_box = box(b'mp4a', sample_entry_payload)
b2758123	130	else:
b2758123 RA	131	sample_entry_payload += u16.pack(0) # pre defined
	132	sample_entry_payload += u16.pack(0) # reserved
	133	sample_entry_payload += u32.pack(0) * 3 # pre defined
	134	sample_entry_payload += u16.pack(width)
	135	sample_entry_payload += u16.pack(height)
	136	sample_entry_payload += u1616.pack(0x48) # horiz resolution 72 dpi
	137	sample_entry_payload += u1616.pack(0x48) # vert resolution 72 dpi
	138	sample_entry_payload += u32.pack(0) # reserved
	139	sample_entry_payload += u16.pack(1) # frame count
	140	sample_entry_payload += u8.pack(0) * 32 # compressor name
	141	sample_entry_payload += u16.pack(0x18) # depth
	142	sample_entry_payload += s16.pack(-1) # pre defined
	143
c989bdbe	144	codec_private_data = binascii.unhexlify(params['codec_private_data'].encode('utf-8'))
b2758123 RA	145	if fourcc in ('H264', 'AVC1'):
	146	sps, pps = codec_private_data.split(u32.pack(1))[1:]
	147	avcc_payload = u8.pack(1) # configuration version
10380e55	148	avcc_payload += sps[1:4] # avc profile indication + profile compatibility + avc level indication
843ad179	149	avcc_payload += u8.pack(0xfc \| (params.get('nal_unit_length_field', 4) - 1)) # complete representation (1) + reserved (11111) + length size minus one
b2758123 RA	150	avcc_payload += u8.pack(1) # reserved (0) + number of sps (0000001)
	151	avcc_payload += u16.pack(len(sps))
	152	avcc_payload += sps
	153	avcc_payload += u8.pack(1) # number of pps
	154	avcc_payload += u16.pack(len(pps))
	155	avcc_payload += pps
	156	sample_entry_payload += box(b'avcC', avcc_payload) # AVC Decoder Configuration Record
9d64e1dc S	157	sample_entry_box = box(b'avc1', sample_entry_payload) # AVC Simple Entry
9d64e1dc S	158	stsd_payload += sample_entry_box
b2758123 RA	159
	160	stbl_payload = full_box(b'stsd', 0, 0, stsd_payload) # Sample Description Box
	161
	162	stts_payload = u32.pack(0) # entry count
	163	stbl_payload += full_box(b'stts', 0, 0, stts_payload) # Decoding Time to Sample Box
	164
	165	stsc_payload = u32.pack(0) # entry count
	166	stbl_payload += full_box(b'stsc', 0, 0, stsc_payload) # Sample To Chunk Box
	167
	168	stco_payload = u32.pack(0) # entry count
	169	stbl_payload += full_box(b'stco', 0, 0, stco_payload) # Chunk Offset Box
	170
	171	minf_payload += box(b'stbl', stbl_payload) # Sample Table Box
	172
	173	mdia_payload += box(b'minf', minf_payload) # Media Information Box
	174
	175	trak_payload += box(b'mdia', mdia_payload) # Media Box
	176
	177	moov_payload += box(b'trak', trak_payload) # Track Box
	178
	179	mehd_payload = u64.pack(duration)
	180	mvex_payload = full_box(b'mehd', 1, 0, mehd_payload) # Movie Extends Header Box
	181
	182	trex_payload = u32.pack(track_id) # track id
	183	trex_payload += u32.pack(1) # default sample description index
	184	trex_payload += u32.pack(0) # default sample duration
	185	trex_payload += u32.pack(0) # default sample size
	186	trex_payload += u32.pack(0) # default sample flags
	187	mvex_payload += full_box(b'trex', 0, 0, trex_payload) # Track Extends Box
	188
	189	moov_payload += box(b'mvex', mvex_payload) # Movie Extends Box
	190	stream.write(box(b'moov', moov_payload)) # Movie Box
	191
	192
	193	def extract_box_data(data, box_sequence):
	194	data_reader = io.BytesIO(data)
	195	while True:
	196	box_size = u32.unpack(data_reader.read(4))[0]
	197	box_type = data_reader.read(4)
	198	if box_type == box_sequence[0]:
	199	box_data = data_reader.read(box_size - 8)
	200	if len(box_sequence) == 1:
	201	return box_data
	202	return extract_box_data(box_data, box_sequence[1:])
	203	data_reader.seek(box_size - 8, 1)
	204
	205
	206	class IsmFD(FragmentFD):
	207	"""
	208	Download segments in a ISM manifest
	209	"""
	210
	211	FD_NAME = 'ism'
	212
	213	def real_download(self, filename, info_dict):
	214	segments = info_dict['fragments'][:1] if self.params.get(
	215	'test', False) else info_dict['fragments']
	216
	217	ctx = {
	218	'filename': filename,
	219	'total_frags': len(segments),
	220	}
	221
	222	self._prepare_and_start_frag_download(ctx)
223
b2758123 RA	224	fragment_retries = self.params.get('fragment_retries', 0)
	225	skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
	226
	227	track_written = False
75a24854	228	frag_index = 0
b2758123	229	for i, segment in enumerate(segments):
75a24854	230	frag_index += 1
3e0304fe	231	if frag_index <= ctx['fragment_index']:
75a24854	232	continue
b2758123 RA	233	count = 0
	234	while count <= fragment_retries:
	235	try:
75a24854	236	success, frag_content = self._download_fragment(ctx, segment['url'], info_dict)
b2758123 RA	237	if not success:
b2758123 RA	238	return False
b2758123	239	if not track_written:
75a24854	240	tfhd_data = extract_box_data(frag_content, [b'moof', b'traf', b'tfhd'])
b2758123 RA	241	info_dict['_download_params']['track_id'] = u32.unpack(tfhd_data[4:8])[0]
	242	write_piff_header(ctx['dest_stream'], info_dict['_download_params'])
	243	track_written = True
75a24854	244	self._append_fragment(ctx, frag_content)
b2758123 RA	245	break
	246	except compat_urllib_error.HTTPError as err:
	247	count += 1
	248	if count <= fragment_retries:
75a24854	249	self.report_retry_fragment(err, frag_index, count, fragment_retries)
b2758123 RA	250	if count > fragment_retries:
b2758123 RA	251	if skip_unavailable_fragments:
75a24854	252	self.report_skip_fragment(frag_index)
b2758123 RA	253	continue
	254	self.report_error('giving up after %s fragment retries' % fragment_retries)
	255	return False
	256
	257	self._finish_frag_download(ctx)
	258
b2758123	259	return True