test/test_InfoExtractor.py

   1 #!/usr/bin/env python
   2
   3 from __future__ import unicode_literals
   4
   5 # Allow direct execution
   6 import io
   7 import os
   8 import sys
   9 import unittest
  10 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  11
  12 from test.helper import FakeYDL, expect_dict, expect_value, http_server_port
  13 from yt_dlp.compat import compat_etree_fromstring, compat_http_server
  14 from yt_dlp.extractor.common import InfoExtractor
  15 from yt_dlp.extractor import YoutubeIE, get_info_extractor
  16 from yt_dlp.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError
  17 import threading
  18
  19
  20 TEAPOT_RESPONSE_STATUS = 418
  21 TEAPOT_RESPONSE_BODY = "<h1>418 I'm a teapot</h1>"
  22
  23
  24 class InfoExtractorTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
  25     def log_message(self, format, *args):
  26         pass
  27
  28     def do_GET(self):
  29         if self.path == '/teapot':
  30             self.send_response(TEAPOT_RESPONSE_STATUS)
  31             self.send_header('Content-Type', 'text/html; charset=utf-8')
  32             self.end_headers()
  33             self.wfile.write(TEAPOT_RESPONSE_BODY.encode())
  34         else:
  35             assert False
  36
  37
  38 class TestIE(InfoExtractor):
  39     pass
  40
  41
  42 class TestInfoExtractor(unittest.TestCase):
  43     def setUp(self):
  44         self.ie = TestIE(FakeYDL())
  45
  46     def test_ie_key(self):
  47         self.assertEqual(get_info_extractor(YoutubeIE.ie_key()), YoutubeIE)
  48
  49     def test_html_search_regex(self):
  50         html = '<p id="foo">Watch this <a href="http://www.youtube.com/watch?v=BaW_jenozKc">video</a></p>'
  51         search = lambda re, *args: self.ie._html_search_regex(re, html, *args)
  52         self.assertEqual(search(r'<p id="foo">(.+?)</p>', 'foo'), 'Watch this video')
  53
  54     def test_opengraph(self):
  55         ie = self.ie
  56         html = '''
  57             <meta name="og:title" content='Foo'/>
  58             <meta content="Some video's description " name="og:description"/>
  59             <meta property='og:image' content='http://domain.com/pic.jpg?key1=val1&amp;key2=val2'/>
  60             <meta content='application/x-shockwave-flash' property='og:video:type'>
  61             <meta content='Foo' property=og:foobar>
  62             <meta name="og:test1" content='foo > < bar'/>
  63             <meta name="og:test2" content="foo >//< bar"/>
  64             <meta property=og-test3 content='Ill-formatted opengraph'/>
  65             '''
  66         self.assertEqual(ie._og_search_title(html), 'Foo')
  67         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
  68         self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
  69         self.assertEqual(ie._og_search_video_url(html, default=None), None)
  70         self.assertEqual(ie._og_search_property('foobar', html), 'Foo')
  71         self.assertEqual(ie._og_search_property('test1', html), 'foo > < bar')
  72         self.assertEqual(ie._og_search_property('test2', html), 'foo >//< bar')
  73         self.assertEqual(ie._og_search_property('test3', html), 'Ill-formatted opengraph')
  74         self.assertEqual(ie._og_search_property(('test0', 'test1'), html), 'foo > < bar')
  75         self.assertRaises(RegexNotFoundError, ie._og_search_property, 'test0', html, None, fatal=True)
  76         self.assertRaises(RegexNotFoundError, ie._og_search_property, ('test0', 'test00'), html, None, fatal=True)
  77
  78     def test_html_search_meta(self):
  79         ie = self.ie
  80         html = '''
  81             <meta name="a" content="1" />
  82             <meta name='b' content='2'>
  83             <meta name="c" content='3'>
  84             <meta name=d content='4'>
  85             <meta property="e" content='5' >
  86             <meta content="6" name="f">
  87         '''
  88
  89         self.assertEqual(ie._html_search_meta('a', html), '1')
  90         self.assertEqual(ie._html_search_meta('b', html), '2')
  91         self.assertEqual(ie._html_search_meta('c', html), '3')
  92         self.assertEqual(ie._html_search_meta('d', html), '4')
  93         self.assertEqual(ie._html_search_meta('e', html), '5')
  94         self.assertEqual(ie._html_search_meta('f', html), '6')
  95         self.assertEqual(ie._html_search_meta(('a', 'b', 'c'), html), '1')
  96         self.assertEqual(ie._html_search_meta(('c', 'b', 'a'), html), '3')
  97         self.assertEqual(ie._html_search_meta(('z', 'x', 'c'), html), '3')
  98         self.assertRaises(RegexNotFoundError, ie._html_search_meta, 'z', html, None, fatal=True)
  99         self.assertRaises(RegexNotFoundError, ie._html_search_meta, ('z', 'x'), html, None, fatal=True)
 100
 101     def test_search_json_ld_realworld(self):
 102         # https://github.com/ytdl-org/youtube-dl/issues/23306
 103         expect_dict(
 104             self,
 105             self.ie._search_json_ld(r'''<script type="application/ld+json">
 106 {
 107 "@context": "http://schema.org/",
 108 "@type": "VideoObject",
 109 "name": "1 On 1 With Kleio",
 110 "url": "https://www.eporner.com/hd-porn/xN49A1cT3eB/1-On-1-With-Kleio/",
 111 "duration": "PT0H12M23S",
 112 "thumbnailUrl": ["https://static-eu-cdn.eporner.com/thumbs/static4/7/78/780/780814/9_360.jpg", "https://imggen.eporner.com/780814/1920/1080/9.jpg"],
 113 "contentUrl": "https://gvideo.eporner.com/xN49A1cT3eB/xN49A1cT3eB.mp4",
 114 "embedUrl": "https://www.eporner.com/embed/xN49A1cT3eB/1-On-1-With-Kleio/",
 115 "image": "https://static-eu-cdn.eporner.com/thumbs/static4/7/78/780/780814/9_360.jpg",
 116 "width": "1920",
 117 "height": "1080",
 118 "encodingFormat": "mp4",
 119 "bitrate": "6617kbps",
 120 "isFamilyFriendly": "False",
 121 "description": "Kleio Valentien",
 122 "uploadDate": "2015-12-05T21:24:35+01:00",
 123 "interactionStatistic": {
 124 "@type": "InteractionCounter",
 125 "interactionType": { "@type": "http://schema.org/WatchAction" },
 126 "userInteractionCount": 1120958
 127 }, "aggregateRating": {
 128 "@type": "AggregateRating",
 129 "ratingValue": "88",
 130 "ratingCount": "630",
 131 "bestRating": "100",
 132 "worstRating": "0"
 133 }, "actor": [{
 134 "@type": "Person",
 135 "name": "Kleio Valentien",
 136 "url": "https://www.eporner.com/pornstar/kleio-valentien/"
 137 }]}
 138 </script>''', None),
 139             {
 140                 'title': '1 On 1 With Kleio',
 141                 'description': 'Kleio Valentien',
 142                 'url': 'https://gvideo.eporner.com/xN49A1cT3eB/xN49A1cT3eB.mp4',
 143                 'timestamp': 1449347075,
 144                 'duration': 743.0,
 145                 'view_count': 1120958,
 146                 'width': 1920,
 147                 'height': 1080,
 148             })
 149
 150     def test_download_json(self):
 151         uri = encode_data_uri(b'{"foo": "blah"}', 'application/json')
 152         self.assertEqual(self.ie._download_json(uri, None), {'foo': 'blah'})
 153         uri = encode_data_uri(b'callback({"foo": "blah"})', 'application/javascript')
 154         self.assertEqual(self.ie._download_json(uri, None, transform_source=strip_jsonp), {'foo': 'blah'})
 155         uri = encode_data_uri(b'{"foo": invalid}', 'application/json')
 156         self.assertRaises(ExtractorError, self.ie._download_json, uri, None)
 157         self.assertEqual(self.ie._download_json(uri, None, fatal=False), None)
 158
 159     def test_parse_html5_media_entries(self):
 160         # inline video tag
 161         expect_dict(
 162             self,
 163             self.ie._parse_html5_media_entries(
 164                 'https://127.0.0.1/video.html',
 165                 r'<html><video src="/vid.mp4" /></html>', None)[0],
 166             {
 167                 'formats': [{
 168                     'url': 'https://127.0.0.1/vid.mp4',
 169                 }],
 170             })
 171
 172         # from https://www.r18.com/
 173         # with kpbs in label
 174         expect_dict(
 175             self,
 176             self.ie._parse_html5_media_entries(
 177                 'https://www.r18.com/',
 178                 r'''
 179                 <video id="samplevideo_amateur" class="js-samplevideo video-js vjs-default-skin vjs-big-play-centered" controls preload="auto" width="400" height="225" poster="//pics.r18.com/digital/amateur/mgmr105/mgmr105jp.jpg">
 180                     <source id="video_source" src="https://awscc3001.r18.com/litevideo/freepv/m/mgm/mgmr105/mgmr105_sm_w.mp4" type="video/mp4"  res="240" label="300kbps">
 181                     <source id="video_source" src="https://awscc3001.r18.com/litevideo/freepv/m/mgm/mgmr105/mgmr105_dm_w.mp4" type="video/mp4"  res="480" label="1000kbps">
 182                     <source id="video_source" src="https://awscc3001.r18.com/litevideo/freepv/m/mgm/mgmr105/mgmr105_dmb_w.mp4" type="video/mp4"  res="740" label="1500kbps">
 183                     <p>Your browser does not support the video tag.</p>
 184                 </video>
 185                 ''', None)[0],
 186             {
 187                 'formats': [{
 188                     'url': 'https://awscc3001.r18.com/litevideo/freepv/m/mgm/mgmr105/mgmr105_sm_w.mp4',
 189                     'ext': 'mp4',
 190                     'format_id': '300kbps',
 191                     'height': 240,
 192                     'tbr': 300,
 193                 }, {
 194                     'url': 'https://awscc3001.r18.com/litevideo/freepv/m/mgm/mgmr105/mgmr105_dm_w.mp4',
 195                     'ext': 'mp4',
 196                     'format_id': '1000kbps',
 197                     'height': 480,
 198                     'tbr': 1000,
 199                 }, {
 200                     'url': 'https://awscc3001.r18.com/litevideo/freepv/m/mgm/mgmr105/mgmr105_dmb_w.mp4',
 201                     'ext': 'mp4',
 202                     'format_id': '1500kbps',
 203                     'height': 740,
 204                     'tbr': 1500,
 205                 }],
 206                 'thumbnail': '//pics.r18.com/digital/amateur/mgmr105/mgmr105jp.jpg'
 207             })
 208
 209         # from https://www.csfd.cz/
 210         # with width and height
 211         expect_dict(
 212             self,
 213             self.ie._parse_html5_media_entries(
 214                 'https://www.csfd.cz/',
 215                 r'''
 216                 <video width="770" height="328" preload="none" controls  poster="https://img.csfd.cz/files/images/film/video/preview/163/344/163344118_748d20.png?h360" >
 217                     <source src="https://video.csfd.cz/files/videos/157/750/157750813/163327358_eac647.mp4" type="video/mp4" width="640" height="360">
 218                     <source src="https://video.csfd.cz/files/videos/157/750/157750813/163327360_3d2646.mp4" type="video/mp4" width="1280" height="720">
 219                     <source src="https://video.csfd.cz/files/videos/157/750/157750813/163327356_91f258.mp4" type="video/mp4" width="1920" height="1080">
 220                     <source src="https://video.csfd.cz/files/videos/157/750/157750813/163327359_962b4a.webm" type="video/webm" width="640" height="360">
 221                     <source src="https://video.csfd.cz/files/videos/157/750/157750813/163327361_6feee0.webm" type="video/webm" width="1280" height="720">
 222                     <source src="https://video.csfd.cz/files/videos/157/750/157750813/163327357_8ab472.webm" type="video/webm" width="1920" height="1080">
 223                     <track src="https://video.csfd.cz/files/subtitles/163/344/163344115_4c388b.srt" type="text/x-srt" kind="subtitles" srclang="cs" label="cs">
 224                 </video>
 225                 ''', None)[0],
 226             {
 227                 'formats': [{
 228                     'url': 'https://video.csfd.cz/files/videos/157/750/157750813/163327358_eac647.mp4',
 229                     'ext': 'mp4',
 230                     'width': 640,
 231                     'height': 360,
 232                 }, {
 233                     'url': 'https://video.csfd.cz/files/videos/157/750/157750813/163327360_3d2646.mp4',
 234                     'ext': 'mp4',
 235                     'width': 1280,
 236                     'height': 720,
 237                 }, {
 238                     'url': 'https://video.csfd.cz/files/videos/157/750/157750813/163327356_91f258.mp4',
 239                     'ext': 'mp4',
 240                     'width': 1920,
 241                     'height': 1080,
 242                 }, {
 243                     'url': 'https://video.csfd.cz/files/videos/157/750/157750813/163327359_962b4a.webm',
 244                     'ext': 'webm',
 245                     'width': 640,
 246                     'height': 360,
 247                 }, {
 248                     'url': 'https://video.csfd.cz/files/videos/157/750/157750813/163327361_6feee0.webm',
 249                     'ext': 'webm',
 250                     'width': 1280,
 251                     'height': 720,
 252                 }, {
 253                     'url': 'https://video.csfd.cz/files/videos/157/750/157750813/163327357_8ab472.webm',
 254                     'ext': 'webm',
 255                     'width': 1920,
 256                     'height': 1080,
 257                 }],
 258                 'subtitles': {
 259                     'cs': [{'url': 'https://video.csfd.cz/files/subtitles/163/344/163344115_4c388b.srt'}]
 260                 },
 261                 'thumbnail': 'https://img.csfd.cz/files/images/film/video/preview/163/344/163344118_748d20.png?h360'
 262             })
 263
 264         # from https://tamasha.com/v/Kkdjw
 265         # with height in label
 266         expect_dict(
 267             self,
 268             self.ie._parse_html5_media_entries(
 269                 'https://tamasha.com/v/Kkdjw',
 270                 r'''
 271                 <video crossorigin="anonymous">
 272                         <source src="https://s-v2.tamasha.com/statics/videos_file/19/8f/Kkdjw_198feff8577d0057536e905cce1fb61438dd64e0_n_240.mp4" type="video/mp4" label="AUTO" res="0"/>
 273                                 <source src="https://s-v2.tamasha.com/statics/videos_file/19/8f/Kkdjw_198feff8577d0057536e905cce1fb61438dd64e0_n_240.mp4" type="video/mp4"
 274                                         label="240p" res="240"/>
 275                                 <source src="https://s-v2.tamasha.com/statics/videos_file/20/00/Kkdjw_200041c66f657fc967db464d156eafbc1ed9fe6f_n_144.mp4" type="video/mp4"
 276                                         label="144p" res="144"/>
 277                 </video>
 278                 ''', None)[0],
 279             {
 280                 'formats': [{
 281                     'url': 'https://s-v2.tamasha.com/statics/videos_file/19/8f/Kkdjw_198feff8577d0057536e905cce1fb61438dd64e0_n_240.mp4',
 282                 }, {
 283                     'url': 'https://s-v2.tamasha.com/statics/videos_file/19/8f/Kkdjw_198feff8577d0057536e905cce1fb61438dd64e0_n_240.mp4',
 284                     'ext': 'mp4',
 285                     'format_id': '240p',
 286                     'height': 240,
 287                 }, {
 288                     'url': 'https://s-v2.tamasha.com/statics/videos_file/20/00/Kkdjw_200041c66f657fc967db464d156eafbc1ed9fe6f_n_144.mp4',
 289                     'ext': 'mp4',
 290                     'format_id': '144p',
 291                     'height': 144,
 292                 }]
 293             })
 294
 295         # from https://www.directvnow.com
 296         # with data-src
 297         expect_dict(
 298             self,
 299             self.ie._parse_html5_media_entries(
 300                 'https://www.directvnow.com',
 301                 r'''
 302                 <video id="vid1" class="header--video-masked active" muted playsinline>
 303                     <source data-src="https://cdn.directv.com/content/dam/dtv/prod/website_directvnow-international/videos/DTVN_hdr_HBO_v3.mp4" type="video/mp4" />
 304                 </video>
 305                 ''', None)[0],
 306             {
 307                 'formats': [{
 308                     'ext': 'mp4',
 309                     'url': 'https://cdn.directv.com/content/dam/dtv/prod/website_directvnow-international/videos/DTVN_hdr_HBO_v3.mp4',
 310                 }]
 311             })
 312
 313         # from https://www.directvnow.com
 314         # with data-src
 315         expect_dict(
 316             self,
 317             self.ie._parse_html5_media_entries(
 318                 'https://www.directvnow.com',
 319                 r'''
 320                 <video id="vid1" class="header--video-masked active" muted playsinline>
 321                     <source data-src="https://cdn.directv.com/content/dam/dtv/prod/website_directvnow-international/videos/DTVN_hdr_HBO_v3.mp4" type="video/mp4" />
 322                 </video>
 323                 ''', None)[0],
 324             {
 325                 'formats': [{
 326                     'url': 'https://cdn.directv.com/content/dam/dtv/prod/website_directvnow-international/videos/DTVN_hdr_HBO_v3.mp4',
 327                     'ext': 'mp4',
 328                 }]
 329             })
 330
 331         # from https://www.klarna.com/uk/
 332         # with data-video-src
 333         expect_dict(
 334             self,
 335             self.ie._parse_html5_media_entries(
 336                 'https://www.directvnow.com',
 337                 r'''
 338                 <video loop autoplay muted class="responsive-video block-kl__video video-on-medium">
 339                     <source src="" data-video-desktop data-video-src="https://www.klarna.com/uk/wp-content/uploads/sites/11/2019/01/KL062_Smooth3_0_DogWalking_5s_920x080_.mp4" type="video/mp4" />
 340                 </video>
 341                 ''', None)[0],
 342             {
 343                 'formats': [{
 344                     'url': 'https://www.klarna.com/uk/wp-content/uploads/sites/11/2019/01/KL062_Smooth3_0_DogWalking_5s_920x080_.mp4',
 345                     'ext': 'mp4',
 346                 }],
 347             })
 348
 349     def test_extract_jwplayer_data_realworld(self):
 350         # from http://www.suffolk.edu/sjc/
 351         expect_dict(
 352             self,
 353             self.ie._extract_jwplayer_data(r'''
 354                 <script type='text/javascript'>
 355                     jwplayer('my-video').setup({
 356                         file: 'rtmp://192.138.214.154/live/sjclive',
 357                         fallback: 'true',
 358                         width: '95%',
 359                       aspectratio: '16:9',
 360                       primary: 'flash',
 361                       mediaid:'XEgvuql4'
 362                     });
 363                 </script>
 364                 ''', None, require_title=False),
 365             {
 366                 'id': 'XEgvuql4',
 367                 'formats': [{
 368                     'url': 'rtmp://192.138.214.154/live/sjclive',
 369                     'ext': 'flv'
 370                 }]
 371             })
 372
 373         # from https://www.pornoxo.com/videos/7564/striptease-from-sexy-secretary/
 374         expect_dict(
 375             self,
 376             self.ie._extract_jwplayer_data(r'''
 377 <script type="text/javascript">
 378     jwplayer("mediaplayer").setup({
 379         'videoid': "7564",
 380         'width': "100%",
 381         'aspectratio': "16:9",
 382         'stretching': "exactfit",
 383         'autostart': 'false',
 384         'flashplayer': "https://t04.vipstreamservice.com/jwplayer/v5.10/player.swf",
 385         'file': "https://cdn.pornoxo.com/key=MF+oEbaxqTKb50P-w9G3nA,end=1489689259,ip=104.199.146.27/ip=104.199.146.27/speed=6573765/buffer=3.0/2009-12/4b2157147afe5efa93ce1978e0265289c193874e02597.flv",
 386         'image': "https://t03.vipstreamservice.com/thumbs/pxo-full/2009-12/14/a4b2157147afe5efa93ce1978e0265289c193874e02597.flv-full-13.jpg",
 387         'filefallback': "https://cdn.pornoxo.com/key=9ZPsTR5EvPLQrBaak2MUGA,end=1489689259,ip=104.199.146.27/ip=104.199.146.27/speed=6573765/buffer=3.0/2009-12/m_4b2157147afe5efa93ce1978e0265289c193874e02597.mp4",
 388         'logo.hide': true,
 389         'skin': "https://t04.vipstreamservice.com/jwplayer/skin/modieus-blk.zip",
 390         'plugins': "https://t04.vipstreamservice.com/jwplayer/dock/dockableskinnableplugin.swf",
 391         'dockableskinnableplugin.piclink': "/index.php?key=ajax-videothumbsn&vid=7564&data=2009-12--14--4b2157147afe5efa93ce1978e0265289c193874e02597.flv--17370",
 392         'controlbar': 'bottom',
 393         'modes': [
 394             {type: 'flash', src: 'https://t04.vipstreamservice.com/jwplayer/v5.10/player.swf'}
 395         ],
 396         'provider': 'http'
 397     });
 398     //noinspection JSAnnotator
 399     invideo.setup({
 400         adsUrl: "/banner-iframe/?zoneId=32",
 401         adsUrl2: "",
 402         autostart: false
 403     });
 404 </script>
 405             ''', 'dummy', require_title=False),
 406             {
 407                 'thumbnail': 'https://t03.vipstreamservice.com/thumbs/pxo-full/2009-12/14/a4b2157147afe5efa93ce1978e0265289c193874e02597.flv-full-13.jpg',
 408                 'formats': [{
 409                     'url': 'https://cdn.pornoxo.com/key=MF+oEbaxqTKb50P-w9G3nA,end=1489689259,ip=104.199.146.27/ip=104.199.146.27/speed=6573765/buffer=3.0/2009-12/4b2157147afe5efa93ce1978e0265289c193874e02597.flv',
 410                     'ext': 'flv'
 411                 }]
 412             })
 413
 414         # from http://www.indiedb.com/games/king-machine/videos
 415         expect_dict(
 416             self,
 417             self.ie._extract_jwplayer_data(r'''
 418 <script>
 419 jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/\/www.indiedb.com\/","displaytitle":false,"autostart":false,"repeat":false,"title":"king machine trailer 1","sharing":{"link":"http:\/\/www.indiedb.com\/games\/king-machine\/videos\/king-machine-trailer-1","code":"<iframe width=\"560\" height=\"315\" src=\"http:\/\/www.indiedb.com\/media\/iframe\/1522983\" frameborder=\"0\" allowfullscreen><\/iframe><br><a href=\"http:\/\/www.indiedb.com\/games\/king-machine\/videos\/king-machine-trailer-1\">king machine trailer 1 - Indie DB<\/a>"},"related":{"file":"http:\/\/rss.indiedb.com\/media\/recommended\/1522983\/feed\/rss.xml","dimensions":"160x120","onclick":"link"},"sources":[{"file":"http:\/\/cdn.dbolical.com\/cache\/videos\/games\/1\/50\/49678\/encode_mp4\/king-machine-trailer.mp4","label":"360p SD","default":"true"},{"file":"http:\/\/cdn.dbolical.com\/cache\/videos\/games\/1\/50\/49678\/encode720p_mp4\/king-machine-trailer.mp4","label":"720p HD"}],"image":"http:\/\/media.indiedb.com\/cache\/images\/games\/1\/50\/49678\/thumb_620x2000\/king-machine-trailer.mp4.jpg","advertising":{"client":"vast","tag":"http:\/\/ads.intergi.com\/adrawdata\/3.0\/5205\/4251742\/0\/1013\/ADTECH;cors=yes;width=560;height=315;referring_url=http:\/\/www.indiedb.com\/games\/king-machine\/videos\/king-machine-trailer-1;content_url=http:\/\/www.indiedb.com\/games\/king-machine\/videos\/king-machine-trailer-1;media_id=1522983;title=king+machine+trailer+1;device=__DEVICE__;model=__MODEL__;os=Windows+OS;osversion=__OSVERSION__;ua=__UA__;ip=109.171.17.81;uniqueid=1522983;tags=__TAGS__;number=58cac25928151;time=1489683033"},"width":620,"height":349}).once("play", function(event) {
 420             videoAnalytics("play");
 421 }).once("complete", function(event) {
 422     videoAnalytics("completed");
 423 });
 424 </script>
 425                 ''', 'dummy'),
 426             {
 427                 'title': 'king machine trailer 1',
 428                 'thumbnail': 'http://media.indiedb.com/cache/images/games/1/50/49678/thumb_620x2000/king-machine-trailer.mp4.jpg',
 429                 'formats': [{
 430                     'url': 'http://cdn.dbolical.com/cache/videos/games/1/50/49678/encode_mp4/king-machine-trailer.mp4',
 431                     'height': 360,
 432                     'ext': 'mp4'
 433                 }, {
 434                     'url': 'http://cdn.dbolical.com/cache/videos/games/1/50/49678/encode720p_mp4/king-machine-trailer.mp4',
 435                     'height': 720,
 436                     'ext': 'mp4'
 437                 }]
 438             })
 439
 440     def test_parse_m3u8_formats(self):
 441         _TEST_CASES = [
 442             (
 443                 # https://github.com/ytdl-org/youtube-dl/issues/11995
 444                 # http://teamcoco.com/video/clueless-gamer-super-bowl-for-honor
 445                 'img_bipbop_adv_example_fmp4',
 446                 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 447                 [{
 448                     'format_id': 'aud1-English',
 449                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/a1/prog_index.m3u8',
 450                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 451                     'language': 'en',
 452                     'ext': 'mp4',
 453                     'protocol': 'm3u8',
 454                     'audio_ext': 'mp4',
 455                 }, {
 456                     'format_id': 'aud2-English',
 457                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/a2/prog_index.m3u8',
 458                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 459                     'language': 'en',
 460                     'ext': 'mp4',
 461                     'protocol': 'm3u8',
 462                     'audio_ext': 'mp4',
 463                 }, {
 464                     'format_id': 'aud3-English',
 465                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/a3/prog_index.m3u8',
 466                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 467                     'language': 'en',
 468                     'ext': 'mp4',
 469                     'protocol': 'm3u8',
 470                     'audio_ext': 'mp4',
 471                 }, {
 472                     'format_id': '530',
 473                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v2/prog_index.m3u8',
 474                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 475                     'ext': 'mp4',
 476                     'protocol': 'm3u8',
 477                     'width': 480,
 478                     'height': 270,
 479                     'vcodec': 'avc1.640015',
 480                 }, {
 481                     'format_id': '561',
 482                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v2/prog_index.m3u8',
 483                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 484                     'ext': 'mp4',
 485                     'protocol': 'm3u8',
 486                     'width': 480,
 487                     'height': 270,
 488                     'vcodec': 'avc1.640015',
 489                 }, {
 490                     'format_id': '753',
 491                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v2/prog_index.m3u8',
 492                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 493                     'ext': 'mp4',
 494                     'protocol': 'm3u8',
 495                     'width': 480,
 496                     'height': 270,
 497                     'vcodec': 'avc1.640015',
 498                 }, {
 499                     'format_id': '895',
 500                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v3/prog_index.m3u8',
 501                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 502                     'ext': 'mp4',
 503                     'protocol': 'm3u8',
 504                     'width': 640,
 505                     'height': 360,
 506                     'vcodec': 'avc1.64001e',
 507                 }, {
 508                     'format_id': '926',
 509                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v3/prog_index.m3u8',
 510                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 511                     'ext': 'mp4',
 512                     'protocol': 'm3u8',
 513                     'width': 640,
 514                     'height': 360,
 515                     'vcodec': 'avc1.64001e',
 516                 }, {
 517                     'format_id': '1118',
 518                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v3/prog_index.m3u8',
 519                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 520                     'ext': 'mp4',
 521                     'protocol': 'm3u8',
 522                     'width': 640,
 523                     'height': 360,
 524                     'vcodec': 'avc1.64001e',
 525                 }, {
 526                     'format_id': '1265',
 527                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v4/prog_index.m3u8',
 528                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 529                     'ext': 'mp4',
 530                     'protocol': 'm3u8',
 531                     'width': 768,
 532                     'height': 432,
 533                     'vcodec': 'avc1.64001e',
 534                 }, {
 535                     'format_id': '1295',
 536                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v4/prog_index.m3u8',
 537                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 538                     'ext': 'mp4',
 539                     'protocol': 'm3u8',
 540                     'width': 768,
 541                     'height': 432,
 542                     'vcodec': 'avc1.64001e',
 543                 }, {
 544                     'format_id': '1487',
 545                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v4/prog_index.m3u8',
 546                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 547                     'ext': 'mp4',
 548                     'protocol': 'm3u8',
 549                     'width': 768,
 550                     'height': 432,
 551                     'vcodec': 'avc1.64001e',
 552                 }, {
 553                     'format_id': '2168',
 554                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v5/prog_index.m3u8',
 555                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 556                     'ext': 'mp4',
 557                     'protocol': 'm3u8',
 558                     'width': 960,
 559                     'height': 540,
 560                     'vcodec': 'avc1.640020',
 561                 }, {
 562                     'format_id': '2198',
 563                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v5/prog_index.m3u8',
 564                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 565                     'ext': 'mp4',
 566                     'protocol': 'm3u8',
 567                     'width': 960,
 568                     'height': 540,
 569                     'vcodec': 'avc1.640020',
 570                 }, {
 571                     'format_id': '2390',
 572                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v5/prog_index.m3u8',
 573                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 574                     'ext': 'mp4',
 575                     'protocol': 'm3u8',
 576                     'width': 960,
 577                     'height': 540,
 578                     'vcodec': 'avc1.640020',
 579                 }, {
 580                     'format_id': '3168',
 581                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v6/prog_index.m3u8',
 582                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 583                     'ext': 'mp4',
 584                     'protocol': 'm3u8',
 585                     'width': 1280,
 586                     'height': 720,
 587                     'vcodec': 'avc1.640020',
 588                 }, {
 589                     'format_id': '3199',
 590                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v6/prog_index.m3u8',
 591                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 592                     'ext': 'mp4',
 593                     'protocol': 'm3u8',
 594                     'width': 1280,
 595                     'height': 720,
 596                     'vcodec': 'avc1.640020',
 597                 }, {
 598                     'format_id': '3391',
 599                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v6/prog_index.m3u8',
 600                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 601                     'ext': 'mp4',
 602                     'protocol': 'm3u8',
 603                     'width': 1280,
 604                     'height': 720,
 605                     'vcodec': 'avc1.640020',
 606                 }, {
 607                     'format_id': '4670',
 608                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v7/prog_index.m3u8',
 609                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 610                     'ext': 'mp4',
 611                     'protocol': 'm3u8',
 612                     'width': 1920,
 613                     'height': 1080,
 614                     'vcodec': 'avc1.64002a',
 615                 }, {
 616                     'format_id': '4701',
 617                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v7/prog_index.m3u8',
 618                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 619                     'ext': 'mp4',
 620                     'protocol': 'm3u8',
 621                     'width': 1920,
 622                     'height': 1080,
 623                     'vcodec': 'avc1.64002a',
 624                 }, {
 625                     'format_id': '4893',
 626                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v7/prog_index.m3u8',
 627                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 628                     'ext': 'mp4',
 629                     'protocol': 'm3u8',
 630                     'width': 1920,
 631                     'height': 1080,
 632                     'vcodec': 'avc1.64002a',
 633                 }, {
 634                     'format_id': '6170',
 635                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v8/prog_index.m3u8',
 636                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 637                     'ext': 'mp4',
 638                     'protocol': 'm3u8',
 639                     'width': 1920,
 640                     'height': 1080,
 641                     'vcodec': 'avc1.64002a',
 642                 }, {
 643                     'format_id': '6200',
 644                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v8/prog_index.m3u8',
 645                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 646                     'ext': 'mp4',
 647                     'protocol': 'm3u8',
 648                     'width': 1920,
 649                     'height': 1080,
 650                     'vcodec': 'avc1.64002a',
 651                 }, {
 652                     'format_id': '6392',
 653                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v8/prog_index.m3u8',
 654                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 655                     'ext': 'mp4',
 656                     'protocol': 'm3u8',
 657                     'width': 1920,
 658                     'height': 1080,
 659                     'vcodec': 'avc1.64002a',
 660                 }, {
 661                     'format_id': '7968',
 662                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v9/prog_index.m3u8',
 663                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 664                     'ext': 'mp4',
 665                     'protocol': 'm3u8',
 666                     'width': 1920,
 667                     'height': 1080,
 668                     'vcodec': 'avc1.64002a',
 669                 }, {
 670                     'format_id': '7998',
 671                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v9/prog_index.m3u8',
 672                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 673                     'ext': 'mp4',
 674                     'protocol': 'm3u8',
 675                     'width': 1920,
 676                     'height': 1080,
 677                     'vcodec': 'avc1.64002a',
 678                 }, {
 679                     'format_id': '8190',
 680                     'url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/v9/prog_index.m3u8',
 681                     'manifest_url': 'https://devstreaming-cdn.apple.com/videos/streaming/examples/img_bipbop_adv_example_fmp4/master.m3u8',
 682                     'ext': 'mp4',
 683                     'protocol': 'm3u8',
 684                     'width': 1920,
 685                     'height': 1080,
 686                     'vcodec': 'avc1.64002a',
 687                 }],
 688                 {}
 689             ),
 690             (
 691                 'bipbop_16x9',
 692                 'https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/bipbop_16x9_variant.m3u8',
 693                 [{
 694                     "format_id": "bipbop_audio-BipBop Audio 2",
 695                     "format_index": None,
 696                     "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/alternate_audio_aac/prog_index.m3u8",
 697                     "manifest_url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/bipbop_16x9_variant.m3u8",
 698                     "language": "eng",
 699                     "ext": "mp4",
 700                     "protocol": "m3u8",
 701                     "preference": None,
 702                     "quality": None,
 703                     "vcodec": "none",
 704                     "audio_ext": "mp4",
 705                     "video_ext": "none",
 706                 }, {
 707                     "format_id": "41",
 708                     "format_index": None,
 709                     "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/gear0/prog_index.m3u8",
 710                     "manifest_url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/bipbop_16x9_variant.m3u8",
 711                     "tbr": 41.457,
 712                     "ext": "mp4",
 713                     "fps": None,
 714                     "protocol": "m3u8",
 715                     "preference": None,
 716                     "quality": None,
 717                     "vcodec": "none",
 718                     "acodec": "mp4a.40.2",
 719                     "audio_ext": "mp4",
 720                     "video_ext": "none",
 721                     "abr": 41.457,
 722                 }, {
 723                     "format_id": "263",
 724                     "format_index": None,
 725                     "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/gear1/prog_index.m3u8",
 726                     "manifest_url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/bipbop_16x9_variant.m3u8",
 727                     "tbr": 263.851,
 728                     "ext": "mp4",
 729                     "fps": None,
 730                     "protocol": "m3u8",
 731                     "preference": None,
 732                     "quality": None,
 733                     "width": 416,
 734                     "height": 234,
 735                     "vcodec": "avc1.4d400d",
 736                     "acodec": "mp4a.40.2",
 737                     "video_ext": "mp4",
 738                     "audio_ext": "none",
 739                     "vbr": 263.851,
 740                     "abr": 0,
 741                 }, {
 742                     "format_id": "577",
 743                     "format_index": None,
 744                     "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/gear2/prog_index.m3u8",
 745                     "manifest_url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/bipbop_16x9_variant.m3u8",
 746                     "tbr": 577.61,
 747                     "ext": "mp4",
 748                     "fps": None,
 749                     "protocol": "m3u8",
 750                     "preference": None,
 751                     "quality": None,
 752                     "width": 640,
 753                     "height": 360,
 754                     "vcodec": "avc1.4d401e",
 755                     "acodec": "mp4a.40.2",
 756                     "video_ext": "mp4",
 757                     "audio_ext": "none",
 758                     "vbr": 577.61,
 759                     "abr": 0,
 760                 }, {
 761                     "format_id": "915",
 762                     "format_index": None,
 763                     "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/gear3/prog_index.m3u8",
 764                     "manifest_url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/bipbop_16x9_variant.m3u8",
 765                     "tbr": 915.905,
 766                     "ext": "mp4",
 767                     "fps": None,
 768                     "protocol": "m3u8",
 769                     "preference": None,
 770                     "quality": None,
 771                     "width": 960,
 772                     "height": 540,
 773                     "vcodec": "avc1.4d401f",
 774                     "acodec": "mp4a.40.2",
 775                     "video_ext": "mp4",
 776                     "audio_ext": "none",
 777                     "vbr": 915.905,
 778                     "abr": 0,
 779                 }, {
 780                     "format_id": "1030",
 781                     "format_index": None,
 782                     "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/gear4/prog_index.m3u8",
 783                     "manifest_url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/bipbop_16x9_variant.m3u8",
 784                     "tbr": 1030.138,
 785                     "ext": "mp4",
 786                     "fps": None,
 787                     "protocol": "m3u8",
 788                     "preference": None,
 789                     "quality": None,
 790                     "width": 1280,
 791                     "height": 720,
 792                     "vcodec": "avc1.4d401f",
 793                     "acodec": "mp4a.40.2",
 794                     "video_ext": "mp4",
 795                     "audio_ext": "none",
 796                     "vbr": 1030.138,
 797                     "abr": 0,
 798                 }, {
 799                     "format_id": "1924",
 800                     "format_index": None,
 801                     "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/gear5/prog_index.m3u8",
 802                     "manifest_url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/bipbop_16x9_variant.m3u8",
 803                     "tbr": 1924.009,
 804                     "ext": "mp4",
 805                     "fps": None,
 806                     "protocol": "m3u8",
 807                     "preference": None,
 808                     "quality": None,
 809                     "width": 1920,
 810                     "height": 1080,
 811                     "vcodec": "avc1.4d401f",
 812                     "acodec": "mp4a.40.2",
 813                     "video_ext": "mp4",
 814                     "audio_ext": "none",
 815                     "vbr": 1924.009,
 816                     "abr": 0,
 817                 }],
 818                 {
 819                     "en": [{
 820                         "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/subtitles/eng/prog_index.m3u8",
 821                         "ext": "vtt",
 822                         "protocol": "m3u8_native"
 823                     }, {
 824                         "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/subtitles/eng_forced/prog_index.m3u8",
 825                         "ext": "vtt",
 826                         "protocol": "m3u8_native"
 827                     }],
 828                     "fr": [{
 829                         "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/subtitles/fra/prog_index.m3u8",
 830                         "ext": "vtt",
 831                         "protocol": "m3u8_native"
 832                     }, {
 833                         "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/subtitles/fra_forced/prog_index.m3u8",
 834                         "ext": "vtt",
 835                         "protocol": "m3u8_native"
 836                     }],
 837                     "es": [{
 838                         "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/subtitles/spa/prog_index.m3u8",
 839                         "ext": "vtt",
 840                         "protocol": "m3u8_native"
 841                     }, {
 842                         "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/subtitles/spa_forced/prog_index.m3u8",
 843                         "ext": "vtt",
 844                         "protocol": "m3u8_native"
 845                     }],
 846                     "ja": [{
 847                         "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/subtitles/jpn/prog_index.m3u8",
 848                         "ext": "vtt",
 849                         "protocol": "m3u8_native"
 850                     }, {
 851                         "url": "https://devstreaming-cdn.apple.com/videos/streaming/examples/bipbop_16x9/subtitles/jpn_forced/prog_index.m3u8",
 852                         "ext": "vtt",
 853                         "protocol": "m3u8_native"
 854                     }],
 855                 }
 856             ),
 857         ]
 858
 859         for m3u8_file, m3u8_url, expected_formats, expected_subs in _TEST_CASES:
 860             with io.open('./test/testdata/m3u8/%s.m3u8' % m3u8_file,
 861                          mode='r', encoding='utf-8') as f:
 862                 formats, subs = self.ie._parse_m3u8_formats_and_subtitles(
 863                     f.read(), m3u8_url, ext='mp4')
 864                 self.ie._sort_formats(formats)
 865                 expect_value(self, formats, expected_formats, None)
 866                 expect_value(self, subs, expected_subs, None)
 867
 868     def test_parse_mpd_formats(self):
 869         _TEST_CASES = [
 870             (
 871                 # https://github.com/ytdl-org/youtube-dl/issues/13919
 872                 # Also tests duplicate representation ids, see
 873                 # https://github.com/ytdl-org/youtube-dl/issues/15111
 874                 'float_duration',
 875                 'http://unknown/manifest.mpd',  # mpd_url
 876                 None,  # mpd_base_url
 877                 [{
 878                     'manifest_url': 'http://unknown/manifest.mpd',
 879                     'ext': 'm4a',
 880                     'format_id': '318597',
 881                     'format_note': 'DASH audio',
 882                     'protocol': 'http_dash_segments',
 883                     'acodec': 'mp4a.40.2',
 884                     'vcodec': 'none',
 885                     'tbr': 61.587,
 886                 }, {
 887                     'manifest_url': 'http://unknown/manifest.mpd',
 888                     'ext': 'mp4',
 889                     'format_id': '318597',
 890                     'format_note': 'DASH video',
 891                     'protocol': 'http_dash_segments',
 892                     'acodec': 'none',
 893                     'vcodec': 'avc1.42001f',
 894                     'tbr': 318.597,
 895                     'width': 340,
 896                     'height': 192,
 897                 }, {
 898                     'manifest_url': 'http://unknown/manifest.mpd',
 899                     'ext': 'mp4',
 900                     'format_id': '638590',
 901                     'format_note': 'DASH video',
 902                     'protocol': 'http_dash_segments',
 903                     'acodec': 'none',
 904                     'vcodec': 'avc1.42001f',
 905                     'tbr': 638.59,
 906                     'width': 512,
 907                     'height': 288,
 908                 }, {
 909                     'manifest_url': 'http://unknown/manifest.mpd',
 910                     'ext': 'mp4',
 911                     'format_id': '1022565',
 912                     'format_note': 'DASH video',
 913                     'protocol': 'http_dash_segments',
 914                     'acodec': 'none',
 915                     'vcodec': 'avc1.4d001f',
 916                     'tbr': 1022.565,
 917                     'width': 688,
 918                     'height': 384,
 919                 }, {
 920                     'manifest_url': 'http://unknown/manifest.mpd',
 921                     'ext': 'mp4',
 922                     'format_id': '2046506',
 923                     'format_note': 'DASH video',
 924                     'protocol': 'http_dash_segments',
 925                     'acodec': 'none',
 926                     'vcodec': 'avc1.4d001f',
 927                     'tbr': 2046.506,
 928                     'width': 1024,
 929                     'height': 576,
 930                 }, {
 931                     'manifest_url': 'http://unknown/manifest.mpd',
 932                     'ext': 'mp4',
 933                     'format_id': '3998017',
 934                     'format_note': 'DASH video',
 935                     'protocol': 'http_dash_segments',
 936                     'acodec': 'none',
 937                     'vcodec': 'avc1.640029',
 938                     'tbr': 3998.017,
 939                     'width': 1280,
 940                     'height': 720,
 941                 }, {
 942                     'manifest_url': 'http://unknown/manifest.mpd',
 943                     'ext': 'mp4',
 944                     'format_id': '5997485',
 945                     'format_note': 'DASH video',
 946                     'protocol': 'http_dash_segments',
 947                     'acodec': 'none',
 948                     'vcodec': 'avc1.640032',
 949                     'tbr': 5997.485,
 950                     'width': 1920,
 951                     'height': 1080,
 952                 }]
 953             ), (
 954                 # https://github.com/ytdl-org/youtube-dl/pull/14844
 955                 'urls_only',
 956                 'http://unknown/manifest.mpd',  # mpd_url
 957                 None,  # mpd_base_url
 958                 [{
 959                     'manifest_url': 'http://unknown/manifest.mpd',
 960                     'ext': 'mp4',
 961                     'format_id': 'h264_aac_144p_m4s',
 962                     'format_note': 'DASH video',
 963                     'protocol': 'http_dash_segments',
 964                     'acodec': 'mp4a.40.2',
 965                     'vcodec': 'avc3.42c01e',
 966                     'tbr': 200,
 967                     'width': 256,
 968                     'height': 144,
 969                 }, {
 970                     'manifest_url': 'http://unknown/manifest.mpd',
 971                     'ext': 'mp4',
 972                     'format_id': 'h264_aac_240p_m4s',
 973                     'format_note': 'DASH video',
 974                     'protocol': 'http_dash_segments',
 975                     'acodec': 'mp4a.40.2',
 976                     'vcodec': 'avc3.42c01e',
 977                     'tbr': 400,
 978                     'width': 424,
 979                     'height': 240,
 980                 }, {
 981                     'manifest_url': 'http://unknown/manifest.mpd',
 982                     'ext': 'mp4',
 983                     'format_id': 'h264_aac_360p_m4s',
 984                     'format_note': 'DASH video',
 985                     'protocol': 'http_dash_segments',
 986                     'acodec': 'mp4a.40.2',
 987                     'vcodec': 'avc3.42c01e',
 988                     'tbr': 800,
 989                     'width': 640,
 990                     'height': 360,
 991                 }, {
 992                     'manifest_url': 'http://unknown/manifest.mpd',
 993                     'ext': 'mp4',
 994                     'format_id': 'h264_aac_480p_m4s',
 995                     'format_note': 'DASH video',
 996                     'protocol': 'http_dash_segments',
 997                     'acodec': 'mp4a.40.2',
 998                     'vcodec': 'avc3.42c01e',
 999                     'tbr': 1200,
1000                     'width': 856,
1001                     'height': 480,
1002                 }, {
1003                     'manifest_url': 'http://unknown/manifest.mpd',
1004                     'ext': 'mp4',
1005                     'format_id': 'h264_aac_576p_m4s',
1006                     'format_note': 'DASH video',
1007                     'protocol': 'http_dash_segments',
1008                     'acodec': 'mp4a.40.2',
1009                     'vcodec': 'avc3.42c01e',
1010                     'tbr': 1600,
1011                     'width': 1024,
1012                     'height': 576,
1013                 }, {
1014                     'manifest_url': 'http://unknown/manifest.mpd',
1015                     'ext': 'mp4',
1016                     'format_id': 'h264_aac_720p_m4s',
1017                     'format_note': 'DASH video',
1018                     'protocol': 'http_dash_segments',
1019                     'acodec': 'mp4a.40.2',
1020                     'vcodec': 'avc3.42c01e',
1021                     'tbr': 2400,
1022                     'width': 1280,
1023                     'height': 720,
1024                 }, {
1025                     'manifest_url': 'http://unknown/manifest.mpd',
1026                     'ext': 'mp4',
1027                     'format_id': 'h264_aac_1080p_m4s',
1028                     'format_note': 'DASH video',
1029                     'protocol': 'http_dash_segments',
1030                     'acodec': 'mp4a.40.2',
1031                     'vcodec': 'avc3.42c01e',
1032                     'tbr': 4400,
1033                     'width': 1920,
1034                     'height': 1080,
1035                 }]
1036             ), (
1037                 # https://github.com/ytdl-org/youtube-dl/issues/20346
1038                 # Media considered unfragmented even though it contains
1039                 # Initialization tag
1040                 'unfragmented',
1041                 'https://v.redd.it/hw1x7rcg7zl21/DASHPlaylist.mpd',  # mpd_url
1042                 'https://v.redd.it/hw1x7rcg7zl21',  # mpd_base_url
1043                 [{
1044                     'url': 'https://v.redd.it/hw1x7rcg7zl21/audio',
1045                     'manifest_url': 'https://v.redd.it/hw1x7rcg7zl21/DASHPlaylist.mpd',
1046                     'ext': 'm4a',
1047                     'format_id': 'AUDIO-1',
1048                     'format_note': 'DASH audio',
1049                     'container': 'm4a_dash',
1050                     'acodec': 'mp4a.40.2',
1051                     'vcodec': 'none',
1052                     'tbr': 129.87,
1053                     'asr': 48000,
1054
1055                 }, {
1056                     'url': 'https://v.redd.it/hw1x7rcg7zl21/DASH_240',
1057                     'manifest_url': 'https://v.redd.it/hw1x7rcg7zl21/DASHPlaylist.mpd',
1058                     'ext': 'mp4',
1059                     'format_id': 'VIDEO-2',
1060                     'format_note': 'DASH video',
1061                     'container': 'mp4_dash',
1062                     'acodec': 'none',
1063                     'vcodec': 'avc1.4d401e',
1064                     'tbr': 608.0,
1065                     'width': 240,
1066                     'height': 240,
1067                     'fps': 30,
1068                 }, {
1069                     'url': 'https://v.redd.it/hw1x7rcg7zl21/DASH_360',
1070                     'manifest_url': 'https://v.redd.it/hw1x7rcg7zl21/DASHPlaylist.mpd',
1071                     'ext': 'mp4',
1072                     'format_id': 'VIDEO-1',
1073                     'format_note': 'DASH video',
1074                     'container': 'mp4_dash',
1075                     'acodec': 'none',
1076                     'vcodec': 'avc1.4d401e',
1077                     'tbr': 804.261,
1078                     'width': 360,
1079                     'height': 360,
1080                     'fps': 30,
1081                 }]
1082             )
1083         ]
1084
1085         for mpd_file, mpd_url, mpd_base_url, expected_formats in _TEST_CASES:
1086             with io.open('./test/testdata/mpd/%s.mpd' % mpd_file,
1087                          mode='r', encoding='utf-8') as f:
1088                 formats = self.ie._parse_mpd_formats(
1089                     compat_etree_fromstring(f.read().encode('utf-8')),
1090                     mpd_base_url=mpd_base_url, mpd_url=mpd_url)
1091                 self.ie._sort_formats(formats)
1092                 expect_value(self, formats, expected_formats, None)
1093
1094     def test_parse_f4m_formats(self):
1095         _TEST_CASES = [
1096             (
1097                 # https://github.com/ytdl-org/youtube-dl/issues/14660
1098                 'custom_base_url',
1099                 'http://api.new.livestream.com/accounts/6115179/events/6764928/videos/144884262.f4m',
1100                 [{
1101                     'manifest_url': 'http://api.new.livestream.com/accounts/6115179/events/6764928/videos/144884262.f4m',
1102                     'ext': 'flv',
1103                     'format_id': '2148',
1104                     'protocol': 'f4m',
1105                     'tbr': 2148,
1106                     'width': 1280,
1107                     'height': 720,
1108                 }]
1109             ),
1110         ]
1111
1112         for f4m_file, f4m_url, expected_formats in _TEST_CASES:
1113             with io.open('./test/testdata/f4m/%s.f4m' % f4m_file,
1114                          mode='r', encoding='utf-8') as f:
1115                 formats = self.ie._parse_f4m_formats(
1116                     compat_etree_fromstring(f.read().encode('utf-8')),
1117                     f4m_url, None)
1118                 self.ie._sort_formats(formats)
1119                 expect_value(self, formats, expected_formats, None)
1120
1121     def test_parse_xspf(self):
1122         _TEST_CASES = [
1123             (
1124                 'foo_xspf',
1125                 'https://example.org/src/foo_xspf.xspf',
1126                 [{
1127                     'id': 'foo_xspf',
1128                     'title': 'Pandemonium',
1129                     'description': 'Visit http://bigbrother404.bandcamp.com',
1130                     'duration': 202.416,
1131                     'formats': [{
1132                         'manifest_url': 'https://example.org/src/foo_xspf.xspf',
1133                         'url': 'https://example.org/src/cd1/track%201.mp3',
1134                     }],
1135                 }, {
1136                     'id': 'foo_xspf',
1137                     'title': 'Final Cartridge (Nichico Twelve Remix)',
1138                     'description': 'Visit http://bigbrother404.bandcamp.com',
1139                     'duration': 255.857,
1140                     'formats': [{
1141                         'manifest_url': 'https://example.org/src/foo_xspf.xspf',
1142                         'url': 'https://example.org/%E3%83%88%E3%83%A9%E3%83%83%E3%82%AF%E3%80%80%EF%BC%92.mp3',
1143                     }],
1144                 }, {
1145                     'id': 'foo_xspf',
1146                     'title': 'Rebuilding Nightingale',
1147                     'description': 'Visit http://bigbrother404.bandcamp.com',
1148                     'duration': 287.915,
1149                     'formats': [{
1150                         'manifest_url': 'https://example.org/src/foo_xspf.xspf',
1151                         'url': 'https://example.org/src/track3.mp3',
1152                     }, {
1153                         'manifest_url': 'https://example.org/src/foo_xspf.xspf',
1154                         'url': 'https://example.com/track3.mp3',
1155                     }]
1156                 }]
1157             ),
1158         ]
1159
1160         for xspf_file, xspf_url, expected_entries in _TEST_CASES:
1161             with io.open('./test/testdata/xspf/%s.xspf' % xspf_file,
1162                          mode='r', encoding='utf-8') as f:
1163                 entries = self.ie._parse_xspf(
1164                     compat_etree_fromstring(f.read().encode('utf-8')),
1165                     xspf_file, xspf_url=xspf_url, xspf_base_url=xspf_url)
1166                 expect_value(self, entries, expected_entries, None)
1167                 for i in range(len(entries)):
1168                     expect_dict(self, entries[i], expected_entries[i])
1169
1170     def test_response_with_expected_status_returns_content(self):
1171         # Checks for mitigations against the effects of
1172         # <https://bugs.python.org/issue15002> that affect Python 3.4.1+, which
1173         # manifest as `_download_webpage`, `_download_xml`, `_download_json`,
1174         # or the underlying `_download_webpage_handle` returning no content
1175         # when a response matches `expected_status`.
1176
1177         httpd = compat_http_server.HTTPServer(
1178             ('127.0.0.1', 0), InfoExtractorTestRequestHandler)
1179         port = http_server_port(httpd)
1180         server_thread = threading.Thread(target=httpd.serve_forever)
1181         server_thread.daemon = True
1182         server_thread.start()
1183
1184         (content, urlh) = self.ie._download_webpage_handle(
1185             'http://127.0.0.1:%d/teapot' % port, None,
1186             expected_status=TEAPOT_RESPONSE_STATUS)
1187         self.assertEqual(content, TEAPOT_RESPONSE_BODY)
1188
1189
1190 if __name__ == '__main__':
1191     unittest.main()