[ie/crunchyroll] Fix stream extraction (#10005)

[yt-dlp.git] / yt_dlp / extractor / box.py

diff --git a/yt_dlp/extractor/box.py b/yt_dlp/extractor/box.py
index aae82d1afff5bc0e5ade7940b0835877beddbdb8..008c011cc8aa233268fcfff0c69e9091472d37ba 100644 (file)
--- a/yt_dlp/extractor/box.py
+++ b/yt_dlp/extractor/box.py
@@ -1,40 +1,61 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
 import json
-import re
+import urllib.parse
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
+    ExtractorError,
     parse_iso8601,
-    # try_get,
     update_url_query,
+    url_or_none,
 )
+from ..utils.traversal import traverse_obj
 
 
 class BoxIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:[^.]+\.)?app\.box\.com/s/(?P<shared_name>[^/]+)/file/(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:[^.]+\.)?app\.box\.com/s/(?P<shared_name>[^/?#]+)(?:/file/(?P<id>\d+))?'
+    _TESTS = [{
         'url': 'https://mlssoccer.app.box.com/s/0evd2o3e08l60lr4ygukepvnkord1o1x/file/510727257538',
         'md5': '1f81b2fd3960f38a40a3b8823e5fcd43',
         'info_dict': {
             'id': '510727257538',
             'ext': 'mp4',
             'title': 'Garber   St. Louis will be 28th MLS team  +scarving.mp4',
-            'uploader': 'MLS Video',
+            'uploader': '',
             'timestamp': 1566320259,
             'upload_date': '20190820',
             'uploader_id': '235196876',
-        }
-    }
+        },
+        'params': {'skip_download': 'dash fragment too small'},
+    }, {
+        'url': 'https://utexas.app.box.com/s/2x6vanv85fdl8j2eqlcxmv0gp1wvps6e',
+        'info_dict': {
+            'id': '787379022466',
+            'ext': 'mp4',
+            'title': 'Webinar recording: Take the Leap!.mp4',
+            'uploader': 'Patricia Mosele',
+            'timestamp': 1615824864,
+            'upload_date': '20210315',
+            'uploader_id': '239068974',
+        },
+        'params': {'skip_download': 'dash fragment too small'},
+    }]
 
     def _real_extract(self, url):
-        shared_name, file_id = re.match(self._VALID_URL, url).groups()
-        webpage = self._download_webpage(url, file_id)
-        request_token = self._parse_json(self._search_regex(
-            r'Box\.config\s*=\s*({.+?});', webpage,
-            'Box config'), file_id)['requestToken']
+        shared_name, file_id = self._match_valid_url(url).groups()
+        webpage = self._download_webpage(url, file_id or shared_name)
+
+        if not file_id:
+            post_stream_data = self._search_json(
+                r'Box\.postStreamData\s*=', webpage, 'Box post-stream data', shared_name)
+            shared_item = traverse_obj(
+                post_stream_data, ('/app-api/enduserapp/shared-item', {dict})) or {}
+            if shared_item.get('itemType') != 'file':
+                raise ExtractorError('The requested resource is not a file', expected=True)
+
+            file_id = str(shared_item['itemID'])
+
+        request_token = self._search_json(
+            r'Box\.config\s*=', webpage, 'Box config', file_id)['requestToken']
         access_token = self._download_json(
             'https://app.box.com/app-api/enduserapp/elements/tokens', file_id,
             'Downloading token JSON metadata',
@@ -62,28 +83,15 @@ def _real_extract(self, url):
 
         formats = []
 
-        # for entry in (try_get(f, lambda x: x['representations']['entries'], list) or []):
-        #     entry_url_template = try_get(
-        #         entry, lambda x: x['content']['url_template'])
-        #     if not entry_url_template:
-        #         continue
-        #     representation = entry.get('representation')
-        #     if representation == 'dash':
-        #         TODO: append query to every fragment URL
-        #         formats.extend(self._extract_mpd_formats(
-        #             entry_url_template.replace('{+asset_path}', 'manifest.mpd'),
-        #             file_id, query=query))
-
-        authenticated_download_url = f.get('authenticated_download_url')
-        if authenticated_download_url and f.get('is_download_available'):
-            formats.append({
-                'ext': f.get('extension') or determine_ext(title),
-                'filesize': f.get('size'),
-                'format_id': 'download',
-                'url': update_url_query(authenticated_download_url, query),
-            })
-
-        self._sort_formats(formats)
+        for url_tmpl in traverse_obj(f, (
+            'representations', 'entries', lambda _, v: v['representation'] == 'dash',
+            'content', 'url_template', {url_or_none}
+        )):
+            manifest_url = update_url_query(url_tmpl.replace('{+asset_path}', 'manifest.mpd'), query)
+            fmts = self._extract_mpd_formats(manifest_url, file_id)
+            for fmt in fmts:
+                fmt['extra_param_to_segment_url'] = urllib.parse.urlparse(manifest_url).query
+            formats.extend(fmts)
 
         creator = f.get('created_by') or {}