]> jfr.im git - yt-dlp.git/blame - yt_dlp/extractor/nfb.py
[ie/youtube] Suppress "Unavailable videos are hidden" warning (#10159)
[yt-dlp.git] / yt_dlp / extractor / nfb.py
CommitLineData
f1657a98 1from .common import InfoExtractor
4b8b0dde 2from ..utils import (
3 int_or_none,
4 join_nonempty,
5 merge_dicts,
6 parse_count,
7 url_or_none,
4b8b0dde 8)
9from ..utils.traversal import traverse_obj
f1657a98 10
11
4b8b0dde 12class NFBBaseIE(InfoExtractor):
13 _VALID_URL_BASE = r'https?://(?:www\.)?(?P<site>nfb|onf)\.ca'
14 _GEO_COUNTRIES = ['CA']
15
16 def _extract_ep_data(self, webpage, video_id, fatal=False):
17 return self._search_json(
0a1a8e30 18 r'episodesData\s*:', webpage, 'episode data', video_id, fatal=fatal) or {}
4b8b0dde 19
20 def _extract_ep_info(self, data, video_id, slug=None):
21 info = traverse_obj(data, (lambda _, v: video_id in v['embed_url'], {
22 'description': ('description', {str}),
23 'thumbnail': ('thumbnail_url', {url_or_none}),
24 'uploader': ('data_layer', 'episodeMaker', {str}),
25 'release_year': ('data_layer', 'episodeYear', {int_or_none}),
26 'episode': ('data_layer', 'episodeTitle', {str}),
27 'season': ('data_layer', 'seasonTitle', {str}),
28 'season_number': ('data_layer', 'seasonTitle', {parse_count}),
29 'series': ('data_layer', 'seriesTitle', {str}),
30 }), get_all=False)
31
32 return {
33 **info,
34 'id': video_id,
35 'title': join_nonempty('series', 'episode', from_dict=info, delim=' - '),
36 'episode_number': int_or_none(self._search_regex(
37 r'[/-]e(?:pisode)?-?(\d+)(?:[/-]|$)', slug or video_id, 'episode number', default=None)),
38 }
39
40
41class NFBIE(NFBBaseIE):
42 IE_NAME = 'nfb'
43 IE_DESC = 'nfb.ca and onf.ca films and episodes'
44 _VALID_URL = [
45 rf'{NFBBaseIE._VALID_URL_BASE}/(?P<type>film)/(?P<id>[^/?#&]+)',
46 rf'{NFBBaseIE._VALID_URL_BASE}/(?P<type>series?)/(?P<id>[^/?#&]+/s(?:ea|ai)son\d+/episode\d+)',
47 ]
f1657a98 48 _TESTS = [{
4b8b0dde 49 'note': 'NFB film',
f1657a98 50 'url': 'https://www.nfb.ca/film/trafficopter/',
51 'info_dict': {
52 'id': 'trafficopter',
53 'ext': 'mp4',
54 'title': 'Trafficopter',
55 'description': 'md5:060228455eb85cf88785c41656776bc0',
56 'thumbnail': r're:^https?://.*\.jpg$',
57 'uploader': 'Barrie Howells',
58 'release_year': 1972,
4b8b0dde 59 'duration': 600.0,
60 },
61 'params': {'skip_download': 'm3u8'},
62 }, {
63 'note': 'ONF film',
64 'url': 'https://www.onf.ca/film/mal-du-siecle/',
65 'info_dict': {
66 'id': 'mal-du-siecle',
67 'ext': 'mp4',
68 'title': 'Le mal du siècle',
69 'description': 'md5:1abf774d77569ebe603419f2d344102b',
70 'thumbnail': r're:^https?://.*\.jpg$',
71 'uploader': 'Catherine Lepage',
72 'release_year': 2019,
73 'duration': 300.0,
74 },
75 'params': {'skip_download': 'm3u8'},
76 }, {
77 'note': 'NFB episode with English title',
78 'url': 'https://www.nfb.ca/series/true-north-inside-the-rise-of-toronto-basketball/season1/episode9/',
79 'info_dict': {
80 'id': 'true-north-episode9-true-north-finale-making-it',
81 'ext': 'mp4',
82 'title': 'True North: Inside the Rise of Toronto Basketball - Finale: Making It',
83 'description': 'We catch up with each player in the midst of their journey as they reflect on their road ahead.',
84 'series': 'True North: Inside the Rise of Toronto Basketball',
85 'release_year': 2018,
86 'season': 'Season 1',
87 'season_number': 1,
88 'episode': 'Finale: Making It',
89 'episode_number': 9,
90 'uploader': 'Ryan Sidhoo',
91 'thumbnail': r're:^https?://.*\.jpg$',
f1657a98 92 },
4b8b0dde 93 'params': {'skip_download': 'm3u8'},
94 }, {
95 'note': 'ONF episode with French title',
96 'url': 'https://www.onf.ca/serie/direction-nord-la-montee-du-basketball-a-toronto/saison1/episode9/',
97 'info_dict': {
98 'id': 'direction-nord-episode-9',
99 'ext': 'mp4',
100 'title': 'Direction nord – La montée du basketball à Toronto - Finale : Réussir',
101 'description': 'md5:349a57419b71432b97bf6083d92b029d',
102 'series': 'Direction nord – La montée du basketball à Toronto',
103 'release_year': 2018,
104 'season': 'Saison 1',
105 'season_number': 1,
106 'episode': 'Finale : Réussir',
107 'episode_number': 9,
108 'uploader': 'Ryan Sidhoo',
109 'thumbnail': r're:^https?://.*\.jpg$',
110 },
111 'params': {'skip_download': 'm3u8'},
112 }, {
113 'note': 'NFB episode with French title (needs geo-bypass)',
114 'url': 'https://www.nfb.ca/series/etoile-du-nord/saison1/episode1/',
115 'info_dict': {
116 'id': 'etoile-du-nord-episode-1-lobservation',
117 'ext': 'mp4',
118 'title': 'Étoile du Nord - L\'observation',
119 'description': 'md5:161a4617260dee3de70f509b2c9dd21b',
120 'series': 'Étoile du Nord',
121 'release_year': 2023,
122 'season': 'Saison 1',
123 'season_number': 1,
124 'episode': 'L\'observation',
125 'episode_number': 1,
126 'uploader': 'Patrick Bossé',
127 'thumbnail': r're:^https?://.*\.jpg$',
128 },
129 'params': {'skip_download': 'm3u8'},
130 }, {
131 'note': 'ONF episode with English title (needs geo-bypass)',
132 'url': 'https://www.onf.ca/serie/north-star/season1/episode1/',
133 'info_dict': {
134 'id': 'north-star-episode-1-observation',
135 'ext': 'mp4',
136 'title': 'North Star - Observation',
137 'description': 'md5:c727f370839d8a817392b9e3f23655c7',
138 'series': 'North Star',
139 'release_year': 2023,
140 'season': 'Season 1',
141 'season_number': 1,
142 'episode': 'Observation',
143 'episode_number': 1,
144 'uploader': 'Patrick Bossé',
145 'thumbnail': r're:^https?://.*\.jpg$',
146 },
147 'params': {'skip_download': 'm3u8'},
148 }, {
149 'note': 'NFB episode with /film/ URL and English title (needs geo-bypass)',
150 'url': 'https://www.nfb.ca/film/north-star-episode-1-observation/',
151 'info_dict': {
152 'id': 'north-star-episode-1-observation',
153 'ext': 'mp4',
154 'title': 'North Star - Observation',
155 'description': 'md5:c727f370839d8a817392b9e3f23655c7',
156 'series': 'North Star',
157 'release_year': 2023,
158 'season': 'Season 1',
159 'season_number': 1,
160 'episode': 'Observation',
161 'episode_number': 1,
162 'uploader': 'Patrick Bossé',
163 'thumbnail': r're:^https?://.*\.jpg$',
164 },
165 'params': {'skip_download': 'm3u8'},
166 }, {
167 'note': 'ONF episode with /film/ URL and French title (needs geo-bypass)',
168 'url': 'https://www.onf.ca/film/etoile-du-nord-episode-1-lobservation/',
169 'info_dict': {
170 'id': 'etoile-du-nord-episode-1-lobservation',
171 'ext': 'mp4',
172 'title': 'Étoile du Nord - L\'observation',
173 'description': 'md5:161a4617260dee3de70f509b2c9dd21b',
174 'series': 'Étoile du Nord',
175 'release_year': 2023,
176 'season': 'Saison 1',
177 'season_number': 1,
178 'episode': 'L\'observation',
179 'episode_number': 1,
180 'uploader': 'Patrick Bossé',
181 'thumbnail': r're:^https?://.*\.jpg$',
182 },
183 'params': {'skip_download': 'm3u8'},
184 }, {
185 'note': 'Season 2 episode w/o episode num in id, extract from json ld',
186 'url': 'https://www.onf.ca/film/liste-des-choses-qui-existent-saison-2-ours',
187 'info_dict': {
188 'id': 'liste-des-choses-qui-existent-saison-2-ours',
189 'ext': 'mp4',
190 'title': 'La liste des choses qui existent - L\'ours en peluche',
191 'description': 'md5:d5e8d8fc5f3a7385a9cf0f509b37e28a',
192 'series': 'La liste des choses qui existent',
193 'release_year': 2022,
194 'season': 'Saison 2',
195 'season_number': 2,
196 'episode': 'L\'ours en peluche',
197 'episode_number': 12,
198 'uploader': 'Francis Papillon',
199 'thumbnail': r're:^https?://.*\.jpg$',
200 },
201 'params': {'skip_download': 'm3u8'},
202 }, {
203 'note': 'NFB film /embed/player/ page',
204 'url': 'https://www.nfb.ca/film/afterlife/embed/player/',
205 'info_dict': {
206 'id': 'afterlife',
207 'ext': 'mp4',
208 'title': 'Afterlife',
209 'description': 'md5:84951394f594f1fb1e62d9c43242fdf5',
210 'release_year': 1978,
211 'duration': 420.0,
212 'uploader': 'Ishu Patel',
213 'thumbnail': r're:^https?://.*\.jpg$',
214 },
215 'params': {'skip_download': 'm3u8'},
f1657a98 216 }]
217
218 def _real_extract(self, url):
4b8b0dde 219 site, type_, slug = self._match_valid_url(url).group('site', 'type', 'id')
220 # Need to construct the URL since we match /embed/player/ URLs as well
221 webpage, urlh = self._download_webpage_handle(f'https://www.{site}.ca/{type_}/{slug}/', slug)
222 # type_ can change from film to serie(s) after redirect; new slug may have episode number
223 type_, slug = self._match_valid_url(urlh.url).group('type', 'id')
f1657a98 224
0a1a8e30 225 player_data = self._search_json(
226 r'window\.PLAYER_OPTIONS\[[^\]]+\]\s*=', webpage, 'player data', slug)
227 video_id = self._match_id(player_data['overlay']['url']) # overlay url always has unique slug
f1657a98 228
4b8b0dde 229 formats, subtitles = self._extract_m3u8_formats_and_subtitles(
0a1a8e30 230 player_data['source'], video_id, 'mp4', m3u8_id='hls')
f1657a98 231
0a1a8e30 232 if dv_source := url_or_none(player_data.get('dvSource')):
4b8b0dde 233 fmts, subs = self._extract_m3u8_formats_and_subtitles(
234 dv_source, video_id, 'mp4', m3u8_id='dv', preference=-2, fatal=False)
235 for fmt in fmts:
236 fmt['format_note'] = 'described video'
237 formats.extend(fmts)
238 self._merge_subtitles(subs, target=subtitles)
f1657a98 239
4b8b0dde 240 info = {
f1657a98 241 'id': video_id,
242 'title': self._html_search_regex(
0a1a8e30 243 r'["\']nfb_version_title["\']\s*:\s*["\']([^"\']+)',
f1657a98 244 webpage, 'title', default=None),
245 'description': self._html_search_regex(
246 r'<[^>]+\bid=["\']tabSynopsis["\'][^>]*>\s*<p[^>]*>\s*([^<]+)',
247 webpage, 'description', default=None),
0a1a8e30 248 'thumbnail': url_or_none(player_data.get('poster')),
f1657a98 249 'uploader': self._html_search_regex(
0a1a8e30 250 r'<[^>]+\bitemprop=["\']director["\'][^>]*>([^<]+)', webpage, 'uploader', default=None),
f1657a98 251 'release_year': int_or_none(self._html_search_regex(
0a1a8e30 252 r'["\']nfb_version_year["\']\s*:\s*["\']([^"\']+)',
f1657a98 253 webpage, 'release_year', default=None)),
4b8b0dde 254 } if type_ == 'film' else self._extract_ep_info(self._extract_ep_data(webpage, video_id, slug), video_id)
255
256 return merge_dicts({
f1657a98 257 'formats': formats,
258 'subtitles': subtitles,
4b8b0dde 259 }, info, self._search_json_ld(webpage, video_id, default={}))
260
261
262class NFBSeriesIE(NFBBaseIE):
263 IE_NAME = 'nfb:series'
264 IE_DESC = 'nfb.ca and onf.ca series'
265 _VALID_URL = rf'{NFBBaseIE._VALID_URL_BASE}/(?P<type>series?)/(?P<id>[^/?#&]+)/?(?:[?#]|$)'
266 _TESTS = [{
267 'url': 'https://www.nfb.ca/series/true-north-inside-the-rise-of-toronto-basketball/',
268 'playlist_mincount': 9,
269 'info_dict': {
270 'id': 'true-north-inside-the-rise-of-toronto-basketball',
271 },
272 }, {
273 'url': 'https://www.onf.ca/serie/la-liste-des-choses-qui-existent-serie/',
274 'playlist_mincount': 26,
275 'info_dict': {
276 'id': 'la-liste-des-choses-qui-existent-serie',
277 },
278 }]
279
280 def _entries(self, episodes):
281 for episode in traverse_obj(episodes, lambda _, v: NFBIE.suitable(v['embed_url'])):
282 mobj = NFBIE._match_valid_url(episode['embed_url'])
283 yield self.url_result(
284 mobj[0], NFBIE, **self._extract_ep_info([episode], mobj.group('id')))
285
286 def _real_extract(self, url):
287 site, type_, series_id = self._match_valid_url(url).group('site', 'type', 'id')
288 season_path = 'saison' if type_ == 'serie' else 'season'
289 webpage = self._download_webpage(
290 f'https://www.{site}.ca/{type_}/{series_id}/{season_path}1/episode1', series_id)
291 episodes = self._extract_ep_data(webpage, series_id, fatal=True)
292
293 return self.playlist_result(self._entries(episodes), series_id)