yt_dlp/extractor/archiveorg.py

   1 import re
   2 import json
   3 from .common import InfoExtractor
   4 from .youtube import YoutubeIE, YoutubeBaseInfoExtractor
   5 from ..compat import (
   6     compat_urllib_parse_unquote,
   7     compat_urllib_parse_unquote_plus,
   8     compat_HTTPError
   9 )
  10 from ..utils import (
  11     bug_reports_message,
  12     clean_html,
  13     dict_get,
  14     extract_attributes,
  15     ExtractorError,
  16     get_element_by_id,
  17     HEADRequest,
  18     int_or_none,
  19     join_nonempty,
  20     KNOWN_EXTENSIONS,
  21     merge_dicts,
  22     mimetype2ext,
  23     orderedSet,
  24     parse_duration,
  25     parse_qs,
  26     str_to_int,
  27     str_or_none,
  28     traverse_obj,
  29     try_get,
  30     unified_strdate,
  31     unified_timestamp,
  32     urlhandle_detect_ext,
  33     url_or_none
  34 )
  35
  36
  37 class ArchiveOrgIE(InfoExtractor):
  38     IE_NAME = 'archive.org'
  39     IE_DESC = 'archive.org video and audio'
  40     _VALID_URL = r'https?://(?:www\.)?archive\.org/(?:details|embed)/(?P<id>[^?#]+)(?:[?].*)?$'
  41     _TESTS = [{
  42         'url': 'http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
  43         'md5': '8af1d4cf447933ed3c7f4871162602db',
  44         'info_dict': {
  45             'id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect',
  46             'ext': 'ogv',
  47             'title': '1968 Demo - FJCC Conference Presentation Reel #1',
  48             'description': 'md5:da45c349df039f1cc8075268eb1b5c25',
  49             'release_date': '19681210',
  50             'timestamp': 1268695290,
  51             'upload_date': '20100315',
  52             'creator': 'SRI International',
  53             'uploader': 'laura@archive.org',
  54         },
  55     }, {
  56         'url': 'https://archive.org/details/Cops1922',
  57         'md5': '0869000b4ce265e8ca62738b336b268a',
  58         'info_dict': {
  59             'id': 'Cops1922',
  60             'ext': 'mp4',
  61             'title': 'Buster Keaton\'s "Cops" (1922)',
  62             'description': 'md5:43a603fd6c5b4b90d12a96b921212b9c',
  63             'uploader': 'yorkmba99@hotmail.com',
  64             'timestamp': 1387699629,
  65             'upload_date': '20131222',
  66         },
  67     }, {
  68         'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
  69         'only_matching': True,
  70     }, {
  71         'url': 'https://archive.org/details/Election_Ads',
  72         'md5': '284180e857160cf866358700bab668a3',
  73         'info_dict': {
  74             'id': 'Election_Ads/Commercial-JFK1960ElectionAdCampaignJingle.mpg',
  75             'title': 'Commercial-JFK1960ElectionAdCampaignJingle.mpg',
  76             'ext': 'mp4',
  77         },
  78     }, {
  79         'url': 'https://archive.org/details/Election_Ads/Commercial-Nixon1960ElectionAdToughonDefense.mpg',
  80         'md5': '7915213ef02559b5501fe630e1a53f59',
  81         'info_dict': {
  82             'id': 'Election_Ads/Commercial-Nixon1960ElectionAdToughonDefense.mpg',
  83             'title': 'Commercial-Nixon1960ElectionAdToughonDefense.mpg',
  84             'ext': 'mp4',
  85             'timestamp': 1205588045,
  86             'uploader': 'mikedavisstripmaster@yahoo.com',
  87             'description': '1960 Presidential Campaign Election Commercials John F Kennedy, Richard M Nixon',
  88             'upload_date': '20080315',
  89         },
  90     }, {
  91         'url': 'https://archive.org/details/gd1977-05-08.shure57.stevenson.29303.flac16',
  92         'md5': '7d07ffb42aba6537c28e053efa4b54c9',
  93         'info_dict': {
  94             'id': 'gd1977-05-08.shure57.stevenson.29303.flac16/gd1977-05-08d01t01.flac',
  95             'title': 'Turning',
  96             'ext': 'flac',
  97         },
  98     }, {
  99         'url': 'https://archive.org/details/gd1977-05-08.shure57.stevenson.29303.flac16/gd1977-05-08d01t07.flac',
 100         'md5': 'a07cd8c6ab4ee1560f8a0021717130f3',
 101         'info_dict': {
 102             'id': 'gd1977-05-08.shure57.stevenson.29303.flac16/gd1977-05-08d01t07.flac',
 103             'title': 'Deal',
 104             'ext': 'flac',
 105             'timestamp': 1205895624,
 106             'uploader': 'mvernon54@yahoo.com',
 107             'description': 'md5:6a31f1996db0aa0fc9da6d6e708a1bb0',
 108             'upload_date': '20080319',
 109             'location': 'Barton Hall - Cornell University',
 110         },
 111     }, {
 112         'url': 'https://archive.org/details/lp_the-music-of-russia_various-artists-a-askaryan-alexander-melik',
 113         'md5': '7cb019baa9b332e82ea7c10403acd180',
 114         'info_dict': {
 115             'id': 'lp_the-music-of-russia_various-artists-a-askaryan-alexander-melik/disc1/01.01. Bells Of Rostov.mp3',
 116             'title': 'Bells Of Rostov',
 117             'ext': 'mp3',
 118         },
 119     }, {
 120         'url': 'https://archive.org/details/lp_the-music-of-russia_various-artists-a-askaryan-alexander-melik/disc1/02.02.+Song+And+Chorus+In+The+Polovetsian+Camp+From+%22Prince+Igor%22+(Act+2%2C+Scene+1).mp3',
 121         'md5': '1d0aabe03edca83ca58d9ed3b493a3c3',
 122         'info_dict': {
 123             'id': 'lp_the-music-of-russia_various-artists-a-askaryan-alexander-melik/disc1/02.02. Song And Chorus In The Polovetsian Camp From "Prince Igor" (Act 2, Scene 1).mp3',
 124             'title': 'Song And Chorus In The Polovetsian Camp From "Prince Igor" (Act 2, Scene 1)',
 125             'ext': 'mp3',
 126             'timestamp': 1569662587,
 127             'uploader': 'associate-joygen-odiongan@archive.org',
 128             'description': 'md5:012b2d668ae753be36896f343d12a236',
 129             'upload_date': '20190928',
 130         },
 131     }]
 132
 133     @staticmethod
 134     def _playlist_data(webpage):
 135         element = re.findall(r'''(?xs)
 136             <input
 137             (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
 138             \s+class=['"]?js-play8-playlist['"]?
 139             (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
 140             \s*/>
 141         ''', webpage)[0]
 142
 143         return json.loads(extract_attributes(element)['value'])
 144
 145     def _real_extract(self, url):
 146         video_id = compat_urllib_parse_unquote_plus(self._match_id(url))
 147         identifier, entry_id = (video_id.split('/', 1) + [None])[:2]
 148
 149         # Archive.org metadata API doesn't clearly demarcate playlist entries
 150         # or subtitle tracks, so we get them from the embeddable player.
 151         embed_page = self._download_webpage(f'https://archive.org/embed/{identifier}', identifier)
 152         playlist = self._playlist_data(embed_page)
 153
 154         entries = {}
 155         for p in playlist:
 156             # If the user specified a playlist entry in the URL, ignore the
 157             # rest of the playlist.
 158             if entry_id and p['orig'] != entry_id:
 159                 continue
 160
 161             entries[p['orig']] = {
 162                 'formats': [],
 163                 'thumbnails': [],
 164                 'artist': p.get('artist'),
 165                 'track': p.get('title'),
 166                 'subtitles': {},
 167             }
 168
 169             for track in p.get('tracks', []):
 170                 if track['kind'] != 'subtitles':
 171                     continue
 172                 entries[p['orig']][track['label']] = {
 173                     'url': 'https://archive.org/' + track['file'].lstrip('/')
 174                 }
 175
 176         metadata = self._download_json('http://archive.org/metadata/' + identifier, identifier)
 177         m = metadata['metadata']
 178         identifier = m['identifier']
 179
 180         info = {
 181             'id': identifier,
 182             'title': m['title'],
 183             'description': clean_html(m.get('description')),
 184             'uploader': dict_get(m, ['uploader', 'adder']),
 185             'creator': m.get('creator'),
 186             'license': m.get('licenseurl'),
 187             'release_date': unified_strdate(m.get('date')),
 188             'timestamp': unified_timestamp(dict_get(m, ['publicdate', 'addeddate'])),
 189             'webpage_url': f'https://archive.org/details/{identifier}',
 190             'location': m.get('venue'),
 191             'release_year': int_or_none(m.get('year'))}
 192
 193         for f in metadata['files']:
 194             if f['name'] in entries:
 195                 entries[f['name']] = merge_dicts(entries[f['name']], {
 196                     'id': identifier + '/' + f['name'],
 197                     'title': f.get('title') or f['name'],
 198                     'display_id': f['name'],
 199                     'description': clean_html(f.get('description')),
 200                     'creator': f.get('creator'),
 201                     'duration': parse_duration(f.get('length')),
 202                     'track_number': int_or_none(f.get('track')),
 203                     'album': f.get('album'),
 204                     'discnumber': int_or_none(f.get('disc')),
 205                     'release_year': int_or_none(f.get('year'))})
 206                 entry = entries[f['name']]
 207             elif traverse_obj(f, 'original', expected_type=str) in entries:
 208                 entry = entries[f['original']]
 209             else:
 210                 continue
 211
 212             if f.get('format') == 'Thumbnail':
 213                 entry['thumbnails'].append({
 214                     'id': f['name'],
 215                     'url': 'https://archive.org/download/' + identifier + '/' + f['name'],
 216                     'width': int_or_none(f.get('width')),
 217                     'height': int_or_none(f.get('width')),
 218                     'filesize': int_or_none(f.get('size'))})
 219
 220             extension = (f['name'].rsplit('.', 1) + [None])[1]
 221             if extension in KNOWN_EXTENSIONS:
 222                 entry['formats'].append({
 223                     'url': 'https://archive.org/download/' + identifier + '/' + f['name'],
 224                     'format': f.get('format'),
 225                     'width': int_or_none(f.get('width')),
 226                     'height': int_or_none(f.get('height')),
 227                     'filesize': int_or_none(f.get('size')),
 228                     'protocol': 'https'})
 229
 230         for entry in entries.values():
 231             self._sort_formats(entry['formats'])
 232
 233         if len(entries) == 1:
 234             # If there's only one item, use it as the main info dict
 235             only_video = next(iter(entries.values()))
 236             if entry_id:
 237                 info = merge_dicts(only_video, info)
 238             else:
 239                 info = merge_dicts(info, only_video)
 240         else:
 241             # Otherwise, we have a playlist.
 242             info['_type'] = 'playlist'
 243             info['entries'] = list(entries.values())
 244
 245         if metadata.get('reviews'):
 246             info['comments'] = []
 247             for review in metadata['reviews']:
 248                 info['comments'].append({
 249                     'id': review.get('review_id'),
 250                     'author': review.get('reviewer'),
 251                     'text': str_or_none(review.get('reviewtitle'), '') + '\n\n' + review.get('reviewbody'),
 252                     'timestamp': unified_timestamp(review.get('createdate')),
 253                     'parent': 'root'})
 254
 255         return info
 256
 257
 258 class YoutubeWebArchiveIE(InfoExtractor):
 259     IE_NAME = 'web.archive:youtube'
 260     IE_DESC = 'web.archive.org saved youtube videos, "ytarchive:" prefix'
 261     _VALID_URL = r'''(?x)(?:(?P<prefix>ytarchive:)|
 262             (?:https?://)?web\.archive\.org/
 263             (?:web/)?(?:(?P<date>[0-9]{14})?[0-9A-Za-z_*]*/)?  # /web and the version index is optional
 264             (?:https?(?::|%3[Aa])//)?(?:
 265                 (?:\w+\.)?youtube\.com(?::(?:80|443))?/watch(?:\.php)?(?:\?|%3[fF])(?:[^\#]+(?:&|%26))?v(?:=|%3[dD])  # Youtube URL
 266                 |(?:wayback-fakeurl\.archive\.org/yt/)  # Or the internal fake url
 267             )
 268         )(?P<id>[0-9A-Za-z_-]{11})
 269         (?(prefix)
 270             (?::(?P<date2>[0-9]{14}))?$|
 271             (?:%26|[#&]|$)
 272         )'''
 273
 274     _TESTS = [
 275         {
 276             'url': 'https://web.archive.org/web/20150415002341/https://www.youtube.com/watch?v=aYAGB11YrSs',
 277             'info_dict': {
 278                 'id': 'aYAGB11YrSs',
 279                 'ext': 'webm',
 280                 'title': 'Team Fortress 2 - Sandviches!',
 281                 'description': 'md5:4984c0f9a07f349fc5d8e82ab7af4eaf',
 282                 'upload_date': '20110926',
 283                 'uploader': 'Zeurel',
 284                 'channel_id': 'UCukCyHaD-bK3in_pKpfH9Eg',
 285                 'duration': 32,
 286                 'uploader_id': 'Zeurel',
 287                 'uploader_url': 'http://www.youtube.com/user/Zeurel'
 288             }
 289         }, {
 290             # Internal link
 291             'url': 'https://web.archive.org/web/2oe/http://wayback-fakeurl.archive.org/yt/97t7Xj_iBv0',
 292             'info_dict': {
 293                 'id': '97t7Xj_iBv0',
 294                 'ext': 'mp4',
 295                 'title': 'Why Machines That Bend Are Better',
 296                 'description': 'md5:00404df2c632d16a674ff8df1ecfbb6c',
 297                 'upload_date': '20190312',
 298                 'uploader': 'Veritasium',
 299                 'channel_id': 'UCHnyfMqiRRG1u-2MsSQLbXA',
 300                 'duration': 771,
 301                 'uploader_id': '1veritasium',
 302                 'uploader_url': 'http://www.youtube.com/user/1veritasium'
 303             }
 304         }, {
 305             # Video from 2012, webm format itag 45. Newest capture is deleted video, with an invalid description.
 306             # Should use the date in the link. Title ends with '- Youtube'. Capture has description in eow-description
 307             'url': 'https://web.archive.org/web/20120712231619/http://www.youtube.com/watch?v=AkhihxRKcrs&gl=US&hl=en',
 308             'info_dict': {
 309                 'id': 'AkhihxRKcrs',
 310                 'ext': 'webm',
 311                 'title': 'Limited Run: Mondo\'s Modern Classic 1 of 3 (SDCC 2012)',
 312                 'upload_date': '20120712',
 313                 'duration': 398,
 314                 'description': 'md5:ff4de6a7980cb65d951c2f6966a4f2f3',
 315                 'uploader_id': 'machinima',
 316                 'uploader_url': 'http://www.youtube.com/user/machinima'
 317             }
 318         }, {
 319             # FLV video. Video file URL does not provide itag information
 320             'url': 'https://web.archive.org/web/20081211103536/http://www.youtube.com/watch?v=jNQXAC9IVRw',
 321             'info_dict': {
 322                 'id': 'jNQXAC9IVRw',
 323                 'ext': 'flv',
 324                 'title': 'Me at the zoo',
 325                 'upload_date': '20050423',
 326                 'channel_id': 'UC4QobU6STFB0P71PMvOGN5A',
 327                 'duration': 19,
 328                 'description': 'md5:10436b12e07ac43ff8df65287a56efb4',
 329                 'uploader_id': 'jawed',
 330                 'uploader_url': 'http://www.youtube.com/user/jawed'
 331             }
 332         }, {
 333             'url': 'https://web.archive.org/web/20110712231407/http://www.youtube.com/watch?v=lTx3G6h2xyA',
 334             'info_dict': {
 335                 'id': 'lTx3G6h2xyA',
 336                 'ext': 'flv',
 337                 'title': 'Madeon - Pop Culture (live mashup)',
 338                 'upload_date': '20110711',
 339                 'uploader': 'Madeon',
 340                 'channel_id': 'UCqMDNf3Pn5L7pcNkuSEeO3w',
 341                 'duration': 204,
 342                 'description': 'md5:f7535343b6eda34a314eff8b85444680',
 343                 'uploader_id': 'itsmadeon',
 344                 'uploader_url': 'http://www.youtube.com/user/itsmadeon'
 345             }
 346         }, {
 347             # First capture is of dead video, second is the oldest from CDX response.
 348             'url': 'https://web.archive.org/https://www.youtube.com/watch?v=1JYutPM8O6E',
 349             'info_dict': {
 350                 'id': '1JYutPM8O6E',
 351                 'ext': 'mp4',
 352                 'title': 'Fake Teen Doctor Strikes AGAIN! - Weekly Weird News',
 353                 'upload_date': '20160218',
 354                 'channel_id': 'UCdIaNUarhzLSXGoItz7BHVA',
 355                 'duration': 1236,
 356                 'description': 'md5:21032bae736421e89c2edf36d1936947',
 357                 'uploader_id': 'MachinimaETC',
 358                 'uploader_url': 'http://www.youtube.com/user/MachinimaETC'
 359             }
 360         }, {
 361             # First capture of dead video, capture date in link links to dead capture.
 362             'url': 'https://web.archive.org/web/20180803221945/https://www.youtube.com/watch?v=6FPhZJGvf4E',
 363             'info_dict': {
 364                 'id': '6FPhZJGvf4E',
 365                 'ext': 'mp4',
 366                 'title': 'WTF: Video Games Still Launch BROKEN?! - T.U.G.S.',
 367                 'upload_date': '20160219',
 368                 'channel_id': 'UCdIaNUarhzLSXGoItz7BHVA',
 369                 'duration': 798,
 370                 'description': 'md5:a1dbf12d9a3bd7cb4c5e33b27d77ffe7',
 371                 'uploader_id': 'MachinimaETC',
 372                 'uploader_url': 'http://www.youtube.com/user/MachinimaETC'
 373             },
 374             'expected_warnings': [
 375                 r'unable to download capture webpage \(it may not be archived\)'
 376             ]
 377         }, {   # Very old YouTube page, has - YouTube in title.
 378             'url': 'http://web.archive.org/web/20070302011044/http://youtube.com/watch?v=-06-KB9XTzg',
 379             'info_dict': {
 380                 'id': '-06-KB9XTzg',
 381                 'ext': 'flv',
 382                 'title': 'New Coin Hack!! 100% Safe!!'
 383             }
 384         }, {
 385             'url': 'web.archive.org/https://www.youtube.com/watch?v=dWW7qP423y8',
 386             'info_dict': {
 387                 'id': 'dWW7qP423y8',
 388                 'ext': 'mp4',
 389                 'title': 'It\'s Bootleg AirPods Time.',
 390                 'upload_date': '20211021',
 391                 'channel_id': 'UC7Jwj9fkrf1adN4fMmTkpug',
 392                 'channel_url': 'http://www.youtube.com/channel/UC7Jwj9fkrf1adN4fMmTkpug',
 393                 'duration': 810,
 394                 'description': 'md5:7b567f898d8237b256f36c1a07d6d7bc',
 395                 'uploader': 'DankPods',
 396                 'uploader_id': 'UC7Jwj9fkrf1adN4fMmTkpug',
 397                 'uploader_url': 'http://www.youtube.com/channel/UC7Jwj9fkrf1adN4fMmTkpug'
 398             }
 399         }, {
 400             # player response contains '};' See: https://github.com/ytdl-org/youtube-dl/issues/27093
 401             'url': 'https://web.archive.org/web/20200827003909if_/http://www.youtube.com/watch?v=6Dh-RL__uN4',
 402             'info_dict': {
 403                 'id': '6Dh-RL__uN4',
 404                 'ext': 'mp4',
 405                 'title': 'bitch lasagna',
 406                 'upload_date': '20181005',
 407                 'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
 408                 'channel_url': 'http://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
 409                 'duration': 135,
 410                 'description': 'md5:2dbe4051feeff2dab5f41f82bb6d11d0',
 411                 'uploader': 'PewDiePie',
 412                 'uploader_id': 'PewDiePie',
 413                 'uploader_url': 'http://www.youtube.com/user/PewDiePie'
 414             }
 415         }, {
 416             'url': 'https://web.archive.org/web/http://www.youtube.com/watch?v=kH-G_aIBlFw',
 417             'only_matching': True
 418         }, {
 419             'url': 'https://web.archive.org/web/20050214000000_if/http://www.youtube.com/watch?v=0altSZ96U4M',
 420             'only_matching': True
 421         }, {
 422             # Video not archived, only capture is unavailable video page
 423             'url': 'https://web.archive.org/web/20210530071008/https://www.youtube.com/watch?v=lHJTf93HL1s&spfreload=10',
 424             'only_matching': True
 425         }, {   # Encoded url
 426             'url': 'https://web.archive.org/web/20120712231619/http%3A//www.youtube.com/watch%3Fgl%3DUS%26v%3DAkhihxRKcrs%26hl%3Den',
 427             'only_matching': True
 428         }, {
 429             'url': 'https://web.archive.org/web/20120712231619/http%3A//www.youtube.com/watch%3Fv%3DAkhihxRKcrs%26gl%3DUS%26hl%3Den',
 430             'only_matching': True
 431         }, {
 432             'url': 'https://web.archive.org/web/20060527081937/http://www.youtube.com:80/watch.php?v=ELTFsLT73fA&amp;search=soccer',
 433             'only_matching': True
 434         }, {
 435             'url': 'https://web.archive.org/http://www.youtube.com:80/watch?v=-05VVye-ffg',
 436             'only_matching': True
 437         }, {
 438             'url': 'ytarchive:BaW_jenozKc:20050214000000',
 439             'only_matching': True
 440         }, {
 441             'url': 'ytarchive:BaW_jenozKc',
 442             'only_matching': True
 443         },
 444     ]
 445     _YT_INITIAL_DATA_RE = YoutubeBaseInfoExtractor._YT_INITIAL_DATA_RE
 446     _YT_INITIAL_PLAYER_RESPONSE_RE = fr'''(?x)
 447         (?:window\s*\[\s*["\']ytInitialPlayerResponse["\']\s*\]|ytInitialPlayerResponse)\s*=[(\s]*|
 448         {YoutubeBaseInfoExtractor._YT_INITIAL_PLAYER_RESPONSE_RE}'''
 449
 450     _YT_DEFAULT_THUMB_SERVERS = ['i.ytimg.com']  # thumbnails most likely archived on these servers
 451     _YT_ALL_THUMB_SERVERS = orderedSet(
 452         _YT_DEFAULT_THUMB_SERVERS + ['img.youtube.com', *[f'{c}{n or ""}.ytimg.com' for c in ('i', 's') for n in (*range(0, 5), 9)]])
 453
 454     _WAYBACK_BASE_URL = 'https://web.archive.org/web/%sif_/'
 455     _OLDEST_CAPTURE_DATE = 20050214000000
 456     _NEWEST_CAPTURE_DATE = 20500101000000
 457
 458     def _call_cdx_api(self, item_id, url, filters: list = None, collapse: list = None, query: dict = None, note=None, fatal=False):
 459         # CDX docs: https://github.com/internetarchive/wayback/blob/master/wayback-cdx-server/README.md
 460         query = {
 461             'url': url,
 462             'output': 'json',
 463             'fl': 'original,mimetype,length,timestamp',
 464             'limit': 500,
 465             'filter': ['statuscode:200'] + (filters or []),
 466             'collapse': collapse or [],
 467             **(query or {})
 468         }
 469         res = self._download_json(
 470             'https://web.archive.org/cdx/search/cdx', item_id,
 471             note or 'Downloading CDX API JSON', query=query, fatal=fatal)
 472         if isinstance(res, list) and len(res) >= 2:
 473             # format response to make it easier to use
 474             return list(dict(zip(res[0], v)) for v in res[1:])
 475         elif not isinstance(res, list) or len(res) != 0:
 476             self.report_warning('Error while parsing CDX API response' + bug_reports_message())
 477
 478     def _extract_webpage_title(self, webpage):
 479         page_title = self._html_extract_title(webpage, default='')
 480         # YouTube video pages appear to always have either 'YouTube -' as prefix or '- YouTube' as suffix.
 481         return self._html_search_regex(
 482             r'(?:YouTube\s*-\s*(.*)$)|(?:(.*)\s*-\s*YouTube$)',
 483             page_title, 'title', default='')
 484
 485     def _extract_metadata(self, video_id, webpage):
 486         search_meta = ((lambda x: self._html_search_meta(x, webpage, default=None)) if webpage else (lambda x: None))
 487         player_response = self._search_json(
 488             self._YT_INITIAL_PLAYER_RESPONSE_RE, webpage, 'initial player response',
 489             video_id, default={})
 490         initial_data = self._search_json(
 491             self._YT_INITIAL_DATA_RE, webpage, 'initial data', video_id, default={})
 492
 493         initial_data_video = traverse_obj(
 494             initial_data, ('contents', 'twoColumnWatchNextResults', 'results', 'results', 'contents', ..., 'videoPrimaryInfoRenderer'),
 495             expected_type=dict, get_all=False, default={})
 496
 497         video_details = traverse_obj(
 498             player_response, 'videoDetails', expected_type=dict, get_all=False, default={})
 499
 500         microformats = traverse_obj(
 501             player_response, ('microformat', 'playerMicroformatRenderer'), expected_type=dict, get_all=False, default={})
 502
 503         video_title = (
 504             video_details.get('title')
 505             or YoutubeBaseInfoExtractor._get_text(microformats, 'title')
 506             or YoutubeBaseInfoExtractor._get_text(initial_data_video, 'title')
 507             or self._extract_webpage_title(webpage)
 508             or search_meta(['og:title', 'twitter:title', 'title']))
 509
 510         channel_id = str_or_none(
 511             video_details.get('channelId')
 512             or microformats.get('externalChannelId')
 513             or search_meta('channelId')
 514             or self._search_regex(
 515                 r'data-channel-external-id=(["\'])(?P<id>(?:(?!\1).)+)\1',  # @b45a9e6
 516                 webpage, 'channel id', default=None, group='id'))
 517         channel_url = f'http://www.youtube.com/channel/{channel_id}' if channel_id else None
 518
 519         duration = int_or_none(
 520             video_details.get('lengthSeconds')
 521             or microformats.get('lengthSeconds')
 522             or parse_duration(search_meta('duration')))
 523         description = (
 524             video_details.get('shortDescription')
 525             or YoutubeBaseInfoExtractor._get_text(microformats, 'description')
 526             or clean_html(get_element_by_id('eow-description', webpage))  # @9e6dd23
 527             or search_meta(['description', 'og:description', 'twitter:description']))
 528
 529         uploader = video_details.get('author')
 530
 531         # Uploader ID and URL
 532         uploader_mobj = re.search(
 533             r'<link itemprop="url" href="(?P<uploader_url>https?://www\.youtube\.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',  # @fd05024
 534             webpage)
 535         if uploader_mobj is not None:
 536             uploader_id, uploader_url = uploader_mobj.group('uploader_id'), uploader_mobj.group('uploader_url')
 537         else:
 538             # @a6211d2
 539             uploader_url = url_or_none(microformats.get('ownerProfileUrl'))
 540             uploader_id = self._search_regex(
 541                 r'(?:user|channel)/([^/]+)', uploader_url or '', 'uploader id', default=None)
 542
 543         upload_date = unified_strdate(
 544             dict_get(microformats, ('uploadDate', 'publishDate'))
 545             or search_meta(['uploadDate', 'datePublished'])
 546             or self._search_regex(
 547                 [r'(?s)id="eow-date.*?>(.*?)</span>',
 548                  r'(?:id="watch-uploader-info".*?>.*?|["\']simpleText["\']\s*:\s*["\'])(?:Published|Uploaded|Streamed live|Started) on (.+?)[<"\']'],  # @7998520
 549                 webpage, 'upload date', default=None))
 550
 551         return {
 552             'title': video_title,
 553             'description': description,
 554             'upload_date': upload_date,
 555             'uploader': uploader,
 556             'channel_id': channel_id,
 557             'channel_url': channel_url,
 558             'duration': duration,
 559             'uploader_url': uploader_url,
 560             'uploader_id': uploader_id,
 561         }
 562
 563     def _extract_thumbnails(self, video_id):
 564         try_all = 'thumbnails' in self._configuration_arg('check_all')
 565         thumbnail_base_urls = ['http://{server}/vi{webp}/{video_id}'.format(
 566             webp='_webp' if ext == 'webp' else '', video_id=video_id, server=server)
 567             for server in (self._YT_ALL_THUMB_SERVERS if try_all else self._YT_DEFAULT_THUMB_SERVERS) for ext in (('jpg', 'webp') if try_all else ('jpg',))]
 568
 569         thumbnails = []
 570         for url in thumbnail_base_urls:
 571             response = self._call_cdx_api(
 572                 video_id, url, filters=['mimetype:image/(?:webp|jpeg)'],
 573                 collapse=['urlkey'], query={'matchType': 'prefix'})
 574             if not response:
 575                 continue
 576             thumbnails.extend(
 577                 {
 578                     'url': (self._WAYBACK_BASE_URL % (int_or_none(thumbnail_dict.get('timestamp')) or self._OLDEST_CAPTURE_DATE)) + thumbnail_dict.get('original'),
 579                     'filesize': int_or_none(thumbnail_dict.get('length')),
 580                     'preference': int_or_none(thumbnail_dict.get('length'))
 581                 } for thumbnail_dict in response)
 582             if not try_all:
 583                 break
 584
 585         self._remove_duplicate_formats(thumbnails)
 586         return thumbnails
 587
 588     def _get_capture_dates(self, video_id, url_date):
 589         capture_dates = []
 590         # Note: CDX API will not find watch pages with extra params in the url.
 591         response = self._call_cdx_api(
 592             video_id, f'https://www.youtube.com/watch?v={video_id}',
 593             filters=['mimetype:text/html'], collapse=['timestamp:6', 'digest'], query={'matchType': 'prefix'}) or []
 594         all_captures = sorted(int_or_none(r['timestamp']) for r in response if int_or_none(r['timestamp']) is not None)
 595
 596         # Prefer the new polymer UI captures as we support extracting more metadata from them
 597         # WBM captures seem to all switch to this layout ~July 2020
 598         modern_captures = [x for x in all_captures if x >= 20200701000000]
 599         if modern_captures:
 600             capture_dates.append(modern_captures[0])
 601         capture_dates.append(url_date)
 602         if all_captures:
 603             capture_dates.append(all_captures[0])
 604
 605         if 'captures' in self._configuration_arg('check_all'):
 606             capture_dates.extend(modern_captures + all_captures)
 607
 608         # Fallbacks if any of the above fail
 609         capture_dates.extend([self._OLDEST_CAPTURE_DATE, self._NEWEST_CAPTURE_DATE])
 610         return orderedSet(filter(None, capture_dates))
 611
 612     def _real_extract(self, url):
 613         video_id, url_date, url_date_2 = self._match_valid_url(url).group('id', 'date', 'date2')
 614         url_date = url_date or url_date_2
 615
 616         urlh = None
 617         try:
 618             urlh = self._request_webpage(
 619                 HEADRequest('https://web.archive.org/web/2oe_/http://wayback-fakeurl.archive.org/yt/%s' % video_id),
 620                 video_id, note='Fetching archived video file url', expected_status=True)
 621         except ExtractorError as e:
 622             # HTTP Error 404 is expected if the video is not saved.
 623             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
 624                 self.raise_no_formats(
 625                     'The requested video is not archived, indexed, or there is an issue with web.archive.org',
 626                     expected=True)
 627             else:
 628                 raise
 629
 630         capture_dates = self._get_capture_dates(video_id, int_or_none(url_date))
 631         self.write_debug('Captures to try: ' + join_nonempty(*capture_dates, delim=', '))
 632         info = {'id': video_id}
 633         for capture in capture_dates:
 634             webpage = self._download_webpage(
 635                 (self._WAYBACK_BASE_URL + 'http://www.youtube.com/watch?v=%s') % (capture, video_id),
 636                 video_id=video_id, fatal=False, errnote='unable to download capture webpage (it may not be archived)',
 637                 note='Downloading capture webpage')
 638             current_info = self._extract_metadata(video_id, webpage or '')
 639             # Try avoid getting deleted video metadata
 640             if current_info.get('title'):
 641                 info = merge_dicts(info, current_info)
 642                 if 'captures' not in self._configuration_arg('check_all'):
 643                     break
 644
 645         info['thumbnails'] = self._extract_thumbnails(video_id)
 646
 647         if urlh:
 648             url = compat_urllib_parse_unquote(urlh.geturl())
 649             video_file_url_qs = parse_qs(url)
 650             # Attempt to recover any ext & format info from playback url & response headers
 651             format = {'url': url, 'filesize': int_or_none(urlh.headers.get('x-archive-orig-content-length'))}
 652             itag = try_get(video_file_url_qs, lambda x: x['itag'][0])
 653             if itag and itag in YoutubeIE._formats:
 654                 format.update(YoutubeIE._formats[itag])
 655                 format.update({'format_id': itag})
 656             else:
 657                 mime = try_get(video_file_url_qs, lambda x: x['mime'][0])
 658                 ext = (mimetype2ext(mime)
 659                        or urlhandle_detect_ext(urlh)
 660                        or mimetype2ext(urlh.headers.get('x-archive-guessed-content-type')))
 661                 format.update({'ext': ext})
 662             info['formats'] = [format]
 663             if not info.get('duration'):
 664                 info['duration'] = str_to_int(try_get(video_file_url_qs, lambda x: x['dur'][0]))
 665
 666         if not info.get('title'):
 667             info['title'] = video_id
 668         return info