]>
jfr.im git - yt-dlp.git/blob - youtube_dl/__init__.py
2 # -*- coding: utf-8 -*-
5 'Ricardo Garcia Gonzalez' ,
13 'Philipp Hagemeister' ,
21 __license__
= 'Public Domain'
22 __version__
= '2012.02.27'
24 UPDATE_URL
= 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
40 from FileDownloader
import *
41 from InfoExtractors
import *
42 from PostProcessing
import *
44 def updateSelf ( downloader
, filename
):
45 ''' Update the program file with the latest version from the repository '''
46 # Note: downloader only used for options
47 if not os
. access ( filename
, os
. W_OK
):
48 sys
. exit ( 'ERROR: no write permissions on %s ' % filename
)
50 downloader
. to_screen ( u
'Updating to latest version...' )
54 urlh
= urllib2
. urlopen ( UPDATE_URL
)
55 newcontent
= urlh
. read ()
57 vmatch
= re
. search ( "__version__ = '([^']+)'" , newcontent
)
58 if vmatch
is not None and vmatch
. group ( 1 ) == __version__
:
59 downloader
. to_screen ( u
'youtube-dl is up-to-date (' + __version__
+ ')' )
63 except ( IOError , OSError ), err
:
64 sys
. exit ( 'ERROR: unable to download latest version' )
67 outf
= open ( filename
, 'wb' )
69 outf
. write ( newcontent
)
72 except ( IOError , OSError ), err
:
73 sys
. exit ( 'ERROR: unable to overwrite current version' )
75 downloader
. to_screen ( u
'Updated youtube-dl. Restart youtube-dl to use the new version.' )
78 def _readOptions ( filename_bytes
):
80 optionf
= open ( filename_bytes
)
82 return [] # silently skip if file is not present
86 res
+= shlex
. split ( l
, comments
= True )
91 def _format_option_string ( option
):
92 ''' ('-o', '--option') -> -o, --format METAVAR'''
96 if option
._ short
_ opts
: opts
. append ( option
._ short
_ opts
[ 0 ])
97 if option
._l ong
_ opts
: opts
. append ( option
._l ong
_ opts
[ 0 ])
98 if len ( opts
) > 1 : opts
. insert ( 1 , ', ' )
100 if option
. takes_value (): opts
. append ( ' %s ' % option
. metavar
)
104 def _find_term_columns ():
105 columns
= os
. environ
. get ( 'COLUMNS' , None )
110 sp
= subprocess
. Popen ([ 'stty' , 'size' ], stdout
= subprocess
. PIPE
, stderr
= subprocess
. PIPE
)
111 out
, err
= sp
. communicate ()
112 return int ( out
. split ()[ 1 ])
118 max_help_position
= 80
120 # No need to wrap help messages if we're on a wide console
121 columns
= _find_term_columns ()
122 if columns
: max_width
= columns
124 fmt
= optparse
. IndentedHelpFormatter ( width
= max_width
, max_help_position
= max_help_position
)
125 fmt
. format_option_strings
= _format_option_string
128 'version' : __version__
,
130 'usage' : '%prog [options] url [url...]' ,
131 'conflict_handler' : 'resolve' ,
134 parser
= optparse
. OptionParser (** kw
)
137 general
= optparse
. OptionGroup ( parser
, 'General Options' )
138 selection
= optparse
. OptionGroup ( parser
, 'Video Selection' )
139 authentication
= optparse
. OptionGroup ( parser
, 'Authentication Options' )
140 video_format
= optparse
. OptionGroup ( parser
, 'Video Format Options' )
141 postproc
= optparse
. OptionGroup ( parser
, 'Post-processing Options' )
142 filesystem
= optparse
. OptionGroup ( parser
, 'Filesystem Options' )
143 verbosity
= optparse
. OptionGroup ( parser
, 'Verbosity / Simulation Options' )
145 general
. add_option ( '-h' , '--help' ,
146 action
= 'help' , help = 'print this help text and exit' )
147 general
. add_option ( '-v' , '--version' ,
148 action
= 'version' , help = 'print program version and exit' )
149 general
. add_option ( '-U' , '--update' ,
150 action
= 'store_true' , dest
= 'update_self' , help = 'update this program to latest version' )
151 general
. add_option ( '-i' , '--ignore-errors' ,
152 action
= 'store_true' , dest
= 'ignoreerrors' , help = 'continue on download errors' , default
= False )
153 general
. add_option ( '-r' , '--rate-limit' ,
154 dest
= 'ratelimit' , metavar
= 'LIMIT' , help = 'download rate limit (e.g. 50k or 44.6m)' )
155 general
. add_option ( '-R' , '--retries' ,
156 dest
= 'retries' , metavar
= 'RETRIES' , help = 'number of retries (default is 10)' , default
= 10 )
157 general
. add_option ( '--dump-user-agent' ,
158 action
= 'store_true' , dest
= 'dump_user_agent' ,
159 help = 'display the current browser identification' , default
= False )
160 general
. add_option ( '--list-extractors' ,
161 action
= 'store_true' , dest
= 'list_extractors' ,
162 help = 'List all supported extractors and the URLs they would handle' , default
= False )
164 selection
. add_option ( '--playlist-start' ,
165 dest
= 'playliststart' , metavar
= 'NUMBER' , help = 'playlist video to start at (default is 1)' , default
= 1 )
166 selection
. add_option ( '--playlist-end' ,
167 dest
= 'playlistend' , metavar
= 'NUMBER' , help = 'playlist video to end at (default is last)' , default
=- 1 )
168 selection
. add_option ( '--match-title' , dest
= 'matchtitle' , metavar
= 'REGEX' , help = 'download only matching titles (regex or caseless sub-string)' )
169 selection
. add_option ( '--reject-title' , dest
= 'rejecttitle' , metavar
= 'REGEX' , help = 'skip download for matching titles (regex or caseless sub-string)' )
170 selection
. add_option ( '--max-downloads' , metavar
= 'NUMBER' , dest
= 'max_downloads' , help = 'Abort after downloading NUMBER files' , default
= None )
172 authentication
. add_option ( '-u' , '--username' ,
173 dest
= 'username' , metavar
= 'USERNAME' , help = 'account username' )
174 authentication
. add_option ( '-p' , '--password' ,
175 dest
= 'password' , metavar
= 'PASSWORD' , help = 'account password' )
176 authentication
. add_option ( '-n' , '--netrc' ,
177 action
= 'store_true' , dest
= 'usenetrc' , help = 'use .netrc authentication data' , default
= False )
180 video_format
. add_option ( '-f' , '--format' ,
181 action
= 'store' , dest
= 'format' , metavar
= 'FORMAT' , help = 'video format code' )
182 video_format
. add_option ( '--all-formats' ,
183 action
= 'store_const' , dest
= 'format' , help = 'download all available video formats' , const
= 'all' )
184 video_format
. add_option ( '--prefer-free-formats' ,
185 action
= 'store_true' , dest
= 'prefer_free_formats' , default
= False , help = 'prefer free video formats unless a specific one is requested' )
186 video_format
. add_option ( '--max-quality' ,
187 action
= 'store' , dest
= 'format_limit' , metavar
= 'FORMAT' , help = 'highest quality format to download' )
188 video_format
. add_option ( '-F' , '--list-formats' ,
189 action
= 'store_true' , dest
= 'listformats' , help = 'list all available formats (currently youtube only)' )
190 video_format
. add_option ( '--write-srt' ,
191 action
= 'store_true' , dest
= 'writesubtitles' ,
192 help = 'write video closed captions to a .srt file (currently youtube only)' , default
= False )
193 video_format
. add_option ( '--srt-lang' ,
194 action
= 'store' , dest
= 'subtitleslang' , metavar
= 'LANG' ,
195 help = 'language of the closed captions to download (optional) use IETF language tags like \' en \' ' )
198 verbosity
. add_option ( '-q' , '--quiet' ,
199 action
= 'store_true' , dest
= 'quiet' , help = 'activates quiet mode' , default
= False )
200 verbosity
. add_option ( '-s' , '--simulate' ,
201 action
= 'store_true' , dest
= 'simulate' , help = 'do not download the video and do not write anything to disk' , default
= False )
202 verbosity
. add_option ( '--skip-download' ,
203 action
= 'store_true' , dest
= 'skip_download' , help = 'do not download the video' , default
= False )
204 verbosity
. add_option ( '-g' , '--get-url' ,
205 action
= 'store_true' , dest
= 'geturl' , help = 'simulate, quiet but print URL' , default
= False )
206 verbosity
. add_option ( '-e' , '--get-title' ,
207 action
= 'store_true' , dest
= 'gettitle' , help = 'simulate, quiet but print title' , default
= False )
208 verbosity
. add_option ( '--get-thumbnail' ,
209 action
= 'store_true' , dest
= 'getthumbnail' ,
210 help = 'simulate, quiet but print thumbnail URL' , default
= False )
211 verbosity
. add_option ( '--get-description' ,
212 action
= 'store_true' , dest
= 'getdescription' ,
213 help = 'simulate, quiet but print video description' , default
= False )
214 verbosity
. add_option ( '--get-filename' ,
215 action
= 'store_true' , dest
= 'getfilename' ,
216 help = 'simulate, quiet but print output filename' , default
= False )
217 verbosity
. add_option ( '--get-format' ,
218 action
= 'store_true' , dest
= 'getformat' ,
219 help = 'simulate, quiet but print output format' , default
= False )
220 verbosity
. add_option ( '--no-progress' ,
221 action
= 'store_true' , dest
= 'noprogress' , help = 'do not print progress bar' , default
= False )
222 verbosity
. add_option ( '--console-title' ,
223 action
= 'store_true' , dest
= 'consoletitle' ,
224 help = 'display progress in console titlebar' , default
= False )
225 verbosity
. add_option ( '-v' , '--verbose' ,
226 action
= 'store_true' , dest
= 'verbose' , help = 'print various debugging information' , default
= False )
229 filesystem
. add_option ( '-t' , '--title' ,
230 action
= 'store_true' , dest
= 'usetitle' , help = 'use title in file name' , default
= False )
231 filesystem
. add_option ( '-l' , '--literal' ,
232 action
= 'store_true' , dest
= 'useliteral' , help = 'use literal title in file name' , default
= False )
233 filesystem
. add_option ( '-A' , '--auto-number' ,
234 action
= 'store_true' , dest
= 'autonumber' ,
235 help = 'number downloaded files starting from 00000' , default
= False )
236 filesystem
. add_option ( '-o' , '--output' ,
237 dest
= 'outtmpl' , metavar
= 'TEMPLATE' , help = 'output filename template. Use %(stitle)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), and %% for a literal percent. Use - to output to stdout.' )
238 filesystem
. add_option ( '-a' , '--batch-file' ,
239 dest
= 'batchfile' , metavar
= 'FILE' , help = 'file containing URLs to download ( \' - \' for stdin)' )
240 filesystem
. add_option ( '-w' , '--no-overwrites' ,
241 action
= 'store_true' , dest
= 'nooverwrites' , help = 'do not overwrite files' , default
= False )
242 filesystem
. add_option ( '-c' , '--continue' ,
243 action
= 'store_true' , dest
= 'continue_dl' , help = 'resume partially downloaded files' , default
= True )
244 filesystem
. add_option ( '--no-continue' ,
245 action
= 'store_false' , dest
= 'continue_dl' ,
246 help = 'do not resume partially downloaded files (restart from beginning)' )
247 filesystem
. add_option ( '--cookies' ,
248 dest
= 'cookiefile' , metavar
= 'FILE' , help = 'file to read cookies from and dump cookie jar in' )
249 filesystem
. add_option ( '--no-part' ,
250 action
= 'store_true' , dest
= 'nopart' , help = 'do not use .part files' , default
= False )
251 filesystem
. add_option ( '--no-mtime' ,
252 action
= 'store_false' , dest
= 'updatetime' ,
253 help = 'do not use the Last-modified header to set the file modification time' , default
= True )
254 filesystem
. add_option ( '--write-description' ,
255 action
= 'store_true' , dest
= 'writedescription' ,
256 help = 'write video description to a .description file' , default
= False )
257 filesystem
. add_option ( '--write-info-json' ,
258 action
= 'store_true' , dest
= 'writeinfojson' ,
259 help = 'write video metadata to a .info.json file' , default
= False )
262 postproc
. add_option ( '--extract-audio' , action
= 'store_true' , dest
= 'extractaudio' , default
= False ,
263 help = 'convert video files to audio-only files (requires ffmpeg and ffprobe)' )
264 postproc
. add_option ( '--audio-format' , metavar
= 'FORMAT' , dest
= 'audioformat' , default
= 'best' ,
265 help = '"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default' )
266 postproc
. add_option ( '--audio-quality' , metavar
= 'QUALITY' , dest
= 'audioquality' , default
= '128K' ,
267 help = 'ffmpeg audio bitrate specification, 128k by default' )
268 postproc
. add_option ( '-k' , '--keep-video' , action
= 'store_true' , dest
= 'keepvideo' , default
= False ,
269 help = 'keeps the video file on disk after the post-processing; the video is erased by default' )
272 parser
. add_option_group ( general
)
273 parser
. add_option_group ( selection
)
274 parser
. add_option_group ( filesystem
)
275 parser
. add_option_group ( verbosity
)
276 parser
. add_option_group ( video_format
)
277 parser
. add_option_group ( authentication
)
278 parser
. add_option_group ( postproc
)
280 xdg_config_home
= os
. environ
. get ( 'XDG_CONFIG_HOME' )
282 userConf
= os
. path
. join ( xdg_config_home
, 'youtube-dl.conf' )
284 userConf
= os
. path
. join ( os
. path
. expanduser ( '~' ), '.config' , 'youtube-dl.conf' )
285 argv
= _readOptions ( '/etc/youtube-dl.conf' ) + _readOptions ( userConf
) + sys
. argv
[ 1 :]
286 opts
, args
= parser
. parse_args ( argv
)
288 return parser
, opts
, args
290 def gen_extractors ():
291 """ Return a list of an instance of every supported extractor.
292 The order does matter; the first extractor matched is the one handling the URL.
294 youtube_ie
= YoutubeIE ()
295 google_ie
= GoogleIE ()
298 YoutubePlaylistIE ( youtube_ie
),
299 YoutubeUserIE ( youtube_ie
),
300 YoutubeSearchIE ( youtube_ie
),
302 MetacafeIE ( youtube_ie
),
305 GoogleSearchIE ( google_ie
),
308 YahooSearchIE ( yahoo_ie
),
321 StanfordOpenClassroomIE (),
328 parser
, opts
, args
= parseOpts ()
330 # Open appropriate CookieJar
331 if opts
. cookiefile
is None :
332 jar
= cookielib
. CookieJar ()
335 jar
= cookielib
. MozillaCookieJar ( opts
. cookiefile
)
336 if os
. path
. isfile ( opts
. cookiefile
) and os
. access ( opts
. cookiefile
, os
. R_OK
):
338 except ( IOError , OSError ), err
:
339 sys
. exit ( u
'ERROR: unable to open cookie file' )
342 if opts
. dump_user_agent
:
343 print std_headers
[ 'User-Agent' ]
346 # Batch file verification
348 if opts
. batchfile
is not None :
350 if opts
. batchfile
== '-' :
353 batchfd
= open ( opts
. batchfile
, 'r' )
354 batchurls
= batchfd
. readlines ()
355 batchurls
= [ x
. strip () for x
in batchurls
]
356 batchurls
= [ x
for x
in batchurls
if len ( x
) > 0 and not re
. search ( r
'^[#/;]' , x
)]
358 sys
. exit ( u
'ERROR: batch file could not be read' )
359 all_urls
= batchurls
+ args
360 all_urls
= map ( lambda url
: url
. strip (), all_urls
)
362 # General configuration
363 cookie_processor
= urllib2
. HTTPCookieProcessor ( jar
)
364 proxy_handler
= urllib2
. ProxyHandler ()
365 opener
= urllib2
. build_opener ( proxy_handler
, cookie_processor
, YoutubeDLHandler ())
366 urllib2
. install_opener ( opener
)
367 socket
. setdefaulttimeout ( 300 ) # 5 minutes should be enough (famous last words)
370 print ( u
'[debug] Proxy map: ' + str ( proxy_handler
. proxies
))
372 extractors
= gen_extractors ()
374 if opts
. list_extractors
:
375 for ie
in extractors
:
377 matchedUrls
= filter ( lambda url
: ie
. suitable ( url
), all_urls
)
378 all_urls
= filter ( lambda url
: url
not in matchedUrls
, all_urls
)
379 for mu
in matchedUrls
:
383 # Conflicting, missing and erroneous options
384 if opts
. usenetrc
and ( opts
. username
is not None or opts
. password
is not None ):
385 parser
. error ( u
'using .netrc conflicts with giving username/password' )
386 if opts
. password
is not None and opts
. username
is None :
387 parser
. error ( u
'account username missing' )
388 if opts
. outtmpl
is not None and ( opts
. useliteral
or opts
. usetitle
or opts
. autonumber
):
389 parser
. error ( u
'using output template conflicts with using title, literal title or auto number' )
390 if opts
. usetitle
and opts
. useliteral
:
391 parser
. error ( u
'using title conflicts with using literal title' )
392 if opts
. username
is not None and opts
. password
is None :
393 opts
. password
= getpass
. getpass ( u
'Type account password and press return:' )
394 if opts
. ratelimit
is not None :
395 numeric_limit
= FileDownloader
. parse_bytes ( opts
. ratelimit
)
396 if numeric_limit
is None :
397 parser
. error ( u
'invalid rate limit specified' )
398 opts
. ratelimit
= numeric_limit
399 if opts
. retries
is not None :
401 opts
. retries
= long ( opts
. retries
)
402 except ( TypeError , ValueError ), err
:
403 parser
. error ( u
'invalid retry count specified' )
405 opts
. playliststart
= int ( opts
. playliststart
)
406 if opts
. playliststart
<= 0 :
407 raise ValueError ( u
'Playlist start must be positive' )
408 except ( TypeError , ValueError ), err
:
409 parser
. error ( u
'invalid playlist start number specified' )
411 opts
. playlistend
= int ( opts
. playlistend
)
412 if opts
. playlistend
!= - 1 and ( opts
. playlistend
<= 0 or opts
. playlistend
< opts
. playliststart
):
413 raise ValueError ( u
'Playlist end must be greater than playlist start' )
414 except ( TypeError , ValueError ), err
:
415 parser
. error ( u
'invalid playlist end number specified' )
416 if opts
. extractaudio
:
417 if opts
. audioformat
not in [ 'best' , 'aac' , 'mp3' , 'vorbis' , 'm4a' , 'wav' ]:
418 parser
. error ( u
'invalid audio format specified' )
421 fd
= FileDownloader ({
422 'usenetrc' : opts
. usenetrc
,
423 'username' : opts
. username
,
424 'password' : opts
. password
,
425 'quiet' : ( opts
. quiet
or opts
. geturl
or opts
. gettitle
or opts
. getthumbnail
or opts
. getdescription
or opts
. getfilename
or opts
. getformat
),
426 'forceurl' : opts
. geturl
,
427 'forcetitle' : opts
. gettitle
,
428 'forcethumbnail' : opts
. getthumbnail
,
429 'forcedescription' : opts
. getdescription
,
430 'forcefilename' : opts
. getfilename
,
431 'forceformat' : opts
. getformat
,
432 'simulate' : opts
. simulate
,
433 'skip_download' : ( opts
. skip_download
or opts
. simulate
or opts
. geturl
or opts
. gettitle
or opts
. getthumbnail
or opts
. getdescription
or opts
. getfilename
or opts
. getformat
),
434 'format' : opts
. format
,
435 'format_limit' : opts
. format_limit
,
436 'listformats' : opts
. listformats
,
437 'outtmpl' : (( opts
. outtmpl
is not None and opts
. outtmpl
. decode ( preferredencoding ()))
438 or ( opts
. format
== '-1' and opts
. usetitle
and u
' %(stitle)s - %(id)s - %(format)s . %(ext)s ' )
439 or ( opts
. format
== '-1' and opts
. useliteral
and u
' %(title)s - %(id)s - %(format)s . %(ext)s ' )
440 or ( opts
. format
== '-1' and u
' %(id)s - %(format)s . %(ext)s ' )
441 or ( opts
. usetitle
and opts
. autonumber
and u
' %(autonumber)s - %(stitle)s - %(id)s . %(ext)s ' )
442 or ( opts
. useliteral
and opts
. autonumber
and u
' %(autonumber)s - %(title)s - %(id)s . %(ext)s ' )
443 or ( opts
. usetitle
and u
' %(stitle)s - %(id)s . %(ext)s ' )
444 or ( opts
. useliteral
and u
' %(title)s - %(id)s . %(ext)s ' )
445 or ( opts
. autonumber
and u
' %(autonumber)s - %(id)s . %(ext)s ' )
446 or u
' %(id)s . %(ext)s ' ),
447 'ignoreerrors' : opts
. ignoreerrors
,
448 'ratelimit' : opts
. ratelimit
,
449 'nooverwrites' : opts
. nooverwrites
,
450 'retries' : opts
. retries
,
451 'continuedl' : opts
. continue_dl
,
452 'noprogress' : opts
. noprogress
,
453 'playliststart' : opts
. playliststart
,
454 'playlistend' : opts
. playlistend
,
455 'logtostderr' : opts
. outtmpl
== '-' ,
456 'consoletitle' : opts
. consoletitle
,
457 'nopart' : opts
. nopart
,
458 'updatetime' : opts
. updatetime
,
459 'writedescription' : opts
. writedescription
,
460 'writeinfojson' : opts
. writeinfojson
,
461 'writesubtitles' : opts
. writesubtitles
,
462 'subtitleslang' : opts
. subtitleslang
,
463 'matchtitle' : opts
. matchtitle
,
464 'rejecttitle' : opts
. rejecttitle
,
465 'max_downloads' : opts
. max_downloads
,
466 'prefer_free_formats' : opts
. prefer_free_formats
,
467 'verbose' : opts
. verbose
,
469 for extractor
in extractors
:
470 fd
. add_info_extractor ( extractor
)
473 if opts
. extractaudio
:
474 fd
. add_post_processor ( FFmpegExtractAudioPP ( preferredcodec
= opts
. audioformat
, preferredquality
= opts
. audioquality
, keepvideo
= opts
. keepvideo
))
478 updateSelf ( fd
, sys
. argv
[ 0 ])
481 if len ( all_urls
) < 1 :
482 if not opts
. update_self
:
483 parser
. error ( u
'you must provide at least one URL' )
488 retcode
= fd
. download ( all_urls
)
489 except MaxDownloadsReached
:
490 fd
. to_screen ( u
'--max-download limit reached, aborting.' )
493 # Dump cookie jar if requested
494 if opts
. cookiefile
is not None :
497 except ( IOError , OSError ), err
:
498 sys
. exit ( u
'ERROR: unable to save cookie jar' )
505 except DownloadError
:
507 except SameFileError
:
508 sys
. exit ( u
'ERROR: fixed output name but more than one file to download' )
509 except KeyboardInterrupt :
510 sys
. exit ( u
' \n ERROR: Interrupted by user' )