release 2013.11.17

[yt-dlp.git] / youtube_dl / FileDownloader.py

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 0b5a5d77d409ca0a193a9ce6bf6890f88767459d..e5a542ed55d2d86f46b95c51da80bb66c17fa947 100644 (file)
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -4,12 +4,16 @@
 import subprocess
 import sys
 import time
 import subprocess
 import sys
 import time

-import traceback

 
 

-if os.name == 'nt':
-    import ctypes
-
-from .utils import *
+from .utils import (
+    compat_urllib_error,
+    compat_urllib_request,
+    ContentTooShortError,
+    determine_ext,
+    encodeFilename,
+    sanitize_open,
+    timeconvert,
+)

 
 
 class FileDownloader(object):
 
 
 class FileDownloader(object):


@@ -77,26 +81,43 @@ def format_seconds(seconds):
     @staticmethod
     def calc_percent(byte_counter, data_len):
         if data_len is None:
     @staticmethod
     def calc_percent(byte_counter, data_len):
         if data_len is None:

+            return None
+        return float(byte_counter) / float(data_len) * 100.0
+
+    @staticmethod
+    def format_percent(percent):
+        if percent is None:

             return '---.-%'
             return '---.-%'

-        return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
+        return '%6s' % ('%3.1f%%' % percent)

 
     @staticmethod
     def calc_eta(start, now, total, current):
         if total is None:
 
     @staticmethod
     def calc_eta(start, now, total, current):
         if total is None:

-            return '--:--'
+            return None

         dif = now - start
         if current == 0 or dif < 0.001: # One millisecond
         dif = now - start
         if current == 0 or dif < 0.001: # One millisecond

-            return '--:--'
+            return None

         rate = float(current) / dif
         rate = float(current) / dif

-        eta = int((float(total) - float(current)) / rate)
+        return int((float(total) - float(current)) / rate)
+
+    @staticmethod
+    def format_eta(eta):
+        if eta is None:
+            return '--:--'

         return FileDownloader.format_seconds(eta)
 
     @staticmethod
     def calc_speed(start, now, bytes):
         dif = now - start
         if bytes == 0 or dif < 0.001: # One millisecond
         return FileDownloader.format_seconds(eta)
 
     @staticmethod
     def calc_speed(start, now, bytes):
         dif = now - start
         if bytes == 0 or dif < 0.001: # One millisecond

+            return None
+        return float(bytes) / dif
+
+    @staticmethod
+    def format_speed(speed):
+        if speed is None:

             return '%10s' % '---b/s'
             return '%10s' % '---b/s'

-        return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
+        return '%10s' % ('%s/s' % FileDownloader.format_bytes(speed))

 
     @staticmethod
     def best_block_size(elapsed_time, bytes):
 
     @staticmethod
     def best_block_size(elapsed_time, bytes):


@@ -127,16 +148,8 @@ def to_screen(self, *args, **kargs):
     def to_stderr(self, message):
         self.ydl.to_screen(message)
 
     def to_stderr(self, message):
         self.ydl.to_screen(message)
 

-    def to_cons_title(self, message):
-        """Set console/terminal window title to message."""
-        if not self.params.get('consoletitle', False):
-            return
-        if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
-            # c_wchar_p() might not be necessary if `message` is
-            # already of type unicode()
-            ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
-        elif 'TERM' in os.environ:
-            self.to_screen('\033]0;%s\007' % message, skip_eol=True)
+    def to_console_title(self, message):
+        self.ydl.to_console_title(message)

 
     def trouble(self, *args, **kargs):
         self.ydl.trouble(*args, **kargs)
 
     def trouble(self, *args, **kargs):
         self.ydl.trouble(*args, **kargs)


@@ -177,7 +190,7 @@ def try_rename(self, old_filename, new_filename):
             if old_filename == new_filename:
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
             if old_filename == new_filename:
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))

-        except (IOError, OSError) as err:
+        except (IOError, OSError):

             self.report_error(u'unable to rename file')
 
     def try_utime(self, filename, last_modified_hdr):
             self.report_error(u'unable to rename file')
 
     def try_utime(self, filename, last_modified_hdr):


@@ -205,18 +218,27 @@ def report_destination(self, filename):
         """Report destination filename."""
         self.to_screen(u'[download] Destination: ' + filename)
 
         """Report destination filename."""
         self.to_screen(u'[download] Destination: ' + filename)
 

-    def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
+    def report_progress(self, percent, data_len_str, speed, eta):

         """Report download progress."""
         if self.params.get('noprogress', False):
             return
         clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')
         """Report download progress."""
         if self.params.get('noprogress', False):
             return
         clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')

+        if eta is not None:
+            eta_str = self.format_eta(eta)
+        else:
+            eta_str = 'Unknown ETA'
+        if percent is not None:
+            percent_str = self.format_percent(percent)
+        else:
+            percent_str = 'Unknown %'
+        speed_str = self.format_speed(speed)

         if self.params.get('progress_with_newline', False):
             self.to_screen(u'[download] %s of %s at %s ETA %s' %
                 (percent_str, data_len_str, speed_str, eta_str))
         else:
             self.to_screen(u'\r%s[download] %s of %s at %s ETA %s' %
                 (clear_line, percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
         if self.params.get('progress_with_newline', False):
             self.to_screen(u'[download] %s of %s at %s ETA %s' %
                 (percent_str, data_len_str, speed_str, eta_str))
         else:
             self.to_screen(u'\r%s[download] %s of %s at %s ETA %s' %
                 (clear_line, percent_str, data_len_str, speed_str, eta_str), skip_eol=True)

-        self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
+        self.to_console_title(u'youtube-dl - %s of %s at %s ETA %s' %

                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 
     def report_resuming_byte(self, resume_len):
                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 
     def report_resuming_byte(self, resume_len):


@@ -231,7 +253,7 @@ def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""
         try:
             self.to_screen(u'[download] %s has already been downloaded' % file_name)
         """Report file has already been fully downloaded."""
         try:
             self.to_screen(u'[download] %s has already been downloaded' % file_name)

-        except (UnicodeEncodeError) as err:
+        except UnicodeEncodeError:

             self.to_screen(u'[download] The file has already been downloaded')
 
     def report_unable_to_resume(self):
             self.to_screen(u'[download] The file has already been downloaded')
 
     def report_unable_to_resume(self):


@@ -247,9 +269,10 @@ def report_finish(self, data_len_str, tot_time):
             self.to_screen(u'\r%s[download] 100%% of %s in %s' %
                 (clear_line, data_len_str, self.format_seconds(tot_time)))
 
             self.to_screen(u'\r%s[download] 100%% of %s in %s' %
                 (clear_line, data_len_str, self.format_seconds(tot_time)))
 

-    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url):
+    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url, live):

         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)

+        test = self.params.get('test', False)

 
         # Check for rtmpdump first
         try:
 
         # Check for rtmpdump first
         try:


@@ -271,6 +294,10 @@ def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path
             basic_args += ['--playpath', play_path]
         if tc_url is not None:
             basic_args += ['--tcUrl', url]
             basic_args += ['--playpath', play_path]
         if tc_url is not None:
             basic_args += ['--tcUrl', url]

+        if test:
+            basic_args += ['--stop', '1']
+        if live:
+            basic_args += ['--live']

         args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)]
         if self.params.get('verbose', False):
             try:
         args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)]
         if self.params.get('verbose', False):
             try:


@@ -280,7 +307,7 @@ def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path
                 shell_quote = repr
             self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
         retval = subprocess.call(args)
                 shell_quote = repr
             self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
         retval = subprocess.call(args)

-        while retval == 2 or retval == 1:
+        while (retval == 2 or retval == 1) and not test:

             prevsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
             time.sleep(5.0) # This seems to be needed
             prevsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
             time.sleep(5.0) # This seems to be needed


@@ -293,7 +320,7 @@ def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path
                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
                 retval = 0
                 break
                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
                 retval = 0
                 break

-        if retval == 0:
+        if retval == 0 or (test and retval == 2):

             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[rtmpdump] %s bytes' % fsize)
             self.try_rename(tmpfilename, filename)
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[rtmpdump] %s bytes' % fsize)
             self.try_rename(tmpfilename, filename)


@@ -343,15 +370,20 @@ def _download_m3u8_with_ffmpeg(self, filename, url):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 

-        args = ['ffmpeg', '-y', '-i', url, '-f', 'mp4', tmpfilename]
-        # Check for ffmpeg first
-        try:
-            subprocess.call(['ffmpeg', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
-        except (OSError, IOError):
-            self.report_error(u'm3u8 download detected but "%s" could not be run' % args[0] )
-            return False
+        args = ['-y', '-i', url, '-f', 'mp4', '-c', 'copy',
+            '-bsf:a', 'aac_adtstoasc', tmpfilename]

 
 

-        retval = subprocess.call(args)
+        for program in ['avconv', 'ffmpeg']:
+            try:
+                subprocess.call([program, '-version'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+                break
+            except (OSError, IOError):
+                pass
+        else:
+            self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found')
+        cmd = [program] + args
+
+        retval = subprocess.call(cmd)

         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize))
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize))


@@ -378,6 +410,7 @@ def _do_download(self, filename, info_dict):
             self._hook_progress({
                 'filename': filename,
                 'status': 'finished',
             self._hook_progress({
                 'filename': filename,
                 'status': 'finished',

+                'total_bytes': os.path.getsize(encodeFilename(filename)),

             })
             return True
 
             })
             return True
 


@@ -387,7 +420,8 @@ def _do_download(self, filename, info_dict):
                                                 info_dict.get('player_url', None),
                                                 info_dict.get('page_url', None),
                                                 info_dict.get('play_path', None),
                                                 info_dict.get('player_url', None),
                                                 info_dict.get('page_url', None),
                                                 info_dict.get('play_path', None),

-                                                info_dict.get('tc_url', None))
+                                                info_dict.get('tc_url', None),
+                                                info_dict.get('rtmp_live', False))

 
         # Attempt to download using mplayer
         if url.startswith('mms') or url.startswith('rtsp'):
 
         # Attempt to download using mplayer
         if url.startswith('mms') or url.startswith('rtsp'):


@@ -524,13 +558,13 @@ def _do_download(self, filename, info_dict):
                 block_size = self.best_block_size(after - before, len(data_block))
 
             # Progress message
                 block_size = self.best_block_size(after - before, len(data_block))
 
             # Progress message

-            speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
+            speed = self.calc_speed(start, time.time(), byte_counter - resume_len)

             if data_len is None:
             if data_len is None:

-                self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
+                eta = percent = None

             else:
             else:

-                percent_str = self.calc_percent(byte_counter, data_len)
-                eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
-                self.report_progress(percent_str, data_len_str, speed_str, eta_str)
+                percent = self.calc_percent(byte_counter, data_len)
+                eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
+            self.report_progress(percent, data_len_str, speed, eta)

 
             self._hook_progress({
                 'downloaded_bytes': byte_counter,
 
             self._hook_progress({
                 'downloaded_bytes': byte_counter,


@@ -538,6 +572,8 @@ def _do_download(self, filename, info_dict):
                 'tmpfilename': tmpfilename,
                 'filename': filename,
                 'status': 'downloading',
                 'tmpfilename': tmpfilename,
                 'filename': filename,
                 'status': 'downloading',

+                'eta': eta,
+                'speed': speed,

             })
 
             # Apply rate limit
             })
 
             # Apply rate limit


@@ -580,6 +616,8 @@ def add_progress_hook(self, ph):
         * downloaded_bytes: Bytes on disks
         * total_bytes: Total bytes, None if unknown
         * tmpfilename: The filename we're currently writing to
         * downloaded_bytes: Bytes on disks
         * total_bytes: Total bytes, None if unknown
         * tmpfilename: The filename we're currently writing to

+        * eta: The estimated time in seconds, None if unknown
+        * speed: The download speed in bytes/second, None if unknown

 
         Hooks are guaranteed to be called at least once (with status "finished")
         if the download is successful.
 
         Hooks are guaranteed to be called at least once (with status "finished")
         if the download is successful.