7 from .common
import FileDownloader
19 class HttpFD(FileDownloader
):
20 def real_download(self
, filename
, info_dict
):
21 url
= info_dict
['url']
22 tmpfilename
= self
.temp_name(filename
)
25 # Do not include the Accept-Encoding header
26 headers
= {'Youtubedl-no-compression': 'True'}
27 if 'user_agent' in info_dict
:
28 headers
['Youtubedl-user-agent'] = info_dict
['user_agent']
29 basic_request
= compat_urllib_request
.Request(url
, None, headers
)
30 request
= compat_urllib_request
.Request(url
, None, headers
)
32 if self
.params
.get('test', False):
33 request
.add_header('Range','bytes=0-10240')
35 # Establish possible resume length
36 if os
.path
.isfile(encodeFilename(tmpfilename
)):
37 resume_len
= os
.path
.getsize(encodeFilename(tmpfilename
))
43 if self
.params
.get('continuedl', False):
44 self
.report_resuming_byte(resume_len
)
45 request
.add_header('Range','bytes=%d-' % resume_len
)
51 retries
= self
.params
.get('retries', 0)
52 while count
<= retries
:
53 # Establish connection
55 if count
== 0 and 'urlhandle' in info_dict
:
56 data
= info_dict
['urlhandle']
57 data
= compat_urllib_request
.urlopen(request
)
59 except (compat_urllib_error
.HTTPError
, ) as err
:
60 if (err
.code
< 500 or err
.code
>= 600) and err
.code
!= 416:
61 # Unexpected HTTP error
64 # Unable to resume (requested range not satisfiable)
66 # Open the connection again without the range header
67 data
= compat_urllib_request
.urlopen(basic_request
)
68 content_length
= data
.info()['Content-Length']
69 except (compat_urllib_error
.HTTPError
, ) as err
:
70 if err
.code
< 500 or err
.code
>= 600:
73 # Examine the reported length
74 if (content_length
is not None and
75 (resume_len
- 100 < int(content_length
) < resume_len
+ 100)):
76 # The file had already been fully downloaded.
77 # Explanation to the above condition: in issue #175 it was revealed that
78 # YouTube sometimes adds or removes a few bytes from the end of the file,
79 # changing the file size slightly and causing problems for some users. So
80 # I decided to implement a suggested change and consider the file
81 # completely downloaded if the file size differs less than 100 bytes from
82 # the one in the hard drive.
83 self
.report_file_already_downloaded(filename
)
84 self
.try_rename(tmpfilename
, filename
)
91 # The length does not match, we start the download over
92 self
.report_unable_to_resume()
98 self
.report_retry(count
, retries
)
101 self
.report_error(u
'giving up after %s retries' % retries
)
104 data_len
= data
.info().get('Content-length', None)
105 if data_len
is not None:
106 data_len
= int(data_len
) + resume_len
107 min_data_len
= self
.params
.get("min_filesize", None)
108 max_data_len
= self
.params
.get("max_filesize", None)
109 if min_data_len
is not None and data_len
< min_data_len
:
110 self
.to_screen(u
'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len
, min_data_len
))
112 if max_data_len
is not None and data_len
> max_data_len
:
113 self
.to_screen(u
'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len
, max_data_len
))
116 data_len_str
= format_bytes(data_len
)
117 byte_counter
= 0 + resume_len
118 block_size
= self
.params
.get('buffersize', 1024)
123 data_block
= data
.read(block_size
)
125 if len(data_block
) == 0:
127 byte_counter
+= len(data_block
)
129 # Open file just in time
132 (stream
, tmpfilename
) = sanitize_open(tmpfilename
, open_mode
)
133 assert stream
is not None
134 filename
= self
.undo_temp_name(tmpfilename
)
135 self
.report_destination(filename
)
136 except (OSError, IOError) as err
:
137 self
.report_error(u
'unable to open for writing: %s' % str(err
))
140 stream
.write(data_block
)
141 except (IOError, OSError):
142 self
.to_stderr(u
"\n")
143 self
.report_error(u
'unable to write data: %s' % str(err
))
145 if not self
.params
.get('noresizebuffer', False):
146 block_size
= self
.best_block_size(after
- before
, len(data_block
))
149 speed
= self
.calc_speed(start
, time
.time(), byte_counter
- resume_len
)
153 percent
= self
.calc_percent(byte_counter
, data_len
)
154 eta
= self
.calc_eta(start
, time
.time(), data_len
- resume_len
, byte_counter
- resume_len
)
155 self
.report_progress(percent
, data_len_str
, speed
, eta
)
157 self
._hook
_progress
({
158 'downloaded_bytes': byte_counter
,
159 'total_bytes': data_len
,
160 'tmpfilename': tmpfilename
,
161 'filename': filename
,
162 'status': 'downloading',
168 self
.slow_down(start
, byte_counter
- resume_len
)
171 self
.to_stderr(u
"\n")
172 self
.report_error(u
'Did not get any data blocks')
175 self
.report_finish(data_len_str
, (time
.time() - start
))
176 if data_len
is not None and byte_counter
!= data_len
:
177 raise ContentTooShortError(byte_counter
, int(data_len
))
178 self
.try_rename(tmpfilename
, filename
)
180 # Update file modification time
181 if self
.params
.get('updatetime', True):
182 info_dict
['filetime'] = self
.try_utime(filename
, data
.info().get('last-modified', None))
184 self
._hook
_progress
({
185 'downloaded_bytes': byte_counter
,
186 'total_bytes': byte_counter
,
187 'filename': filename
,
188 'status': 'finished',