]>
jfr.im git - yt-dlp.git/blob - yt_dlp/extractor/openload.py
8 from ..compat
import compat_urlparse
20 def cookie_to_dict(cookie
):
23 'value': cookie
.value
,
25 if cookie
.port_specified
:
26 cookie_dict
['port'] = cookie
.port
27 if cookie
.domain_specified
:
28 cookie_dict
['domain'] = cookie
.domain
29 if cookie
.path_specified
:
30 cookie_dict
['path'] = cookie
.path
31 if cookie
.expires
is not None:
32 cookie_dict
['expires'] = cookie
.expires
33 if cookie
.secure
is not None:
34 cookie_dict
['secure'] = cookie
.secure
35 if cookie
.discard
is not None:
36 cookie_dict
['discard'] = cookie
.discard
37 with contextlib
.suppress(TypeError):
38 if (cookie
.has_nonstandard_attr('httpOnly')
39 or cookie
.has_nonstandard_attr('httponly')
40 or cookie
.has_nonstandard_attr('HttpOnly')):
41 cookie_dict
['httponly'] = True
45 def cookie_jar_to_list(cookie_jar
):
46 return [cookie_to_dict(cookie
) for cookie
in cookie_jar
]
49 class PhantomJSwrapper
:
50 """PhantomJS wrapper class
52 This class is experimental.
56 phantom.onError = function(msg, trace) {{
57 var msgStack = ['PHANTOM ERROR: ' + msg];
58 if(trace && trace.length) {{
59 msgStack.push('TRACE:');
60 trace.forEach(function(t) {{
61 msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
62 + (t.function ? ' (in function ' + t.function +')' : ''));
65 console.error(msgStack.join('\n'));
71 var page = require('webpage').create();
72 var fs = require('fs');
73 var read = {{ mode: 'r', charset: 'utf-8' }};
74 var write = {{ mode: 'w', charset: 'utf-8' }};
75 JSON.parse(fs.read("{cookies}", read)).forEach(function(x) {{
78 page.settings.resourceTimeout = {timeout};
79 page.settings.userAgent = "{ua}";
80 page.onLoadStarted = function() {{
81 page.evaluate(function() {{
82 delete window._phantom;
83 delete window.callPhantom;
86 var saveAndExit = function() {{
87 fs.write("{html}", page.content, write);
88 fs.write("{cookies}", JSON.stringify(phantom.cookies), write);
91 page.onLoadFinished = function(status) {{
92 if(page.url === "") {{
93 page.setContent(fs.read("{html}", read), "{url}");
102 _TMP_FILE_NAMES
= ['script', 'html', 'cookies']
106 return get_exe_version('phantomjs', version_re
=r
'([0-9.]+)')
108 def __init__(self
, extractor
, required_version
=None, timeout
=10000):
111 self
.exe
= check_executable('phantomjs', ['-v'])
113 raise ExtractorError(
114 'PhantomJS not found, Please download it from https://phantomjs.org/download.html', expected
=True)
116 self
.extractor
= extractor
119 version
= self
._version
()
120 if is_outdated_version(version
, required_version
):
121 self
.extractor
._downloader
.report_warning(
122 'Your copy of PhantomJS is outdated, update it to version '
123 '%s or newer if you encounter any errors.' % required_version
)
125 for name
in self
._TMP
_FILE
_NAMES
:
126 tmp
= tempfile
.NamedTemporaryFile(delete
=False)
128 self
._TMP
_FILES
[name
] = tmp
130 self
.options
= collections
.ChainMap({
133 x
: self
._TMP
_FILES
[x
].name
.replace('\\', '\\\\').replace('"', '\\"')
134 for x
in self
._TMP
_FILE
_NAMES
138 for name
in self
._TMP
_FILE
_NAMES
:
139 with contextlib
.suppress(OSError, KeyError):
140 os
.remove(self
._TMP
_FILES
[name
].name
)
142 def _save_cookies(self
, url
):
143 cookies
= cookie_jar_to_list(self
.extractor
.cookiejar
)
144 for cookie
in cookies
:
145 if 'path' not in cookie
:
147 if 'domain' not in cookie
:
148 cookie
['domain'] = compat_urlparse
.urlparse(url
).netloc
149 with open(self
._TMP
_FILES
['cookies'].name
, 'wb') as f
:
150 f
.write(json
.dumps(cookies
).encode('utf-8'))
152 def _load_cookies(self
):
153 with open(self
._TMP
_FILES
['cookies'].name
, 'rb') as f
:
154 cookies
= json
.loads(f
.read().decode('utf-8'))
155 for cookie
in cookies
:
156 if cookie
['httponly'] is True:
157 cookie
['rest'] = {'httpOnly': None}
158 if 'expiry' in cookie
:
159 cookie
['expire_time'] = cookie
['expiry']
160 self
.extractor
._set
_cookie
(**cookie
)
162 def get(self
, url
, html
=None, video_id
=None, note
=None, note2
='Executing JS on webpage', headers
={}, jscode
='saveAndExit();'):
164 Downloads webpage (if needed) and executes JS
168 html: optional, html code of website
170 note: optional, displayed when downloading webpage
171 note2: optional, displayed when executing JS
172 headers: custom http headers
173 jscode: code to be executed when page is loaded
176 * downloaded website (after JS execution)
177 * anything you print with `console.log` (but not inside `page.execute`!)
179 In most cases you don't need to add any `jscode`.
180 It is executed in `page.onLoadFinished`.
181 `saveAndExit();` is mandatory, use it instead of `phantom.exit()`
182 It is possible to wait for some element on the webpage, e.g.
183 var check = function() {
184 var elementFound = page.evaluate(function() {
185 return document.querySelector('#b.done') !== null;
190 window.setTimeout(check, 500);
193 page.evaluate(function(){
194 document.querySelector('#a').click();
198 if 'saveAndExit();' not in jscode
:
199 raise ExtractorError('`saveAndExit();` not found in `jscode`')
201 html
= self
.extractor
._download
_webpage
(url
, video_id
, note
=note
, headers
=headers
)
202 with open(self
._TMP
_FILES
['html'].name
, 'wb') as f
:
203 f
.write(html
.encode('utf-8'))
205 self
._save
_cookies
(url
)
207 user_agent
= headers
.get('User-Agent') or self
.extractor
.get_param('http_headers')['User-Agent']
208 jscode
= self
._TEMPLATE
.format_map(self
.options
.new_child({
210 'ua': user_agent
.replace('"', '\\"'),
214 stdout
= self
.execute(jscode
, video_id
, note2
)
216 with open(self
._TMP
_FILES
['html'].name
, 'rb') as f
:
217 html
= f
.read().decode('utf-8')
222 def execute(self
, jscode
, video_id
=None, *, note
='Executing JS'):
223 """Execute JS and return stdout"""
224 if 'phantom.exit();' not in jscode
:
225 jscode
+= ';\nphantom.exit();'
226 jscode
= self
._BASE
_JS
+ jscode
228 with open(self
._TMP
_FILES
['script'].name
, 'w', encoding
='utf-8') as f
:
230 self
.extractor
.to_screen(f
'{format_field(video_id, None, "%s: ")}{note}')
232 cmd
= [self
.exe
, '--ssl-protocol=any', self
._TMP
_FILES
['script'].name
]
233 self
.extractor
.write_debug(f
'PhantomJS command line: {shell_quote(cmd)}')
235 stdout
, stderr
, returncode
= Popen
.run(cmd
, timeout
=self
.options
['timeout'] / 1000,
236 text
=True, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
237 except Exception as e
:
238 raise ExtractorError(f
'{note} failed: Unable to run PhantomJS binary', cause
=e
)
240 raise ExtractorError(f
'{note} failed:\n{stderr.strip()}')