]> jfr.im git - yt-dlp.git/blame - devscripts/buildserver.py
[devscripts/run_tests] Use markers to filter tests (#1258)
[yt-dlp.git] / devscripts / buildserver.py
CommitLineData
9d83ad93 1# UNUSED
2
71cedb3c 3#!/usr/bin/python3
83de7942 4
71cedb3c
PH
5import argparse
6import ctypes
22b50ecb 7import functools
f574103d
S
8import shutil
9import subprocess
71cedb3c 10import sys
f574103d 11import tempfile
71cedb3c 12import threading
22b50ecb 13import traceback
71cedb3c 14import os.path
83de7942 15
f574103d 16sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__)))))
7a5c1cfe 17from yt_dlp.compat import (
e92b552a 18 compat_input,
f574103d
S
19 compat_http_server,
20 compat_str,
21 compat_urlparse,
22)
23
24# These are not used outside of buildserver.py thus not in compat.py
25
26try:
27 import winreg as compat_winreg
28except ImportError: # Python 2
29 import _winreg as compat_winreg
30
31try:
32 import socketserver as compat_socketserver
33except ImportError: # Python 2
34 import SocketServer as compat_socketserver
83de7942 35
f574103d
S
36
37class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer):
83de7942
PH
38 allow_reuse_address = True
39
40
71cedb3c 41advapi32 = ctypes.windll.advapi32
83de7942 42
71cedb3c
PH
43SC_MANAGER_ALL_ACCESS = 0xf003f
44SC_MANAGER_CREATE_SERVICE = 0x02
45SERVICE_WIN32_OWN_PROCESS = 0x10
46SERVICE_AUTO_START = 0x2
47SERVICE_ERROR_NORMAL = 0x1
48DELETE = 0x00010000
22b50ecb
PH
49SERVICE_STATUS_START_PENDING = 0x00000002
50SERVICE_STATUS_RUNNING = 0x00000004
51SERVICE_ACCEPT_STOP = 0x1
52
53SVCNAME = 'youtubedl_builder'
54
55LPTSTR = ctypes.c_wchar_p
56START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR))
57
58
59class SERVICE_TABLE_ENTRY(ctypes.Structure):
60 _fields_ = [
61 ('lpServiceName', LPTSTR),
62 ('lpServiceProc', START_CALLBACK)
63 ]
64
65
66HandlerEx = ctypes.WINFUNCTYPE(
67 ctypes.c_int, # return
68 ctypes.c_int, # dwControl
69 ctypes.c_int, # dwEventType
70 ctypes.c_void_p, # lpEventData,
71 ctypes.c_void_p, # lpContext,
72)
73
74
75def _ctypes_array(c_type, py_array):
76 ar = (c_type * len(py_array))()
77 ar[:] = py_array
78 return ar
83de7942 79
83de7942 80
71cedb3c 81def win_OpenSCManager():
22b50ecb 82 res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS)
71cedb3c
PH
83 if not res:
84 raise Exception('Opening service manager failed - '
85 'are you running this as administrator?')
86 return res
87
88
89def win_install_service(service_name, cmdline):
90 manager = win_OpenSCManager()
91 try:
22b50ecb 92 h = advapi32.CreateServiceW(
71cedb3c
PH
93 manager, service_name, None,
94 SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS,
95 SERVICE_AUTO_START, SERVICE_ERROR_NORMAL,
96 cmdline, None, None, None, None, None)
97 if not h:
98 raise OSError('Service creation failed: %s' % ctypes.FormatError())
99
100 advapi32.CloseServiceHandle(h)
101 finally:
102 advapi32.CloseServiceHandle(manager)
103
104
105def win_uninstall_service(service_name):
106 manager = win_OpenSCManager()
107 try:
22b50ecb 108 h = advapi32.OpenServiceW(manager, service_name, DELETE)
71cedb3c
PH
109 if not h:
110 raise OSError('Could not find service %s: %s' % (
111 service_name, ctypes.FormatError()))
112
113 try:
114 if not advapi32.DeleteService(h):
115 raise OSError('Deletion failed: %s' % ctypes.FormatError())
116 finally:
117 advapi32.CloseServiceHandle(h)
118 finally:
119 advapi32.CloseServiceHandle(manager)
120
121
22b50ecb
PH
122def win_service_report_event(service_name, msg, is_error=True):
123 with open('C:/sshkeys/log', 'a', encoding='utf-8') as f:
124 f.write(msg + '\n')
125
126 event_log = advapi32.RegisterEventSourceW(None, service_name)
127 if not event_log:
128 raise OSError('Could not report event: %s' % ctypes.FormatError())
129
130 try:
131 type_id = 0x0001 if is_error else 0x0004
132 event_id = 0xc0000000 if is_error else 0x40000000
133 lines = _ctypes_array(LPTSTR, [msg])
134
135 if not advapi32.ReportEventW(
136 event_log, type_id, 0, event_id, None, len(lines), 0,
137 lines, None):
138 raise OSError('Event reporting failed: %s' % ctypes.FormatError())
139 finally:
140 advapi32.DeregisterEventSource(event_log)
141
142
143def win_service_handler(stop_event, *args):
144 try:
145 raise ValueError('Handler called with args ' + repr(args))
146 TODO
147 except Exception as e:
148 tb = traceback.format_exc()
149 msg = str(e) + '\n' + tb
150 win_service_report_event(service_name, msg, is_error=True)
151 raise
152
153
154def win_service_set_status(handle, status_code):
155 svcStatus = SERVICE_STATUS()
156 svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS
157 svcStatus.dwCurrentState = status_code
158 svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP
159
160 svcStatus.dwServiceSpecificExitCode = 0
71cedb3c 161
22b50ecb
PH
162 if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)):
163 raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError())
71cedb3c 164
71cedb3c 165
22b50ecb
PH
166def win_service_main(service_name, real_main, argc, argv_raw):
167 try:
9e1a5b84 168 # args = [argv_raw[i].value for i in range(argc)]
22b50ecb
PH
169 stop_event = threading.Event()
170 handler = HandlerEx(functools.partial(stop_event, win_service_handler))
171 h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None)
172 if not h:
173 raise OSError('Handler registration failed: %s' %
174 ctypes.FormatError())
175
176 TODO
177 except Exception as e:
178 tb = traceback.format_exc()
179 msg = str(e) + '\n' + tb
180 win_service_report_event(service_name, msg, is_error=True)
181 raise
71cedb3c 182
22b50ecb
PH
183
184def win_service_start(service_name, real_main):
185 try:
186 cb = START_CALLBACK(
187 functools.partial(win_service_main, service_name, real_main))
188 dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [
189 SERVICE_TABLE_ENTRY(
190 service_name,
191 cb
192 ),
193 SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK))
194 ])
195
196 if not advapi32.StartServiceCtrlDispatcherW(dispatch_table):
197 raise OSError('ctypes start failed: %s' % ctypes.FormatError())
198 except Exception as e:
199 tb = traceback.format_exc()
200 msg = str(e) + '\n' + tb
201 win_service_report_event(service_name, msg, is_error=True)
202 raise
203
204
205def main(args=None):
71cedb3c
PH
206 parser = argparse.ArgumentParser()
207 parser.add_argument('-i', '--install',
208 action='store_const', dest='action', const='install',
209 help='Launch at Windows startup')
210 parser.add_argument('-u', '--uninstall',
211 action='store_const', dest='action', const='uninstall',
212 help='Remove Windows service')
213 parser.add_argument('-s', '--service',
22b50ecb 214 action='store_const', dest='action', const='service',
71cedb3c
PH
215 help='Run as a Windows service')
216 parser.add_argument('-b', '--bind', metavar='<host:port>',
56bd028a 217 action='store', default='0.0.0.0:8142',
71cedb3c 218 help='Bind to host:port (default %default)')
22b50ecb 219 options = parser.parse_args(args=args)
71cedb3c
PH
220
221 if options.action == 'install':
22b50ecb
PH
222 fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox')
223 cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind)
224 win_install_service(SVCNAME, cmdline)
225 return
71cedb3c
PH
226
227 if options.action == 'uninstall':
22b50ecb
PH
228 win_uninstall_service(SVCNAME)
229 return
230
231 if options.action == 'service':
232 win_service_start(SVCNAME, main)
233 return
71cedb3c
PH
234
235 host, port_str = options.bind.split(':')
236 port = int(port_str)
237
238 print('Listening on %s:%d' % (host, port))
83de7942
PH
239 srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler)
240 thr = threading.Thread(target=srv.serve_forever)
241 thr.start()
f574103d 242 compat_input('Press ENTER to shut down')
83de7942
PH
243 srv.shutdown()
244 thr.join()
245
246
247def rmtree(path):
248 for name in os.listdir(path):
249 fname = os.path.join(path, name)
250 if os.path.isdir(fname):
251 rmtree(fname)
252 else:
71cedb3c 253 os.chmod(fname, 0o666)
83de7942
PH
254 os.remove(fname)
255 os.rmdir(path)
256
5f6a1245 257
83de7942
PH
258class BuildError(Exception):
259 def __init__(self, output, code=500):
260 self.output = output
261 self.code = code
262
263 def __str__(self):
264 return self.output
265
266
267class HTTPError(BuildError):
268 pass
269
270
271class PythonBuilder(object):
272 def __init__(self, **kwargs):
f574103d 273 python_version = kwargs.pop('python', '3.4')
165e3561
S
274 python_path = None
275 for node in ('Wow6432Node\\', ''):
83de7942 276 try:
165e3561
S
277 key = compat_winreg.OpenKey(
278 compat_winreg.HKEY_LOCAL_MACHINE,
279 r'SOFTWARE\%sPython\PythonCore\%s\InstallPath' % (node, python_version))
280 try:
281 python_path, _ = compat_winreg.QueryValueEx(key, '')
282 finally:
283 compat_winreg.CloseKey(key)
284 break
285 except Exception:
286 pass
287
288 if not python_path:
f574103d 289 raise BuildError('No such Python version: %s' % python_version)
83de7942 290
165e3561
S
291 self.pythonPath = python_path
292
83de7942
PH
293 super(PythonBuilder, self).__init__(**kwargs)
294
295
296class GITInfoBuilder(object):
297 def __init__(self, **kwargs):
298 try:
299 self.user, self.repoName = kwargs['path'][:2]
300 self.rev = kwargs.pop('rev')
301 except ValueError:
302 raise BuildError('Invalid path')
303 except KeyError as e:
304 raise BuildError('Missing mandatory parameter "%s"' % e.args[0])
305
306 path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user)
307 if not os.path.exists(path):
308 os.makedirs(path)
309 self.basePath = tempfile.mkdtemp(dir=path)
310 self.buildPath = os.path.join(self.basePath, 'build')
311
312 super(GITInfoBuilder, self).__init__(**kwargs)
313
314
315class GITBuilder(GITInfoBuilder):
316 def build(self):
317 try:
318 subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath])
319 subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath)
320 except subprocess.CalledProcessError as e:
321 raise BuildError(e.output)
322
323 super(GITBuilder, self).build()
324
325
326class YoutubeDLBuilder(object):
067aa17e 327 authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile', 'ytdl-org']
83de7942
PH
328
329 def __init__(self, **kwargs):
7a5c1cfe 330 if self.repoName != 'yt-dlp':
83de7942
PH
331 raise BuildError('Invalid repository "%s"' % self.repoName)
332 if self.user not in self.authorizedUsers:
333 raise HTTPError('Unauthorized user "%s"' % self.user, 401)
334
335 super(YoutubeDLBuilder, self).__init__(**kwargs)
336
337 def build(self):
338 try:
f574103d
S
339 proc = subprocess.Popen([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], stdin=subprocess.PIPE, cwd=self.buildPath)
340 proc.wait()
341 #subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'],
342 # cwd=self.buildPath)
83de7942
PH
343 except subprocess.CalledProcessError as e:
344 raise BuildError(e.output)
345
346 super(YoutubeDLBuilder, self).build()
347
348
349class DownloadBuilder(object):
350 def __init__(self, **kwargs):
351 self.handler = kwargs.pop('handler')
352 self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:]))
353 self.srcPath = os.path.abspath(os.path.normpath(self.srcPath))
354 if not self.srcPath.startswith(self.buildPath):
355 raise HTTPError(self.srcPath, 401)
356
357 super(DownloadBuilder, self).__init__(**kwargs)
358
359 def build(self):
360 if not os.path.exists(self.srcPath):
361 raise HTTPError('No such file', 404)
362 if os.path.isdir(self.srcPath):
363 raise HTTPError('Is a directory: %s' % self.srcPath, 401)
364
365 self.handler.send_response(200)
366 self.handler.send_header('Content-Type', 'application/octet-stream')
367 self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1])
368 self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size))
369 self.handler.end_headers()
370
371 with open(self.srcPath, 'rb') as src:
372 shutil.copyfileobj(src, self.handler.wfile)
373
374 super(DownloadBuilder, self).build()
375
376
377class CleanupTempDir(object):
378 def build(self):
379 try:
380 rmtree(self.basePath)
381 except Exception as e:
71cedb3c 382 print('WARNING deleting "%s": %s' % (self.basePath, e))
83de7942
PH
383
384 super(CleanupTempDir, self).build()
385
386
387class Null(object):
388 def __init__(self, **kwargs):
389 pass
390
391 def start(self):
392 pass
393
394 def close(self):
395 pass
396
397 def build(self):
398 pass
399
400
401class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null):
402 pass
403
404
f574103d 405class BuildHTTPRequestHandler(compat_http_server.BaseHTTPRequestHandler):
5f6a1245 406 actionDict = {'build': Builder, 'download': Builder} # They're the same, no more caching.
83de7942
PH
407
408 def do_GET(self):
f574103d
S
409 path = compat_urlparse.urlparse(self.path)
410 paramDict = dict([(key, value[0]) for key, value in compat_urlparse.parse_qs(path.query).items()])
83de7942
PH
411 action, _, path = path.path.strip('/').partition('/')
412 if path:
413 path = path.split('/')
414 if action in self.actionDict:
415 try:
416 builder = self.actionDict[action](path=path, handler=self, **paramDict)
417 builder.start()
418 try:
419 builder.build()
420 finally:
421 builder.close()
422 except BuildError as e:
423 self.send_response(e.code)
f574103d 424 msg = compat_str(e).encode('UTF-8')
83de7942
PH
425 self.send_header('Content-Type', 'text/plain; charset=UTF-8')
426 self.send_header('Content-Length', len(msg))
427 self.end_headers()
428 self.wfile.write(msg)
83de7942
PH
429 else:
430 self.send_response(500, 'Unknown build method "%s"' % action)
431 else:
432 self.send_response(500, 'Malformed URL')
433
83de7942 434if __name__ == '__main__':
22b50ecb 435 main()