]> jfr.im git - yt-dlp.git/blame - devscripts/buildserver.py
[youtube] Make cache ID a tuple of lengths instead of just the whole length
[yt-dlp.git] / devscripts / buildserver.py
CommitLineData
71cedb3c 1#!/usr/bin/python3
83de7942 2
71cedb3c
PH
3from http.server import HTTPServer, BaseHTTPRequestHandler
4from socketserver import ThreadingMixIn
5import argparse
6import ctypes
22b50ecb 7import functools
71cedb3c
PH
8import sys
9import threading
22b50ecb 10import traceback
71cedb3c 11import os.path
83de7942
PH
12
13
14class BuildHTTPServer(ThreadingMixIn, HTTPServer):
15 allow_reuse_address = True
16
17
71cedb3c 18advapi32 = ctypes.windll.advapi32
83de7942 19
71cedb3c
PH
20SC_MANAGER_ALL_ACCESS = 0xf003f
21SC_MANAGER_CREATE_SERVICE = 0x02
22SERVICE_WIN32_OWN_PROCESS = 0x10
23SERVICE_AUTO_START = 0x2
24SERVICE_ERROR_NORMAL = 0x1
25DELETE = 0x00010000
22b50ecb
PH
26SERVICE_STATUS_START_PENDING = 0x00000002
27SERVICE_STATUS_RUNNING = 0x00000004
28SERVICE_ACCEPT_STOP = 0x1
29
30SVCNAME = 'youtubedl_builder'
31
32LPTSTR = ctypes.c_wchar_p
33START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR))
34
35
36class SERVICE_TABLE_ENTRY(ctypes.Structure):
37 _fields_ = [
38 ('lpServiceName', LPTSTR),
39 ('lpServiceProc', START_CALLBACK)
40 ]
41
42
43HandlerEx = ctypes.WINFUNCTYPE(
44 ctypes.c_int, # return
45 ctypes.c_int, # dwControl
46 ctypes.c_int, # dwEventType
47 ctypes.c_void_p, # lpEventData,
48 ctypes.c_void_p, # lpContext,
49)
50
51
52def _ctypes_array(c_type, py_array):
53 ar = (c_type * len(py_array))()
54 ar[:] = py_array
55 return ar
83de7942 56
83de7942 57
71cedb3c 58def win_OpenSCManager():
22b50ecb 59 res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS)
71cedb3c
PH
60 if not res:
61 raise Exception('Opening service manager failed - '
62 'are you running this as administrator?')
63 return res
64
65
66def win_install_service(service_name, cmdline):
67 manager = win_OpenSCManager()
68 try:
22b50ecb 69 h = advapi32.CreateServiceW(
71cedb3c
PH
70 manager, service_name, None,
71 SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS,
72 SERVICE_AUTO_START, SERVICE_ERROR_NORMAL,
73 cmdline, None, None, None, None, None)
74 if not h:
75 raise OSError('Service creation failed: %s' % ctypes.FormatError())
76
77 advapi32.CloseServiceHandle(h)
78 finally:
79 advapi32.CloseServiceHandle(manager)
80
81
82def win_uninstall_service(service_name):
83 manager = win_OpenSCManager()
84 try:
22b50ecb 85 h = advapi32.OpenServiceW(manager, service_name, DELETE)
71cedb3c
PH
86 if not h:
87 raise OSError('Could not find service %s: %s' % (
88 service_name, ctypes.FormatError()))
89
90 try:
91 if not advapi32.DeleteService(h):
92 raise OSError('Deletion failed: %s' % ctypes.FormatError())
93 finally:
94 advapi32.CloseServiceHandle(h)
95 finally:
96 advapi32.CloseServiceHandle(manager)
97
98
22b50ecb
PH
99def win_service_report_event(service_name, msg, is_error=True):
100 with open('C:/sshkeys/log', 'a', encoding='utf-8') as f:
101 f.write(msg + '\n')
102
103 event_log = advapi32.RegisterEventSourceW(None, service_name)
104 if not event_log:
105 raise OSError('Could not report event: %s' % ctypes.FormatError())
106
107 try:
108 type_id = 0x0001 if is_error else 0x0004
109 event_id = 0xc0000000 if is_error else 0x40000000
110 lines = _ctypes_array(LPTSTR, [msg])
111
112 if not advapi32.ReportEventW(
113 event_log, type_id, 0, event_id, None, len(lines), 0,
114 lines, None):
115 raise OSError('Event reporting failed: %s' % ctypes.FormatError())
116 finally:
117 advapi32.DeregisterEventSource(event_log)
118
119
120def win_service_handler(stop_event, *args):
121 try:
122 raise ValueError('Handler called with args ' + repr(args))
123 TODO
124 except Exception as e:
125 tb = traceback.format_exc()
126 msg = str(e) + '\n' + tb
127 win_service_report_event(service_name, msg, is_error=True)
128 raise
129
130
131def win_service_set_status(handle, status_code):
132 svcStatus = SERVICE_STATUS()
133 svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS
134 svcStatus.dwCurrentState = status_code
135 svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP
136
137 svcStatus.dwServiceSpecificExitCode = 0
71cedb3c 138
22b50ecb
PH
139 if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)):
140 raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError())
71cedb3c 141
71cedb3c 142
22b50ecb
PH
143def win_service_main(service_name, real_main, argc, argv_raw):
144 try:
145 #args = [argv_raw[i].value for i in range(argc)]
146 stop_event = threading.Event()
147 handler = HandlerEx(functools.partial(stop_event, win_service_handler))
148 h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None)
149 if not h:
150 raise OSError('Handler registration failed: %s' %
151 ctypes.FormatError())
152
153 TODO
154 except Exception as e:
155 tb = traceback.format_exc()
156 msg = str(e) + '\n' + tb
157 win_service_report_event(service_name, msg, is_error=True)
158 raise
71cedb3c 159
22b50ecb
PH
160
161def win_service_start(service_name, real_main):
162 try:
163 cb = START_CALLBACK(
164 functools.partial(win_service_main, service_name, real_main))
165 dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [
166 SERVICE_TABLE_ENTRY(
167 service_name,
168 cb
169 ),
170 SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK))
171 ])
172
173 if not advapi32.StartServiceCtrlDispatcherW(dispatch_table):
174 raise OSError('ctypes start failed: %s' % ctypes.FormatError())
175 except Exception as e:
176 tb = traceback.format_exc()
177 msg = str(e) + '\n' + tb
178 win_service_report_event(service_name, msg, is_error=True)
179 raise
180
181
182def main(args=None):
71cedb3c
PH
183 parser = argparse.ArgumentParser()
184 parser.add_argument('-i', '--install',
185 action='store_const', dest='action', const='install',
186 help='Launch at Windows startup')
187 parser.add_argument('-u', '--uninstall',
188 action='store_const', dest='action', const='uninstall',
189 help='Remove Windows service')
190 parser.add_argument('-s', '--service',
22b50ecb 191 action='store_const', dest='action', const='service',
71cedb3c
PH
192 help='Run as a Windows service')
193 parser.add_argument('-b', '--bind', metavar='<host:port>',
194 action='store', default='localhost:8142',
195 help='Bind to host:port (default %default)')
22b50ecb 196 options = parser.parse_args(args=args)
71cedb3c
PH
197
198 if options.action == 'install':
22b50ecb
PH
199 fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox')
200 cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind)
201 win_install_service(SVCNAME, cmdline)
202 return
71cedb3c
PH
203
204 if options.action == 'uninstall':
22b50ecb
PH
205 win_uninstall_service(SVCNAME)
206 return
207
208 if options.action == 'service':
209 win_service_start(SVCNAME, main)
210 return
71cedb3c
PH
211
212 host, port_str = options.bind.split(':')
213 port = int(port_str)
214
215 print('Listening on %s:%d' % (host, port))
83de7942
PH
216 srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler)
217 thr = threading.Thread(target=srv.serve_forever)
218 thr.start()
71cedb3c 219 input('Press ENTER to shut down')
83de7942
PH
220 srv.shutdown()
221 thr.join()
222
223
224def rmtree(path):
225 for name in os.listdir(path):
226 fname = os.path.join(path, name)
227 if os.path.isdir(fname):
228 rmtree(fname)
229 else:
71cedb3c 230 os.chmod(fname, 0o666)
83de7942
PH
231 os.remove(fname)
232 os.rmdir(path)
233
234#==============================================================================
235
236class BuildError(Exception):
237 def __init__(self, output, code=500):
238 self.output = output
239 self.code = code
240
241 def __str__(self):
242 return self.output
243
244
245class HTTPError(BuildError):
246 pass
247
248
249class PythonBuilder(object):
250 def __init__(self, **kwargs):
251 pythonVersion = kwargs.pop('python', '2.7')
252 try:
253 key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\Python\PythonCore\%s\InstallPath' % pythonVersion)
254 try:
255 self.pythonPath, _ = _winreg.QueryValueEx(key, '')
256 finally:
257 _winreg.CloseKey(key)
258 except Exception:
259 raise BuildError('No such Python version: %s' % pythonVersion)
260
261 super(PythonBuilder, self).__init__(**kwargs)
262
263
264class GITInfoBuilder(object):
265 def __init__(self, **kwargs):
266 try:
267 self.user, self.repoName = kwargs['path'][:2]
268 self.rev = kwargs.pop('rev')
269 except ValueError:
270 raise BuildError('Invalid path')
271 except KeyError as e:
272 raise BuildError('Missing mandatory parameter "%s"' % e.args[0])
273
274 path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user)
275 if not os.path.exists(path):
276 os.makedirs(path)
277 self.basePath = tempfile.mkdtemp(dir=path)
278 self.buildPath = os.path.join(self.basePath, 'build')
279
280 super(GITInfoBuilder, self).__init__(**kwargs)
281
282
283class GITBuilder(GITInfoBuilder):
284 def build(self):
285 try:
286 subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath])
287 subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath)
288 except subprocess.CalledProcessError as e:
289 raise BuildError(e.output)
290
291 super(GITBuilder, self).build()
292
293
294class YoutubeDLBuilder(object):
295 authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile']
296
297 def __init__(self, **kwargs):
298 if self.repoName != 'youtube-dl':
299 raise BuildError('Invalid repository "%s"' % self.repoName)
300 if self.user not in self.authorizedUsers:
301 raise HTTPError('Unauthorized user "%s"' % self.user, 401)
302
303 super(YoutubeDLBuilder, self).__init__(**kwargs)
304
305 def build(self):
306 try:
307 subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'],
308 cwd=self.buildPath)
309 except subprocess.CalledProcessError as e:
310 raise BuildError(e.output)
311
312 super(YoutubeDLBuilder, self).build()
313
314
315class DownloadBuilder(object):
316 def __init__(self, **kwargs):
317 self.handler = kwargs.pop('handler')
318 self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:]))
319 self.srcPath = os.path.abspath(os.path.normpath(self.srcPath))
320 if not self.srcPath.startswith(self.buildPath):
321 raise HTTPError(self.srcPath, 401)
322
323 super(DownloadBuilder, self).__init__(**kwargs)
324
325 def build(self):
326 if not os.path.exists(self.srcPath):
327 raise HTTPError('No such file', 404)
328 if os.path.isdir(self.srcPath):
329 raise HTTPError('Is a directory: %s' % self.srcPath, 401)
330
331 self.handler.send_response(200)
332 self.handler.send_header('Content-Type', 'application/octet-stream')
333 self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1])
334 self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size))
335 self.handler.end_headers()
336
337 with open(self.srcPath, 'rb') as src:
338 shutil.copyfileobj(src, self.handler.wfile)
339
340 super(DownloadBuilder, self).build()
341
342
343class CleanupTempDir(object):
344 def build(self):
345 try:
346 rmtree(self.basePath)
347 except Exception as e:
71cedb3c 348 print('WARNING deleting "%s": %s' % (self.basePath, e))
83de7942
PH
349
350 super(CleanupTempDir, self).build()
351
352
353class Null(object):
354 def __init__(self, **kwargs):
355 pass
356
357 def start(self):
358 pass
359
360 def close(self):
361 pass
362
363 def build(self):
364 pass
365
366
367class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null):
368 pass
369
370
371class BuildHTTPRequestHandler(BaseHTTPRequestHandler):
372 actionDict = { 'build': Builder, 'download': Builder } # They're the same, no more caching.
373
374 def do_GET(self):
375 path = urlparse.urlparse(self.path)
376 paramDict = dict([(key, value[0]) for key, value in urlparse.parse_qs(path.query).items()])
377 action, _, path = path.path.strip('/').partition('/')
378 if path:
379 path = path.split('/')
380 if action in self.actionDict:
381 try:
382 builder = self.actionDict[action](path=path, handler=self, **paramDict)
383 builder.start()
384 try:
385 builder.build()
386 finally:
387 builder.close()
388 except BuildError as e:
389 self.send_response(e.code)
390 msg = unicode(e).encode('UTF-8')
391 self.send_header('Content-Type', 'text/plain; charset=UTF-8')
392 self.send_header('Content-Length', len(msg))
393 self.end_headers()
394 self.wfile.write(msg)
395 except HTTPError as e:
396 self.send_response(e.code, str(e))
397 else:
398 self.send_response(500, 'Unknown build method "%s"' % action)
399 else:
400 self.send_response(500, 'Malformed URL')
401
402#==============================================================================
403
404if __name__ == '__main__':
22b50ecb 405 main()