venv/lib/python3.11/site-packages/pip/_vendor/pygments/lexers/python.py

   1 """
   2     pygments.lexers.python
   3     ~~~~~~~~~~~~~~~~~~~~~~
   4
   5     Lexers for Python and related languages.
   6
   7     :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
   8     :license: BSD, see LICENSE for details.
   9 """
  10
  11 import re
  12 import keyword
  13
  14 from pip._vendor.pygments.lexer import DelegatingLexer, Lexer, RegexLexer, include, \
  15     bygroups, using, default, words, combined, do_insertions, this, line_re
  16 from pip._vendor.pygments.util import get_bool_opt, shebang_matches
  17 from pip._vendor.pygments.token import Text, Comment, Operator, Keyword, Name, String, \
  18     Number, Punctuation, Generic, Other, Error, Whitespace
  19 from pip._vendor.pygments import unistring as uni
  20
  21 __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer',
  22            'Python2Lexer', 'Python2TracebackLexer',
  23            'CythonLexer', 'DgLexer', 'NumPyLexer']
  24
  25
  26 class PythonLexer(RegexLexer):
  27     """
  28     For Python source code (version 3.x).
  29
  30     .. versionadded:: 0.10
  31
  32     .. versionchanged:: 2.5
  33        This is now the default ``PythonLexer``.  It is still available as the
  34        alias ``Python3Lexer``.
  35     """
  36
  37     name = 'Python'
  38     url = 'http://www.python.org'
  39     aliases = ['python', 'py', 'sage', 'python3', 'py3']
  40     filenames = [
  41         '*.py',
  42         '*.pyw',
  43         # Type stubs
  44         '*.pyi',
  45         # Jython
  46         '*.jy',
  47         # Sage
  48         '*.sage',
  49         # SCons
  50         '*.sc',
  51         'SConstruct',
  52         'SConscript',
  53         # Skylark/Starlark (used by Bazel, Buck, and Pants)
  54         '*.bzl',
  55         'BUCK',
  56         'BUILD',
  57         'BUILD.bazel',
  58         'WORKSPACE',
  59         # Twisted Application infrastructure
  60         '*.tac',
  61     ]
  62     mimetypes = ['text/x-python', 'application/x-python',
  63                  'text/x-python3', 'application/x-python3']
  64
  65     uni_name = "[%s][%s]*" % (uni.xid_start, uni.xid_continue)
  66
  67     def innerstring_rules(ttype):
  68         return [
  69             # the old style '%s' % (...) string formatting (still valid in Py3)
  70             (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
  71              '[hlL]?[E-GXc-giorsaux%]', String.Interpol),
  72             # the new style '{}'.format(...) string formatting
  73             (r'\{'
  74              r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?'  # field name
  75              r'(\![sra])?'                       # conversion
  76              r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?'
  77              r'\}', String.Interpol),
  78
  79             # backslashes, quotes and formatting signs must be parsed one at a time
  80             (r'[^\\\'"%{\n]+', ttype),
  81             (r'[\'"\\]', ttype),
  82             # unhandled string formatting sign
  83             (r'%|(\{{1,2})', ttype)
  84             # newlines are an error (use "nl" state)
  85         ]
  86
  87     def fstring_rules(ttype):
  88         return [
  89             # Assuming that a '}' is the closing brace after format specifier.
  90             # Sadly, this means that we won't detect syntax error. But it's
  91             # more important to parse correct syntax correctly, than to
  92             # highlight invalid syntax.
  93             (r'\}', String.Interpol),
  94             (r'\{', String.Interpol, 'expr-inside-fstring'),
  95             # backslashes, quotes and formatting signs must be parsed one at a time
  96             (r'[^\\\'"{}\n]+', ttype),
  97             (r'[\'"\\]', ttype),
  98             # newlines are an error (use "nl" state)
  99         ]
 100
 101     tokens = {
 102         'root': [
 103             (r'\n', Whitespace),
 104             (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
 105              bygroups(Whitespace, String.Affix, String.Doc)),
 106             (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
 107              bygroups(Whitespace, String.Affix, String.Doc)),
 108             (r'\A#!.+$', Comment.Hashbang),
 109             (r'#.*$', Comment.Single),
 110             (r'\\\n', Text),
 111             (r'\\', Text),
 112             include('keywords'),
 113             include('soft-keywords'),
 114             (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'),
 115             (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'),
 116             (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
 117              'fromimport'),
 118             (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
 119              'import'),
 120             include('expr'),
 121         ],
 122         'expr': [
 123             # raw f-strings
 124             ('(?i)(rf|fr)(""")',
 125              bygroups(String.Affix, String.Double),
 126              combined('rfstringescape', 'tdqf')),
 127             ("(?i)(rf|fr)(''')",
 128              bygroups(String.Affix, String.Single),
 129              combined('rfstringescape', 'tsqf')),
 130             ('(?i)(rf|fr)(")',
 131              bygroups(String.Affix, String.Double),
 132              combined('rfstringescape', 'dqf')),
 133             ("(?i)(rf|fr)(')",
 134              bygroups(String.Affix, String.Single),
 135              combined('rfstringescape', 'sqf')),
 136             # non-raw f-strings
 137             ('([fF])(""")', bygroups(String.Affix, String.Double),
 138              combined('fstringescape', 'tdqf')),
 139             ("([fF])(''')", bygroups(String.Affix, String.Single),
 140              combined('fstringescape', 'tsqf')),
 141             ('([fF])(")', bygroups(String.Affix, String.Double),
 142              combined('fstringescape', 'dqf')),
 143             ("([fF])(')", bygroups(String.Affix, String.Single),
 144              combined('fstringescape', 'sqf')),
 145             # raw bytes and strings
 146             ('(?i)(rb|br|r)(""")',
 147              bygroups(String.Affix, String.Double), 'tdqs'),
 148             ("(?i)(rb|br|r)(''')",
 149              bygroups(String.Affix, String.Single), 'tsqs'),
 150             ('(?i)(rb|br|r)(")',
 151              bygroups(String.Affix, String.Double), 'dqs'),
 152             ("(?i)(rb|br|r)(')",
 153              bygroups(String.Affix, String.Single), 'sqs'),
 154             # non-raw strings
 155             ('([uU]?)(""")', bygroups(String.Affix, String.Double),
 156              combined('stringescape', 'tdqs')),
 157             ("([uU]?)(''')", bygroups(String.Affix, String.Single),
 158              combined('stringescape', 'tsqs')),
 159             ('([uU]?)(")', bygroups(String.Affix, String.Double),
 160              combined('stringescape', 'dqs')),
 161             ("([uU]?)(')", bygroups(String.Affix, String.Single),
 162              combined('stringescape', 'sqs')),
 163             # non-raw bytes
 164             ('([bB])(""")', bygroups(String.Affix, String.Double),
 165              combined('bytesescape', 'tdqs')),
 166             ("([bB])(''')", bygroups(String.Affix, String.Single),
 167              combined('bytesescape', 'tsqs')),
 168             ('([bB])(")', bygroups(String.Affix, String.Double),
 169              combined('bytesescape', 'dqs')),
 170             ("([bB])(')", bygroups(String.Affix, String.Single),
 171              combined('bytesescape', 'sqs')),
 172
 173             (r'[^\S\n]+', Text),
 174             include('numbers'),
 175             (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|.]', Operator),
 176             (r'[]{}:(),;[]', Punctuation),
 177             (r'(in|is|and|or|not)\b', Operator.Word),
 178             include('expr-keywords'),
 179             include('builtins'),
 180             include('magicfuncs'),
 181             include('magicvars'),
 182             include('name'),
 183         ],
 184         'expr-inside-fstring': [
 185             (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
 186             # without format specifier
 187             (r'(=\s*)?'         # debug (https://bugs.python.org/issue36817)
 188              r'(\![sraf])?'     # conversion
 189              r'\}', String.Interpol, '#pop'),
 190             # with format specifier
 191             # we'll catch the remaining '}' in the outer scope
 192             (r'(=\s*)?'         # debug (https://bugs.python.org/issue36817)
 193              r'(\![sraf])?'     # conversion
 194              r':', String.Interpol, '#pop'),
 195             (r'\s+', Whitespace),  # allow new lines
 196             include('expr'),
 197         ],
 198         'expr-inside-fstring-inner': [
 199             (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
 200             (r'[])}]', Punctuation, '#pop'),
 201             (r'\s+', Whitespace),  # allow new lines
 202             include('expr'),
 203         ],
 204         'expr-keywords': [
 205             # Based on https://docs.python.org/3/reference/expressions.html
 206             (words((
 207                 'async for', 'await', 'else', 'for', 'if', 'lambda',
 208                 'yield', 'yield from'), suffix=r'\b'),
 209              Keyword),
 210             (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
 211         ],
 212         'keywords': [
 213             (words((
 214                 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif',
 215                 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda',
 216                 'pass', 'raise', 'nonlocal', 'return', 'try', 'while', 'yield',
 217                 'yield from', 'as', 'with'), suffix=r'\b'),
 218              Keyword),
 219             (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
 220         ],
 221         'soft-keywords': [
 222             # `match`, `case` and `_` soft keywords
 223             (r'(^[ \t]*)'              # at beginning of line + possible indentation
 224              r'(match|case)\b'         # a possible keyword
 225              r'(?![ \t]*(?:'           # not followed by...
 226              r'[:,;=^&|@~)\]}]|(?:' +  # characters and keywords that mean this isn't
 227              r'|'.join(keyword.kwlist) + r')\b))',                 # pattern matching
 228              bygroups(Text, Keyword), 'soft-keywords-inner'),
 229         ],
 230         'soft-keywords-inner': [
 231             # optional `_` keyword
 232             (r'(\s+)([^\n_]*)(_\b)', bygroups(Whitespace, using(this), Keyword)),
 233             default('#pop')
 234         ],
 235         'builtins': [
 236             (words((
 237                 '__import__', 'abs', 'aiter', 'all', 'any', 'bin', 'bool', 'bytearray',
 238                 'breakpoint', 'bytes', 'callable', 'chr', 'classmethod', 'compile',
 239                 'complex', 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval',
 240                 'filter', 'float', 'format', 'frozenset', 'getattr', 'globals',
 241                 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'isinstance',
 242                 'issubclass', 'iter', 'len', 'list', 'locals', 'map', 'max',
 243                 'memoryview', 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow',
 244                 'print', 'property', 'range', 'repr', 'reversed', 'round', 'set',
 245                 'setattr', 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super',
 246                 'tuple', 'type', 'vars', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'),
 247              Name.Builtin),
 248             (r'(?<!\.)(self|Ellipsis|NotImplemented|cls)\b', Name.Builtin.Pseudo),
 249             (words((
 250                 'ArithmeticError', 'AssertionError', 'AttributeError',
 251                 'BaseException', 'BufferError', 'BytesWarning', 'DeprecationWarning',
 252                 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError',
 253                 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError',
 254                 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError',
 255                 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError',
 256                 'NotImplementedError', 'OSError', 'OverflowError',
 257                 'PendingDeprecationWarning', 'ReferenceError', 'ResourceWarning',
 258                 'RuntimeError', 'RuntimeWarning', 'StopIteration',
 259                 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit',
 260                 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
 261                 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
 262                 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError',
 263                 'Warning', 'WindowsError', 'ZeroDivisionError',
 264                 # new builtin exceptions from PEP 3151
 265                 'BlockingIOError', 'ChildProcessError', 'ConnectionError',
 266                 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError',
 267                 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError',
 268                 'InterruptedError', 'IsADirectoryError', 'NotADirectoryError',
 269                 'PermissionError', 'ProcessLookupError', 'TimeoutError',
 270                 # others new in Python 3
 271                 'StopAsyncIteration', 'ModuleNotFoundError', 'RecursionError',
 272                 'EncodingWarning'),
 273                 prefix=r'(?<!\.)', suffix=r'\b'),
 274              Name.Exception),
 275         ],
 276         'magicfuncs': [
 277             (words((
 278                 '__abs__', '__add__', '__aenter__', '__aexit__', '__aiter__',
 279                 '__and__', '__anext__', '__await__', '__bool__', '__bytes__',
 280                 '__call__', '__complex__', '__contains__', '__del__', '__delattr__',
 281                 '__delete__', '__delitem__', '__dir__', '__divmod__', '__enter__',
 282                 '__eq__', '__exit__', '__float__', '__floordiv__', '__format__',
 283                 '__ge__', '__get__', '__getattr__', '__getattribute__',
 284                 '__getitem__', '__gt__', '__hash__', '__iadd__', '__iand__',
 285                 '__ifloordiv__', '__ilshift__', '__imatmul__', '__imod__',
 286                 '__imul__', '__index__', '__init__', '__instancecheck__',
 287                 '__int__', '__invert__', '__ior__', '__ipow__', '__irshift__',
 288                 '__isub__', '__iter__', '__itruediv__', '__ixor__', '__le__',
 289                 '__len__', '__length_hint__', '__lshift__', '__lt__', '__matmul__',
 290                 '__missing__', '__mod__', '__mul__', '__ne__', '__neg__',
 291                 '__new__', '__next__', '__or__', '__pos__', '__pow__',
 292                 '__prepare__', '__radd__', '__rand__', '__rdivmod__', '__repr__',
 293                 '__reversed__', '__rfloordiv__', '__rlshift__', '__rmatmul__',
 294                 '__rmod__', '__rmul__', '__ror__', '__round__', '__rpow__',
 295                 '__rrshift__', '__rshift__', '__rsub__', '__rtruediv__',
 296                 '__rxor__', '__set__', '__setattr__', '__setitem__', '__str__',
 297                 '__sub__', '__subclasscheck__', '__truediv__',
 298                 '__xor__'), suffix=r'\b'),
 299              Name.Function.Magic),
 300         ],
 301         'magicvars': [
 302             (words((
 303                 '__annotations__', '__bases__', '__class__', '__closure__',
 304                 '__code__', '__defaults__', '__dict__', '__doc__', '__file__',
 305                 '__func__', '__globals__', '__kwdefaults__', '__module__',
 306                 '__mro__', '__name__', '__objclass__', '__qualname__',
 307                 '__self__', '__slots__', '__weakref__'), suffix=r'\b'),
 308              Name.Variable.Magic),
 309         ],
 310         'numbers': [
 311             (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)'
 312              r'([eE][+-]?\d(?:_?\d)*)?', Number.Float),
 313             (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float),
 314             (r'0[oO](?:_?[0-7])+', Number.Oct),
 315             (r'0[bB](?:_?[01])+', Number.Bin),
 316             (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex),
 317             (r'\d(?:_?\d)*', Number.Integer),
 318         ],
 319         'name': [
 320             (r'@' + uni_name, Name.Decorator),
 321             (r'@', Operator),  # new matrix multiplication operator
 322             (uni_name, Name),
 323         ],
 324         'funcname': [
 325             include('magicfuncs'),
 326             (uni_name, Name.Function, '#pop'),
 327             default('#pop'),
 328         ],
 329         'classname': [
 330             (uni_name, Name.Class, '#pop'),
 331         ],
 332         'import': [
 333             (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)),
 334             (r'\.', Name.Namespace),
 335             (uni_name, Name.Namespace),
 336             (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)),
 337             default('#pop')  # all else: go back
 338         ],
 339         'fromimport': [
 340             (r'(\s+)(import)\b', bygroups(Text, Keyword.Namespace), '#pop'),
 341             (r'\.', Name.Namespace),
 342             # if None occurs here, it's "raise x from None", since None can
 343             # never be a module name
 344             (r'None\b', Keyword.Constant, '#pop'),
 345             (uni_name, Name.Namespace),
 346             default('#pop'),
 347         ],
 348         'rfstringescape': [
 349             (r'\{\{', String.Escape),
 350             (r'\}\}', String.Escape),
 351         ],
 352         'fstringescape': [
 353             include('rfstringescape'),
 354             include('stringescape'),
 355         ],
 356         'bytesescape': [
 357             (r'\\([\\abfnrtv"\']|\n|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
 358         ],
 359         'stringescape': [
 360             (r'\\(N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8})', String.Escape),
 361             include('bytesescape')
 362         ],
 363         'fstrings-single': fstring_rules(String.Single),
 364         'fstrings-double': fstring_rules(String.Double),
 365         'strings-single': innerstring_rules(String.Single),
 366         'strings-double': innerstring_rules(String.Double),
 367         'dqf': [
 368             (r'"', String.Double, '#pop'),
 369             (r'\\\\|\\"|\\\n', String.Escape),  # included here for raw strings
 370             include('fstrings-double')
 371         ],
 372         'sqf': [
 373             (r"'", String.Single, '#pop'),
 374             (r"\\\\|\\'|\\\n", String.Escape),  # included here for raw strings
 375             include('fstrings-single')
 376         ],
 377         'dqs': [
 378             (r'"', String.Double, '#pop'),
 379             (r'\\\\|\\"|\\\n', String.Escape),  # included here for raw strings
 380             include('strings-double')
 381         ],
 382         'sqs': [
 383             (r"'", String.Single, '#pop'),
 384             (r"\\\\|\\'|\\\n", String.Escape),  # included here for raw strings
 385             include('strings-single')
 386         ],
 387         'tdqf': [
 388             (r'"""', String.Double, '#pop'),
 389             include('fstrings-double'),
 390             (r'\n', String.Double)
 391         ],
 392         'tsqf': [
 393             (r"'''", String.Single, '#pop'),
 394             include('fstrings-single'),
 395             (r'\n', String.Single)
 396         ],
 397         'tdqs': [
 398             (r'"""', String.Double, '#pop'),
 399             include('strings-double'),
 400             (r'\n', String.Double)
 401         ],
 402         'tsqs': [
 403             (r"'''", String.Single, '#pop'),
 404             include('strings-single'),
 405             (r'\n', String.Single)
 406         ],
 407     }
 408
 409     def analyse_text(text):
 410         return shebang_matches(text, r'pythonw?(3(\.\d)?)?') or \
 411             'import ' in text[:1000]
 412
 413
 414 Python3Lexer = PythonLexer
 415
 416
 417 class Python2Lexer(RegexLexer):
 418     """
 419     For Python 2.x source code.
 420
 421     .. versionchanged:: 2.5
 422        This class has been renamed from ``PythonLexer``.  ``PythonLexer`` now
 423        refers to the Python 3 variant.  File name patterns like ``*.py`` have
 424        been moved to Python 3 as well.
 425     """
 426
 427     name = 'Python 2.x'
 428     url = 'http://www.python.org'
 429     aliases = ['python2', 'py2']
 430     filenames = []  # now taken over by PythonLexer (3.x)
 431     mimetypes = ['text/x-python2', 'application/x-python2']
 432
 433     def innerstring_rules(ttype):
 434         return [
 435             # the old style '%s' % (...) string formatting
 436             (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
 437              '[hlL]?[E-GXc-giorsux%]', String.Interpol),
 438             # backslashes, quotes and formatting signs must be parsed one at a time
 439             (r'[^\\\'"%\n]+', ttype),
 440             (r'[\'"\\]', ttype),
 441             # unhandled string formatting sign
 442             (r'%', ttype),
 443             # newlines are an error (use "nl" state)
 444         ]
 445
 446     tokens = {
 447         'root': [
 448             (r'\n', Whitespace),
 449             (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
 450              bygroups(Whitespace, String.Affix, String.Doc)),
 451             (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
 452              bygroups(Whitespace, String.Affix, String.Doc)),
 453             (r'[^\S\n]+', Text),
 454             (r'\A#!.+$', Comment.Hashbang),
 455             (r'#.*$', Comment.Single),
 456             (r'[]{}:(),;[]', Punctuation),
 457             (r'\\\n', Text),
 458             (r'\\', Text),
 459             (r'(in|is|and|or|not)\b', Operator.Word),
 460             (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator),
 461             include('keywords'),
 462             (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'),
 463             (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'),
 464             (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
 465              'fromimport'),
 466             (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
 467              'import'),
 468             include('builtins'),
 469             include('magicfuncs'),
 470             include('magicvars'),
 471             include('backtick'),
 472             ('([rR]|[uUbB][rR]|[rR][uUbB])(""")',
 473              bygroups(String.Affix, String.Double), 'tdqs'),
 474             ("([rR]|[uUbB][rR]|[rR][uUbB])(''')",
 475              bygroups(String.Affix, String.Single), 'tsqs'),
 476             ('([rR]|[uUbB][rR]|[rR][uUbB])(")',
 477              bygroups(String.Affix, String.Double), 'dqs'),
 478             ("([rR]|[uUbB][rR]|[rR][uUbB])(')",
 479              bygroups(String.Affix, String.Single), 'sqs'),
 480             ('([uUbB]?)(""")', bygroups(String.Affix, String.Double),
 481              combined('stringescape', 'tdqs')),
 482             ("([uUbB]?)(''')", bygroups(String.Affix, String.Single),
 483              combined('stringescape', 'tsqs')),
 484             ('([uUbB]?)(")', bygroups(String.Affix, String.Double),
 485              combined('stringescape', 'dqs')),
 486             ("([uUbB]?)(')", bygroups(String.Affix, String.Single),
 487              combined('stringescape', 'sqs')),
 488             include('name'),
 489             include('numbers'),
 490         ],
 491         'keywords': [
 492             (words((
 493                 'assert', 'break', 'continue', 'del', 'elif', 'else', 'except',
 494                 'exec', 'finally', 'for', 'global', 'if', 'lambda', 'pass',
 495                 'print', 'raise', 'return', 'try', 'while', 'yield',
 496                 'yield from', 'as', 'with'), suffix=r'\b'),
 497              Keyword),
 498         ],
 499         'builtins': [
 500             (words((
 501                 '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin',
 502                 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod',
 503                 'cmp', 'coerce', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod',
 504                 'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float',
 505                 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id',
 506                 'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', 'len',
 507                 'list', 'locals', 'long', 'map', 'max', 'min', 'next', 'object',
 508                 'oct', 'open', 'ord', 'pow', 'property', 'range', 'raw_input', 'reduce',
 509                 'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice',
 510                 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type',
 511                 'unichr', 'unicode', 'vars', 'xrange', 'zip'),
 512                 prefix=r'(?<!\.)', suffix=r'\b'),
 513              Name.Builtin),
 514             (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|cls'
 515              r')\b', Name.Builtin.Pseudo),
 516             (words((
 517                 'ArithmeticError', 'AssertionError', 'AttributeError',
 518                 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError',
 519                 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit',
 520                 'IOError', 'ImportError', 'ImportWarning', 'IndentationError',
 521                 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError',
 522                 'MemoryError', 'NameError',
 523                 'NotImplementedError', 'OSError', 'OverflowError', 'OverflowWarning',
 524                 'PendingDeprecationWarning', 'ReferenceError',
 525                 'RuntimeError', 'RuntimeWarning', 'StandardError', 'StopIteration',
 526                 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit',
 527                 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
 528                 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
 529                 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError', 'Warning',
 530                 'WindowsError', 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
 531              Name.Exception),
 532         ],
 533         'magicfuncs': [
 534             (words((
 535                 '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__',
 536                 '__complex__', '__contains__', '__del__', '__delattr__', '__delete__',
 537                 '__delitem__', '__delslice__', '__div__', '__divmod__', '__enter__',
 538                 '__eq__', '__exit__', '__float__', '__floordiv__', '__ge__', '__get__',
 539                 '__getattr__', '__getattribute__', '__getitem__', '__getslice__', '__gt__',
 540                 '__hash__', '__hex__', '__iadd__', '__iand__', '__idiv__', '__ifloordiv__',
 541                 '__ilshift__', '__imod__', '__imul__', '__index__', '__init__',
 542                 '__instancecheck__', '__int__', '__invert__', '__iop__', '__ior__',
 543                 '__ipow__', '__irshift__', '__isub__', '__iter__', '__itruediv__',
 544                 '__ixor__', '__le__', '__len__', '__long__', '__lshift__', '__lt__',
 545                 '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', '__new__',
 546                 '__nonzero__', '__oct__', '__op__', '__or__', '__pos__', '__pow__',
 547                 '__radd__', '__rand__', '__rcmp__', '__rdiv__', '__rdivmod__', '__repr__',
 548                 '__reversed__', '__rfloordiv__', '__rlshift__', '__rmod__', '__rmul__',
 549                 '__rop__', '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__',
 550                 '__rtruediv__', '__rxor__', '__set__', '__setattr__', '__setitem__',
 551                 '__setslice__', '__str__', '__sub__', '__subclasscheck__', '__truediv__',
 552                 '__unicode__', '__xor__'), suffix=r'\b'),
 553              Name.Function.Magic),
 554         ],
 555         'magicvars': [
 556             (words((
 557                 '__bases__', '__class__', '__closure__', '__code__', '__defaults__',
 558                 '__dict__', '__doc__', '__file__', '__func__', '__globals__',
 559                 '__metaclass__', '__module__', '__mro__', '__name__', '__self__',
 560                 '__slots__', '__weakref__'),
 561                 suffix=r'\b'),
 562              Name.Variable.Magic),
 563         ],
 564         'numbers': [
 565             (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
 566             (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
 567             (r'0[0-7]+j?', Number.Oct),
 568             (r'0[bB][01]+', Number.Bin),
 569             (r'0[xX][a-fA-F0-9]+', Number.Hex),
 570             (r'\d+L', Number.Integer.Long),
 571             (r'\d+j?', Number.Integer)
 572         ],
 573         'backtick': [
 574             ('`.*?`', String.Backtick),
 575         ],
 576         'name': [
 577             (r'@[\w.]+', Name.Decorator),
 578             (r'[a-zA-Z_]\w*', Name),
 579         ],
 580         'funcname': [
 581             include('magicfuncs'),
 582             (r'[a-zA-Z_]\w*', Name.Function, '#pop'),
 583             default('#pop'),
 584         ],
 585         'classname': [
 586             (r'[a-zA-Z_]\w*', Name.Class, '#pop')
 587         ],
 588         'import': [
 589             (r'(?:[ \t]|\\\n)+', Text),
 590             (r'as\b', Keyword.Namespace),
 591             (r',', Operator),
 592             (r'[a-zA-Z_][\w.]*', Name.Namespace),
 593             default('#pop')  # all else: go back
 594         ],
 595         'fromimport': [
 596             (r'(?:[ \t]|\\\n)+', Text),
 597             (r'import\b', Keyword.Namespace, '#pop'),
 598             # if None occurs here, it's "raise x from None", since None can
 599             # never be a module name
 600             (r'None\b', Name.Builtin.Pseudo, '#pop'),
 601             # sadly, in "raise x from y" y will be highlighted as namespace too
 602             (r'[a-zA-Z_.][\w.]*', Name.Namespace),
 603             # anything else here also means "raise x from y" and is therefore
 604             # not an error
 605             default('#pop'),
 606         ],
 607         'stringescape': [
 608             (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
 609              r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
 610         ],
 611         'strings-single': innerstring_rules(String.Single),
 612         'strings-double': innerstring_rules(String.Double),
 613         'dqs': [
 614             (r'"', String.Double, '#pop'),
 615             (r'\\\\|\\"|\\\n', String.Escape),  # included here for raw strings
 616             include('strings-double')
 617         ],
 618         'sqs': [
 619             (r"'", String.Single, '#pop'),
 620             (r"\\\\|\\'|\\\n", String.Escape),  # included here for raw strings
 621             include('strings-single')
 622         ],
 623         'tdqs': [
 624             (r'"""', String.Double, '#pop'),
 625             include('strings-double'),
 626             (r'\n', String.Double)
 627         ],
 628         'tsqs': [
 629             (r"'''", String.Single, '#pop'),
 630             include('strings-single'),
 631             (r'\n', String.Single)
 632         ],
 633     }
 634
 635     def analyse_text(text):
 636         return shebang_matches(text, r'pythonw?2(\.\d)?')
 637
 638 class _PythonConsoleLexerBase(RegexLexer):
 639     name = 'Python console session'
 640     aliases = ['pycon']
 641     mimetypes = ['text/x-python-doctest']
 642
 643     """Auxiliary lexer for `PythonConsoleLexer`.
 644
 645     Code tokens are output as ``Token.Other.Code``, traceback tokens as
 646     ``Token.Other.Traceback``.
 647     """
 648     tokens = {
 649         'root': [
 650             (r'(>>> )(.*\n)', bygroups(Generic.Prompt, Other.Code), 'continuations'),
 651             # This happens, e.g., when tracebacks are embedded in documentation;
 652             # trailing whitespaces are often stripped in such contexts.
 653             (r'(>>>)(\n)', bygroups(Generic.Prompt, Whitespace)),
 654             (r'(\^C)?Traceback \(most recent call last\):\n', Other.Traceback, 'traceback'),
 655             # SyntaxError starts with this
 656             (r'  File "[^"]+", line \d+', Other.Traceback, 'traceback'),
 657             (r'.*\n', Generic.Output),
 658         ],
 659         'continuations': [
 660             (r'(\.\.\. )(.*\n)', bygroups(Generic.Prompt, Other.Code)),
 661             # See above.
 662             (r'(\.\.\.)(\n)', bygroups(Generic.Prompt, Whitespace)),
 663             default('#pop'),
 664         ],
 665         'traceback': [
 666             # As soon as we see a traceback, consume everything until the next
 667             # >>> prompt.
 668             (r'(?=>>>( |$))', Text, '#pop'),
 669             (r'(KeyboardInterrupt)(\n)', bygroups(Name.Class, Whitespace)),
 670             (r'.*\n', Other.Traceback),
 671         ],
 672     }
 673
 674 class PythonConsoleLexer(DelegatingLexer):
 675     """
 676     For Python console output or doctests, such as:
 677
 678     .. sourcecode:: pycon
 679
 680         >>> a = 'foo'
 681         >>> print(a)
 682         foo
 683         >>> 1 / 0
 684         Traceback (most recent call last):
 685           File "<stdin>", line 1, in <module>
 686         ZeroDivisionError: integer division or modulo by zero
 687
 688     Additional options:
 689
 690     `python3`
 691         Use Python 3 lexer for code.  Default is ``True``.
 692
 693         .. versionadded:: 1.0
 694         .. versionchanged:: 2.5
 695            Now defaults to ``True``.
 696     """
 697
 698     name = 'Python console session'
 699     aliases = ['pycon']
 700     mimetypes = ['text/x-python-doctest']
 701
 702     def __init__(self, **options):
 703         python3 = get_bool_opt(options, 'python3', True)
 704         if python3:
 705             pylexer = PythonLexer
 706             tblexer = PythonTracebackLexer
 707         else:
 708             pylexer = Python2Lexer
 709             tblexer = Python2TracebackLexer
 710         # We have two auxiliary lexers. Use DelegatingLexer twice with
 711         # different tokens.  TODO: DelegatingLexer should support this
 712         # directly, by accepting a tuplet of auxiliary lexers and a tuple of
 713         # distinguishing tokens. Then we wouldn't need this intermediary
 714         # class.
 715         class _ReplaceInnerCode(DelegatingLexer):
 716             def __init__(self, **options):
 717                 super().__init__(pylexer, _PythonConsoleLexerBase, Other.Code, **options)
 718         super().__init__(tblexer, _ReplaceInnerCode, Other.Traceback, **options)
 719
 720 class PythonTracebackLexer(RegexLexer):
 721     """
 722     For Python 3.x tracebacks, with support for chained exceptions.
 723
 724     .. versionadded:: 1.0
 725
 726     .. versionchanged:: 2.5
 727        This is now the default ``PythonTracebackLexer``.  It is still available
 728        as the alias ``Python3TracebackLexer``.
 729     """
 730
 731     name = 'Python Traceback'
 732     aliases = ['pytb', 'py3tb']
 733     filenames = ['*.pytb', '*.py3tb']
 734     mimetypes = ['text/x-python-traceback', 'text/x-python3-traceback']
 735
 736     tokens = {
 737         'root': [
 738             (r'\n', Whitespace),
 739             (r'^(\^C)?Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'),
 740             (r'^During handling of the above exception, another '
 741              r'exception occurred:\n\n', Generic.Traceback),
 742             (r'^The above exception was the direct cause of the '
 743              r'following exception:\n\n', Generic.Traceback),
 744             (r'^(?=  File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
 745             (r'^.*\n', Other),
 746         ],
 747         'intb': [
 748             (r'^(  File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
 749              bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)),
 750             (r'^(  File )("[^"]+")(, line )(\d+)(\n)',
 751              bygroups(Text, Name.Builtin, Text, Number, Whitespace)),
 752             (r'^(    )(.+)(\n)',
 753              bygroups(Whitespace, using(PythonLexer), Whitespace), 'markers'),
 754             (r'^([ \t]*)(\.\.\.)(\n)',
 755              bygroups(Whitespace, Comment, Whitespace)),  # for doctests...
 756             (r'^([^:]+)(: )(.+)(\n)',
 757              bygroups(Generic.Error, Text, Name, Whitespace), '#pop'),
 758             (r'^([a-zA-Z_][\w.]*)(:?\n)',
 759              bygroups(Generic.Error, Whitespace), '#pop'),
 760             default('#pop'),
 761         ],
 762         'markers': [
 763             # Either `PEP 657 <https://www.python.org/dev/peps/pep-0657/>`
 764             # error locations in Python 3.11+, or single-caret markers
 765             # for syntax errors before that.
 766             (r'^( {4,})([~^]+)(\n)',
 767              bygroups(Whitespace, Punctuation.Marker, Whitespace),
 768              '#pop'),
 769             default('#pop'),
 770         ],
 771     }
 772
 773
 774 Python3TracebackLexer = PythonTracebackLexer
 775
 776
 777 class Python2TracebackLexer(RegexLexer):
 778     """
 779     For Python tracebacks.
 780
 781     .. versionadded:: 0.7
 782
 783     .. versionchanged:: 2.5
 784        This class has been renamed from ``PythonTracebackLexer``.
 785        ``PythonTracebackLexer`` now refers to the Python 3 variant.
 786     """
 787
 788     name = 'Python 2.x Traceback'
 789     aliases = ['py2tb']
 790     filenames = ['*.py2tb']
 791     mimetypes = ['text/x-python2-traceback']
 792
 793     tokens = {
 794         'root': [
 795             # Cover both (most recent call last) and (innermost last)
 796             # The optional ^C allows us to catch keyboard interrupt signals.
 797             (r'^(\^C)?(Traceback.*\n)',
 798              bygroups(Text, Generic.Traceback), 'intb'),
 799             # SyntaxError starts with this.
 800             (r'^(?=  File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
 801             (r'^.*\n', Other),
 802         ],
 803         'intb': [
 804             (r'^(  File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
 805              bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)),
 806             (r'^(  File )("[^"]+")(, line )(\d+)(\n)',
 807              bygroups(Text, Name.Builtin, Text, Number, Whitespace)),
 808             (r'^(    )(.+)(\n)',
 809              bygroups(Text, using(Python2Lexer), Whitespace), 'marker'),
 810             (r'^([ \t]*)(\.\.\.)(\n)',
 811              bygroups(Text, Comment, Whitespace)),  # for doctests...
 812             (r'^([^:]+)(: )(.+)(\n)',
 813              bygroups(Generic.Error, Text, Name, Whitespace), '#pop'),
 814             (r'^([a-zA-Z_]\w*)(:?\n)',
 815              bygroups(Generic.Error, Whitespace), '#pop')
 816         ],
 817         'marker': [
 818             # For syntax errors.
 819             (r'( {4,})(\^)', bygroups(Text, Punctuation.Marker), '#pop'),
 820             default('#pop'),
 821         ],
 822     }
 823
 824
 825 class CythonLexer(RegexLexer):
 826     """
 827     For Pyrex and Cython source code.
 828
 829     .. versionadded:: 1.1
 830     """
 831
 832     name = 'Cython'
 833     url = 'http://cython.org'
 834     aliases = ['cython', 'pyx', 'pyrex']
 835     filenames = ['*.pyx', '*.pxd', '*.pxi']
 836     mimetypes = ['text/x-cython', 'application/x-cython']
 837
 838     tokens = {
 839         'root': [
 840             (r'\n', Whitespace),
 841             (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Whitespace, String.Doc)),
 842             (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Whitespace, String.Doc)),
 843             (r'[^\S\n]+', Text),
 844             (r'#.*$', Comment),
 845             (r'[]{}:(),;[]', Punctuation),
 846             (r'\\\n', Whitespace),
 847             (r'\\', Text),
 848             (r'(in|is|and|or|not)\b', Operator.Word),
 849             (r'(<)([a-zA-Z0-9.?]+)(>)',
 850              bygroups(Punctuation, Keyword.Type, Punctuation)),
 851             (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator),
 852             (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)',
 853              bygroups(Keyword, Number.Integer, Operator, Name, Operator,
 854                       Name, Punctuation)),
 855             include('keywords'),
 856             (r'(def|property)(\s+)', bygroups(Keyword, Text), 'funcname'),
 857             (r'(cp?def)(\s+)', bygroups(Keyword, Text), 'cdef'),
 858             # (should actually start a block with only cdefs)
 859             (r'(cdef)(:)', bygroups(Keyword, Punctuation)),
 860             (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'classname'),
 861             (r'(from)(\s+)', bygroups(Keyword, Text), 'fromimport'),
 862             (r'(c?import)(\s+)', bygroups(Keyword, Text), 'import'),
 863             include('builtins'),
 864             include('backtick'),
 865             ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'),
 866             ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'),
 867             ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'),
 868             ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'),
 869             ('[uU]?"""', String, combined('stringescape', 'tdqs')),
 870             ("[uU]?'''", String, combined('stringescape', 'tsqs')),
 871             ('[uU]?"', String, combined('stringescape', 'dqs')),
 872             ("[uU]?'", String, combined('stringescape', 'sqs')),
 873             include('name'),
 874             include('numbers'),
 875         ],
 876         'keywords': [
 877             (words((
 878                 'assert', 'async', 'await', 'break', 'by', 'continue', 'ctypedef', 'del', 'elif',
 879                 'else', 'except', 'except?', 'exec', 'finally', 'for', 'fused', 'gil',
 880                 'global', 'if', 'include', 'lambda', 'nogil', 'pass', 'print',
 881                 'raise', 'return', 'try', 'while', 'yield', 'as', 'with'), suffix=r'\b'),
 882              Keyword),
 883             (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc),
 884         ],
 885         'builtins': [
 886             (words((
 887                 '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', 'bint',
 888                 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr',
 889                 'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'delattr',
 890                 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'execfile', 'exit',
 891                 'file', 'filter', 'float', 'frozenset', 'getattr', 'globals',
 892                 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'intern', 'isinstance',
 893                 'issubclass', 'iter', 'len', 'list', 'locals', 'long', 'map', 'max',
 894                 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property', 'Py_ssize_t',
 895                 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed',
 896                 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod',
 897                 'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode', 'unsigned',
 898                 'vars', 'xrange', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'),
 899              Name.Builtin),
 900             (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|NULL'
 901              r')\b', Name.Builtin.Pseudo),
 902             (words((
 903                 'ArithmeticError', 'AssertionError', 'AttributeError',
 904                 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError',
 905                 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit',
 906                 'IOError', 'ImportError', 'ImportWarning', 'IndentationError',
 907                 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError',
 908                 'MemoryError', 'NameError', 'NotImplemented', 'NotImplementedError',
 909                 'OSError', 'OverflowError', 'OverflowWarning',
 910                 'PendingDeprecationWarning', 'ReferenceError', 'RuntimeError',
 911                 'RuntimeWarning', 'StandardError', 'StopIteration', 'SyntaxError',
 912                 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError',
 913                 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
 914                 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
 915                 'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning',
 916                 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
 917              Name.Exception),
 918         ],
 919         'numbers': [
 920             (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
 921             (r'0\d+', Number.Oct),
 922             (r'0[xX][a-fA-F0-9]+', Number.Hex),
 923             (r'\d+L', Number.Integer.Long),
 924             (r'\d+', Number.Integer)
 925         ],
 926         'backtick': [
 927             ('`.*?`', String.Backtick),
 928         ],
 929         'name': [
 930             (r'@\w+', Name.Decorator),
 931             (r'[a-zA-Z_]\w*', Name),
 932         ],
 933         'funcname': [
 934             (r'[a-zA-Z_]\w*', Name.Function, '#pop')
 935         ],
 936         'cdef': [
 937             (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved),
 938             (r'(struct|enum|union|class)\b', Keyword),
 939             (r'([a-zA-Z_]\w*)(\s*)(?=[(:#=]|$)',
 940              bygroups(Name.Function, Text), '#pop'),
 941             (r'([a-zA-Z_]\w*)(\s*)(,)',
 942              bygroups(Name.Function, Text, Punctuation)),
 943             (r'from\b', Keyword, '#pop'),
 944             (r'as\b', Keyword),
 945             (r':', Punctuation, '#pop'),
 946             (r'(?=["\'])', Text, '#pop'),
 947             (r'[a-zA-Z_]\w*', Keyword.Type),
 948             (r'.', Text),
 949         ],
 950         'classname': [
 951             (r'[a-zA-Z_]\w*', Name.Class, '#pop')
 952         ],
 953         'import': [
 954             (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)),
 955             (r'[a-zA-Z_][\w.]*', Name.Namespace),
 956             (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)),
 957             default('#pop')  # all else: go back
 958         ],
 959         'fromimport': [
 960             (r'(\s+)(c?import)\b', bygroups(Text, Keyword), '#pop'),
 961             (r'[a-zA-Z_.][\w.]*', Name.Namespace),
 962             # ``cdef foo from "header"``, or ``for foo from 0 < i < 10``
 963             default('#pop'),
 964         ],
 965         'stringescape': [
 966             (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
 967              r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
 968         ],
 969         'strings': [
 970             (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
 971              '[hlL]?[E-GXc-giorsux%]', String.Interpol),
 972             (r'[^\\\'"%\n]+', String),
 973             # quotes, percents and backslashes must be parsed one at a time
 974             (r'[\'"\\]', String),
 975             # unhandled string formatting sign
 976             (r'%', String)
 977             # newlines are an error (use "nl" state)
 978         ],
 979         'nl': [
 980             (r'\n', String)
 981         ],
 982         'dqs': [
 983             (r'"', String, '#pop'),
 984             (r'\\\\|\\"|\\\n', String.Escape),  # included here again for raw strings
 985             include('strings')
 986         ],
 987         'sqs': [
 988             (r"'", String, '#pop'),
 989             (r"\\\\|\\'|\\\n", String.Escape),  # included here again for raw strings
 990             include('strings')
 991         ],
 992         'tdqs': [
 993             (r'"""', String, '#pop'),
 994             include('strings'),
 995             include('nl')
 996         ],
 997         'tsqs': [
 998             (r"'''", String, '#pop'),
 999             include('strings'),
1000             include('nl')
1001         ],
1002     }
1003
1004
1005 class DgLexer(RegexLexer):
1006     """
1007     Lexer for dg,
1008     a functional and object-oriented programming language
1009     running on the CPython 3 VM.
1010
1011     .. versionadded:: 1.6
1012     """
1013     name = 'dg'
1014     aliases = ['dg']
1015     filenames = ['*.dg']
1016     mimetypes = ['text/x-dg']
1017
1018     tokens = {
1019         'root': [
1020             (r'\s+', Text),
1021             (r'#.*?$', Comment.Single),
1022
1023             (r'(?i)0b[01]+', Number.Bin),
1024             (r'(?i)0o[0-7]+', Number.Oct),
1025             (r'(?i)0x[0-9a-f]+', Number.Hex),
1026             (r'(?i)[+-]?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?j?', Number.Float),
1027             (r'(?i)[+-]?[0-9]+e[+-]?\d+j?', Number.Float),
1028             (r'(?i)[+-]?[0-9]+j?', Number.Integer),
1029
1030             (r"(?i)(br|r?b?)'''", String, combined('stringescape', 'tsqs', 'string')),
1031             (r'(?i)(br|r?b?)"""', String, combined('stringescape', 'tdqs', 'string')),
1032             (r"(?i)(br|r?b?)'", String, combined('stringescape', 'sqs', 'string')),
1033             (r'(?i)(br|r?b?)"', String, combined('stringescape', 'dqs', 'string')),
1034
1035             (r"`\w+'*`", Operator),
1036             (r'\b(and|in|is|or|where)\b', Operator.Word),
1037             (r'[!$%&*+\-./:<-@\\^|~;,]+', Operator),
1038
1039             (words((
1040                 'bool', 'bytearray', 'bytes', 'classmethod', 'complex', 'dict', 'dict\'',
1041                 'float', 'frozenset', 'int', 'list', 'list\'', 'memoryview', 'object',
1042                 'property', 'range', 'set', 'set\'', 'slice', 'staticmethod', 'str',
1043                 'super', 'tuple', 'tuple\'', 'type'),
1044                    prefix=r'(?<!\.)', suffix=r'(?![\'\w])'),
1045              Name.Builtin),
1046             (words((
1047                 '__import__', 'abs', 'all', 'any', 'bin', 'bind', 'chr', 'cmp', 'compile',
1048                 'complex', 'delattr', 'dir', 'divmod', 'drop', 'dropwhile', 'enumerate',
1049                 'eval', 'exhaust', 'filter', 'flip', 'foldl1?', 'format', 'fst',
1050                 'getattr', 'globals', 'hasattr', 'hash', 'head', 'hex', 'id', 'init',
1051                 'input', 'isinstance', 'issubclass', 'iter', 'iterate', 'last', 'len',
1052                 'locals', 'map', 'max', 'min', 'next', 'oct', 'open', 'ord', 'pow',
1053                 'print', 'repr', 'reversed', 'round', 'setattr', 'scanl1?', 'snd',
1054                 'sorted', 'sum', 'tail', 'take', 'takewhile', 'vars', 'zip'),
1055                    prefix=r'(?<!\.)', suffix=r'(?![\'\w])'),
1056              Name.Builtin),
1057             (r"(?<!\.)(self|Ellipsis|NotImplemented|None|True|False)(?!['\w])",
1058              Name.Builtin.Pseudo),
1059
1060             (r"(?<!\.)[A-Z]\w*(Error|Exception|Warning)'*(?!['\w])",
1061              Name.Exception),
1062             (r"(?<!\.)(Exception|GeneratorExit|KeyboardInterrupt|StopIteration|"
1063              r"SystemExit)(?!['\w])", Name.Exception),
1064
1065             (r"(?<![\w.])(except|finally|for|if|import|not|otherwise|raise|"
1066              r"subclass|while|with|yield)(?!['\w])", Keyword.Reserved),
1067
1068             (r"[A-Z_]+'*(?!['\w])", Name),
1069             (r"[A-Z]\w+'*(?!['\w])", Keyword.Type),
1070             (r"\w+'*", Name),
1071
1072             (r'[()]', Punctuation),
1073             (r'.', Error),
1074         ],
1075         'stringescape': [
1076             (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
1077              r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
1078         ],
1079         'string': [
1080             (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
1081              '[hlL]?[E-GXc-giorsux%]', String.Interpol),
1082             (r'[^\\\'"%\n]+', String),
1083             # quotes, percents and backslashes must be parsed one at a time
1084             (r'[\'"\\]', String),
1085             # unhandled string formatting sign
1086             (r'%', String),
1087             (r'\n', String)
1088         ],
1089         'dqs': [
1090             (r'"', String, '#pop')
1091         ],
1092         'sqs': [
1093             (r"'", String, '#pop')
1094         ],
1095         'tdqs': [
1096             (r'"""', String, '#pop')
1097         ],
1098         'tsqs': [
1099             (r"'''", String, '#pop')
1100         ],
1101     }
1102
1103
1104 class NumPyLexer(PythonLexer):
1105     """
1106     A Python lexer recognizing Numerical Python builtins.
1107
1108     .. versionadded:: 0.10
1109     """
1110
1111     name = 'NumPy'
1112     url = 'https://numpy.org/'
1113     aliases = ['numpy']
1114
1115     # override the mimetypes to not inherit them from python
1116     mimetypes = []
1117     filenames = []
1118
1119     EXTRA_KEYWORDS = {
1120         'abs', 'absolute', 'accumulate', 'add', 'alen', 'all', 'allclose',
1121         'alltrue', 'alterdot', 'amax', 'amin', 'angle', 'any', 'append',
1122         'apply_along_axis', 'apply_over_axes', 'arange', 'arccos', 'arccosh',
1123         'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'argmax', 'argmin',
1124         'argsort', 'argwhere', 'around', 'array', 'array2string', 'array_equal',
1125         'array_equiv', 'array_repr', 'array_split', 'array_str', 'arrayrange',
1126         'asanyarray', 'asarray', 'asarray_chkfinite', 'ascontiguousarray',
1127         'asfarray', 'asfortranarray', 'asmatrix', 'asscalar', 'astype',
1128         'atleast_1d', 'atleast_2d', 'atleast_3d', 'average', 'bartlett',
1129         'base_repr', 'beta', 'binary_repr', 'bincount', 'binomial',
1130         'bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor', 'blackman',
1131         'bmat', 'broadcast', 'byte_bounds', 'bytes', 'byteswap', 'c_',
1132         'can_cast', 'ceil', 'choose', 'clip', 'column_stack', 'common_type',
1133         'compare_chararrays', 'compress', 'concatenate', 'conj', 'conjugate',
1134         'convolve', 'copy', 'corrcoef', 'correlate', 'cos', 'cosh', 'cov',
1135         'cross', 'cumprod', 'cumproduct', 'cumsum', 'delete', 'deprecate',
1136         'diag', 'diagflat', 'diagonal', 'diff', 'digitize', 'disp', 'divide',
1137         'dot', 'dsplit', 'dstack', 'dtype', 'dump', 'dumps', 'ediff1d', 'empty',
1138         'empty_like', 'equal', 'exp', 'expand_dims', 'expm1', 'extract', 'eye',
1139         'fabs', 'fastCopyAndTranspose', 'fft', 'fftfreq', 'fftshift', 'fill',
1140         'finfo', 'fix', 'flat', 'flatnonzero', 'flatten', 'fliplr', 'flipud',
1141         'floor', 'floor_divide', 'fmod', 'frexp', 'fromarrays', 'frombuffer',
1142         'fromfile', 'fromfunction', 'fromiter', 'frompyfunc', 'fromstring',
1143         'generic', 'get_array_wrap', 'get_include', 'get_numarray_include',
1144         'get_numpy_include', 'get_printoptions', 'getbuffer', 'getbufsize',
1145         'geterr', 'geterrcall', 'geterrobj', 'getfield', 'gradient', 'greater',
1146         'greater_equal', 'gumbel', 'hamming', 'hanning', 'histogram',
1147         'histogram2d', 'histogramdd', 'hsplit', 'hstack', 'hypot', 'i0',
1148         'identity', 'ifft', 'imag', 'index_exp', 'indices', 'inf', 'info',
1149         'inner', 'insert', 'int_asbuffer', 'interp', 'intersect1d',
1150         'intersect1d_nu', 'inv', 'invert', 'iscomplex', 'iscomplexobj',
1151         'isfinite', 'isfortran', 'isinf', 'isnan', 'isneginf', 'isposinf',
1152         'isreal', 'isrealobj', 'isscalar', 'issctype', 'issubclass_',
1153         'issubdtype', 'issubsctype', 'item', 'itemset', 'iterable', 'ix_',
1154         'kaiser', 'kron', 'ldexp', 'left_shift', 'less', 'less_equal', 'lexsort',
1155         'linspace', 'load', 'loads', 'loadtxt', 'log', 'log10', 'log1p', 'log2',
1156         'logical_and', 'logical_not', 'logical_or', 'logical_xor', 'logspace',
1157         'lstsq', 'mat', 'matrix', 'max', 'maximum', 'maximum_sctype',
1158         'may_share_memory', 'mean', 'median', 'meshgrid', 'mgrid', 'min',
1159         'minimum', 'mintypecode', 'mod', 'modf', 'msort', 'multiply', 'nan',
1160         'nan_to_num', 'nanargmax', 'nanargmin', 'nanmax', 'nanmin', 'nansum',
1161         'ndenumerate', 'ndim', 'ndindex', 'negative', 'newaxis', 'newbuffer',
1162         'newbyteorder', 'nonzero', 'not_equal', 'obj2sctype', 'ogrid', 'ones',
1163         'ones_like', 'outer', 'permutation', 'piecewise', 'pinv', 'pkgload',
1164         'place', 'poisson', 'poly', 'poly1d', 'polyadd', 'polyder', 'polydiv',
1165         'polyfit', 'polyint', 'polymul', 'polysub', 'polyval', 'power', 'prod',
1166         'product', 'ptp', 'put', 'putmask', 'r_', 'randint', 'random_integers',
1167         'random_sample', 'ranf', 'rank', 'ravel', 'real', 'real_if_close',
1168         'recarray', 'reciprocal', 'reduce', 'remainder', 'repeat', 'require',
1169         'reshape', 'resize', 'restoredot', 'right_shift', 'rint', 'roll',
1170         'rollaxis', 'roots', 'rot90', 'round', 'round_', 'row_stack', 's_',
1171         'sample', 'savetxt', 'sctype2char', 'searchsorted', 'seed', 'select',
1172         'set_numeric_ops', 'set_printoptions', 'set_string_function',
1173         'setbufsize', 'setdiff1d', 'seterr', 'seterrcall', 'seterrobj',
1174         'setfield', 'setflags', 'setmember1d', 'setxor1d', 'shape',
1175         'show_config', 'shuffle', 'sign', 'signbit', 'sin', 'sinc', 'sinh',
1176         'size', 'slice', 'solve', 'sometrue', 'sort', 'sort_complex', 'source',
1177         'split', 'sqrt', 'square', 'squeeze', 'standard_normal', 'std',
1178         'subtract', 'sum', 'svd', 'swapaxes', 'take', 'tan', 'tanh', 'tensordot',
1179         'test', 'tile', 'tofile', 'tolist', 'tostring', 'trace', 'transpose',
1180         'trapz', 'tri', 'tril', 'trim_zeros', 'triu', 'true_divide', 'typeDict',
1181         'typename', 'uniform', 'union1d', 'unique', 'unique1d', 'unravel_index',
1182         'unwrap', 'vander', 'var', 'vdot', 'vectorize', 'view', 'vonmises',
1183         'vsplit', 'vstack', 'weibull', 'where', 'who', 'zeros', 'zeros_like'
1184     }
1185
1186     def get_tokens_unprocessed(self, text):
1187         for index, token, value in \
1188                 PythonLexer.get_tokens_unprocessed(self, text):
1189             if token is Name and value in self.EXTRA_KEYWORDS:
1190                 yield index, Keyword.Pseudo, value
1191             else:
1192                 yield index, token, value
1193
1194     def analyse_text(text):
1195         ltext = text[:1000]
1196         return (shebang_matches(text, r'pythonw?(3(\.\d)?)?') or
1197                 'import ' in ltext) \
1198             and ('import numpy' in ltext or 'from numpy import' in ltext)