venv/lib/python3.11/site-packages/jinja2/parser.py

   1 """Parse tokens from the lexer into nodes for the compiler."""
   2 import typing
   3 import typing as t
   4
   5 from . import nodes
   6 from .exceptions import TemplateAssertionError
   7 from .exceptions import TemplateSyntaxError
   8 from .lexer import describe_token
   9 from .lexer import describe_token_expr
  10
  11 if t.TYPE_CHECKING:
  12     import typing_extensions as te
  13     from .environment import Environment
  14
  15 _ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include)
  16 _MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock)
  17
  18 _statement_keywords = frozenset(
  19     [
  20         "for",
  21         "if",
  22         "block",
  23         "extends",
  24         "print",
  25         "macro",
  26         "include",
  27         "from",
  28         "import",
  29         "set",
  30         "with",
  31         "autoescape",
  32     ]
  33 )
  34 _compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])
  35
  36 _math_nodes: t.Dict[str, t.Type[nodes.Expr]] = {
  37     "add": nodes.Add,
  38     "sub": nodes.Sub,
  39     "mul": nodes.Mul,
  40     "div": nodes.Div,
  41     "floordiv": nodes.FloorDiv,
  42     "mod": nodes.Mod,
  43 }
  44
  45
  46 class Parser:
  47     """This is the central parsing class Jinja uses.  It's passed to
  48     extensions and can be used to parse expressions or statements.
  49     """
  50
  51     def __init__(
  52         self,
  53         environment: "Environment",
  54         source: str,
  55         name: t.Optional[str] = None,
  56         filename: t.Optional[str] = None,
  57         state: t.Optional[str] = None,
  58     ) -> None:
  59         self.environment = environment
  60         self.stream = environment._tokenize(source, name, filename, state)
  61         self.name = name
  62         self.filename = filename
  63         self.closed = False
  64         self.extensions: t.Dict[
  65             str, t.Callable[["Parser"], t.Union[nodes.Node, t.List[nodes.Node]]]
  66         ] = {}
  67         for extension in environment.iter_extensions():
  68             for tag in extension.tags:
  69                 self.extensions[tag] = extension.parse
  70         self._last_identifier = 0
  71         self._tag_stack: t.List[str] = []
  72         self._end_token_stack: t.List[t.Tuple[str, ...]] = []
  73
  74     def fail(
  75         self,
  76         msg: str,
  77         lineno: t.Optional[int] = None,
  78         exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError,
  79     ) -> "te.NoReturn":
  80         """Convenience method that raises `exc` with the message, passed
  81         line number or last line number as well as the current name and
  82         filename.
  83         """
  84         if lineno is None:
  85             lineno = self.stream.current.lineno
  86         raise exc(msg, lineno, self.name, self.filename)
  87
  88     def _fail_ut_eof(
  89         self,
  90         name: t.Optional[str],
  91         end_token_stack: t.List[t.Tuple[str, ...]],
  92         lineno: t.Optional[int],
  93     ) -> "te.NoReturn":
  94         expected: t.Set[str] = set()
  95         for exprs in end_token_stack:
  96             expected.update(map(describe_token_expr, exprs))
  97         if end_token_stack:
  98             currently_looking: t.Optional[str] = " or ".join(
  99                 map(repr, map(describe_token_expr, end_token_stack[-1]))
 100             )
 101         else:
 102             currently_looking = None
 103
 104         if name is None:
 105             message = ["Unexpected end of template."]
 106         else:
 107             message = [f"Encountered unknown tag {name!r}."]
 108
 109         if currently_looking:
 110             if name is not None and name in expected:
 111                 message.append(
 112                     "You probably made a nesting mistake. Jinja is expecting this tag,"
 113                     f" but currently looking for {currently_looking}."
 114                 )
 115             else:
 116                 message.append(
 117                     f"Jinja was looking for the following tags: {currently_looking}."
 118                 )
 119
 120         if self._tag_stack:
 121             message.append(
 122                 "The innermost block that needs to be closed is"
 123                 f" {self._tag_stack[-1]!r}."
 124             )
 125
 126         self.fail(" ".join(message), lineno)
 127
 128     def fail_unknown_tag(
 129         self, name: str, lineno: t.Optional[int] = None
 130     ) -> "te.NoReturn":
 131         """Called if the parser encounters an unknown tag.  Tries to fail
 132         with a human readable error message that could help to identify
 133         the problem.
 134         """
 135         self._fail_ut_eof(name, self._end_token_stack, lineno)
 136
 137     def fail_eof(
 138         self,
 139         end_tokens: t.Optional[t.Tuple[str, ...]] = None,
 140         lineno: t.Optional[int] = None,
 141     ) -> "te.NoReturn":
 142         """Like fail_unknown_tag but for end of template situations."""
 143         stack = list(self._end_token_stack)
 144         if end_tokens is not None:
 145             stack.append(end_tokens)
 146         self._fail_ut_eof(None, stack, lineno)
 147
 148     def is_tuple_end(
 149         self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None
 150     ) -> bool:
 151         """Are we at the end of a tuple?"""
 152         if self.stream.current.type in ("variable_end", "block_end", "rparen"):
 153             return True
 154         elif extra_end_rules is not None:
 155             return self.stream.current.test_any(extra_end_rules)  # type: ignore
 156         return False
 157
 158     def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName:
 159         """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
 160         self._last_identifier += 1
 161         rv = object.__new__(nodes.InternalName)
 162         nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno)
 163         return rv
 164
 165     def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]:
 166         """Parse a single statement."""
 167         token = self.stream.current
 168         if token.type != "name":
 169             self.fail("tag name expected", token.lineno)
 170         self._tag_stack.append(token.value)
 171         pop_tag = True
 172         try:
 173             if token.value in _statement_keywords:
 174                 f = getattr(self, f"parse_{self.stream.current.value}")
 175                 return f()  # type: ignore
 176             if token.value == "call":
 177                 return self.parse_call_block()
 178             if token.value == "filter":
 179                 return self.parse_filter_block()
 180             ext = self.extensions.get(token.value)
 181             if ext is not None:
 182                 return ext(self)
 183
 184             # did not work out, remove the token we pushed by accident
 185             # from the stack so that the unknown tag fail function can
 186             # produce a proper error message.
 187             self._tag_stack.pop()
 188             pop_tag = False
 189             self.fail_unknown_tag(token.value, token.lineno)
 190         finally:
 191             if pop_tag:
 192                 self._tag_stack.pop()
 193
 194     def parse_statements(
 195         self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False
 196     ) -> t.List[nodes.Node]:
 197         """Parse multiple statements into a list until one of the end tokens
 198         is reached.  This is used to parse the body of statements as it also
 199         parses template data if appropriate.  The parser checks first if the
 200         current token is a colon and skips it if there is one.  Then it checks
 201         for the block end and parses until if one of the `end_tokens` is
 202         reached.  Per default the active token in the stream at the end of
 203         the call is the matched end token.  If this is not wanted `drop_needle`
 204         can be set to `True` and the end token is removed.
 205         """
 206         # the first token may be a colon for python compatibility
 207         self.stream.skip_if("colon")
 208
 209         # in the future it would be possible to add whole code sections
 210         # by adding some sort of end of statement token and parsing those here.
 211         self.stream.expect("block_end")
 212         result = self.subparse(end_tokens)
 213
 214         # we reached the end of the template too early, the subparser
 215         # does not check for this, so we do that now
 216         if self.stream.current.type == "eof":
 217             self.fail_eof(end_tokens)
 218
 219         if drop_needle:
 220             next(self.stream)
 221         return result
 222
 223     def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]:
 224         """Parse an assign statement."""
 225         lineno = next(self.stream).lineno
 226         target = self.parse_assign_target(with_namespace=True)
 227         if self.stream.skip_if("assign"):
 228             expr = self.parse_tuple()
 229             return nodes.Assign(target, expr, lineno=lineno)
 230         filter_node = self.parse_filter(None)
 231         body = self.parse_statements(("name:endset",), drop_needle=True)
 232         return nodes.AssignBlock(target, filter_node, body, lineno=lineno)
 233
 234     def parse_for(self) -> nodes.For:
 235         """Parse a for loop."""
 236         lineno = self.stream.expect("name:for").lineno
 237         target = self.parse_assign_target(extra_end_rules=("name:in",))
 238         self.stream.expect("name:in")
 239         iter = self.parse_tuple(
 240             with_condexpr=False, extra_end_rules=("name:recursive",)
 241         )
 242         test = None
 243         if self.stream.skip_if("name:if"):
 244             test = self.parse_expression()
 245         recursive = self.stream.skip_if("name:recursive")
 246         body = self.parse_statements(("name:endfor", "name:else"))
 247         if next(self.stream).value == "endfor":
 248             else_ = []
 249         else:
 250             else_ = self.parse_statements(("name:endfor",), drop_needle=True)
 251         return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)
 252
 253     def parse_if(self) -> nodes.If:
 254         """Parse an if construct."""
 255         node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)
 256         while True:
 257             node.test = self.parse_tuple(with_condexpr=False)
 258             node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))
 259             node.elif_ = []
 260             node.else_ = []
 261             token = next(self.stream)
 262             if token.test("name:elif"):
 263                 node = nodes.If(lineno=self.stream.current.lineno)
 264                 result.elif_.append(node)
 265                 continue
 266             elif token.test("name:else"):
 267                 result.else_ = self.parse_statements(("name:endif",), drop_needle=True)
 268             break
 269         return result
 270
 271     def parse_with(self) -> nodes.With:
 272         node = nodes.With(lineno=next(self.stream).lineno)
 273         targets: t.List[nodes.Expr] = []
 274         values: t.List[nodes.Expr] = []
 275         while self.stream.current.type != "block_end":
 276             if targets:
 277                 self.stream.expect("comma")
 278             target = self.parse_assign_target()
 279             target.set_ctx("param")
 280             targets.append(target)
 281             self.stream.expect("assign")
 282             values.append(self.parse_expression())
 283         node.targets = targets
 284         node.values = values
 285         node.body = self.parse_statements(("name:endwith",), drop_needle=True)
 286         return node
 287
 288     def parse_autoescape(self) -> nodes.Scope:
 289         node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)
 290         node.options = [nodes.Keyword("autoescape", self.parse_expression())]
 291         node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)
 292         return nodes.Scope([node])
 293
 294     def parse_block(self) -> nodes.Block:
 295         node = nodes.Block(lineno=next(self.stream).lineno)
 296         node.name = self.stream.expect("name").value
 297         node.scoped = self.stream.skip_if("name:scoped")
 298         node.required = self.stream.skip_if("name:required")
 299
 300         # common problem people encounter when switching from django
 301         # to jinja.  we do not support hyphens in block names, so let's
 302         # raise a nicer error message in that case.
 303         if self.stream.current.type == "sub":
 304             self.fail(
 305                 "Block names in Jinja have to be valid Python identifiers and may not"
 306                 " contain hyphens, use an underscore instead."
 307             )
 308
 309         node.body = self.parse_statements(("name:endblock",), drop_needle=True)
 310
 311         # enforce that required blocks only contain whitespace or comments
 312         # by asserting that the body, if not empty, is just TemplateData nodes
 313         # with whitespace data
 314         if node.required and not all(
 315             isinstance(child, nodes.TemplateData) and child.data.isspace()
 316             for body in node.body
 317             for child in body.nodes  # type: ignore
 318         ):
 319             self.fail("Required blocks can only contain comments or whitespace")
 320
 321         self.stream.skip_if("name:" + node.name)
 322         return node
 323
 324     def parse_extends(self) -> nodes.Extends:
 325         node = nodes.Extends(lineno=next(self.stream).lineno)
 326         node.template = self.parse_expression()
 327         return node
 328
 329     def parse_import_context(
 330         self, node: _ImportInclude, default: bool
 331     ) -> _ImportInclude:
 332         if self.stream.current.test_any(
 333             "name:with", "name:without"
 334         ) and self.stream.look().test("name:context"):
 335             node.with_context = next(self.stream).value == "with"
 336             self.stream.skip()
 337         else:
 338             node.with_context = default
 339         return node
 340
 341     def parse_include(self) -> nodes.Include:
 342         node = nodes.Include(lineno=next(self.stream).lineno)
 343         node.template = self.parse_expression()
 344         if self.stream.current.test("name:ignore") and self.stream.look().test(
 345             "name:missing"
 346         ):
 347             node.ignore_missing = True
 348             self.stream.skip(2)
 349         else:
 350             node.ignore_missing = False
 351         return self.parse_import_context(node, True)
 352
 353     def parse_import(self) -> nodes.Import:
 354         node = nodes.Import(lineno=next(self.stream).lineno)
 355         node.template = self.parse_expression()
 356         self.stream.expect("name:as")
 357         node.target = self.parse_assign_target(name_only=True).name
 358         return self.parse_import_context(node, False)
 359
 360     def parse_from(self) -> nodes.FromImport:
 361         node = nodes.FromImport(lineno=next(self.stream).lineno)
 362         node.template = self.parse_expression()
 363         self.stream.expect("name:import")
 364         node.names = []
 365
 366         def parse_context() -> bool:
 367             if self.stream.current.value in {
 368                 "with",
 369                 "without",
 370             } and self.stream.look().test("name:context"):
 371                 node.with_context = next(self.stream).value == "with"
 372                 self.stream.skip()
 373                 return True
 374             return False
 375
 376         while True:
 377             if node.names:
 378                 self.stream.expect("comma")
 379             if self.stream.current.type == "name":
 380                 if parse_context():
 381                     break
 382                 target = self.parse_assign_target(name_only=True)
 383                 if target.name.startswith("_"):
 384                     self.fail(
 385                         "names starting with an underline can not be imported",
 386                         target.lineno,
 387                         exc=TemplateAssertionError,
 388                     )
 389                 if self.stream.skip_if("name:as"):
 390                     alias = self.parse_assign_target(name_only=True)
 391                     node.names.append((target.name, alias.name))
 392                 else:
 393                     node.names.append(target.name)
 394                 if parse_context() or self.stream.current.type != "comma":
 395                     break
 396             else:
 397                 self.stream.expect("name")
 398         if not hasattr(node, "with_context"):
 399             node.with_context = False
 400         return node
 401
 402     def parse_signature(self, node: _MacroCall) -> None:
 403         args = node.args = []
 404         defaults = node.defaults = []
 405         self.stream.expect("lparen")
 406         while self.stream.current.type != "rparen":
 407             if args:
 408                 self.stream.expect("comma")
 409             arg = self.parse_assign_target(name_only=True)
 410             arg.set_ctx("param")
 411             if self.stream.skip_if("assign"):
 412                 defaults.append(self.parse_expression())
 413             elif defaults:
 414                 self.fail("non-default argument follows default argument")
 415             args.append(arg)
 416         self.stream.expect("rparen")
 417
 418     def parse_call_block(self) -> nodes.CallBlock:
 419         node = nodes.CallBlock(lineno=next(self.stream).lineno)
 420         if self.stream.current.type == "lparen":
 421             self.parse_signature(node)
 422         else:
 423             node.args = []
 424             node.defaults = []
 425
 426         call_node = self.parse_expression()
 427         if not isinstance(call_node, nodes.Call):
 428             self.fail("expected call", node.lineno)
 429         node.call = call_node
 430         node.body = self.parse_statements(("name:endcall",), drop_needle=True)
 431         return node
 432
 433     def parse_filter_block(self) -> nodes.FilterBlock:
 434         node = nodes.FilterBlock(lineno=next(self.stream).lineno)
 435         node.filter = self.parse_filter(None, start_inline=True)  # type: ignore
 436         node.body = self.parse_statements(("name:endfilter",), drop_needle=True)
 437         return node
 438
 439     def parse_macro(self) -> nodes.Macro:
 440         node = nodes.Macro(lineno=next(self.stream).lineno)
 441         node.name = self.parse_assign_target(name_only=True).name
 442         self.parse_signature(node)
 443         node.body = self.parse_statements(("name:endmacro",), drop_needle=True)
 444         return node
 445
 446     def parse_print(self) -> nodes.Output:
 447         node = nodes.Output(lineno=next(self.stream).lineno)
 448         node.nodes = []
 449         while self.stream.current.type != "block_end":
 450             if node.nodes:
 451                 self.stream.expect("comma")
 452             node.nodes.append(self.parse_expression())
 453         return node
 454
 455     @typing.overload
 456     def parse_assign_target(
 457         self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ...
 458     ) -> nodes.Name:
 459         ...
 460
 461     @typing.overload
 462     def parse_assign_target(
 463         self,
 464         with_tuple: bool = True,
 465         name_only: bool = False,
 466         extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
 467         with_namespace: bool = False,
 468     ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:
 469         ...
 470
 471     def parse_assign_target(
 472         self,
 473         with_tuple: bool = True,
 474         name_only: bool = False,
 475         extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
 476         with_namespace: bool = False,
 477     ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:
 478         """Parse an assignment target.  As Jinja allows assignments to
 479         tuples, this function can parse all allowed assignment targets.  Per
 480         default assignments to tuples are parsed, that can be disable however
 481         by setting `with_tuple` to `False`.  If only assignments to names are
 482         wanted `name_only` can be set to `True`.  The `extra_end_rules`
 483         parameter is forwarded to the tuple parsing function.  If
 484         `with_namespace` is enabled, a namespace assignment may be parsed.
 485         """
 486         target: nodes.Expr
 487
 488         if with_namespace and self.stream.look().type == "dot":
 489             token = self.stream.expect("name")
 490             next(self.stream)  # dot
 491             attr = self.stream.expect("name")
 492             target = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
 493         elif name_only:
 494             token = self.stream.expect("name")
 495             target = nodes.Name(token.value, "store", lineno=token.lineno)
 496         else:
 497             if with_tuple:
 498                 target = self.parse_tuple(
 499                     simplified=True, extra_end_rules=extra_end_rules
 500                 )
 501             else:
 502                 target = self.parse_primary()
 503
 504             target.set_ctx("store")
 505
 506         if not target.can_assign():
 507             self.fail(
 508                 f"can't assign to {type(target).__name__.lower()!r}", target.lineno
 509             )
 510
 511         return target  # type: ignore
 512
 513     def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr:
 514         """Parse an expression.  Per default all expressions are parsed, if
 515         the optional `with_condexpr` parameter is set to `False` conditional
 516         expressions are not parsed.
 517         """
 518         if with_condexpr:
 519             return self.parse_condexpr()
 520         return self.parse_or()
 521
 522     def parse_condexpr(self) -> nodes.Expr:
 523         lineno = self.stream.current.lineno
 524         expr1 = self.parse_or()
 525         expr3: t.Optional[nodes.Expr]
 526
 527         while self.stream.skip_if("name:if"):
 528             expr2 = self.parse_or()
 529             if self.stream.skip_if("name:else"):
 530                 expr3 = self.parse_condexpr()
 531             else:
 532                 expr3 = None
 533             expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
 534             lineno = self.stream.current.lineno
 535         return expr1
 536
 537     def parse_or(self) -> nodes.Expr:
 538         lineno = self.stream.current.lineno
 539         left = self.parse_and()
 540         while self.stream.skip_if("name:or"):
 541             right = self.parse_and()
 542             left = nodes.Or(left, right, lineno=lineno)
 543             lineno = self.stream.current.lineno
 544         return left
 545
 546     def parse_and(self) -> nodes.Expr:
 547         lineno = self.stream.current.lineno
 548         left = self.parse_not()
 549         while self.stream.skip_if("name:and"):
 550             right = self.parse_not()
 551             left = nodes.And(left, right, lineno=lineno)
 552             lineno = self.stream.current.lineno
 553         return left
 554
 555     def parse_not(self) -> nodes.Expr:
 556         if self.stream.current.test("name:not"):
 557             lineno = next(self.stream).lineno
 558             return nodes.Not(self.parse_not(), lineno=lineno)
 559         return self.parse_compare()
 560
 561     def parse_compare(self) -> nodes.Expr:
 562         lineno = self.stream.current.lineno
 563         expr = self.parse_math1()
 564         ops = []
 565         while True:
 566             token_type = self.stream.current.type
 567             if token_type in _compare_operators:
 568                 next(self.stream)
 569                 ops.append(nodes.Operand(token_type, self.parse_math1()))
 570             elif self.stream.skip_if("name:in"):
 571                 ops.append(nodes.Operand("in", self.parse_math1()))
 572             elif self.stream.current.test("name:not") and self.stream.look().test(
 573                 "name:in"
 574             ):
 575                 self.stream.skip(2)
 576                 ops.append(nodes.Operand("notin", self.parse_math1()))
 577             else:
 578                 break
 579             lineno = self.stream.current.lineno
 580         if not ops:
 581             return expr
 582         return nodes.Compare(expr, ops, lineno=lineno)
 583
 584     def parse_math1(self) -> nodes.Expr:
 585         lineno = self.stream.current.lineno
 586         left = self.parse_concat()
 587         while self.stream.current.type in ("add", "sub"):
 588             cls = _math_nodes[self.stream.current.type]
 589             next(self.stream)
 590             right = self.parse_concat()
 591             left = cls(left, right, lineno=lineno)
 592             lineno = self.stream.current.lineno
 593         return left
 594
 595     def parse_concat(self) -> nodes.Expr:
 596         lineno = self.stream.current.lineno
 597         args = [self.parse_math2()]
 598         while self.stream.current.type == "tilde":
 599             next(self.stream)
 600             args.append(self.parse_math2())
 601         if len(args) == 1:
 602             return args[0]
 603         return nodes.Concat(args, lineno=lineno)
 604
 605     def parse_math2(self) -> nodes.Expr:
 606         lineno = self.stream.current.lineno
 607         left = self.parse_pow()
 608         while self.stream.current.type in ("mul", "div", "floordiv", "mod"):
 609             cls = _math_nodes[self.stream.current.type]
 610             next(self.stream)
 611             right = self.parse_pow()
 612             left = cls(left, right, lineno=lineno)
 613             lineno = self.stream.current.lineno
 614         return left
 615
 616     def parse_pow(self) -> nodes.Expr:
 617         lineno = self.stream.current.lineno
 618         left = self.parse_unary()
 619         while self.stream.current.type == "pow":
 620             next(self.stream)
 621             right = self.parse_unary()
 622             left = nodes.Pow(left, right, lineno=lineno)
 623             lineno = self.stream.current.lineno
 624         return left
 625
 626     def parse_unary(self, with_filter: bool = True) -> nodes.Expr:
 627         token_type = self.stream.current.type
 628         lineno = self.stream.current.lineno
 629         node: nodes.Expr
 630
 631         if token_type == "sub":
 632             next(self.stream)
 633             node = nodes.Neg(self.parse_unary(False), lineno=lineno)
 634         elif token_type == "add":
 635             next(self.stream)
 636             node = nodes.Pos(self.parse_unary(False), lineno=lineno)
 637         else:
 638             node = self.parse_primary()
 639         node = self.parse_postfix(node)
 640         if with_filter:
 641             node = self.parse_filter_expr(node)
 642         return node
 643
 644     def parse_primary(self) -> nodes.Expr:
 645         token = self.stream.current
 646         node: nodes.Expr
 647         if token.type == "name":
 648             if token.value in ("true", "false", "True", "False"):
 649                 node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)
 650             elif token.value in ("none", "None"):
 651                 node = nodes.Const(None, lineno=token.lineno)
 652             else:
 653                 node = nodes.Name(token.value, "load", lineno=token.lineno)
 654             next(self.stream)
 655         elif token.type == "string":
 656             next(self.stream)
 657             buf = [token.value]
 658             lineno = token.lineno
 659             while self.stream.current.type == "string":
 660                 buf.append(self.stream.current.value)
 661                 next(self.stream)
 662             node = nodes.Const("".join(buf), lineno=lineno)
 663         elif token.type in ("integer", "float"):
 664             next(self.stream)
 665             node = nodes.Const(token.value, lineno=token.lineno)
 666         elif token.type == "lparen":
 667             next(self.stream)
 668             node = self.parse_tuple(explicit_parentheses=True)
 669             self.stream.expect("rparen")
 670         elif token.type == "lbracket":
 671             node = self.parse_list()
 672         elif token.type == "lbrace":
 673             node = self.parse_dict()
 674         else:
 675             self.fail(f"unexpected {describe_token(token)!r}", token.lineno)
 676         return node
 677
 678     def parse_tuple(
 679         self,
 680         simplified: bool = False,
 681         with_condexpr: bool = True,
 682         extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
 683         explicit_parentheses: bool = False,
 684     ) -> t.Union[nodes.Tuple, nodes.Expr]:
 685         """Works like `parse_expression` but if multiple expressions are
 686         delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
 687         This method could also return a regular expression instead of a tuple
 688         if no commas where found.
 689
 690         The default parsing mode is a full tuple.  If `simplified` is `True`
 691         only names and literals are parsed.  The `no_condexpr` parameter is
 692         forwarded to :meth:`parse_expression`.
 693
 694         Because tuples do not require delimiters and may end in a bogus comma
 695         an extra hint is needed that marks the end of a tuple.  For example
 696         for loops support tuples between `for` and `in`.  In that case the
 697         `extra_end_rules` is set to ``['name:in']``.
 698
 699         `explicit_parentheses` is true if the parsing was triggered by an
 700         expression in parentheses.  This is used to figure out if an empty
 701         tuple is a valid expression or not.
 702         """
 703         lineno = self.stream.current.lineno
 704         if simplified:
 705             parse = self.parse_primary
 706         elif with_condexpr:
 707             parse = self.parse_expression
 708         else:
 709
 710             def parse() -> nodes.Expr:
 711                 return self.parse_expression(with_condexpr=False)
 712
 713         args: t.List[nodes.Expr] = []
 714         is_tuple = False
 715
 716         while True:
 717             if args:
 718                 self.stream.expect("comma")
 719             if self.is_tuple_end(extra_end_rules):
 720                 break
 721             args.append(parse())
 722             if self.stream.current.type == "comma":
 723                 is_tuple = True
 724             else:
 725                 break
 726             lineno = self.stream.current.lineno
 727
 728         if not is_tuple:
 729             if args:
 730                 return args[0]
 731
 732             # if we don't have explicit parentheses, an empty tuple is
 733             # not a valid expression.  This would mean nothing (literally
 734             # nothing) in the spot of an expression would be an empty
 735             # tuple.
 736             if not explicit_parentheses:
 737                 self.fail(
 738                     "Expected an expression,"
 739                     f" got {describe_token(self.stream.current)!r}"
 740                 )
 741
 742         return nodes.Tuple(args, "load", lineno=lineno)
 743
 744     def parse_list(self) -> nodes.List:
 745         token = self.stream.expect("lbracket")
 746         items: t.List[nodes.Expr] = []
 747         while self.stream.current.type != "rbracket":
 748             if items:
 749                 self.stream.expect("comma")
 750             if self.stream.current.type == "rbracket":
 751                 break
 752             items.append(self.parse_expression())
 753         self.stream.expect("rbracket")
 754         return nodes.List(items, lineno=token.lineno)
 755
 756     def parse_dict(self) -> nodes.Dict:
 757         token = self.stream.expect("lbrace")
 758         items: t.List[nodes.Pair] = []
 759         while self.stream.current.type != "rbrace":
 760             if items:
 761                 self.stream.expect("comma")
 762             if self.stream.current.type == "rbrace":
 763                 break
 764             key = self.parse_expression()
 765             self.stream.expect("colon")
 766             value = self.parse_expression()
 767             items.append(nodes.Pair(key, value, lineno=key.lineno))
 768         self.stream.expect("rbrace")
 769         return nodes.Dict(items, lineno=token.lineno)
 770
 771     def parse_postfix(self, node: nodes.Expr) -> nodes.Expr:
 772         while True:
 773             token_type = self.stream.current.type
 774             if token_type == "dot" or token_type == "lbracket":
 775                 node = self.parse_subscript(node)
 776             # calls are valid both after postfix expressions (getattr
 777             # and getitem) as well as filters and tests
 778             elif token_type == "lparen":
 779                 node = self.parse_call(node)
 780             else:
 781                 break
 782         return node
 783
 784     def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr:
 785         while True:
 786             token_type = self.stream.current.type
 787             if token_type == "pipe":
 788                 node = self.parse_filter(node)  # type: ignore
 789             elif token_type == "name" and self.stream.current.value == "is":
 790                 node = self.parse_test(node)
 791             # calls are valid both after postfix expressions (getattr
 792             # and getitem) as well as filters and tests
 793             elif token_type == "lparen":
 794                 node = self.parse_call(node)
 795             else:
 796                 break
 797         return node
 798
 799     def parse_subscript(
 800         self, node: nodes.Expr
 801     ) -> t.Union[nodes.Getattr, nodes.Getitem]:
 802         token = next(self.stream)
 803         arg: nodes.Expr
 804
 805         if token.type == "dot":
 806             attr_token = self.stream.current
 807             next(self.stream)
 808             if attr_token.type == "name":
 809                 return nodes.Getattr(
 810                     node, attr_token.value, "load", lineno=token.lineno
 811                 )
 812             elif attr_token.type != "integer":
 813                 self.fail("expected name or number", attr_token.lineno)
 814             arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
 815             return nodes.Getitem(node, arg, "load", lineno=token.lineno)
 816         if token.type == "lbracket":
 817             args: t.List[nodes.Expr] = []
 818             while self.stream.current.type != "rbracket":
 819                 if args:
 820                     self.stream.expect("comma")
 821                 args.append(self.parse_subscribed())
 822             self.stream.expect("rbracket")
 823             if len(args) == 1:
 824                 arg = args[0]
 825             else:
 826                 arg = nodes.Tuple(args, "load", lineno=token.lineno)
 827             return nodes.Getitem(node, arg, "load", lineno=token.lineno)
 828         self.fail("expected subscript expression", token.lineno)
 829
 830     def parse_subscribed(self) -> nodes.Expr:
 831         lineno = self.stream.current.lineno
 832         args: t.List[t.Optional[nodes.Expr]]
 833
 834         if self.stream.current.type == "colon":
 835             next(self.stream)
 836             args = [None]
 837         else:
 838             node = self.parse_expression()
 839             if self.stream.current.type != "colon":
 840                 return node
 841             next(self.stream)
 842             args = [node]
 843
 844         if self.stream.current.type == "colon":
 845             args.append(None)
 846         elif self.stream.current.type not in ("rbracket", "comma"):
 847             args.append(self.parse_expression())
 848         else:
 849             args.append(None)
 850
 851         if self.stream.current.type == "colon":
 852             next(self.stream)
 853             if self.stream.current.type not in ("rbracket", "comma"):
 854                 args.append(self.parse_expression())
 855             else:
 856                 args.append(None)
 857         else:
 858             args.append(None)
 859
 860         return nodes.Slice(lineno=lineno, *args)
 861
 862     def parse_call_args(self) -> t.Tuple:
 863         token = self.stream.expect("lparen")
 864         args = []
 865         kwargs = []
 866         dyn_args = None
 867         dyn_kwargs = None
 868         require_comma = False
 869
 870         def ensure(expr: bool) -> None:
 871             if not expr:
 872                 self.fail("invalid syntax for function call expression", token.lineno)
 873
 874         while self.stream.current.type != "rparen":
 875             if require_comma:
 876                 self.stream.expect("comma")
 877
 878                 # support for trailing comma
 879                 if self.stream.current.type == "rparen":
 880                     break
 881
 882             if self.stream.current.type == "mul":
 883                 ensure(dyn_args is None and dyn_kwargs is None)
 884                 next(self.stream)
 885                 dyn_args = self.parse_expression()
 886             elif self.stream.current.type == "pow":
 887                 ensure(dyn_kwargs is None)
 888                 next(self.stream)
 889                 dyn_kwargs = self.parse_expression()
 890             else:
 891                 if (
 892                     self.stream.current.type == "name"
 893                     and self.stream.look().type == "assign"
 894                 ):
 895                     # Parsing a kwarg
 896                     ensure(dyn_kwargs is None)
 897                     key = self.stream.current.value
 898                     self.stream.skip(2)
 899                     value = self.parse_expression()
 900                     kwargs.append(nodes.Keyword(key, value, lineno=value.lineno))
 901                 else:
 902                     # Parsing an arg
 903                     ensure(dyn_args is None and dyn_kwargs is None and not kwargs)
 904                     args.append(self.parse_expression())
 905
 906             require_comma = True
 907
 908         self.stream.expect("rparen")
 909         return args, kwargs, dyn_args, dyn_kwargs
 910
 911     def parse_call(self, node: nodes.Expr) -> nodes.Call:
 912         # The lparen will be expected in parse_call_args, but the lineno
 913         # needs to be recorded before the stream is advanced.
 914         token = self.stream.current
 915         args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
 916         return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)
 917
 918     def parse_filter(
 919         self, node: t.Optional[nodes.Expr], start_inline: bool = False
 920     ) -> t.Optional[nodes.Expr]:
 921         while self.stream.current.type == "pipe" or start_inline:
 922             if not start_inline:
 923                 next(self.stream)
 924             token = self.stream.expect("name")
 925             name = token.value
 926             while self.stream.current.type == "dot":
 927                 next(self.stream)
 928                 name += "." + self.stream.expect("name").value
 929             if self.stream.current.type == "lparen":
 930                 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
 931             else:
 932                 args = []
 933                 kwargs = []
 934                 dyn_args = dyn_kwargs = None
 935             node = nodes.Filter(
 936                 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
 937             )
 938             start_inline = False
 939         return node
 940
 941     def parse_test(self, node: nodes.Expr) -> nodes.Expr:
 942         token = next(self.stream)
 943         if self.stream.current.test("name:not"):
 944             next(self.stream)
 945             negated = True
 946         else:
 947             negated = False
 948         name = self.stream.expect("name").value
 949         while self.stream.current.type == "dot":
 950             next(self.stream)
 951             name += "." + self.stream.expect("name").value
 952         dyn_args = dyn_kwargs = None
 953         kwargs = []
 954         if self.stream.current.type == "lparen":
 955             args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
 956         elif self.stream.current.type in {
 957             "name",
 958             "string",
 959             "integer",
 960             "float",
 961             "lparen",
 962             "lbracket",
 963             "lbrace",
 964         } and not self.stream.current.test_any("name:else", "name:or", "name:and"):
 965             if self.stream.current.test("name:is"):
 966                 self.fail("You cannot chain multiple tests with is")
 967             arg_node = self.parse_primary()
 968             arg_node = self.parse_postfix(arg_node)
 969             args = [arg_node]
 970         else:
 971             args = []
 972         node = nodes.Test(
 973             node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
 974         )
 975         if negated:
 976             node = nodes.Not(node, lineno=token.lineno)
 977         return node
 978
 979     def subparse(
 980         self, end_tokens: t.Optional[t.Tuple[str, ...]] = None
 981     ) -> t.List[nodes.Node]:
 982         body: t.List[nodes.Node] = []
 983         data_buffer: t.List[nodes.Node] = []
 984         add_data = data_buffer.append
 985
 986         if end_tokens is not None:
 987             self._end_token_stack.append(end_tokens)
 988
 989         def flush_data() -> None:
 990             if data_buffer:
 991                 lineno = data_buffer[0].lineno
 992                 body.append(nodes.Output(data_buffer[:], lineno=lineno))
 993                 del data_buffer[:]
 994
 995         try:
 996             while self.stream:
 997                 token = self.stream.current
 998                 if token.type == "data":
 999                     if token.value:
1000                         add_data(nodes.TemplateData(token.value, lineno=token.lineno))
1001                     next(self.stream)
1002                 elif token.type == "variable_begin":
1003                     next(self.stream)
1004                     add_data(self.parse_tuple(with_condexpr=True))
1005                     self.stream.expect("variable_end")
1006                 elif token.type == "block_begin":
1007                     flush_data()
1008                     next(self.stream)
1009                     if end_tokens is not None and self.stream.current.test_any(
1010                         *end_tokens
1011                     ):
1012                         return body
1013                     rv = self.parse_statement()
1014                     if isinstance(rv, list):
1015                         body.extend(rv)
1016                     else:
1017                         body.append(rv)
1018                     self.stream.expect("block_end")
1019                 else:
1020                     raise AssertionError("internal parsing error")
1021
1022             flush_data()
1023         finally:
1024             if end_tokens is not None:
1025                 self._end_token_stack.pop()
1026         return body
1027
1028     def parse(self) -> nodes.Template:
1029         """Parse the whole template into a `Template` node."""
1030         result = nodes.Template(self.subparse(), lineno=1)
1031         result.set_environment(self.environment)
1032         return result