Method tokeniter

src/jinja2/lexer.py:669–868

This method tokenizes the text and returns the tokens in a generator. Use this method if you just want to tokenize a template. .. versionchanged:: 3.0 Only ``\\n``, ``\\r\\n`` and ``\\r`` are treated as line breaks.

(
        self,
        source: str,
        name: t.Optional[str],
        filename: t.Optional[str] = None,
        state: t.Optional[str] = None,
    )

            yield Token(lineno, token, value)

    def tokeniter(
        self,
        source: str,
        name: t.Optional[str],
        filename: t.Optional[str] = None,
        state: t.Optional[str] = None,
    ) -> t.Iterator[t.Tuple[int, str, str]]:
        """This method tokenizes the text and returns the tokens in a
        generator. Use this method if you just want to tokenize a template.

        .. versionchanged:: 3.0
            Only ``\\n``, ``\\r\\n`` and ``\\r`` are treated as line
            breaks.
        """
        lines = newline_re.split(source)[::2]

        if not self.keep_trailing_newline and lines[-1] == "":
            del lines[-1]

        source = "\n".join(lines)
        pos = 0
        lineno = 1
        stack = ["root"]

        if state is not None and state != "root":
            assert state in ("variable", "block"), "invalid state"
            stack.append(state + "_begin")

        statetokens = self.rules[stack[-1]]
        source_length = len(source)
        balancing_stack: t.List[str] = []
        newlines_stripped = 0
        line_starting = True

        while True:
            # tokenizer loop
            for regex, tokens, new_state in statetokens:
                m = regex.match(source, pos)

                # if no match we try again with the next rule
                if m is None:
                    continue

                # we only match blocks and variables if braces / parentheses
                # are balanced. continue parsing with the lower rule which
                # is the operator rule. do this only if the end tags look
                # like operators
                if balancing_stack and tokens in (
                    TOKEN_VARIABLE_END,
                    TOKEN_BLOCK_END,
                    TOKEN_LINESTATEMENT_END,
                ):
                    continue

                # tuples support more options
                if isinstance(tokens, tuple):
                    groups: t.Sequence[str] = m.groups()