Method wrap

src/jinja2/lexer.py:615–667

This is called with the stream as returned by `tokenize` and wraps every token in a :class:`Token` and converts the value.

(
        self,
        stream: t.Iterable[t.Tuple[int, str, str]],
        name: t.Optional[str] = None,
        filename: t.Optional[str] = None,
    )

        return TokenStream(self.wrap(stream, name, filename), name, filename)

    def wrap(
        self,
        stream: t.Iterable[t.Tuple[int, str, str]],
        name: t.Optional[str] = None,
        filename: t.Optional[str] = None,
    ) -> t.Iterator[Token]:
        """This is called with the stream as returned by `tokenize` and wraps
        every token in a :class:`Token` and converts the value.
        """
        for lineno, token, value_str in stream:
            if token in ignored_tokens:
                continue

            value: t.Any = value_str

            if token == TOKEN_LINESTATEMENT_BEGIN:
                token = TOKEN_BLOCK_BEGIN
            elif token == TOKEN_LINESTATEMENT_END:
                token = TOKEN_BLOCK_END
            # we are not interested in those tokens in the parser
            elif token in (TOKEN_RAW_BEGIN, TOKEN_RAW_END):
                continue
            elif token == TOKEN_DATA:
                value = self._normalize_newlines(value_str)
            elif token == "keyword":
                token = value_str
            elif token == TOKEN_NAME:
                value = value_str

                if not value.isidentifier():
                    raise TemplateSyntaxError(
                        "Invalid character in identifier", lineno, name, filename
                    )
            elif token == TOKEN_STRING:
                # try to unescape string
                try:
                    value = (
                        self._normalize_newlines(value_str[1:-1])
                        .encode("ascii", "backslashreplace")
                        .decode("unicode-escape")
                    )
                except Exception as e:
                    msg = str(e).split(":")[-1].strip()
                    raise TemplateSyntaxError(msg, lineno, name, filename) from e
            elif token == TOKEN_INTEGER:
                value = int(value_str.replace("_", ""), 0)
            elif token == TOKEN_FLOAT:
                # remove all "_" first to support more Python versions
                value = literal_eval(value_str.replace("_", ""))
            elif token == TOKEN_OPERATOR:
                token = operators[value_str]

            yield Token(lineno, token, value)

    def tokeniter(
        self,