Calls tokeniter + tokenize and wraps it in a token stream.
(
self,
source: str,
name: t.Optional[str] = None,
filename: t.Optional[str] = None,
state: t.Optional[str] = None,
)
| 602 | return newline_re.sub(self.newline_sequence, value) |
| 603 | |
| 604 | def tokenize( |
| 605 | self, |
| 606 | source: str, |
| 607 | name: t.Optional[str] = None, |
| 608 | filename: t.Optional[str] = None, |
| 609 | state: t.Optional[str] = None, |
| 610 | ) -> TokenStream: |
| 611 | """Calls tokeniter + tokenize and wraps it in a token stream.""" |
| 612 | stream = self.tokeniter(source, name, filename, state) |
| 613 | return TokenStream(self.wrap(stream, name, filename), name, filename) |
| 614 | |
| 615 | def wrap( |
| 616 | self, |