Method split_on_punctuation

cmd2/parsing.py:698–757

Further splits tokens from a command line using punctuation characters. Punctuation characters are treated as word breaks when they are in unquoted strings. Each run of punctuation characters is treated as a single token. :param tokens: the tokens as parsed by shlex

(self, tokens: list[str])

        return command, args

    def split_on_punctuation(self, tokens: list[str]) -> list[str]:
        """Further splits tokens from a command line using punctuation characters.

        Punctuation characters are treated as word breaks when they are in
        unquoted strings. Each run of punctuation characters is treated as a
        single token.

        :param tokens: the tokens as parsed by shlex
        :return: a new list of tokens, further split using punctuation
        """
        # Using a set for faster lookups
        punctuation = {*self.terminators, *constants.REDIRECTION_CHARS}

        punctuated_tokens = []

        for cur_initial_token in tokens:
            # Save tokens up to 1 character in length or quoted tokens. No need to parse these.
            if len(cur_initial_token) <= 1 or cur_initial_token[0] in constants.QUOTES:
                punctuated_tokens.append(cur_initial_token)
                continue

            # Iterate over each character in this token
            cur_index = 0
            cur_char = cur_initial_token[cur_index]

            # Keep track of the token we are building
            new_token = ""

            while True:
                if cur_char not in punctuation:
                    # Keep appending to new_token until we hit a punctuation char
                    while cur_char not in punctuation:
                        new_token += cur_char
                        cur_index += 1
                        if cur_index < len(cur_initial_token):
                            cur_char = cur_initial_token[cur_index]
                        else:
                            break

                else:
                    cur_punc = cur_char

                    # Keep appending to new_token until we hit something other than cur_punc
                    while cur_char == cur_punc:
                        new_token += cur_char
                        cur_index += 1
                        if cur_index < len(cur_initial_token):
                            cur_char = cur_initial_token[cur_index]
                        else:
                            break

                # Save the new token
                punctuated_tokens.append(new_token)
                new_token = ""

                # Check if we've viewed all characters
                if cur_index >= len(cur_initial_token):
                    break