Estimate the number of tokens in a text string. Uses provider-specific character-per-token ratios and applies a multiplier for SQL-heavy content. Args: text: The text to estimate tokens for. provider: The LLM provider name for ratio selection. Returns: Esti
(text: str, provider: str = 'openai')
| 70 | |
| 71 | |
| 72 | def estimate_tokens(text: str, provider: str = 'openai') -> int: |
| 73 | """Estimate the number of tokens in a text string. |
| 74 | |
| 75 | Uses provider-specific character-per-token ratios and applies |
| 76 | a multiplier for SQL-heavy content. |
| 77 | |
| 78 | Args: |
| 79 | text: The text to estimate tokens for. |
| 80 | provider: The LLM provider name for ratio selection. |
| 81 | |
| 82 | Returns: |
| 83 | Estimated token count. |
| 84 | """ |
| 85 | if not text: |
| 86 | return 0 |
| 87 | |
| 88 | chars_per_token = CHARS_PER_TOKEN.get(provider, 4.0) |
| 89 | base_tokens = len(text) / chars_per_token |
| 90 | |
| 91 | # Apply SQL multiplier if content looks like SQL |
| 92 | if re.search(r'\b(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER)\b', |
| 93 | text, re.IGNORECASE): |
| 94 | base_tokens *= SQL_TOKEN_MULTIPLIER |
| 95 | |
| 96 | return int(base_tokens) + MESSAGE_OVERHEAD_TOKENS |
| 97 | |
| 98 | |
| 99 | def estimate_message_tokens(message: Message, provider: str = 'openai') -> int: |