Function approximateMessageTokens

src/utils/analyzeContext.ts:869–938

(
  messages: Message[],
)

}

async function approximateMessageTokens(
  messages: Message[],
): Promise<MessageBreakdown> {
  const microcompactResult = await microcompactMessages(messages)

  // Initialize tracking
  const breakdown: MessageBreakdown = {
    totalTokens: 0,
    toolCallTokens: 0,
    toolResultTokens: 0,
    attachmentTokens: 0,
    assistantMessageTokens: 0,
    userMessageTokens: 0,
    toolCallsByType: new Map<string, number>(),
    toolResultsByType: new Map<string, number>(),
    attachmentsByType: new Map<string, number>(),
  }

  // Build a map of tool_use_id to tool_name for easier lookup
  const toolUseIdToName = new Map<string, string>()
  for (const msg of microcompactResult.messages) {
    if (msg.type === 'assistant' && Array.isArray(msg.message!.content)) {
      for (const block of msg.message!.content) {
        if (
          typeof block !== 'string' &&
          'type' in block &&
          block.type === 'tool_use'
        ) {
          const toolUseId = 'id' in block ? (block.id as string) : undefined
          const toolName =
            (('name' in block ? block.name : undefined) as
              | string
              | undefined) || 'unknown'
          if (toolUseId) {
            toolUseIdToName.set(toolUseId, toolName)
          }
        }
      }
    }
  }

  // Process each message for detailed breakdown
  for (const msg of microcompactResult.messages) {
    if (msg.type === 'assistant') {
      processAssistantMessage(msg as AssistantMessage, breakdown)
    } else if (msg.type === 'user') {
      processUserMessage(msg as UserMessage, breakdown, toolUseIdToName)
    } else if (msg.type === 'attachment') {
      processAttachment(msg as AttachmentMessage, breakdown)
    }
  }

  // Calculate total tokens using the API for accuracy
  const approximateMessageTokens = await countTokensWithFallback(
    normalizeMessagesForAPI(microcompactResult.messages).map(_ => {
      if (_.type === 'assistant') {
        return {
          // Important: strip out fields like id, etc. -- the counting API errors if they're present