| 117 | const dedupSet = new Set() |
| 118 | |
| 119 | const appendChunk = (chunk, kbName) => { |
| 120 | if (!chunk || typeof chunk !== 'object') return |
| 121 | const content = typeof chunk.content === 'string' ? chunk.content.trim() : '' |
| 122 | if (!content) return |
| 123 | |
| 124 | const metadata = chunk.metadata && typeof chunk.metadata === 'object' ? chunk.metadata : {} |
| 125 | const dedupKey = |
| 126 | metadata.chunk_id && typeof metadata.chunk_id === 'string' |
| 127 | ? `${kbName}::${metadata.chunk_id}` |
| 128 | : `${kbName}::${content}` |
| 129 | if (dedupSet.has(dedupKey)) return |
| 130 | dedupSet.add(dedupKey) |
| 131 | |
| 132 | const score = typeof chunk.score === 'number' ? chunk.score : null |
| 133 | normalizedChunks.push({ |
| 134 | kb_name: kbName, |
| 135 | content, |
| 136 | score, |
| 137 | metadata: { |
| 138 | source: metadata.source || '', |
| 139 | file_id: metadata.file_id || '', |
| 140 | chunk_id: metadata.chunk_id || '', |
| 141 | chunk_index: metadata.chunk_index |
| 142 | } |
| 143 | }) |
| 144 | } |
| 145 | |
| 146 | const parseToolResultContent = (content) => { |
| 147 | if (Array.isArray(content)) return content |