(tokens [][]byte)
| 66 | } |
| 67 | |
| 68 | func extractQueryTrigrams(tokens [][]byte) []uint32 { |
| 69 | seen := make(map[uint32]struct{}) |
| 70 | for _, tok := range tokens { |
| 71 | if len(tok) < 3 { |
| 72 | continue |
| 73 | } |
| 74 | for i := 0; i <= len(tok)-3; i++ { |
| 75 | seen[packTrigram(tok[i], tok[i+1], tok[i+2])] = struct{}{} |
| 76 | } |
| 77 | } |
| 78 | if len(seen) == 0 { |
| 79 | return nil |
| 80 | } |
| 81 | result := make([]uint32, 0, len(seen)) |
| 82 | for g := range seen { |
| 83 | result = append(result, g) |
| 84 | } |
| 85 | return result |
| 86 | } |
| 87 | |
| 88 | func packTrigram(a, b, c byte) uint32 { |
| 89 | return uint32(toLowerASCII(a))<<16 | uint32(toLowerASCII(b))<<8 | uint32(toLowerASCII(c)) |
no test coverage detected