(ip_lens, tokenizer, random_seed)
| 140 | |
| 141 | |
| 142 | def gen_random_tokens(ip_lens, tokenizer, random_seed): |
| 143 | |
| 144 | def get_sample_from_population(population_range, sample_size): |
| 145 | # random.sample can not sample a value more than once. hence the check |
| 146 | if sample_size < len(population_range): |
| 147 | sample = random.sample(population_range, sample_size) |
| 148 | else: |
| 149 | sample = random.choices(population_range, k=sample_size) |
| 150 | |
| 151 | return sample |
| 152 | |
| 153 | input_ids = [] |
| 154 | random.seed(random_seed) |
| 155 | for ip_len in ip_lens: |
| 156 | start_ids = get_sample_from_population(range(0, tokenizer.vocab_size), |
| 157 | ip_len) |
| 158 | # Make sure it does not contain EOS token |
| 159 | eos_id = tokenizer.encode(tokenizer.eos_token, add_special_tokens=False) |
| 160 | while set(eos_id).issubset(start_ids): |
| 161 | tmp_id = (eos_id[0] + 1) % tokenizer.vocab_size |
| 162 | start_ids = [ |
| 163 | tmp_id if element == eos_id[0] else element |
| 164 | for element in start_ids |
| 165 | ] |
| 166 | input_ids.append(start_ids) |
| 167 | |
| 168 | return input_ids |
no test coverage detected