Perform the transformations to produce input_ids given untokenized strings. Given a Tensor of untokenized strings and a tokenization function, perform all the necessary transformation for producing the input_ids input used in BERT. This includes tokenization (token and wordpiece), and concat
(inputs,
tokenize_fn,
cls_token=_CLS_TOKEN,
sep_token=_SEP_TOKEN,
pad_token=_PAD_TOKEN)
| 144 | |
| 145 | |
| 146 | def construct_input_ids(inputs, |
| 147 | tokenize_fn, |
| 148 | cls_token=_CLS_TOKEN, |
| 149 | sep_token=_SEP_TOKEN, |
| 150 | pad_token=_PAD_TOKEN): |
| 151 | """Perform the transformations to produce input_ids given untokenized strings. |
| 152 | |
| 153 | Given a Tensor of untokenized strings and a tokenization function, perform all |
| 154 | the necessary transformation for producing the input_ids input used in BERT. |
| 155 | This includes tokenization (token and wordpiece), and concat of special |
| 156 | tokens. |
| 157 | |
| 158 | Args: |
| 159 | inputs: [<tf.string>] a list of string Tensors each containing an untokenized |
| 160 | string segment. |
| 161 | tokenize_fn: a function that takes a tensor of strings and returns a |
| 162 | RaggedTensor (w/ ragged_rank=2 of tokenized wordpieces). |
| 163 | cls_token: string, the string used to represent the CLS token, default: |
| 164 | '[CLS]' |
| 165 | sep_token: string, the string used to represent the SEP token, default: |
| 166 | '[SEP]' |
| 167 | pad_token: string or int, the value used to pad tensors. Default: '[PAD]' |
| 168 | |
| 169 | Returns: |
| 170 | a Tensor of input_ids w/ necessary padding. |
| 171 | """ |
| 172 | assert isinstance(inputs, list) and inputs |
| 173 | assert isinstance(cls_token, str) |
| 174 | assert isinstance(sep_token, str) |
| 175 | |
| 176 | # TODO(hterry): add truncate budget/logic |
| 177 | |
| 178 | wordpieces = [tokenize_fn(segment) for segment in inputs] |
| 179 | |
| 180 | # Collapse the wordpiece dimension |
| 181 | collapsed_wordpieces = [collapse_dims(rt) for rt in wordpieces] |
| 182 | |
| 183 | # Concat the [CLS], [SEP] tokens to get the input to BERT model_fn |
| 184 | cls_tokens = tf.reshape( |
| 185 | tf.tile([cls_token], [collapsed_wordpieces[0].nrows()]), |
| 186 | [collapsed_wordpieces[0].nrows(), 1]) |
| 187 | sep_tokens = tf.reshape( |
| 188 | tf.tile([sep_token], [collapsed_wordpieces[0].nrows()]), |
| 189 | [collapsed_wordpieces[0].nrows(), 1]) |
| 190 | result = [cls_tokens] |
| 191 | for rt in collapsed_wordpieces: |
| 192 | result.append(rt) |
| 193 | result.append(sep_tokens) |
| 194 | return tf.concat(result, axis=1).to_tensor(pad_token) |
| 195 | |
| 196 | |
| 197 | def safe_zip(*iterables): |
nothing calls this directly
no test coverage detected
searching dependent graphs…