Chapter 3 questions

mewerain
Sat, 07 Mar 2026 12:40:28 +0000

Getting following errors ( Processing the data - Hugging Face LLM Course ) on executing this, any help

from transformers import AutoTokenizer

checkpoint = “bert-base-uncased”
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
tokenized_sentences_1 = tokenizer(raw_datasets[“train”][“sentence1”])
tokenized_sentences_2 = tokenizer(raw_datasets[“train”][“sentence2”])

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[30], line 7
      3 checkpoint = "bert-base-uncased"
      4 tokenizer = AutoTokenizer.from_pretrained(checkpoint)
----> 7 tokenized_sentences_1 = tokenizer(raw_datasets["train"]["sentence1"])
      8 tokenized_sentences_2 = tokenizer(raw_datasets["train"]["sentence2"])

File ~/dev/meetme2meat/course/.venv/lib/python3.14/site-packages/transformers/tokenization_utils_base.py:2456, in PreTrainedTokenizerBase.__call__(self, text, text_pair, text_target, text_pair_target, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, tokenizer_kwargs, **kwargs)
   2454     if not self._in_target_context_manager and hasattr(self, "_switch_to_input_mode"):
   2455         self._switch_to_input_mode()
-> 2456     encodings = self._encode_plus(
   2457         text=text,
   2458         text_pair=text_pair,
   2459         padding_strategy=padding_strategy,
   2460         truncation_strategy=truncation_strategy,
   2461         max_length=max_length,
   2462         **all_kwargs,
   2463     )
   2464 if text_target is not None:
   2465     if hasattr(self, "_switch_to_target_mode"):

File ~/dev/meetme2meat/course/.venv/lib/python3.14/site-packages/transformers/tokenization_utils_tokenizers.py:804, in TokenizersBackend._encode_plus(self, text, text_pair, add_special_tokens, padding_strategy, truncation_strategy, max_length, stride, is_split_into_words, pad_to_multiple_of, padding_side, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, split_special_tokens, **kwargs)
    801         return False
    803 if not _is_valid_text_input(text):
--> 804     raise ValueError(
    805         "text input must be of type `str` (single example), `list[str]` (batch or single pretokenized example) "
    806         "or `list[list[str]]` (batch of pretokenized examples) or `list[tuple[list[str], list[str]]]` (batch of pretokenized sequence pairs)."
    807     )
    809 if text_pair is not None and not _is_valid_text_input(text_pair):
    810     raise ValueError(
    811         "text input must be of type `str` (single example), `list[str]` (batch or single pretokenized example) "
    812         "or `list[list[str]]` (batch of pretokenized examples) or `list[tuple[list[str], list[str]]]` (batch of pretokenized sequence pairs)."
    813     )

ValueError: text input must be of type `str` (single example), `list[str]` (batch or single pretokenized example) or `list[list[str]]` (batch of pretokenized examples) or `list[tuple[list[str], list[str]]]` (batch of pretokenized sequence pairs).