LLM fine-tuning step: Tokenizing

Tokenization is a crucial phase in fine-tuning the LLM, requiring us to:

1. Encode the input, which is typically a form of guiding text, such as a question.
2. Decode the output, usually it’s a courteous response after the “generate()” method has been invoked.

What is kind Tokenizing

In simple terms, it means using a dictionary (an instance of the tokenizer class) to transform words into something that computers can better understand, that is, digitization. Of course, this is a simplified explanation, as shown in the figure:

Boilerplate codes

In fact, the vast majority of work is boilerplate code, which is repeated and reused in daily work.

Example (just encode and decode)

from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("chat gpt")
text = "Hi, how are you?"
encoded_text = tokenizer(text)["input_ids"]
decoded_text = tokenizer.decode(encoded_text)
print("Decoded tokens back into text: ", decoded_text)

Example(preprocessing after load HF dataset api)

import datasets

def tokenizing(example):
    #
    # Callback for each examples
    # examples["question"] is a list containing str.
    # 
    if "question" in example and "answer" in example:
      text = example["question"][0] + example["answer"][0]
    elif "input" in example and "output" in example:
      text = example["input"][0] + example["output"][0]
    else:
      text = example["text"][0]

    tokenizer.pad_token = tokenizer.eos_token
    tokenized_inputs = tokenizer(
        text,
        return_tensors="np",
        padding=True,
    )

    max_length = min(
        tokenized_inputs["input_ids"].shape[1],
        2048
    )
    tokenizer.truncation_side = "left"
    tokenized_inputs = tokenizer(
        text,
        return_tensors="np",
        truncation=True,
        max_length=max_length
    )

    return tokenized_inputs

finetuning_dataset_loaded = datasets.load_dataset("json", data_files=filename, split="train")

tokenized_dataset = finetuning_dataset_loaded.map(
    tokenizing,
    batched=True,
    batch_size=1,
    drop_last_batch=True
)

print(tokenized_dataset)

'''
output:
Dataset({
    features: ['question', 'answer', 'input_ids', 'attention_mask'],
    num_rows: 1400
})
'''

Example (prompting)

from transformers import AutoTokenizer

def infer(text, model, tokenizer, max_input_tokens=1000, max_output_tokens=100):
  # Tokenize
  input_ids = tokenizer.encode(
          text,
          return_tensors="pt",
          truncation=True,
          max_length=max_input_tokens
  )

  # Generate
  device = model.device
  generated_tokens_with_prompt = model.generate(
    input_ids=input_ids.to(device),
    max_length=max_output_tokens
  )

  # Decode
  generated_text_with_prompt = tokenizer.batch_decode(generated_tokens_with_prompt, skip_special_tokens=True)

  # Strip the prompt
  generated_text_answer = generated_text_with_prompt[0][len(text):]

  return generated_text_answer

tokenizer = AutoTokenizer.from_pretrained("chat gpt")
model = AutoModelForCausalLM.from_pretrained("chat gpt")

infer("hello, how to write loop in Python?", model, tokenizer)