Comment implémenter DPO pour aligner un LLM en 2026

pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121
pip install transformers==4.44.0 datasets==2.21.0
pip install trl==0.9.6 accelerate==0.33.0 peft==0.12.0
pip install bitsandbytes==0.43.1 safetensors==0.4.3

from datasets import load_dataset
import torch

# Charger le dataset DPO prêt-à-l'emploi
dataset = load_dataset("lvwerra/stack-exchange-paired", split="train")

# Prendre un petit sous-ensemble pour test rapide (1000 exemples)
dataset = dataset.shuffle(seed=42).select(range(1000))

# Vérifier la structure
dataset[0]
print(f"Dataset size: {len(dataset)}")
print("Exemple prompt:", dataset[0]['prompt'])
print("Exemple choisi:", dataset[0]['chosen'])
print("Exemple rejeté:", dataset[0]['rejected'])

# Sauvegarder localement pour réutilisation
dataset.save_to_disk("./dpo_dataset")
print("Dataset prêt ! Exécutez ce script d'abord.")

from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
from datasets import load_from_disk

# Charger tokenizer et modèle base (GPT-2 petit)
model_name = "gpt2"
tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "left"

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.bfloat16,
    device_map="auto",
    trust_remote_code=True
)

# Charger dataset préparé
dataset = load_from_disk("./dpo_dataset")

print("Modèle et dataset chargés. Prêt pour DPO.")
print(f"Modèle params: {model.num_parameters():,}")
print(f"Vocab size: {tokenizer.vocab_size}")

from trl import DPOConfig, DPOTrainer
from transformers import TrainingArguments, AutoTokenizer
from peft import LoraConfig, get_peft_model, TaskType
from datasets import load_from_disk
import torch

# Charger tokenizer, model, dataset (de scripts précédents)
model_name = "gpt2"
tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "left"
model = AutoModelForCausalLM.from_pretrained(
    model_name, torch_dtype=torch.bfloat16, device_map="auto"
)
dataset = load_from_disk("./dpo_dataset")

# Config LoRA pour efficiency (r=16, alpha=32)
peft_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=16,
    lora_alpha=32,
    lora_dropout=0.1,
    target_modules=["c_attn", "c_proj"]
)
model = get_peft_model(model, peft_config)

# Config DPO
beta = 0.1
dpo_config = DPOConfig(
    beta=beta,
    output_dir="./dpo-gpt2",
    max_length=256,
    max_prompt_length=128,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
    num_train_epochs=1,
    learning_rate=1e-5,
    logging_steps=10,
    save_steps=100,
    remove_unused_columns=False,
)

trainer = DPOTrainer(
    model=model,
    ref_model=None,  # Auto-approximation
    args=dpo_config,
    train_dataset=dataset,
    tokenizer=tokenizer,
)

print("Trainer configuré. Lancez trainer.train() ensuite.")
trainer.model.print_trainable_parameters()

from setup_trainer import trainer  # Import du trainer configuré

# Lancer l'entraînement
train_result = trainer.train()

# Sauvegarder le modèle fine-tuné
trainer.save_model("./dpo-gpt2-final")
trainer.tokenizer.save_pretrained("./dpo-gpt2-final")

print("Entraînement terminé ! Modèle sauvé dans ./dpo-gpt2-final")
print(f"Résultats: {train_result.metrics}")

from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

# Charger modèle DPO fine-tuné
model_path = "./dpo-gpt2-final"
tokenizer = AutoTokenizer.from_pretrained(model_path)
model = AutoModelForCausalLM.from_pretrained(
    model_path, torch_dtype=torch.bfloat16, device_map="auto"
)

prompt = "Question: What is the best way to learn Python? Answer:"
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)

with torch.no_grad():
    outputs = model.generate(**inputs, max_new_tokens=50, do_sample=True, temperature=0.7)
response_dpo = tokenizer.decode(outputs[0], skip_special_tokens=True)

print("Réponse DPO:", response_dpo[len(prompt):])

# Comparaison base (téléchargez gpt2 si besoin)
model_base = AutoModelForCausalLM.from_pretrained("gpt2", torch_dtype=torch.bfloat16, device_map="auto")
outputs_base = model_base.generate(**inputs, max_new_tokens=50, do_sample=True, temperature=0.7)
response_base = tokenizer.decode(outputs_base[0], skip_special_tokens=True)
print("Réponse base:", response_base[len(prompt):])