# DPO Chat Example
# Train a chat model with Direct Preference Optimization
# Uses Llama2-7B and preference data (chosen vs rejected)

model: meta-llama/Llama-2-7b-chat-hf
data:
  path: examples/data/chat_preferences.jsonl
  format: sharegpt
task: dpo
backend: transformers
quantization: int8
lora_r: 64
lora_alpha: 128
lora_dropout: 0.05
lora_target_modules:
  - q_proj
  - v_proj
  - k_proj
  - out_proj
batch_size: 8
gradient_accumulation_steps: 2
num_epochs: 2
learning_rate: 5e-4
lr_scheduler_type: cosine
warmup_ratio: 0.1
weight_decay: 0.01
max_seq_length: 2048
output_dir: ./output_dpo_chat/
seed: 42
logging_steps: 10
save_steps: 100
eval_steps: 100
eval_strategy: steps
load_best_model_at_end: true
dpo_beta: 0.1