Fine-tuning LoRA z Hugging Face PEFT

Dlaczego LoRA zamiast pełnego dostrajania?

# LoRA math intuition:
# Full fine-tuning: update W (large matrix, e.g., 4096 x 4096 = 16.7M parameters)
# LoRA: instead train W = W_0 + A @ B where:
#   A has shape (4096, r) - only r*4096 params
#   B has shape (r, 4096) - only r*4096 params
#   r (rank) is typically 4, 8, or 16 - much smaller than 4096

# Memory comparison for 7B model:
# Full fine-tuning: ~80GB GPU RAM
# LoRA (rank=8): ~12GB GPU RAM - fits on a single A100 or 3090
print('LoRA makes fine-tuning accessible without massive GPU clusters')

Instalowanie wymaganych bibliotek

# pip install transformers peft trl accelerate bitsandbytes datasets

from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model, TaskType
from trl import SFTTrainer, SFTConfig
from datasets import Dataset
import torch

print('Libraries imported successfully')
print(f'GPU available: {torch.cuda.is_available()}')
if torch.cuda.is_available():
    print(f'GPU: {torch.cuda.get_device_name(0)}')
    print(f'GPU memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB')

Ładowanie modelu bazowego z kwantyzacją

from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
import torch

MODEL_NAME = 'mistralai/Mistral-7B-Instruct-v0.2'  # or 'meta-llama/Llama-3.2-3B-Instruct'

# 4-bit quantization config (QLoRA)
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,                         # quantize base model to 4-bit
    bnb_4bit_quant_type='nf4',                 # NF4 quantization type
    bnb_4bit_compute_dtype=torch.float16,      # compute in float16
    bnb_4bit_use_double_quant=True             # double quantization for extra savings
)

# Load quantized model
model = AutoModelForCausalLM.from_pretrained(
    MODEL_NAME,
    quantization_config=bnb_config,
    device_map='auto',                         # automatically distribute across GPUs
    trust_remote_code=True
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
tokenizer.pad_token = tokenizer.eos_token    # required for training batches
print(f'Model loaded. Parameters: {model.num_parameters():,}')

Konfigurowanie hiperparametrów LoRA

from peft import LoraConfig, TaskType

lora_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,         # causal language modeling
    r=8,                                   # rank: 4, 8, 16, 32 — higher = more params
    lora_alpha=16,                         # scaling: usually 2*r
    lora_dropout=0.1,                      # dropout on LoRA layers for regularization
    target_modules=['q_proj', 'v_proj',    # which weight matrices to adapt
                    'k_proj', 'o_proj',    # common to target all attention projections
                    'gate_proj', 'up_proj', 'down_proj'],  # and FFN layers
    bias='none',                           # do not adapt bias parameters
)

# Wrap the model with LoRA adapters
from peft import get_peft_model, prepare_model_for_kbit_training

model = prepare_model_for_kbit_training(model)  # prepares quantized model for training
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# Output: trainable params: 4,194,304 || all params: 3,752,071,168 || trainable%: 0.11%

Przygotowywanie zbioru danych

from datasets import Dataset
import json

def load_and_format_dataset(jsonl_path: str, tokenizer) -> Dataset:
    examples = []
    with open(jsonl_path) as f:
        for line in f:
            ex = json.loads(line.strip())
            # Apply chat template to format as expected by the model
            formatted = tokenizer.apply_chat_template(
                ex['messages'],
                tokenize=False,
                add_generation_prompt=False
            )
            examples.append({'text': formatted})
    
    return Dataset.from_list(examples)

# Load training and validation datasets
train_dataset = load_and_format_dataset('train.jsonl', tokenizer)
val_dataset = load_and_format_dataset('validation.jsonl', tokenizer)

print(f'Train examples: {len(train_dataset)}')
print(f'Validation examples: {len(val_dataset)}')
print('Sample formatted text:')
print(train_dataset[0]['text'][:300])

Trening za pomocą SFTTrainer

from trl import SFTTrainer, SFTConfig

training_args = SFTConfig(
    output_dir='./fine-tuned-model',
    num_train_epochs=2,                    # 2-3 epochs for instruction fine-tuning
    per_device_train_batch_size=4,         # increase if GPU memory allows
    gradient_accumulation_steps=4,         # effective batch size = 4*4 = 16
    learning_rate=2e-4,                    # typical LoRA learning rate
    warmup_ratio=0.05,                     # warmup for 5% of steps
    lr_scheduler_type='cosine',            # cosine decay learning rate schedule
    logging_steps=10,
    eval_strategy='steps',
    eval_steps=50,                         # evaluate on validation set every 50 steps
    save_steps=100,
    max_seq_length=2048,                   # max token length per example
    fp16=True,                             # mixed precision training
    report_to='none'                       # or 'wandb' for experiment tracking
)

trainer = SFTTrainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=val_dataset,
)

print('Starting LoRA fine-tuning...')
trainer.train()

Monitorowanie postępów treningu

# Reading training logs
# Training step logs look like:
# {'loss': 1.4523, 'grad_norm': 0.85, 'learning_rate': 0.0002, 'epoch': 0.2, 'step': 20}
# {'loss': 1.2341, 'grad_norm': 0.72, 'learning_rate': 0.00018, 'epoch': 0.4, 'step': 40}

# Validation results look like:
# {'eval_loss': 1.1823, 'eval_runtime': 12.3, 'eval_samples_per_second': 8.1, 'step': 50}
# {'eval_loss': 1.0923, 'eval_runtime': 12.1, 'eval_samples_per_second': 8.3, 'step': 100}
# {'eval_loss': 1.1234, 'eval_runtime': 12.4, 'eval_samples_per_second': 8.2, 'step': 150}
# ^ validation loss went UP at step 150 - overfitting starting

# Load the best checkpoint (lowest validation loss)
from transformers import TrainerCallback

print('Best model checkpoint is saved automatically by SFTTrainer (load_best_model_at_end=True)')

Zapisywanie i scalanie adapterów LoRA

# Save only the LoRA adapter (tiny - typically 10-100MB)
model.save_pretrained('./lora-adapter-only')
tokenizer.save_pretrained('./lora-adapter-only')

# Load the adapter for inference (requires base model + peft)
from peft import PeftModel

base_model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, torch_dtype=torch.float16, device_map='auto')
model_with_adapter = PeftModel.from_pretrained(base_model, './lora-adapter-only')

# Alternative: Merge adapter into base model (no PEFT needed at inference)
print('Merging LoRA weights into base model...')
merged_model = model_with_adapter.merge_and_unload()  # creates a regular model
merged_model.save_pretrained('./merged-model')
print('Merged model saved - can be used with standard transformers, no PEFT needed')

Wnioskowanie za pomocą dostrojonego modelu

def generate(model, tokenizer, prompt: str, max_new_tokens=512) -> str:
    inputs = tokenizer(prompt, return_tensors='pt').to(model.device)
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_new_tokens,
            temperature=0.1,
            do_sample=True,
            pad_token_id=tokenizer.eos_token_id
        )
    # Decode only the new tokens (not the input prompt)
    new_tokens = outputs[0][inputs['input_ids'].shape[1]:]
    return tokenizer.decode(new_tokens, skip_special_tokens=True)

# Compare base vs fine-tuned
test_prompt = 'Extract JSON from: "Alice Johnson, 28, software engineer in NYC since 2021"'
print('=== BASE MODEL ===')
print(generate(base_model, tokenizer, test_prompt))
print('\n=== FINE-TUNED MODEL ===')
print(generate(merged_model, tokenizer, test_prompt))

Podsumowanie kluczowych hiperparametrów LoRA

Kiedy zamiast tego użyć dostrajania OpenAI

from openai import OpenAI

client = OpenAI()

# Upload training file
train_file = client.files.create(
    file=open('train.jsonl', 'rb'),
    purpose='fine-tune'
)
val_file = client.files.create(
    file=open('validation.jsonl', 'rb'),
    purpose='fine-tune'
)

# Create fine-tuning job
job = client.fine_tuning.jobs.create(
    training_file=train_file.id,
    validation_file=val_file.id,
    model='gpt-4o-mini',  # base model to fine-tune
    hyperparameters={
        'n_epochs': 3,
        'batch_size': 'auto',
        'learning_rate_multiplier': 'auto'
    }
)
print(f'Fine-tuning job created: {job.id}')
# Monitor: client.fine_tuning.jobs.retrieve(job.id)
# Use: client.chat.completions.create(model=job.fine_tuned_model, ...)

Szybkie sprawdzenie

Podsumowanie lekcji