LoRA 与 QLoRA 实战

LoRA（Low-Rank Adaptation）是目前最实用的参数高效微调方法——用 5% 的成本获得 95% 的效果。

LoRA 原理

LoRA vs QLoRA 对比

完整训练流程

"""
LoRA / QLoRA 微调完整配置
"""
from dataclasses import dataclass, field
from enum import Enum
from typing import Any
class LoRATarget(Enum):
"""LoRA 目标模块"""
QKV = "q_proj,k_proj,v_proj"
QKVO = "q_proj,k_proj,v_proj,o_proj"
ALL_LINEAR = "all_linear"           # 所有线性层
@dataclass
class LoRAConfig:
"""LoRA 配置"""
r: int = 16                       # 低秩维度
lora_alpha: int = 32              # 缩放因子
lora_dropout: float = 0.05
target_modules: LoRATarget = LoRATarget.QKVO
use_qlora: bool = False           # 是否使用 QLoRA
bnb_4bit_quant_type: str = "nf4"  # QLoRA 量化类型
bnb_4bit_compute_dtype: str = "bfloat16"
@dataclass
class TrainingConfig:
"""训练配置"""
model_name: str = "meta-llama/Llama-3.1-8B"
output_dir: str = "./output"
# 训练超参数
num_epochs: int = 3
batch_size: int = 4
gradient_accumulation_steps: int = 4  # 等效 batch_size = 16
learning_rate: float = 2e-4
warmup_ratio: float = 0.03
weight_decay: float = 0.01
max_seq_length: int = 2048
# LoRA 配置
lora: LoRAConfig = field(default_factory=LoRAConfig)
# 优化
fp16: bool = False
bf16: bool = True
gradient_checkpointing: bool = True  # 省显存
class LoRATrainer:
"""LoRA 微调训练器（伪代码示意）"""
def __init__(self, config: TrainingConfig):
self.config = config
def prepare_model(self):
"""准备模型"""
steps = [
"1. 加载基座模型",
"2. 应用 QLoRA 4-bit 量化" if self.config.lora.use_qlora else "2. FP16 精度",
f"3. 注入 LoRA 适配器 (r={self.config.lora.r}, alpha={self.config.lora.lora_alpha})",
f"4. 目标模块: {self.config.lora.target_modules.value}",
"5. 冻结原始权重，只训练 LoRA 参数",
]
return steps
def estimate_resources(self) -> dict:
"""预估资源需求"""
model_size = self._parse_model_size()
if self.config.lora.use_qlora:
vram_gb = model_size * 0.8  # 4-bit 基座 + LoRA
else:
vram_gb = model_size * 2.2  # FP16 基座 + LoRA
trainable_params = model_size * 1e9 * 0.02  # LoRA ≈ 2% 参数
return {
"estimated_vram_gb": round(vram_gb, 1),
"trainable_params_m": round(trainable_params / 1e6, 1),
"total_params_b": model_size,
"trainable_ratio_pct": 2.0,
}
def _parse_model_size(self) -> float:
"""从模型名推断参数量（B）"""
name = self.config.model_name.lower()
for size in ["70b", "34b", "13b", "8b", "7b", "3b", "1.5b"]:
if size in name:
return float(size.replace("b", ""))
return 7.0
# 推荐配置
RECOMMENDED_CONFIGS = {
"7B-LoRA": LoRAConfig(r=16, lora_alpha=32, use_qlora=False),
"7B-QLoRA": LoRAConfig(r=16, lora_alpha=32, use_qlora=True),
"13B-QLoRA": LoRAConfig(r=32, lora_alpha=64, use_qlora=True),
"70B-QLoRA": LoRAConfig(r=64, lora_alpha=128, use_qlora=True),
}

r 值选择指南

常见问题排查

本章小结

下一章：分布式训练与监控