微调安全与对齐保护

微调可能破坏基座模型的安全对齐——几百条恶意数据就能让模型"越狱"。如何在微调中保持安全？

安全风险

安全训练数据审查

"""
微调数据安全审查
"""
from dataclasses import dataclass
from enum import Enum
class SafetyLevel(Enum):
SAFE = "safe"
WARNING = "warning"
DANGEROUS = "dangerous"
@dataclass
class SafetyCheckResult:
"""安全检查结果"""
level: SafetyLevel
issues: list[str]
sample_index: int
class TrainingDataAuditor:
"""训练数据安全审计器"""
# 高风险关键词（简化示例）
DANGEROUS_PATTERNS = [
"如何制作", "怎么攻击", "绕过安全",
"ignore previous instructions",
"jailbreak", "DAN mode",
]
ALIGNMENT_KEYWORDS = [
"我不能", "我无法帮助", "这违反了",
"I can't", "I cannot help",
]
def audit_sample(self, instruction: str, output: str, index: int) -> SafetyCheckResult:
"""审查单条样本"""
issues = []
# 检查指令是否包含危险请求
for pattern in self.DANGEROUS_PATTERNS:
if pattern.lower() in instruction.lower():
issues.append(f"危险指令模式: '{pattern}'")
# 检查输出是否移除了安全对齐
# 即：危险问题 + 正常回答（而非拒绝）→ 可能破坏对齐
has_danger = any(p.lower() in instruction.lower() for p in self.DANGEROUS_PATTERNS)
has_refusal = any(k in output for k in self.ALIGNMENT_KEYWORDS)
if has_danger and not has_refusal:
issues.append("危险问题但输出未拒绝 — 可能破坏对齐")
level = SafetyLevel.SAFE
if issues:
level = SafetyLevel.DANGEROUS if has_danger else SafetyLevel.WARNING
return SafetyCheckResult(level=level, issues=issues, sample_index=index)
def audit_dataset(self, samples: list[dict]) -> dict:
"""审查整个数据集"""
results = {
SafetyLevel.SAFE: 0,
SafetyLevel.WARNING: 0,
SafetyLevel.DANGEROUS: 0,
}
dangerous_indices = []
for i, sample in enumerate(samples):
result = self.audit_sample(
instruction=sample.get("instruction", ""),
output=sample.get("output", ""),
index=i,
)
results[result.level] += 1
if result.level == SafetyLevel.DANGEROUS:
dangerous_indices.append(i)
return {
"total": len(samples),
"safe": results[SafetyLevel.SAFE],
"warning": results[SafetyLevel.WARNING],
"dangerous": results[SafetyLevel.DANGEROUS],
"dangerous_indices": dangerous_indices,
"safe_ratio": results[SafetyLevel.SAFE] / len(samples) if samples else 0,
}

安全微调策略

"""
安全约束微调
"""
from dataclasses import dataclass, field
@dataclass
class SafetyConstrainedConfig:
"""安全约束训练配置"""
# 混入安全样本
safety_data_ratio: float = 0.1     # 10% 安全对齐样本
safety_data_path: str = ""
# 对齐保护
kl_penalty: float = 0.01           # KL 散度惩罚（防偏离基座）
max_gradient_norm: float = 1.0     # 梯度裁剪
# 安全评估门槛
max_harmful_rate: float = 0.01     # 有害输出率 < 1%
min_refusal_rate: float = 0.95     # 危险问题拒绝率 > 95%
SAFETY_BEST_PRACTICES = {
"数据审查": "上线前 100% 自动审查 + 5% 人工抽检",
"对齐保护": "混入 10% 安全对齐样本（拒绝回答危险问题）",
"KL 惩罚": "防止模型过度偏离基座，保留安全 behavior",
"持续监控": "上线后监控 拒绝率 和 有害输出率",
"红队测试": "每次微调后运行标准化红队测试集",
}

安全评估对比

安全微调检查清单

本章小结

下一章：客服模型微调